pmap.c revision 132487
119914Swollman/* 250476Speter * Copyright (c) 1991 Regents of the University of California. 31558Srgrimes * All rights reserved. 4156905Sru * Copyright (c) 1994 John S. Dyson 5156905Sru * All rights reserved. 61558Srgrimes * Copyright (c) 1994 David Greenman 725451Speter * All rights reserved. 8138593Ssam * Copyright (c) 1998,2000 Doug Rabson 925451Speter * All rights reserved. 10138593Ssam * 11138593Ssam * This code is derived from software contributed to Berkeley by 12138593Ssam * the Systems Programming Group of the University of Utah Computer 13138593Ssam * Science Department and William Jolitz of UUNET Technologies Inc. 14138593Ssam * 15138593Ssam * Redistribution and use in source and binary forms, with or without 16138593Ssam * modification, are permitted provided that the following conditions 17138593Ssam * are met: 18138593Ssam * 1. Redistributions of source code must retain the above copyright 19138593Ssam * notice, this list of conditions and the following disclaimer. 20138593Ssam * 2. Redistributions in binary form must reproduce the above copyright 21197138Shrs * notice, this list of conditions and the following disclaimer in the 2244764Swpaul * documentation and/or other materials provided with the distribution. 23138593Ssam * 3. All advertising materials mentioning features or use of this software 24138593Ssam * must display the following acknowledgement: 25138593Ssam * This product includes software developed by the University of 26138593Ssam * California, Berkeley and its contributors. 27181224Sthompsa * 4. Neither the name of the University nor the names of its contributors 28193664Shrs * may be used to endorse or promote products derived from this software 2977217Sphk * without specific prior written permission. 30178354Ssam * 31204329Sru * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 32194871Sjamie * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 33178354Ssam * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 34142215Sglebius * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 35159781Smlaier * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36142215Sglebius * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37142215Sglebius * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38146987Sthompsa * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 39168793Sthompsa * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 40146987Sthompsa * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 41156905Sru * SUCH DAMAGE. 42138593Ssam * 43178354Ssam * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 44178354Ssam * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp 4580057Sobrien * with some ideas from NetBSD's alpha pmap 4680057Sobrien */ 47138593Ssam 48138593Ssam#include <sys/cdefs.h> 49144130Sbrooks__FBSDID("$FreeBSD: head/sys/ia64/ia64/pmap.c 132487 2004-07-21 07:01:48Z alc $"); 50202532Sed 511558Srgrimes#include <sys/param.h> 521558Srgrimes#include <sys/kernel.h> 53#include <sys/lock.h> 54#include <sys/mman.h> 55#include <sys/mutex.h> 56#include <sys/proc.h> 57#include <sys/smp.h> 58#include <sys/sysctl.h> 59#include <sys/systm.h> 60 61#include <vm/vm.h> 62#include <vm/vm_page.h> 63#include <vm/vm_map.h> 64#include <vm/vm_object.h> 65#include <vm/vm_pageout.h> 66#include <vm/uma.h> 67 68#include <machine/md_var.h> 69#include <machine/pal.h> 70 71/* 72 * Manages physical address maps. 73 * 74 * In addition to hardware address maps, this 75 * module is called upon to provide software-use-only 76 * maps which may or may not be stored in the same 77 * form as hardware maps. These pseudo-maps are 78 * used to store intermediate results from copy 79 * operations to and from address spaces. 80 * 81 * Since the information managed by this module is 82 * also stored by the logical address mapping module, 83 * this module may throw away valid virtual-to-physical 84 * mappings at almost any time. However, invalidations 85 * of virtual-to-physical mappings must be done as 86 * requested. 87 * 88 * In order to cope with hardware architectures which 89 * make virtual-to-physical map invalidates expensive, 90 * this module may delay invalidate or reduced protection 91 * operations until such time as they are actually 92 * necessary. This module is given full information as 93 * to which processors are currently using which maps, 94 * and to when physical maps must be made correct. 95 */ 96 97/* 98 * Following the Linux model, region IDs are allocated in groups of 99 * eight so that a single region ID can be used for as many RRs as we 100 * want by encoding the RR number into the low bits of the ID. 101 * 102 * We reserve region ID 0 for the kernel and allocate the remaining 103 * IDs for user pmaps. 104 * 105 * Region 0..4 106 * User virtually mapped 107 * 108 * Region 5 109 * Kernel virtually mapped 110 * 111 * Region 6 112 * Kernel physically mapped uncacheable 113 * 114 * Region 7 115 * Kernel physically mapped cacheable 116 */ 117 118/* XXX move to a header. */ 119extern u_int64_t ia64_gateway_page[]; 120 121MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures"); 122 123#ifndef PMAP_SHPGPERPROC 124#define PMAP_SHPGPERPROC 200 125#endif 126 127#if defined(DIAGNOSTIC) 128#define PMAP_DIAGNOSTIC 129#endif 130 131#define MINPV 2048 /* Preallocate at least this many */ 132 133#if 0 134#define PMAP_DIAGNOSTIC 135#define PMAP_DEBUG 136#endif 137 138#if !defined(PMAP_DIAGNOSTIC) 139#define PMAP_INLINE __inline 140#else 141#define PMAP_INLINE 142#endif 143 144/* 145 * Get PDEs and PTEs for user/kernel address space 146 */ 147#define pmap_pte_w(pte) ((pte)->pte_ig & PTE_IG_WIRED) 148#define pmap_pte_managed(pte) ((pte)->pte_ig & PTE_IG_MANAGED) 149#define pmap_pte_v(pte) ((pte)->pte_p) 150#define pmap_pte_pa(pte) (((pte)->pte_ppn) << 12) 151#define pmap_pte_prot(pte) (((pte)->pte_ar << 2) | (pte)->pte_pl) 152 153#define pmap_pte_set_w(pte, v) ((v)?((pte)->pte_ig |= PTE_IG_WIRED) \ 154 :((pte)->pte_ig &= ~PTE_IG_WIRED)) 155#define pmap_pte_set_prot(pte, v) do { \ 156 (pte)->pte_ar = v >> 2; \ 157 (pte)->pte_pl = v & 3; \ 158} while (0) 159 160/* 161 * Given a map and a machine independent protection code, 162 * convert to an ia64 protection code. 163 */ 164#define pte_prot(m, p) (protection_codes[m == kernel_pmap ? 0 : 1][p]) 165#define pte_prot_pl(m, p) (pte_prot(m, p) & 3) 166#define pte_prot_ar(m, p) (pte_prot(m, p) >> 2) 167int protection_codes[2][8]; 168 169/* 170 * Return non-zero if this pmap is currently active 171 */ 172#define pmap_isactive(pmap) (pmap->pm_active) 173 174/* 175 * Statically allocated kernel pmap 176 */ 177struct pmap kernel_pmap_store; 178 179vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 180vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 181static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 182 183vm_offset_t vhpt_base, vhpt_size; 184struct mtx pmap_vhptmutex; 185 186/* 187 * We use an object to own the kernel's 'page tables'. For simplicity, 188 * we use one page directory to index a set of pages containing 189 * ia64_lptes. This gives us up to 2Gb of kernel virtual space. 190 */ 191static int nkpt; 192struct ia64_lpte **ia64_kptdir; 193#define KPTE_DIR_INDEX(va) \ 194 ((va >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1)) 195#define KPTE_PTE_INDEX(va) \ 196 ((va >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1)) 197#define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte)) 198 199vm_offset_t kernel_vm_end; 200 201/* Values for ptc.e. XXX values for SKI. */ 202static u_int64_t pmap_ptc_e_base = 0x100000000; 203static u_int64_t pmap_ptc_e_count1 = 3; 204static u_int64_t pmap_ptc_e_count2 = 2; 205static u_int64_t pmap_ptc_e_stride1 = 0x2000; 206static u_int64_t pmap_ptc_e_stride2 = 0x100000000; 207 208/* 209 * Data for the RID allocator 210 */ 211static int pmap_ridcount; 212static int pmap_rididx; 213static int pmap_ridmapsz; 214static int pmap_ridmax; 215static u_int64_t *pmap_ridmap; 216struct mtx pmap_ridmutex; 217 218/* 219 * Data for the pv entry allocation mechanism 220 */ 221static uma_zone_t pvzone; 222static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 223int pmap_pagedaemon_waken; 224 225/* 226 * Data for allocating PTEs for user processes. 227 */ 228static uma_zone_t ptezone; 229 230/* 231 * VHPT instrumentation. 232 */ 233static int pmap_vhpt_inserts; 234static int pmap_vhpt_collisions; 235static int pmap_vhpt_resident; 236SYSCTL_DECL(_vm_stats); 237SYSCTL_NODE(_vm_stats, OID_AUTO, vhpt, CTLFLAG_RD, 0, ""); 238SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, inserts, CTLFLAG_RD, 239 &pmap_vhpt_inserts, 0, ""); 240SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, collisions, CTLFLAG_RD, 241 &pmap_vhpt_collisions, 0, ""); 242SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, resident, CTLFLAG_RD, 243 &pmap_vhpt_resident, 0, ""); 244 245static PMAP_INLINE void free_pv_entry(pv_entry_t pv); 246static pv_entry_t get_pv_entry(void); 247static void ia64_protection_init(void); 248 249static pmap_t pmap_install(pmap_t); 250static void pmap_invalidate_all(pmap_t pmap); 251 252vm_offset_t 253pmap_steal_memory(vm_size_t size) 254{ 255 vm_size_t bank_size; 256 vm_offset_t pa, va; 257 258 size = round_page(size); 259 260 bank_size = phys_avail[1] - phys_avail[0]; 261 while (size > bank_size) { 262 int i; 263 for (i = 0; phys_avail[i+2]; i+= 2) { 264 phys_avail[i] = phys_avail[i+2]; 265 phys_avail[i+1] = phys_avail[i+3]; 266 } 267 phys_avail[i] = 0; 268 phys_avail[i+1] = 0; 269 if (!phys_avail[0]) 270 panic("pmap_steal_memory: out of memory"); 271 bank_size = phys_avail[1] - phys_avail[0]; 272 } 273 274 pa = phys_avail[0]; 275 phys_avail[0] += size; 276 277 va = IA64_PHYS_TO_RR7(pa); 278 bzero((caddr_t) va, size); 279 return va; 280} 281 282/* 283 * Bootstrap the system enough to run with virtual memory. 284 */ 285void 286pmap_bootstrap() 287{ 288 int i, j, count, ridbits; 289 struct ia64_pal_result res; 290 291 /* 292 * Query the PAL Code to find the loop parameters for the 293 * ptc.e instruction. 294 */ 295 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0); 296 if (res.pal_status != 0) 297 panic("Can't configure ptc.e parameters"); 298 pmap_ptc_e_base = res.pal_result[0]; 299 pmap_ptc_e_count1 = res.pal_result[1] >> 32; 300 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1); 301 pmap_ptc_e_stride1 = res.pal_result[2] >> 32; 302 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1); 303 if (bootverbose) 304 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, " 305 "stride1=0x%lx, stride2=0x%lx\n", 306 pmap_ptc_e_base, 307 pmap_ptc_e_count1, 308 pmap_ptc_e_count2, 309 pmap_ptc_e_stride1, 310 pmap_ptc_e_stride2); 311 312 /* 313 * Setup RIDs. RIDs 0..7 are reserved for the kernel. 314 * 315 * We currently need at least 19 bits in the RID because PID_MAX 316 * can only be encoded in 17 bits and we need RIDs for 5 regions 317 * per process. With PID_MAX equalling 99999 this means that we 318 * need to be able to encode 499995 (=5*PID_MAX). 319 * The Itanium processor only has 18 bits and the architected 320 * minimum is exactly that. So, we cannot use a PID based scheme 321 * in those cases. Enter pmap_ridmap... 322 * We should avoid the map when running on a processor that has 323 * implemented enough bits. This means that we should pass the 324 * process/thread ID to pmap. This we currently don't do, so we 325 * use the map anyway. However, we don't want to allocate a map 326 * that is large enough to cover the range dictated by the number 327 * of bits in the RID, because that may result in a RID map of 328 * 2MB in size for a 24-bit RID. A 64KB map is enough. 329 * The bottomline: we create a 32KB map when the processor only 330 * implements 18 bits (or when we can't figure it out). Otherwise 331 * we create a 64KB map. 332 */ 333 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 334 if (res.pal_status != 0) { 335 if (bootverbose) 336 printf("Can't read VM Summary - assuming 18 Region ID bits\n"); 337 ridbits = 18; /* guaranteed minimum */ 338 } else { 339 ridbits = (res.pal_result[1] >> 8) & 0xff; 340 if (bootverbose) 341 printf("Processor supports %d Region ID bits\n", 342 ridbits); 343 } 344 if (ridbits > 19) 345 ridbits = 19; 346 347 pmap_ridmax = (1 << ridbits); 348 pmap_ridmapsz = pmap_ridmax / 64; 349 pmap_ridmap = (u_int64_t *)pmap_steal_memory(pmap_ridmax / 8); 350 pmap_ridmap[0] |= 0xff; 351 pmap_rididx = 0; 352 pmap_ridcount = 8; 353 mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF); 354 355 /* 356 * Allocate some memory for initial kernel 'page tables'. 357 */ 358 ia64_kptdir = (void *)pmap_steal_memory(PAGE_SIZE); 359 for (i = 0; i < NKPT; i++) { 360 ia64_kptdir[i] = (void*)pmap_steal_memory(PAGE_SIZE); 361 } 362 nkpt = NKPT; 363 kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS - 364 VM_GATEWAY_SIZE; 365 366 for (i = 0; phys_avail[i+2]; i+= 2) ; 367 count = i+2; 368 369 /* 370 * Figure out a useful size for the VHPT, based on the size of 371 * physical memory and try to locate a region which is large 372 * enough to contain the VHPT (which must be a power of two in 373 * size and aligned to a natural boundary). 374 */ 375 vhpt_size = 15; 376 while ((1<<vhpt_size) < Maxmem * 32) 377 vhpt_size++; 378 379 vhpt_base = 0; 380 while (!vhpt_base) { 381 vm_offset_t mask; 382 if (bootverbose) 383 printf("Trying VHPT size 0x%lx\n", (1L<<vhpt_size)); 384 mask = (1L << vhpt_size) - 1; 385 for (i = 0; i < count; i += 2) { 386 vm_offset_t base, limit; 387 base = (phys_avail[i] + mask) & ~mask; 388 limit = base + (1L << vhpt_size); 389 if (limit <= phys_avail[i+1]) 390 /* 391 * VHPT can fit in this region 392 */ 393 break; 394 } 395 if (!phys_avail[i]) { 396 /* 397 * Can't fit, try next smaller size. 398 */ 399 vhpt_size--; 400 } else { 401 vhpt_base = (phys_avail[i] + mask) & ~mask; 402 } 403 } 404 if (vhpt_size < 15) 405 panic("Can't find space for VHPT"); 406 407 if (bootverbose) 408 printf("Putting VHPT at %p\n", (void *) vhpt_base); 409 if (vhpt_base != phys_avail[i]) { 410 /* 411 * Split this region. 412 */ 413 if (bootverbose) 414 printf("Splitting [%p-%p]\n", 415 (void *) phys_avail[i], 416 (void *) phys_avail[i+1]); 417 for (j = count; j > i; j -= 2) { 418 phys_avail[j] = phys_avail[j-2]; 419 phys_avail[j+1] = phys_avail[j-2+1]; 420 } 421 phys_avail[count+2] = 0; 422 phys_avail[count+3] = 0; 423 phys_avail[i+1] = vhpt_base; 424 phys_avail[i+2] = vhpt_base + (1L << vhpt_size); 425 } else { 426 phys_avail[i] = vhpt_base + (1L << vhpt_size); 427 } 428 429 vhpt_base = IA64_PHYS_TO_RR7(vhpt_base); 430 bzero((void *) vhpt_base, (1L << vhpt_size)); 431 432 mtx_init(&pmap_vhptmutex, "VHPT collision chain lock", NULL, MTX_DEF); 433 434 __asm __volatile("mov cr.pta=%0;; srlz.i;;" 435 :: "r" (vhpt_base + (1<<8) + (vhpt_size<<2) + 1)); 436 437 virtual_avail = VM_MIN_KERNEL_ADDRESS; 438 virtual_end = VM_MAX_KERNEL_ADDRESS; 439 440 /* 441 * Initialize protection array. 442 */ 443 ia64_protection_init(); 444 445 /* 446 * Initialize the kernel pmap (which is statically allocated). 447 */ 448 PMAP_LOCK_INIT(kernel_pmap); 449 for (i = 0; i < 5; i++) 450 kernel_pmap->pm_rid[i] = 0; 451 kernel_pmap->pm_active = 1; 452 TAILQ_INIT(&kernel_pmap->pm_pvlist); 453 PCPU_SET(current_pmap, kernel_pmap); 454 455 /* 456 * Region 5 is mapped via the vhpt. 457 */ 458 ia64_set_rr(IA64_RR_BASE(5), 459 (5 << 8) | (PAGE_SHIFT << 2) | 1); 460 461 /* 462 * Region 6 is direct mapped UC and region 7 is direct mapped 463 * WC. The details of this is controlled by the Alt {I,D}TLB 464 * handlers. Here we just make sure that they have the largest 465 * possible page size to minimise TLB usage. 466 */ 467 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (IA64_ID_PAGE_SHIFT << 2)); 468 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (IA64_ID_PAGE_SHIFT << 2)); 469 470 /* 471 * Clear out any random TLB entries left over from booting. 472 */ 473 pmap_invalidate_all(kernel_pmap); 474 475 map_gateway_page(); 476} 477 478/* 479 * Initialize the pmap module. 480 * Called by vm_init, to initialize any structures that the pmap 481 * system needs to map virtual memory. 482 * pmap_init has been enhanced to support in a fairly consistant 483 * way, discontiguous physical memory. 484 */ 485void 486pmap_init(void) 487{ 488 int i; 489 490 /* 491 * Allocate memory for random pmap data structures. Includes the 492 * pv_head_table. 493 */ 494 495 for(i = 0; i < vm_page_array_size; i++) { 496 vm_page_t m; 497 498 m = &vm_page_array[i]; 499 TAILQ_INIT(&m->md.pv_list); 500 m->md.pv_list_count = 0; 501 } 502 503 /* 504 * Init the pv free list and the PTE free list. 505 */ 506 pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), 507 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE); 508 uma_prealloc(pvzone, MINPV); 509 510 ptezone = uma_zcreate("PT ENTRY", sizeof (struct ia64_lpte), 511 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE); 512 uma_prealloc(ptezone, MINPV); 513 514 /* 515 * Now it is safe to enable pv_table recording. 516 */ 517 pmap_initialized = TRUE; 518} 519 520/* 521 * Initialize the address space (zone) for the pv_entries. Set a 522 * high water mark so that the system can recover from excessive 523 * numbers of pv entries. 524 */ 525void 526pmap_init2() 527{ 528 int shpgperproc = PMAP_SHPGPERPROC; 529 530 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 531 pv_entry_max = shpgperproc * maxproc + vm_page_array_size; 532 pv_entry_high_water = 9 * (pv_entry_max / 10); 533} 534 535 536/*************************************************** 537 * Manipulate TLBs for a pmap 538 ***************************************************/ 539 540static void 541pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 542{ 543 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 544 ("invalidating TLB for non-current pmap")); 545 ia64_ptc_g(va, PAGE_SHIFT << 2); 546} 547 548static void 549pmap_invalidate_all_1(void *arg) 550{ 551 u_int64_t addr; 552 int i, j; 553 register_t psr; 554 555 psr = intr_disable(); 556 addr = pmap_ptc_e_base; 557 for (i = 0; i < pmap_ptc_e_count1; i++) { 558 for (j = 0; j < pmap_ptc_e_count2; j++) { 559 ia64_ptc_e(addr); 560 addr += pmap_ptc_e_stride2; 561 } 562 addr += pmap_ptc_e_stride1; 563 } 564 intr_restore(psr); 565} 566 567static void 568pmap_invalidate_all(pmap_t pmap) 569{ 570 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 571 ("invalidating TLB for non-current pmap")); 572 573 574#ifdef SMP 575 smp_rendezvous(0, pmap_invalidate_all_1, 0, 0); 576#else 577 pmap_invalidate_all_1(0); 578#endif 579} 580 581static u_int32_t 582pmap_allocate_rid(void) 583{ 584 uint64_t bit, bits; 585 int rid; 586 587 mtx_lock(&pmap_ridmutex); 588 if (pmap_ridcount == pmap_ridmax) 589 panic("pmap_allocate_rid: All Region IDs used"); 590 591 /* Find an index with a free bit. */ 592 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) { 593 pmap_rididx++; 594 if (pmap_rididx == pmap_ridmapsz) 595 pmap_rididx = 0; 596 } 597 rid = pmap_rididx * 64; 598 599 /* Find a free bit. */ 600 bit = 1UL; 601 while (bits & bit) { 602 rid++; 603 bit <<= 1; 604 } 605 606 pmap_ridmap[pmap_rididx] |= bit; 607 pmap_ridcount++; 608 mtx_unlock(&pmap_ridmutex); 609 610 return rid; 611} 612 613static void 614pmap_free_rid(u_int32_t rid) 615{ 616 uint64_t bit; 617 int idx; 618 619 idx = rid / 64; 620 bit = ~(1UL << (rid & 63)); 621 622 mtx_lock(&pmap_ridmutex); 623 pmap_ridmap[idx] &= bit; 624 pmap_ridcount--; 625 mtx_unlock(&pmap_ridmutex); 626} 627 628/*************************************************** 629 * Low level helper routines..... 630 ***************************************************/ 631 632/* 633 * Install a pte into the VHPT 634 */ 635static PMAP_INLINE void 636pmap_install_pte(struct ia64_lpte *vhpte, struct ia64_lpte *pte) 637{ 638 u_int64_t *vhp, *p; 639 640 vhp = (u_int64_t *)vhpte; 641 p = (u_int64_t *)pte; 642 643 critical_enter(); 644 645 /* Invalidate the tag so the VHPT walker will not match this entry. */ 646 vhp[2] = 1UL << 63; 647 ia64_mf(); 648 649 vhp[0] = p[0]; 650 vhp[1] = p[1]; 651 ia64_mf(); 652 653 /* Install a proper tag now that we're done. */ 654 vhp[2] = p[2]; 655 ia64_mf(); 656 657 critical_exit(); 658} 659 660/* 661 * Compare essential parts of pte. 662 */ 663static PMAP_INLINE int 664pmap_equal_pte(struct ia64_lpte *pte1, struct ia64_lpte *pte2) 665{ 666 return *(u_int64_t *) pte1 == *(u_int64_t *) pte2; 667} 668 669/* 670 * this routine defines the region(s) of memory that should 671 * not be tested for the modified bit. 672 */ 673static PMAP_INLINE int 674pmap_track_modified(vm_offset_t va) 675{ 676 if ((va < kmi.clean_sva) || (va >= kmi.clean_eva)) 677 return 1; 678 else 679 return 0; 680} 681 682/*************************************************** 683 * Page table page management routines..... 684 ***************************************************/ 685 686void 687pmap_pinit0(struct pmap *pmap) 688{ 689 /* kernel_pmap is the same as any other pmap. */ 690 pmap_pinit(pmap); 691} 692 693/* 694 * Initialize a preallocated and zeroed pmap structure, 695 * such as one in a vmspace structure. 696 */ 697void 698pmap_pinit(struct pmap *pmap) 699{ 700 int i; 701 702 PMAP_LOCK_INIT(pmap); 703 for (i = 0; i < 5; i++) 704 pmap->pm_rid[i] = pmap_allocate_rid(); 705 pmap->pm_active = 0; 706 TAILQ_INIT(&pmap->pm_pvlist); 707 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 708} 709 710/*************************************************** 711 * Pmap allocation/deallocation routines. 712 ***************************************************/ 713 714/* 715 * Release any resources held by the given physical map. 716 * Called when a pmap initialized by pmap_pinit is being released. 717 * Should only be called if the map contains no valid mappings. 718 */ 719void 720pmap_release(pmap_t pmap) 721{ 722 int i; 723 724 for (i = 0; i < 5; i++) 725 if (pmap->pm_rid[i]) 726 pmap_free_rid(pmap->pm_rid[i]); 727 PMAP_LOCK_DESTROY(pmap); 728} 729 730/* 731 * grow the number of kernel page table entries, if needed 732 */ 733void 734pmap_growkernel(vm_offset_t addr) 735{ 736 struct ia64_lpte *ptepage; 737 vm_page_t nkpg; 738 739 if (kernel_vm_end >= addr) 740 return; 741 742 critical_enter(); 743 744 while (kernel_vm_end < addr) { 745 /* We could handle more by increasing the size of kptdir. */ 746 if (nkpt == MAXKPT) 747 panic("pmap_growkernel: out of kernel address space"); 748 749 nkpg = vm_page_alloc(NULL, nkpt, 750 VM_ALLOC_NOOBJ | VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 751 if (!nkpg) 752 panic("pmap_growkernel: no memory to grow kernel"); 753 754 ptepage = (struct ia64_lpte *) 755 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg)); 756 bzero(ptepage, PAGE_SIZE); 757 ia64_kptdir[KPTE_DIR_INDEX(kernel_vm_end)] = ptepage; 758 759 nkpt++; 760 kernel_vm_end += PAGE_SIZE * NKPTEPG; 761 } 762 763 critical_exit(); 764} 765 766/*************************************************** 767 * page management routines. 768 ***************************************************/ 769 770/* 771 * free the pv_entry back to the free list 772 */ 773static PMAP_INLINE void 774free_pv_entry(pv_entry_t pv) 775{ 776 pv_entry_count--; 777 uma_zfree(pvzone, pv); 778} 779 780/* 781 * get a new pv_entry, allocating a block from the system 782 * when needed. 783 * the memory allocation is performed bypassing the malloc code 784 * because of the possibility of allocations at interrupt time. 785 */ 786static pv_entry_t 787get_pv_entry(void) 788{ 789 pv_entry_count++; 790 if (pv_entry_high_water && 791 (pv_entry_count > pv_entry_high_water) && 792 (pmap_pagedaemon_waken == 0)) { 793 pmap_pagedaemon_waken = 1; 794 wakeup (&vm_pages_needed); 795 } 796 return uma_zalloc(pvzone, M_NOWAIT); 797} 798 799/* 800 * Add an ia64_lpte to the VHPT. 801 */ 802static void 803pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va) 804{ 805 struct ia64_lpte *vhpte; 806 807 pmap_vhpt_inserts++; 808 pmap_vhpt_resident++; 809 810 vhpte = (struct ia64_lpte *) ia64_thash(va); 811 812 if (vhpte->pte_chain) 813 pmap_vhpt_collisions++; 814 815 mtx_lock(&pmap_vhptmutex); 816 817 pte->pte_chain = vhpte->pte_chain; 818 ia64_mf(); 819 vhpte->pte_chain = ia64_tpa((vm_offset_t)pte); 820 ia64_mf(); 821 822 if (!vhpte->pte_p && pte->pte_p) 823 pmap_install_pte(vhpte, pte); 824 825 mtx_unlock(&pmap_vhptmutex); 826} 827 828/* 829 * Update VHPT after a pte has changed. 830 */ 831static void 832pmap_update_vhpt(struct ia64_lpte *pte, vm_offset_t va) 833{ 834 struct ia64_lpte *vhpte; 835 836 vhpte = (struct ia64_lpte *)ia64_thash(va); 837 838 mtx_lock(&pmap_vhptmutex); 839 840 if ((!vhpte->pte_p || vhpte->pte_tag == pte->pte_tag) && pte->pte_p) 841 pmap_install_pte(vhpte, pte); 842 843 mtx_unlock(&pmap_vhptmutex); 844} 845 846/* 847 * Remove the ia64_lpte matching va from the VHPT. Return zero if it 848 * worked or an appropriate error code otherwise. 849 */ 850static int 851pmap_remove_vhpt(vm_offset_t va) 852{ 853 struct ia64_lpte *pte; 854 struct ia64_lpte *lpte; 855 struct ia64_lpte *vhpte; 856 u_int64_t tag; 857 858 vhpte = (struct ia64_lpte *)ia64_thash(va); 859 860 /* 861 * If the VHPTE is invalid, there can't be a collision chain. 862 */ 863 if (!vhpte->pte_p) { 864 KASSERT(!vhpte->pte_chain, ("bad vhpte")); 865 return (ENOENT); 866 } 867 868 lpte = vhpte; 869 tag = ia64_ttag(va); 870 871 mtx_lock(&pmap_vhptmutex); 872 873 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(vhpte->pte_chain); 874 KASSERT(pte != NULL, ("foo")); 875 876 while (pte->pte_tag != tag) { 877 lpte = pte; 878 if (pte->pte_chain == 0) { 879 mtx_unlock(&pmap_vhptmutex); 880 return (ENOENT); 881 } 882 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->pte_chain); 883 } 884 885 /* Snip this pv_entry out of the collision chain. */ 886 lpte->pte_chain = pte->pte_chain; 887 ia64_mf(); 888 889 /* 890 * If the VHPTE matches as well, change it to map the first 891 * element from the chain if there is one. 892 */ 893 if (vhpte->pte_tag == tag) { 894 if (vhpte->pte_chain) { 895 pte = (void*)IA64_PHYS_TO_RR7(vhpte->pte_chain); 896 pmap_install_pte(vhpte, pte); 897 } else 898 vhpte->pte_p = 0; 899 } 900 901 mtx_unlock(&pmap_vhptmutex); 902 pmap_vhpt_resident--; 903 return (0); 904} 905 906/* 907 * Find the ia64_lpte for the given va, if any. 908 */ 909static struct ia64_lpte * 910pmap_find_vhpt(vm_offset_t va) 911{ 912 struct ia64_lpte *pte; 913 u_int64_t tag; 914 915 tag = ia64_ttag(va); 916 pte = (struct ia64_lpte *)ia64_thash(va); 917 if (pte->pte_chain == 0) 918 return (NULL); 919 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->pte_chain); 920 while (pte->pte_tag != tag) { 921 if (pte->pte_chain == 0) 922 return (NULL); 923 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->pte_chain); 924 } 925 return (pte); 926} 927 928/* 929 * Remove an entry from the list of managed mappings. 930 */ 931static int 932pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv) 933{ 934 if (!pv) { 935 if (m->md.pv_list_count < pmap->pm_stats.resident_count) { 936 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 937 if (pmap == pv->pv_pmap && va == pv->pv_va) 938 break; 939 } 940 } else { 941 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 942 if (va == pv->pv_va) 943 break; 944 } 945 } 946 } 947 948 if (pv) { 949 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 950 m->md.pv_list_count--; 951 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 952 vm_page_flag_clear(m, PG_WRITEABLE); 953 954 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 955 free_pv_entry(pv); 956 return 0; 957 } else { 958 return ENOENT; 959 } 960} 961 962/* 963 * Create a pv entry for page at pa for 964 * (pmap, va). 965 */ 966static void 967pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) 968{ 969 pv_entry_t pv; 970 971 pv = get_pv_entry(); 972 pv->pv_pmap = pmap; 973 pv->pv_va = va; 974 975 vm_page_lock_queues(); 976 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 977 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 978 m->md.pv_list_count++; 979 vm_page_unlock_queues(); 980} 981 982/* 983 * Routine: pmap_extract 984 * Function: 985 * Extract the physical page address associated 986 * with the given map/virtual_address pair. 987 */ 988vm_paddr_t 989pmap_extract(pmap_t pmap, vm_offset_t va) 990{ 991 struct ia64_lpte *pte; 992 pmap_t oldpmap; 993 vm_paddr_t pa; 994 995 pa = 0; 996 if (pmap == NULL) 997 return (pa); 998 PMAP_LOCK(pmap); 999 oldpmap = pmap_install(pmap); 1000 pte = pmap_find_vhpt(va); 1001 if (pte != NULL && pmap_pte_v(pte)) 1002 pa = pmap_pte_pa(pte); 1003 pmap_install(oldpmap); 1004 PMAP_UNLOCK(pmap); 1005 return (pa); 1006} 1007 1008/* 1009 * Routine: pmap_extract_and_hold 1010 * Function: 1011 * Atomically extract and hold the physical page 1012 * with the given pmap and virtual address pair 1013 * if that mapping permits the given protection. 1014 */ 1015vm_page_t 1016pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 1017{ 1018 vm_paddr_t pa; 1019 vm_page_t m; 1020 1021 m = NULL; 1022 mtx_lock(&Giant); 1023 if ((pa = pmap_extract(pmap, va)) != 0) { 1024 m = PHYS_TO_VM_PAGE(pa); 1025 vm_page_lock_queues(); 1026 vm_page_hold(m); 1027 vm_page_unlock_queues(); 1028 } 1029 mtx_unlock(&Giant); 1030 return (m); 1031} 1032 1033/*************************************************** 1034 * Low level mapping routines..... 1035 ***************************************************/ 1036 1037/* 1038 * Find the kernel lpte for mapping the given virtual address, which 1039 * must be in the part of region 5 which we can cover with our kernel 1040 * 'page tables'. 1041 */ 1042static struct ia64_lpte * 1043pmap_find_kpte(vm_offset_t va) 1044{ 1045 KASSERT((va >> 61) == 5, 1046 ("kernel mapping 0x%lx not in region 5", va)); 1047 KASSERT(IA64_RR_MASK(va) < (nkpt * PAGE_SIZE * NKPTEPG), 1048 ("kernel mapping 0x%lx out of range", va)); 1049 return (&ia64_kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)]); 1050} 1051 1052/* 1053 * Find a pte suitable for mapping a user-space address. If one exists 1054 * in the VHPT, that one will be returned, otherwise a new pte is 1055 * allocated. 1056 */ 1057static struct ia64_lpte * 1058pmap_find_pte(vm_offset_t va) 1059{ 1060 struct ia64_lpte *pte; 1061 1062 if (va >= VM_MAXUSER_ADDRESS) 1063 return pmap_find_kpte(va); 1064 1065 pte = pmap_find_vhpt(va); 1066 if (!pte) { 1067 pte = uma_zalloc(ptezone, M_WAITOK); 1068 pte->pte_p = 0; 1069 } 1070 return pte; 1071} 1072 1073/* 1074 * Free a pte which is now unused. This simply returns it to the zone 1075 * allocator if it is a user mapping. For kernel mappings, clear the 1076 * valid bit to make it clear that the mapping is not currently used. 1077 */ 1078static void 1079pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va) 1080{ 1081 if (va < VM_MAXUSER_ADDRESS) 1082 uma_zfree(ptezone, pte); 1083 else 1084 pte->pte_p = 0; 1085} 1086 1087/* 1088 * Set a pte to contain a valid mapping and enter it in the VHPT. If 1089 * the pte was orginally valid, then its assumed to already be in the 1090 * VHPT. 1091 */ 1092static void 1093pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa, 1094 int ig, int pl, int ar) 1095{ 1096 int wasvalid = pte->pte_p; 1097 1098 pte->pte_p = 1; 1099 pte->pte_ma = PTE_MA_WB; 1100 if (ig & PTE_IG_MANAGED) { 1101 pte->pte_a = 0; 1102 pte->pte_d = 0; 1103 } else { 1104 pte->pte_a = 1; 1105 pte->pte_d = 1; 1106 } 1107 pte->pte_pl = pl; 1108 pte->pte_ar = ar; 1109 pte->pte_ppn = pa >> 12; 1110 pte->pte_ed = 0; 1111 pte->pte_ig = ig; 1112 1113 pte->pte_ps = PAGE_SHIFT; 1114 pte->pte_key = 0; 1115 1116 pte->pte_tag = ia64_ttag(va); 1117 1118 if (wasvalid) { 1119 pmap_update_vhpt(pte, va); 1120 } else { 1121 pmap_enter_vhpt(pte, va); 1122 } 1123} 1124 1125/* 1126 * If a pte contains a valid mapping, clear it and update the VHPT. 1127 */ 1128static void 1129pmap_clear_pte(struct ia64_lpte *pte, vm_offset_t va) 1130{ 1131 if (pte->pte_p) { 1132 pmap_remove_vhpt(va); 1133 ia64_ptc_g(va, PAGE_SHIFT << 2); 1134 pte->pte_p = 0; 1135 } 1136} 1137 1138/* 1139 * Remove the (possibly managed) mapping represented by pte from the 1140 * given pmap. 1141 */ 1142static int 1143pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va, 1144 pv_entry_t pv, int freepte) 1145{ 1146 int error; 1147 vm_page_t m; 1148 1149 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 1150 ("removing pte for non-current pmap")); 1151 1152 /* 1153 * First remove from the VHPT. 1154 */ 1155 error = pmap_remove_vhpt(va); 1156 if (error) 1157 return error; 1158 1159 /* 1160 * Make sure pmap_set_pte() knows it isn't in the VHPT. 1161 */ 1162 pte->pte_p = 0; 1163 1164 if (pte->pte_ig & PTE_IG_WIRED) 1165 pmap->pm_stats.wired_count -= 1; 1166 1167 pmap->pm_stats.resident_count -= 1; 1168 if (pte->pte_ig & PTE_IG_MANAGED) { 1169 m = PHYS_TO_VM_PAGE(pmap_pte_pa(pte)); 1170 if (pte->pte_d) 1171 if (pmap_track_modified(va)) 1172 vm_page_dirty(m); 1173 if (pte->pte_a) 1174 vm_page_flag_set(m, PG_REFERENCED); 1175 1176 if (freepte) 1177 pmap_free_pte(pte, va); 1178 return pmap_remove_entry(pmap, m, va, pv); 1179 } else { 1180 if (freepte) 1181 pmap_free_pte(pte, va); 1182 return 0; 1183 } 1184} 1185 1186/* 1187 * Extract the physical page address associated with a kernel 1188 * virtual address. 1189 */ 1190vm_paddr_t 1191pmap_kextract(vm_offset_t va) 1192{ 1193 struct ia64_lpte *pte; 1194 vm_offset_t gwpage; 1195 1196 KASSERT(va >= IA64_RR_BASE(5), ("Must be kernel VA")); 1197 1198 /* Regions 6 and 7 are direct mapped. */ 1199 if (va >= IA64_RR_BASE(6)) 1200 return (IA64_RR_MASK(va)); 1201 1202 /* EPC gateway page? */ 1203 gwpage = (vm_offset_t)ia64_get_k5(); 1204 if (va >= gwpage && va < gwpage + VM_GATEWAY_SIZE) 1205 return (IA64_RR_MASK((vm_offset_t)ia64_gateway_page)); 1206 1207 /* Bail out if the virtual address is beyond our limits. */ 1208 if (IA64_RR_MASK(va) >= nkpt * PAGE_SIZE * NKPTEPG) 1209 return (0); 1210 1211 pte = pmap_find_kpte(va); 1212 if (!pte->pte_p) 1213 return (0); 1214 return ((pte->pte_ppn << 12) | (va & PAGE_MASK)); 1215} 1216 1217/* 1218 * Add a list of wired pages to the kva 1219 * this routine is only used for temporary 1220 * kernel mappings that do not need to have 1221 * page modification or references recorded. 1222 * Note that old mappings are simply written 1223 * over. The page *must* be wired. 1224 */ 1225void 1226pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 1227{ 1228 int i; 1229 struct ia64_lpte *pte; 1230 1231 for (i = 0; i < count; i++) { 1232 vm_offset_t tva = va + i * PAGE_SIZE; 1233 int wasvalid; 1234 pte = pmap_find_kpte(tva); 1235 wasvalid = pte->pte_p; 1236 pmap_set_pte(pte, tva, VM_PAGE_TO_PHYS(m[i]), 1237 0, PTE_PL_KERN, PTE_AR_RWX); 1238 if (wasvalid) 1239 ia64_ptc_g(tva, PAGE_SHIFT << 2); 1240 } 1241} 1242 1243/* 1244 * this routine jerks page mappings from the 1245 * kernel -- it is meant only for temporary mappings. 1246 */ 1247void 1248pmap_qremove(vm_offset_t va, int count) 1249{ 1250 int i; 1251 struct ia64_lpte *pte; 1252 1253 for (i = 0; i < count; i++) { 1254 pte = pmap_find_kpte(va); 1255 pmap_clear_pte(pte, va); 1256 va += PAGE_SIZE; 1257 } 1258} 1259 1260/* 1261 * Add a wired page to the kva. 1262 */ 1263void 1264pmap_kenter(vm_offset_t va, vm_offset_t pa) 1265{ 1266 struct ia64_lpte *pte; 1267 int wasvalid; 1268 1269 pte = pmap_find_kpte(va); 1270 wasvalid = pte->pte_p; 1271 pmap_set_pte(pte, va, pa, 0, PTE_PL_KERN, PTE_AR_RWX); 1272 if (wasvalid) 1273 ia64_ptc_g(va, PAGE_SHIFT << 2); 1274} 1275 1276/* 1277 * Remove a page from the kva 1278 */ 1279void 1280pmap_kremove(vm_offset_t va) 1281{ 1282 struct ia64_lpte *pte; 1283 1284 pte = pmap_find_kpte(va); 1285 pmap_clear_pte(pte, va); 1286} 1287 1288/* 1289 * Used to map a range of physical addresses into kernel 1290 * virtual address space. 1291 * 1292 * The value passed in '*virt' is a suggested virtual address for 1293 * the mapping. Architectures which can support a direct-mapped 1294 * physical to virtual region can return the appropriate address 1295 * within that region, leaving '*virt' unchanged. Other 1296 * architectures should map the pages starting at '*virt' and 1297 * update '*virt' with the first usable address after the mapped 1298 * region. 1299 */ 1300vm_offset_t 1301pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 1302{ 1303 return IA64_PHYS_TO_RR7(start); 1304} 1305 1306/* 1307 * Remove a single page from a process address space 1308 */ 1309static void 1310pmap_remove_page(pmap_t pmap, vm_offset_t va) 1311{ 1312 struct ia64_lpte *pte; 1313 1314 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 1315 ("removing page for non-current pmap")); 1316 1317 pte = pmap_find_vhpt(va); 1318 if (pte) { 1319 pmap_remove_pte(pmap, pte, va, 0, 1); 1320 pmap_invalidate_page(pmap, va); 1321 } 1322 return; 1323} 1324 1325/* 1326 * Remove the given range of addresses from the specified map. 1327 * 1328 * It is assumed that the start and end are properly 1329 * rounded to the page size. 1330 */ 1331void 1332pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1333{ 1334 pmap_t oldpmap; 1335 vm_offset_t va; 1336 pv_entry_t npv, pv; 1337 struct ia64_lpte *pte; 1338 1339 if (pmap == NULL) 1340 return; 1341 1342 if (pmap->pm_stats.resident_count == 0) 1343 return; 1344 1345 vm_page_lock_queues(); 1346 PMAP_LOCK(pmap); 1347 oldpmap = pmap_install(pmap); 1348 1349 /* 1350 * special handling of removing one page. a very 1351 * common operation and easy to short circuit some 1352 * code. 1353 */ 1354 if (sva + PAGE_SIZE == eva) { 1355 pmap_remove_page(pmap, sva); 1356 goto out; 1357 } 1358 1359 if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) { 1360 TAILQ_FOREACH_SAFE(pv, &pmap->pm_pvlist, pv_plist, npv) { 1361 va = pv->pv_va; 1362 if (va >= sva && va < eva) { 1363 pte = pmap_find_vhpt(va); 1364 KASSERT(pte != NULL, ("pte")); 1365 pmap_remove_pte(pmap, pte, va, pv, 1); 1366 pmap_invalidate_page(pmap, va); 1367 } 1368 } 1369 1370 } else { 1371 for (va = sva; va < eva; va = va += PAGE_SIZE) { 1372 pte = pmap_find_vhpt(va); 1373 if (pte) { 1374 pmap_remove_pte(pmap, pte, va, 0, 1); 1375 pmap_invalidate_page(pmap, va); 1376 } 1377 } 1378 } 1379 1380out: 1381 pmap_install(oldpmap); 1382 PMAP_UNLOCK(pmap); 1383 vm_page_unlock_queues(); 1384} 1385 1386/* 1387 * Routine: pmap_remove_all 1388 * Function: 1389 * Removes this physical page from 1390 * all physical maps in which it resides. 1391 * Reflects back modify bits to the pager. 1392 * 1393 * Notes: 1394 * Original versions of this routine were very 1395 * inefficient because they iteratively called 1396 * pmap_remove (slow...) 1397 */ 1398 1399void 1400pmap_remove_all(vm_page_t m) 1401{ 1402 pmap_t oldpmap; 1403 pv_entry_t pv; 1404 int s; 1405 1406#if defined(PMAP_DIAGNOSTIC) 1407 /* 1408 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 1409 * pages! 1410 */ 1411 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) { 1412 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", VM_PAGE_TO_PHYS(m)); 1413 } 1414#endif 1415 1416 s = splvm(); 1417 1418 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1419 struct ia64_lpte *pte; 1420 pmap_t pmap = pv->pv_pmap; 1421 vm_offset_t va = pv->pv_va; 1422 1423 PMAP_LOCK(pmap); 1424 oldpmap = pmap_install(pmap); 1425 pte = pmap_find_vhpt(va); 1426 KASSERT(pte != NULL, ("pte")); 1427 if (pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(m)) 1428 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m)); 1429 pmap_remove_pte(pmap, pte, va, pv, 1); 1430 pmap_invalidate_page(pmap, va); 1431 pmap_install(oldpmap); 1432 PMAP_UNLOCK(pmap); 1433 } 1434 1435 vm_page_flag_clear(m, PG_WRITEABLE); 1436 1437 splx(s); 1438 return; 1439} 1440 1441/* 1442 * Set the physical protection on the 1443 * specified range of this map as requested. 1444 */ 1445void 1446pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1447{ 1448 pmap_t oldpmap; 1449 struct ia64_lpte *pte; 1450 int newprot; 1451 1452 if (pmap == NULL) 1453 return; 1454 1455 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1456 pmap_remove(pmap, sva, eva); 1457 return; 1458 } 1459 1460 if (prot & VM_PROT_WRITE) 1461 return; 1462 1463 newprot = pte_prot(pmap, prot); 1464 1465 if ((sva & PAGE_MASK) || (eva & PAGE_MASK)) 1466 panic("pmap_protect: unaligned addresses"); 1467 1468 vm_page_lock_queues(); 1469 PMAP_LOCK(pmap); 1470 oldpmap = pmap_install(pmap); 1471 while (sva < eva) { 1472 /* 1473 * If page is invalid, skip this page 1474 */ 1475 pte = pmap_find_vhpt(sva); 1476 if (!pte) { 1477 sva += PAGE_SIZE; 1478 continue; 1479 } 1480 1481 if (pmap_pte_prot(pte) != newprot) { 1482 if (pte->pte_ig & PTE_IG_MANAGED) { 1483 vm_offset_t pa = pmap_pte_pa(pte); 1484 vm_page_t m = PHYS_TO_VM_PAGE(pa); 1485 if (pte->pte_d) { 1486 if (pmap_track_modified(sva)) 1487 vm_page_dirty(m); 1488 pte->pte_d = 0; 1489 } 1490 if (pte->pte_a) { 1491 vm_page_flag_set(m, PG_REFERENCED); 1492 pte->pte_a = 0; 1493 } 1494 } 1495 pmap_pte_set_prot(pte, newprot); 1496 pmap_update_vhpt(pte, sva); 1497 pmap_invalidate_page(pmap, sva); 1498 } 1499 1500 sva += PAGE_SIZE; 1501 } 1502 pmap_install(oldpmap); 1503 PMAP_UNLOCK(pmap); 1504 vm_page_unlock_queues(); 1505} 1506 1507/* 1508 * Insert the given physical page (p) at 1509 * the specified virtual address (v) in the 1510 * target physical map with the protection requested. 1511 * 1512 * If specified, the page will be wired down, meaning 1513 * that the related pte can not be reclaimed. 1514 * 1515 * NB: This is the only routine which MAY NOT lazy-evaluate 1516 * or lose information. That is, this routine must actually 1517 * insert this page into the given map NOW. 1518 */ 1519void 1520pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1521 boolean_t wired) 1522{ 1523 pmap_t oldpmap; 1524 vm_offset_t pa; 1525 vm_offset_t opa; 1526 struct ia64_lpte origpte; 1527 struct ia64_lpte *pte; 1528 int managed; 1529 1530 if (pmap == NULL) 1531 return; 1532 1533 oldpmap = pmap_install(pmap); 1534 1535 va &= ~PAGE_MASK; 1536#ifdef PMAP_DIAGNOSTIC 1537 if (va > VM_MAX_KERNEL_ADDRESS) 1538 panic("pmap_enter: toobig"); 1539#endif 1540 1541 /* 1542 * Find (or create) a pte for the given mapping. 1543 */ 1544 pte = pmap_find_pte(va); 1545 origpte = *pte; 1546 1547 if (origpte.pte_p) 1548 opa = pmap_pte_pa(&origpte); 1549 else 1550 opa = 0; 1551 managed = 0; 1552 1553 pa = VM_PAGE_TO_PHYS(m) & ~PAGE_MASK; 1554 1555 /* 1556 * Mapping has not changed, must be protection or wiring change. 1557 */ 1558 if (origpte.pte_p && (opa == pa)) { 1559 /* 1560 * Wiring change, just update stats. We don't worry about 1561 * wiring PT pages as they remain resident as long as there 1562 * are valid mappings in them. Hence, if a user page is wired, 1563 * the PT page will be also. 1564 */ 1565 if (wired && ((origpte.pte_ig & PTE_IG_WIRED) == 0)) 1566 pmap->pm_stats.wired_count++; 1567 else if (!wired && (origpte.pte_ig & PTE_IG_WIRED)) 1568 pmap->pm_stats.wired_count--; 1569 1570 /* 1571 * We might be turning off write access to the page, 1572 * so we go ahead and sense modify status. 1573 */ 1574 if (origpte.pte_ig & PTE_IG_MANAGED) { 1575 if (origpte.pte_d && pmap_track_modified(va)) { 1576 vm_page_t om; 1577 om = PHYS_TO_VM_PAGE(opa); 1578 vm_page_dirty(om); 1579 } 1580 } 1581 1582 managed = origpte.pte_ig & PTE_IG_MANAGED; 1583 goto validate; 1584 } 1585 /* 1586 * Mapping has changed, invalidate old range and fall 1587 * through to handle validating new mapping. 1588 */ 1589 if (opa) { 1590 int error; 1591 vm_page_lock_queues(); 1592 error = pmap_remove_pte(pmap, pte, va, 0, 0); 1593 vm_page_unlock_queues(); 1594 if (error) 1595 panic("pmap_enter: pte vanished, va: 0x%lx", va); 1596 } 1597 1598 /* 1599 * Enter on the PV list if part of our managed memory. 1600 */ 1601 if (pmap_initialized && 1602 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) { 1603 pmap_insert_entry(pmap, va, m); 1604 managed |= PTE_IG_MANAGED; 1605 } 1606 1607 /* 1608 * Increment counters 1609 */ 1610 pmap->pm_stats.resident_count++; 1611 if (wired) 1612 pmap->pm_stats.wired_count++; 1613 1614validate: 1615 1616 /* 1617 * Now validate mapping with desired protection/wiring. This 1618 * adds the pte to the VHPT if necessary. 1619 */ 1620 pmap_set_pte(pte, va, pa, managed | (wired ? PTE_IG_WIRED : 0), 1621 pte_prot_pl(pmap, prot), pte_prot_ar(pmap, prot)); 1622 1623 /* 1624 * if the mapping or permission bits are different, we need 1625 * to invalidate the page. 1626 */ 1627 if (!pmap_equal_pte(&origpte, pte)) 1628 pmap_invalidate_page(pmap, va); 1629 1630 pmap_install(oldpmap); 1631} 1632 1633/* 1634 * this code makes some *MAJOR* assumptions: 1635 * 1. Current pmap & pmap exists. 1636 * 2. Not wired. 1637 * 3. Read access. 1638 * 4. No page table pages. 1639 * 5. Tlbflush is deferred to calling procedure. 1640 * 6. Page IS managed. 1641 * but is *MUCH* faster than pmap_enter... 1642 */ 1643 1644vm_page_t 1645pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) 1646{ 1647 struct ia64_lpte *pte; 1648 pmap_t oldpmap; 1649 int managed; 1650 1651 oldpmap = pmap_install(pmap); 1652 1653 pte = pmap_find_pte(va); 1654 if (pte->pte_p) 1655 goto reinstall; 1656 managed = 0; 1657 1658 /* 1659 * Enter on the PV list since its part of our managed memory. 1660 */ 1661 if (pmap_initialized && 1662 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) { 1663 pmap_insert_entry(pmap, va, m); 1664 managed |= PTE_IG_MANAGED; 1665 } 1666 1667 /* 1668 * Increment counters 1669 */ 1670 pmap->pm_stats.resident_count++; 1671 1672 /* 1673 * Initialise PTE with read-only protection and enter into VHPT. 1674 */ 1675 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), managed, 1676 PTE_PL_USER, PTE_AR_R); 1677reinstall: 1678 pmap_install(oldpmap); 1679 return (NULL); 1680} 1681 1682/* 1683 * pmap_object_init_pt preloads the ptes for a given object 1684 * into the specified pmap. This eliminates the blast of soft 1685 * faults on process startup and immediately after an mmap. 1686 */ 1687void 1688pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 1689 vm_object_t object, vm_pindex_t pindex, 1690 vm_size_t size) 1691{ 1692 1693 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1694 KASSERT(object->type == OBJT_DEVICE, 1695 ("pmap_object_init_pt: non-device object")); 1696} 1697 1698/* 1699 * Routine: pmap_change_wiring 1700 * Function: Change the wiring attribute for a map/virtual-address 1701 * pair. 1702 * In/out conditions: 1703 * The mapping must already exist in the pmap. 1704 */ 1705void 1706pmap_change_wiring(pmap, va, wired) 1707 register pmap_t pmap; 1708 vm_offset_t va; 1709 boolean_t wired; 1710{ 1711 pmap_t oldpmap; 1712 struct ia64_lpte *pte; 1713 1714 if (pmap == NULL) 1715 return; 1716 1717 PMAP_LOCK(pmap); 1718 oldpmap = pmap_install(pmap); 1719 1720 pte = pmap_find_vhpt(va); 1721 KASSERT(pte != NULL, ("pte")); 1722 if (wired && !pmap_pte_w(pte)) 1723 pmap->pm_stats.wired_count++; 1724 else if (!wired && pmap_pte_w(pte)) 1725 pmap->pm_stats.wired_count--; 1726 1727 /* 1728 * Wiring is not a hardware characteristic so there is no need to 1729 * invalidate TLB. 1730 */ 1731 pmap_pte_set_w(pte, wired); 1732 1733 pmap_install(oldpmap); 1734 PMAP_UNLOCK(pmap); 1735} 1736 1737 1738 1739/* 1740 * Copy the range specified by src_addr/len 1741 * from the source map to the range dst_addr/len 1742 * in the destination map. 1743 * 1744 * This routine is only advisory and need not do anything. 1745 */ 1746 1747void 1748pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 1749 vm_offset_t src_addr) 1750{ 1751} 1752 1753 1754/* 1755 * pmap_zero_page zeros the specified hardware page by 1756 * mapping it into virtual memory and using bzero to clear 1757 * its contents. 1758 */ 1759 1760void 1761pmap_zero_page(vm_page_t m) 1762{ 1763 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 1764 bzero((caddr_t) va, PAGE_SIZE); 1765} 1766 1767 1768/* 1769 * pmap_zero_page_area zeros the specified hardware page by 1770 * mapping it into virtual memory and using bzero to clear 1771 * its contents. 1772 * 1773 * off and size must reside within a single page. 1774 */ 1775 1776void 1777pmap_zero_page_area(vm_page_t m, int off, int size) 1778{ 1779 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 1780 bzero((char *)(caddr_t)va + off, size); 1781} 1782 1783 1784/* 1785 * pmap_zero_page_idle zeros the specified hardware page by 1786 * mapping it into virtual memory and using bzero to clear 1787 * its contents. This is for the vm_idlezero process. 1788 */ 1789 1790void 1791pmap_zero_page_idle(vm_page_t m) 1792{ 1793 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 1794 bzero((caddr_t) va, PAGE_SIZE); 1795} 1796 1797 1798/* 1799 * pmap_copy_page copies the specified (machine independent) 1800 * page by mapping the page into virtual memory and using 1801 * bcopy to copy the page, one machine dependent page at a 1802 * time. 1803 */ 1804void 1805pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1806{ 1807 vm_offset_t src = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(msrc)); 1808 vm_offset_t dst = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(mdst)); 1809 bcopy((caddr_t) src, (caddr_t) dst, PAGE_SIZE); 1810} 1811 1812/* 1813 * Returns true if the pmap's pv is one of the first 1814 * 16 pvs linked to from this page. This count may 1815 * be changed upwards or downwards in the future; it 1816 * is only necessary that true be returned for a small 1817 * subset of pmaps for proper page aging. 1818 */ 1819boolean_t 1820pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 1821{ 1822 pv_entry_t pv; 1823 int loops = 0; 1824 int s; 1825 1826 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 1827 return FALSE; 1828 1829 s = splvm(); 1830 1831 /* 1832 * Not found, check current mappings returning immediately if found. 1833 */ 1834 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1835 if (pv->pv_pmap == pmap) { 1836 splx(s); 1837 return TRUE; 1838 } 1839 loops++; 1840 if (loops >= 16) 1841 break; 1842 } 1843 splx(s); 1844 return (FALSE); 1845} 1846 1847#define PMAP_REMOVE_PAGES_CURPROC_ONLY 1848/* 1849 * Remove all pages from specified address space 1850 * this aids process exit speeds. Also, this code 1851 * is special cased for current process only, but 1852 * can have the more generic (and slightly slower) 1853 * mode enabled. This is much faster than pmap_remove 1854 * in the case of running down an entire address space. 1855 */ 1856void 1857pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1858{ 1859 pv_entry_t pv, npv; 1860 1861#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY 1862 if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))) { 1863 printf("warning: pmap_remove_pages called with non-current pmap\n"); 1864 return; 1865 } 1866#endif 1867 1868 vm_page_lock_queues(); 1869 PMAP_LOCK(pmap); 1870 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); 1871 pv; 1872 pv = npv) { 1873 struct ia64_lpte *pte; 1874 1875 npv = TAILQ_NEXT(pv, pv_plist); 1876 1877 if (pv->pv_va >= eva || pv->pv_va < sva) { 1878 continue; 1879 } 1880 1881 pte = pmap_find_vhpt(pv->pv_va); 1882 KASSERT(pte != NULL, ("pte")); 1883 if (pte->pte_ig & PTE_IG_WIRED) 1884 continue; 1885 1886 pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1); 1887 } 1888 1889 pmap_invalidate_all(pmap); 1890 PMAP_UNLOCK(pmap); 1891 vm_page_unlock_queues(); 1892} 1893 1894/* 1895 * pmap_page_protect: 1896 * 1897 * Lower the permission for all mappings to a given page. 1898 */ 1899void 1900pmap_page_protect(vm_page_t m, vm_prot_t prot) 1901{ 1902 struct ia64_lpte *pte; 1903 pmap_t oldpmap; 1904 pv_entry_t pv; 1905 1906 if ((prot & VM_PROT_WRITE) != 0) 1907 return; 1908 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 1909 if ((m->flags & PG_WRITEABLE) == 0) 1910 return; 1911 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1912 int newprot = pte_prot(pv->pv_pmap, prot); 1913 PMAP_LOCK(pv->pv_pmap); 1914 oldpmap = pmap_install(pv->pv_pmap); 1915 pte = pmap_find_vhpt(pv->pv_va); 1916 KASSERT(pte != NULL, ("pte")); 1917 pmap_pte_set_prot(pte, newprot); 1918 pmap_update_vhpt(pte, pv->pv_va); 1919 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1920 pmap_install(oldpmap); 1921 PMAP_UNLOCK(pv->pv_pmap); 1922 } 1923 vm_page_flag_clear(m, PG_WRITEABLE); 1924 } else { 1925 pmap_remove_all(m); 1926 } 1927} 1928 1929/* 1930 * pmap_ts_referenced: 1931 * 1932 * Return a count of reference bits for a page, clearing those bits. 1933 * It is not necessary for every reference bit to be cleared, but it 1934 * is necessary that 0 only be returned when there are truly no 1935 * reference bits set. 1936 * 1937 * XXX: The exact number of bits to check and clear is a matter that 1938 * should be tested and standardized at some point in the future for 1939 * optimal aging of shared pages. 1940 */ 1941int 1942pmap_ts_referenced(vm_page_t m) 1943{ 1944 struct ia64_lpte *pte; 1945 pmap_t oldpmap; 1946 pv_entry_t pv; 1947 int count = 0; 1948 1949 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 1950 return 0; 1951 1952 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1953 PMAP_LOCK(pv->pv_pmap); 1954 oldpmap = pmap_install(pv->pv_pmap); 1955 pte = pmap_find_vhpt(pv->pv_va); 1956 KASSERT(pte != NULL, ("pte")); 1957 if (pte->pte_a) { 1958 count++; 1959 pte->pte_a = 0; 1960 pmap_update_vhpt(pte, pv->pv_va); 1961 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1962 } 1963 pmap_install(oldpmap); 1964 PMAP_UNLOCK(pv->pv_pmap); 1965 } 1966 1967 return count; 1968} 1969 1970#if 0 1971/* 1972 * pmap_is_referenced: 1973 * 1974 * Return whether or not the specified physical page was referenced 1975 * in any physical maps. 1976 */ 1977static boolean_t 1978pmap_is_referenced(vm_page_t m) 1979{ 1980 pv_entry_t pv; 1981 1982 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 1983 return FALSE; 1984 1985 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1986 pmap_t oldpmap = pmap_install(pv->pv_pmap); 1987 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 1988 pmap_install(oldpmap); 1989 KASSERT(pte != NULL, ("pte")); 1990 if (pte->pte_a) 1991 return 1; 1992 } 1993 1994 return 0; 1995} 1996#endif 1997 1998/* 1999 * pmap_is_modified: 2000 * 2001 * Return whether or not the specified physical page was modified 2002 * in any physical maps. 2003 */ 2004boolean_t 2005pmap_is_modified(vm_page_t m) 2006{ 2007 struct ia64_lpte *pte; 2008 pmap_t oldpmap; 2009 pv_entry_t pv; 2010 boolean_t rv; 2011 2012 rv = FALSE; 2013 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2014 return (rv); 2015 2016 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2017 PMAP_LOCK(pv->pv_pmap); 2018 oldpmap = pmap_install(pv->pv_pmap); 2019 pte = pmap_find_vhpt(pv->pv_va); 2020 pmap_install(oldpmap); 2021 KASSERT(pte != NULL, ("pte")); 2022 rv = pte->pte_d != 0; 2023 PMAP_UNLOCK(pv->pv_pmap); 2024 if (rv) 2025 break; 2026 } 2027 2028 return (rv); 2029} 2030 2031/* 2032 * pmap_is_prefaultable: 2033 * 2034 * Return whether or not the specified virtual address is elgible 2035 * for prefault. 2036 */ 2037boolean_t 2038pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2039{ 2040 struct ia64_lpte *pte; 2041 2042 pte = pmap_find_vhpt(addr); 2043 if (pte && pte->pte_p) 2044 return (FALSE); 2045 return (TRUE); 2046} 2047 2048/* 2049 * Clear the modify bits on the specified physical page. 2050 */ 2051void 2052pmap_clear_modify(vm_page_t m) 2053{ 2054 struct ia64_lpte *pte; 2055 pmap_t oldpmap; 2056 pv_entry_t pv; 2057 2058 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2059 return; 2060 2061 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2062 PMAP_LOCK(pv->pv_pmap); 2063 oldpmap = pmap_install(pv->pv_pmap); 2064 pte = pmap_find_vhpt(pv->pv_va); 2065 KASSERT(pte != NULL, ("pte")); 2066 if (pte->pte_d) { 2067 pte->pte_d = 0; 2068 pmap_update_vhpt(pte, pv->pv_va); 2069 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2070 } 2071 pmap_install(oldpmap); 2072 PMAP_UNLOCK(pv->pv_pmap); 2073 } 2074} 2075 2076/* 2077 * pmap_clear_reference: 2078 * 2079 * Clear the reference bit on the specified physical page. 2080 */ 2081void 2082pmap_clear_reference(vm_page_t m) 2083{ 2084 struct ia64_lpte *pte; 2085 pmap_t oldpmap; 2086 pv_entry_t pv; 2087 2088 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2089 return; 2090 2091 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2092 PMAP_LOCK(pv->pv_pmap); 2093 oldpmap = pmap_install(pv->pv_pmap); 2094 pte = pmap_find_vhpt(pv->pv_va); 2095 KASSERT(pte != NULL, ("pte")); 2096 if (pte->pte_a) { 2097 pte->pte_a = 0; 2098 pmap_update_vhpt(pte, pv->pv_va); 2099 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2100 } 2101 pmap_install(oldpmap); 2102 PMAP_UNLOCK(pv->pv_pmap); 2103 } 2104} 2105 2106/* 2107 * Miscellaneous support routines follow 2108 */ 2109 2110static void 2111ia64_protection_init() 2112{ 2113 int prot, *kp, *up; 2114 2115 kp = protection_codes[0]; 2116 up = protection_codes[1]; 2117 2118 for (prot = 0; prot < 8; prot++) { 2119 switch (prot) { 2120 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 2121 *kp++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2122 *up++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2123 break; 2124 2125 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 2126 *kp++ = (PTE_AR_X_RX << 2) | PTE_PL_KERN; 2127 *up++ = (PTE_AR_X_RX << 2) | PTE_PL_USER; 2128 break; 2129 2130 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 2131 *kp++ = (PTE_AR_RW << 2) | PTE_PL_KERN; 2132 *up++ = (PTE_AR_RW << 2) | PTE_PL_USER; 2133 break; 2134 2135 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 2136 *kp++ = (PTE_AR_RWX << 2) | PTE_PL_KERN; 2137 *up++ = (PTE_AR_RWX << 2) | PTE_PL_USER; 2138 break; 2139 2140 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 2141 *kp++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2142 *up++ = (PTE_AR_R << 2) | PTE_PL_USER; 2143 break; 2144 2145 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 2146 *kp++ = (PTE_AR_RX << 2) | PTE_PL_KERN; 2147 *up++ = (PTE_AR_RX << 2) | PTE_PL_USER; 2148 break; 2149 2150 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 2151 *kp++ = (PTE_AR_RW << 2) | PTE_PL_KERN; 2152 *up++ = (PTE_AR_RW << 2) | PTE_PL_USER; 2153 break; 2154 2155 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 2156 *kp++ = (PTE_AR_RWX << 2) | PTE_PL_KERN; 2157 *up++ = (PTE_AR_RWX << 2) | PTE_PL_USER; 2158 break; 2159 } 2160 } 2161} 2162 2163/* 2164 * Map a set of physical memory pages into the kernel virtual 2165 * address space. Return a pointer to where it is mapped. This 2166 * routine is intended to be used for mapping device memory, 2167 * NOT real memory. 2168 */ 2169void * 2170pmap_mapdev(vm_offset_t pa, vm_size_t size) 2171{ 2172 return (void*) IA64_PHYS_TO_RR6(pa); 2173} 2174 2175/* 2176 * 'Unmap' a range mapped by pmap_mapdev(). 2177 */ 2178void 2179pmap_unmapdev(vm_offset_t va, vm_size_t size) 2180{ 2181 return; 2182} 2183 2184/* 2185 * perform the pmap work for mincore 2186 */ 2187int 2188pmap_mincore(pmap_t pmap, vm_offset_t addr) 2189{ 2190 pmap_t oldpmap; 2191 struct ia64_lpte *pte; 2192 int val = 0; 2193 2194 PMAP_LOCK(pmap); 2195 oldpmap = pmap_install(pmap); 2196 pte = pmap_find_vhpt(addr); 2197 pmap_install(oldpmap); 2198 PMAP_UNLOCK(pmap); 2199 2200 if (!pte) 2201 return 0; 2202 2203 if (pmap_pte_v(pte)) { 2204 vm_page_t m; 2205 vm_offset_t pa; 2206 2207 val = MINCORE_INCORE; 2208 if ((pte->pte_ig & PTE_IG_MANAGED) == 0) 2209 return val; 2210 2211 pa = pmap_pte_pa(pte); 2212 2213 m = PHYS_TO_VM_PAGE(pa); 2214 2215 /* 2216 * Modified by us 2217 */ 2218 if (pte->pte_d) 2219 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER; 2220 else { 2221 /* 2222 * Modified by someone 2223 */ 2224 vm_page_lock_queues(); 2225 if (pmap_is_modified(m)) 2226 val |= MINCORE_MODIFIED_OTHER; 2227 vm_page_unlock_queues(); 2228 } 2229 /* 2230 * Referenced by us 2231 */ 2232 if (pte->pte_a) 2233 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER; 2234 else { 2235 /* 2236 * Referenced by someone 2237 */ 2238 vm_page_lock_queues(); 2239 if (pmap_ts_referenced(m)) { 2240 val |= MINCORE_REFERENCED_OTHER; 2241 vm_page_flag_set(m, PG_REFERENCED); 2242 } 2243 vm_page_unlock_queues(); 2244 } 2245 } 2246 return val; 2247} 2248 2249void 2250pmap_activate(struct thread *td) 2251{ 2252 pmap_install(vmspace_pmap(td->td_proc->p_vmspace)); 2253} 2254 2255pmap_t 2256pmap_switch(pmap_t pm) 2257{ 2258 pmap_t prevpm; 2259 int i; 2260 2261 mtx_assert(&sched_lock, MA_OWNED); 2262 2263 prevpm = PCPU_GET(current_pmap); 2264 if (prevpm == pm) 2265 return (prevpm); 2266 if (prevpm != NULL) 2267 atomic_clear_32(&prevpm->pm_active, PCPU_GET(cpumask)); 2268 if (pm == NULL) { 2269 for (i = 0; i < 5; i++) { 2270 ia64_set_rr(IA64_RR_BASE(i), 2271 (i << 8)|(PAGE_SHIFT << 2)|1); 2272 } 2273 } else { 2274 for (i = 0; i < 5; i++) { 2275 ia64_set_rr(IA64_RR_BASE(i), 2276 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1); 2277 } 2278 atomic_set_32(&pm->pm_active, PCPU_GET(cpumask)); 2279 } 2280 PCPU_SET(current_pmap, pm); 2281 __asm __volatile("srlz.d"); 2282 return (prevpm); 2283} 2284 2285static pmap_t 2286pmap_install(pmap_t pm) 2287{ 2288 pmap_t prevpm; 2289 2290 mtx_lock_spin(&sched_lock); 2291 prevpm = pmap_switch(pm); 2292 mtx_unlock_spin(&sched_lock); 2293 return (prevpm); 2294} 2295 2296vm_offset_t 2297pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) 2298{ 2299 2300 return addr; 2301} 2302 2303#include "opt_ddb.h" 2304 2305#ifdef DDB 2306 2307#include <ddb/ddb.h> 2308 2309static const char* psnames[] = { 2310 "1B", "2B", "4B", "8B", 2311 "16B", "32B", "64B", "128B", 2312 "256B", "512B", "1K", "2K", 2313 "4K", "8K", "16K", "32K", 2314 "64K", "128K", "256K", "512K", 2315 "1M", "2M", "4M", "8M", 2316 "16M", "32M", "64M", "128M", 2317 "256M", "512M", "1G", "2G" 2318}; 2319 2320static void 2321print_trs(int type) 2322{ 2323 struct ia64_pal_result res; 2324 int i, maxtr; 2325 struct { 2326 struct ia64_pte pte; 2327 struct ia64_itir itir; 2328 struct ia64_ifa ifa; 2329 struct ia64_rr rr; 2330 } buf; 2331 static const char* manames[] = { 2332 "WB", "bad", "bad", "bad", 2333 "UC", "UCE", "WC", "NaT", 2334 2335 }; 2336 2337 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 2338 if (res.pal_status != 0) { 2339 db_printf("Can't get VM summary\n"); 2340 return; 2341 } 2342 2343 if (type == 0) 2344 maxtr = (res.pal_result[0] >> 40) & 0xff; 2345 else 2346 maxtr = (res.pal_result[0] >> 32) & 0xff; 2347 2348 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n"); 2349 for (i = 0; i <= maxtr; i++) { 2350 bzero(&buf, sizeof(buf)); 2351 res = ia64_call_pal_stacked_physical 2352 (PAL_VM_TR_READ, i, type, ia64_tpa((u_int64_t) &buf)); 2353 if (!(res.pal_result[0] & 1)) 2354 buf.pte.pte_ar = 0; 2355 if (!(res.pal_result[0] & 2)) 2356 buf.pte.pte_pl = 0; 2357 if (!(res.pal_result[0] & 4)) 2358 buf.pte.pte_d = 0; 2359 if (!(res.pal_result[0] & 8)) 2360 buf.pte.pte_ma = 0; 2361 db_printf( 2362 "%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s %d %06x\n", 2363 buf.ifa.ifa_ig & 1, 2364 buf.rr.rr_rid, 2365 buf.ifa.ifa_vpn, 2366 buf.pte.pte_ppn, 2367 psnames[buf.itir.itir_ps], 2368 buf.pte.pte_ed, 2369 buf.pte.pte_ar, 2370 buf.pte.pte_pl, 2371 buf.pte.pte_d, 2372 buf.pte.pte_a, 2373 manames[buf.pte.pte_ma], 2374 buf.pte.pte_p, 2375 buf.itir.itir_key); 2376 } 2377} 2378 2379DB_COMMAND(itr, db_itr) 2380{ 2381 print_trs(0); 2382} 2383 2384DB_COMMAND(dtr, db_dtr) 2385{ 2386 print_trs(1); 2387} 2388 2389DB_COMMAND(rr, db_rr) 2390{ 2391 int i; 2392 u_int64_t t; 2393 struct ia64_rr rr; 2394 2395 printf("RR RID PgSz VE\n"); 2396 for (i = 0; i < 8; i++) { 2397 __asm __volatile ("mov %0=rr[%1]" 2398 : "=r"(t) 2399 : "r"(IA64_RR_BASE(i))); 2400 *(u_int64_t *) &rr = t; 2401 printf("%d %06x %4s %d\n", 2402 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve); 2403 } 2404} 2405 2406DB_COMMAND(thash, db_thash) 2407{ 2408 if (!have_addr) 2409 return; 2410 2411 db_printf("%p\n", (void *) ia64_thash(addr)); 2412} 2413 2414DB_COMMAND(ttag, db_ttag) 2415{ 2416 if (!have_addr) 2417 return; 2418 2419 db_printf("0x%lx\n", ia64_ttag(addr)); 2420} 2421 2422#endif 2423