pmap.c revision 126716
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * Copyright (c) 1998,2000 Doug Rabson 9 * All rights reserved. 10 * 11 * This code is derived from software contributed to Berkeley by 12 * the Systems Programming Group of the University of Utah Computer 13 * Science Department and William Jolitz of UUNET Technologies Inc. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. All advertising materials mentioning features or use of this software 24 * must display the following acknowledgement: 25 * This product includes software developed by the University of 26 * California, Berkeley and its contributors. 27 * 4. Neither the name of the University nor the names of its contributors 28 * may be used to endorse or promote products derived from this software 29 * without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 41 * SUCH DAMAGE. 42 * 43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 44 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp 45 * with some ideas from NetBSD's alpha pmap 46 */ 47 48#include <sys/cdefs.h> 49__FBSDID("$FreeBSD: head/sys/ia64/ia64/pmap.c 126716 2004-03-07 07:43:13Z alc $"); 50 51#include <sys/param.h> 52#include <sys/kernel.h> 53#include <sys/lock.h> 54#include <sys/mman.h> 55#include <sys/mutex.h> 56#include <sys/proc.h> 57#include <sys/smp.h> 58#include <sys/sysctl.h> 59#include <sys/systm.h> 60 61#include <vm/vm.h> 62#include <vm/vm_page.h> 63#include <vm/vm_map.h> 64#include <vm/vm_object.h> 65#include <vm/vm_pageout.h> 66#include <vm/uma.h> 67 68#include <machine/md_var.h> 69#include <machine/pal.h> 70 71/* 72 * Manages physical address maps. 73 * 74 * In addition to hardware address maps, this 75 * module is called upon to provide software-use-only 76 * maps which may or may not be stored in the same 77 * form as hardware maps. These pseudo-maps are 78 * used to store intermediate results from copy 79 * operations to and from address spaces. 80 * 81 * Since the information managed by this module is 82 * also stored by the logical address mapping module, 83 * this module may throw away valid virtual-to-physical 84 * mappings at almost any time. However, invalidations 85 * of virtual-to-physical mappings must be done as 86 * requested. 87 * 88 * In order to cope with hardware architectures which 89 * make virtual-to-physical map invalidates expensive, 90 * this module may delay invalidate or reduced protection 91 * operations until such time as they are actually 92 * necessary. This module is given full information as 93 * to which processors are currently using which maps, 94 * and to when physical maps must be made correct. 95 */ 96 97/* 98 * Following the Linux model, region IDs are allocated in groups of 99 * eight so that a single region ID can be used for as many RRs as we 100 * want by encoding the RR number into the low bits of the ID. 101 * 102 * We reserve region ID 0 for the kernel and allocate the remaining 103 * IDs for user pmaps. 104 * 105 * Region 0..4 106 * User virtually mapped 107 * 108 * Region 5 109 * Kernel virtually mapped 110 * 111 * Region 6 112 * Kernel physically mapped uncacheable 113 * 114 * Region 7 115 * Kernel physically mapped cacheable 116 */ 117 118/* XXX move to a header. */ 119extern u_int64_t ia64_gateway_page[]; 120 121MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures"); 122 123#ifndef PMAP_SHPGPERPROC 124#define PMAP_SHPGPERPROC 200 125#endif 126 127#if defined(DIAGNOSTIC) 128#define PMAP_DIAGNOSTIC 129#endif 130 131#define MINPV 2048 /* Preallocate at least this many */ 132#define MAXPV 20480 /* But no more than this */ 133 134#if 0 135#define PMAP_DIAGNOSTIC 136#define PMAP_DEBUG 137#endif 138 139#if !defined(PMAP_DIAGNOSTIC) 140#define PMAP_INLINE __inline 141#else 142#define PMAP_INLINE 143#endif 144 145/* 146 * Get PDEs and PTEs for user/kernel address space 147 */ 148#define pmap_pte_w(pte) ((pte)->pte_ig & PTE_IG_WIRED) 149#define pmap_pte_managed(pte) ((pte)->pte_ig & PTE_IG_MANAGED) 150#define pmap_pte_v(pte) ((pte)->pte_p) 151#define pmap_pte_pa(pte) (((pte)->pte_ppn) << 12) 152#define pmap_pte_prot(pte) (((pte)->pte_ar << 2) | (pte)->pte_pl) 153 154#define pmap_pte_set_w(pte, v) ((v)?((pte)->pte_ig |= PTE_IG_WIRED) \ 155 :((pte)->pte_ig &= ~PTE_IG_WIRED)) 156#define pmap_pte_set_prot(pte, v) do { \ 157 (pte)->pte_ar = v >> 2; \ 158 (pte)->pte_pl = v & 3; \ 159} while (0) 160 161/* 162 * Given a map and a machine independent protection code, 163 * convert to an ia64 protection code. 164 */ 165#define pte_prot(m, p) (protection_codes[m == kernel_pmap ? 0 : 1][p]) 166#define pte_prot_pl(m, p) (pte_prot(m, p) & 3) 167#define pte_prot_ar(m, p) (pte_prot(m, p) >> 2) 168int protection_codes[2][8]; 169 170/* 171 * Return non-zero if this pmap is currently active 172 */ 173#define pmap_isactive(pmap) (pmap->pm_active) 174 175/* 176 * Statically allocated kernel pmap 177 */ 178struct pmap kernel_pmap_store; 179 180vm_offset_t avail_start; /* PA of first available physical page */ 181vm_offset_t avail_end; /* PA of last available physical page */ 182vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 183vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 184static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 185 186vm_offset_t vhpt_base, vhpt_size; 187struct mtx pmap_vhptmutex; 188 189/* 190 * We use an object to own the kernel's 'page tables'. For simplicity, 191 * we use one page directory to index a set of pages containing 192 * ia64_lptes. This gives us up to 2Gb of kernel virtual space. 193 */ 194static int nkpt; 195struct ia64_lpte **ia64_kptdir; 196#define KPTE_DIR_INDEX(va) \ 197 ((va >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1)) 198#define KPTE_PTE_INDEX(va) \ 199 ((va >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1)) 200#define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte)) 201 202vm_offset_t kernel_vm_end; 203 204/* Values for ptc.e. XXX values for SKI. */ 205static u_int64_t pmap_ptc_e_base = 0x100000000; 206static u_int64_t pmap_ptc_e_count1 = 3; 207static u_int64_t pmap_ptc_e_count2 = 2; 208static u_int64_t pmap_ptc_e_stride1 = 0x2000; 209static u_int64_t pmap_ptc_e_stride2 = 0x100000000; 210 211/* 212 * Data for the RID allocator 213 */ 214static int pmap_ridcount; 215static int pmap_rididx; 216static int pmap_ridmapsz; 217static int pmap_ridmax; 218static u_int64_t *pmap_ridmap; 219struct mtx pmap_ridmutex; 220 221/* 222 * Data for the pv entry allocation mechanism 223 */ 224static uma_zone_t pvzone; 225static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 226int pmap_pagedaemon_waken; 227static struct pv_entry *pvbootentries; 228static int pvbootnext, pvbootmax; 229 230/* 231 * Data for allocating PTEs for user processes. 232 */ 233static uma_zone_t ptezone; 234 235/* 236 * VHPT instrumentation. 237 */ 238static int pmap_vhpt_inserts; 239static int pmap_vhpt_collisions; 240static int pmap_vhpt_resident; 241SYSCTL_DECL(_vm_stats); 242SYSCTL_NODE(_vm_stats, OID_AUTO, vhpt, CTLFLAG_RD, 0, ""); 243SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, inserts, CTLFLAG_RD, 244 &pmap_vhpt_inserts, 0, ""); 245SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, collisions, CTLFLAG_RD, 246 &pmap_vhpt_collisions, 0, ""); 247SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, resident, CTLFLAG_RD, 248 &pmap_vhpt_resident, 0, ""); 249 250static PMAP_INLINE void free_pv_entry(pv_entry_t pv); 251static pv_entry_t get_pv_entry(void); 252static void ia64_protection_init(void); 253 254static pmap_t pmap_install(pmap_t); 255static void pmap_invalidate_all(pmap_t pmap); 256 257vm_offset_t 258pmap_steal_memory(vm_size_t size) 259{ 260 vm_size_t bank_size; 261 vm_offset_t pa, va; 262 263 size = round_page(size); 264 265 bank_size = phys_avail[1] - phys_avail[0]; 266 while (size > bank_size) { 267 int i; 268 for (i = 0; phys_avail[i+2]; i+= 2) { 269 phys_avail[i] = phys_avail[i+2]; 270 phys_avail[i+1] = phys_avail[i+3]; 271 } 272 phys_avail[i] = 0; 273 phys_avail[i+1] = 0; 274 if (!phys_avail[0]) 275 panic("pmap_steal_memory: out of memory"); 276 bank_size = phys_avail[1] - phys_avail[0]; 277 } 278 279 pa = phys_avail[0]; 280 phys_avail[0] += size; 281 282 va = IA64_PHYS_TO_RR7(pa); 283 bzero((caddr_t) va, size); 284 return va; 285} 286 287/* 288 * Bootstrap the system enough to run with virtual memory. 289 */ 290void 291pmap_bootstrap() 292{ 293 int i, j, count, ridbits; 294 struct ia64_pal_result res; 295 296 /* 297 * Query the PAL Code to find the loop parameters for the 298 * ptc.e instruction. 299 */ 300 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0); 301 if (res.pal_status != 0) 302 panic("Can't configure ptc.e parameters"); 303 pmap_ptc_e_base = res.pal_result[0]; 304 pmap_ptc_e_count1 = res.pal_result[1] >> 32; 305 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1); 306 pmap_ptc_e_stride1 = res.pal_result[2] >> 32; 307 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1); 308 if (bootverbose) 309 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, " 310 "stride1=0x%lx, stride2=0x%lx\n", 311 pmap_ptc_e_base, 312 pmap_ptc_e_count1, 313 pmap_ptc_e_count2, 314 pmap_ptc_e_stride1, 315 pmap_ptc_e_stride2); 316 317 /* 318 * Setup RIDs. RIDs 0..7 are reserved for the kernel. 319 * 320 * We currently need at least 19 bits in the RID because PID_MAX 321 * can only be encoded in 17 bits and we need RIDs for 5 regions 322 * per process. With PID_MAX equalling 99999 this means that we 323 * need to be able to encode 499995 (=5*PID_MAX). 324 * The Itanium processor only has 18 bits and the architected 325 * minimum is exactly that. So, we cannot use a PID based scheme 326 * in those cases. Enter pmap_ridmap... 327 * We should avoid the map when running on a processor that has 328 * implemented enough bits. This means that we should pass the 329 * process/thread ID to pmap. This we currently don't do, so we 330 * use the map anyway. However, we don't want to allocate a map 331 * that is large enough to cover the range dictated by the number 332 * of bits in the RID, because that may result in a RID map of 333 * 2MB in size for a 24-bit RID. A 64KB map is enough. 334 * The bottomline: we create a 32KB map when the processor only 335 * implements 18 bits (or when we can't figure it out). Otherwise 336 * we create a 64KB map. 337 */ 338 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 339 if (res.pal_status != 0) { 340 if (bootverbose) 341 printf("Can't read VM Summary - assuming 18 Region ID bits\n"); 342 ridbits = 18; /* guaranteed minimum */ 343 } else { 344 ridbits = (res.pal_result[1] >> 8) & 0xff; 345 if (bootverbose) 346 printf("Processor supports %d Region ID bits\n", 347 ridbits); 348 } 349 if (ridbits > 19) 350 ridbits = 19; 351 352 pmap_ridmax = (1 << ridbits); 353 pmap_ridmapsz = pmap_ridmax / 64; 354 pmap_ridmap = (u_int64_t *)pmap_steal_memory(pmap_ridmax / 8); 355 pmap_ridmap[0] |= 0xff; 356 pmap_rididx = 0; 357 pmap_ridcount = 8; 358 mtx_init(&pmap_ridmutex, "RID allocator lock", NULL, MTX_DEF); 359 360 /* 361 * Allocate some memory for initial kernel 'page tables'. 362 */ 363 ia64_kptdir = (void *)pmap_steal_memory(PAGE_SIZE); 364 for (i = 0; i < NKPT; i++) { 365 ia64_kptdir[i] = (void*)pmap_steal_memory(PAGE_SIZE); 366 } 367 nkpt = NKPT; 368 kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS - 369 VM_GATEWAY_SIZE; 370 371 avail_start = phys_avail[0]; 372 for (i = 0; phys_avail[i+2]; i+= 2) ; 373 avail_end = phys_avail[i+1]; 374 count = i+2; 375 376 /* 377 * Figure out a useful size for the VHPT, based on the size of 378 * physical memory and try to locate a region which is large 379 * enough to contain the VHPT (which must be a power of two in 380 * size and aligned to a natural boundary). 381 * Don't use the difference between avail_start and avail_end 382 * as a measure for memory size. The address space is often 383 * enough sparse, causing us to (try to) create a huge VHPT. 384 */ 385 vhpt_size = 15; 386 while ((1<<vhpt_size) < Maxmem * 32) 387 vhpt_size++; 388 389 vhpt_base = 0; 390 while (!vhpt_base) { 391 vm_offset_t mask; 392 if (bootverbose) 393 printf("Trying VHPT size 0x%lx\n", (1L<<vhpt_size)); 394 mask = (1L << vhpt_size) - 1; 395 for (i = 0; i < count; i += 2) { 396 vm_offset_t base, limit; 397 base = (phys_avail[i] + mask) & ~mask; 398 limit = base + (1L << vhpt_size); 399 if (limit <= phys_avail[i+1]) 400 /* 401 * VHPT can fit in this region 402 */ 403 break; 404 } 405 if (!phys_avail[i]) { 406 /* 407 * Can't fit, try next smaller size. 408 */ 409 vhpt_size--; 410 } else { 411 vhpt_base = (phys_avail[i] + mask) & ~mask; 412 } 413 } 414 if (vhpt_size < 15) 415 panic("Can't find space for VHPT"); 416 417 if (bootverbose) 418 printf("Putting VHPT at %p\n", (void *) vhpt_base); 419 if (vhpt_base != phys_avail[i]) { 420 /* 421 * Split this region. 422 */ 423 if (bootverbose) 424 printf("Splitting [%p-%p]\n", 425 (void *) phys_avail[i], 426 (void *) phys_avail[i+1]); 427 for (j = count; j > i; j -= 2) { 428 phys_avail[j] = phys_avail[j-2]; 429 phys_avail[j+1] = phys_avail[j-2+1]; 430 } 431 phys_avail[count+2] = 0; 432 phys_avail[count+3] = 0; 433 phys_avail[i+1] = vhpt_base; 434 phys_avail[i+2] = vhpt_base + (1L << vhpt_size); 435 } else { 436 phys_avail[i] = vhpt_base + (1L << vhpt_size); 437 } 438 439 vhpt_base = IA64_PHYS_TO_RR7(vhpt_base); 440 bzero((void *) vhpt_base, (1L << vhpt_size)); 441 442 mtx_init(&pmap_vhptmutex, "VHPT collision chain lock", NULL, MTX_DEF); 443 444 __asm __volatile("mov cr.pta=%0;; srlz.i;;" 445 :: "r" (vhpt_base + (1<<8) + (vhpt_size<<2) + 1)); 446 447 virtual_avail = VM_MIN_KERNEL_ADDRESS; 448 virtual_end = VM_MAX_KERNEL_ADDRESS; 449 450 /* 451 * Initialize protection array. 452 */ 453 ia64_protection_init(); 454 455 /* 456 * Initialize the kernel pmap (which is statically allocated). 457 */ 458 for (i = 0; i < 5; i++) 459 kernel_pmap->pm_rid[i] = 0; 460 kernel_pmap->pm_active = 1; 461 TAILQ_INIT(&kernel_pmap->pm_pvlist); 462 PCPU_SET(current_pmap, kernel_pmap); 463 464 /* 465 * Region 5 is mapped via the vhpt. 466 */ 467 ia64_set_rr(IA64_RR_BASE(5), 468 (5 << 8) | (PAGE_SHIFT << 2) | 1); 469 470 /* 471 * Region 6 is direct mapped UC and region 7 is direct mapped 472 * WC. The details of this is controlled by the Alt {I,D}TLB 473 * handlers. Here we just make sure that they have the largest 474 * possible page size to minimise TLB usage. 475 */ 476 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (IA64_ID_PAGE_SHIFT << 2)); 477 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (IA64_ID_PAGE_SHIFT << 2)); 478 479 /* 480 * Reserve some memory for allocating pvs while bootstrapping 481 * the pv allocator. We need to have enough to cover mapping 482 * the kmem_alloc region used to allocate the initial_pvs in 483 * pmap_init. In general, the size of this region is 484 * approximately (# physical pages) * (size of pv entry). 485 */ 486 pvbootmax = ((physmem * sizeof(struct pv_entry)) >> PAGE_SHIFT) + 128; 487 pvbootentries = (struct pv_entry *) 488 pmap_steal_memory(pvbootmax * sizeof(struct pv_entry)); 489 pvbootnext = 0; 490 491 /* 492 * Clear out any random TLB entries left over from booting. 493 */ 494 pmap_invalidate_all(kernel_pmap); 495 496 map_gateway_page(); 497} 498 499/* 500 * Initialize the pmap module. 501 * Called by vm_init, to initialize any structures that the pmap 502 * system needs to map virtual memory. 503 * pmap_init has been enhanced to support in a fairly consistant 504 * way, discontiguous physical memory. 505 */ 506void 507pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) 508{ 509 int i; 510 int initial_pvs; 511 512 /* 513 * Allocate memory for random pmap data structures. Includes the 514 * pv_head_table. 515 */ 516 517 for(i = 0; i < vm_page_array_size; i++) { 518 vm_page_t m; 519 520 m = &vm_page_array[i]; 521 TAILQ_INIT(&m->md.pv_list); 522 m->md.pv_list_count = 0; 523 } 524 525 /* 526 * Init the pv free list and the PTE free list. 527 */ 528 initial_pvs = vm_page_array_size; 529 if (initial_pvs < MINPV) 530 initial_pvs = MINPV; 531 if (initial_pvs > MAXPV) 532 initial_pvs = MAXPV; 533 pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), 534 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE); 535 uma_prealloc(pvzone, initial_pvs); 536 537 ptezone = uma_zcreate("PT ENTRY", sizeof (struct ia64_lpte), 538 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE); 539 uma_prealloc(ptezone, initial_pvs); 540 541 /* 542 * Now it is safe to enable pv_table recording. 543 */ 544 pmap_initialized = TRUE; 545} 546 547/* 548 * Initialize the address space (zone) for the pv_entries. Set a 549 * high water mark so that the system can recover from excessive 550 * numbers of pv entries. 551 */ 552void 553pmap_init2() 554{ 555 int shpgperproc = PMAP_SHPGPERPROC; 556 557 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 558 pv_entry_max = shpgperproc * maxproc + vm_page_array_size; 559 pv_entry_high_water = 9 * (pv_entry_max / 10); 560} 561 562 563/*************************************************** 564 * Manipulate TLBs for a pmap 565 ***************************************************/ 566 567static void 568pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 569{ 570 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 571 ("invalidating TLB for non-current pmap")); 572 ia64_ptc_g(va, PAGE_SHIFT << 2); 573} 574 575static void 576pmap_invalidate_all_1(void *arg) 577{ 578 u_int64_t addr; 579 int i, j; 580 register_t psr; 581 582 psr = intr_disable(); 583 addr = pmap_ptc_e_base; 584 for (i = 0; i < pmap_ptc_e_count1; i++) { 585 for (j = 0; j < pmap_ptc_e_count2; j++) { 586 ia64_ptc_e(addr); 587 addr += pmap_ptc_e_stride2; 588 } 589 addr += pmap_ptc_e_stride1; 590 } 591 intr_restore(psr); 592} 593 594static void 595pmap_invalidate_all(pmap_t pmap) 596{ 597 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 598 ("invalidating TLB for non-current pmap")); 599 600 601#ifdef SMP 602 smp_rendezvous(0, pmap_invalidate_all_1, 0, 0); 603#else 604 pmap_invalidate_all_1(0); 605#endif 606} 607 608static u_int32_t 609pmap_allocate_rid(void) 610{ 611 uint64_t bit, bits; 612 int rid; 613 614 mtx_lock(&pmap_ridmutex); 615 if (pmap_ridcount == pmap_ridmax) 616 panic("pmap_allocate_rid: All Region IDs used"); 617 618 /* Find an index with a free bit. */ 619 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) { 620 pmap_rididx++; 621 if (pmap_rididx == pmap_ridmapsz) 622 pmap_rididx = 0; 623 } 624 rid = pmap_rididx * 64; 625 626 /* Find a free bit. */ 627 bit = 1UL; 628 while (bits & bit) { 629 rid++; 630 bit <<= 1; 631 } 632 633 pmap_ridmap[pmap_rididx] |= bit; 634 pmap_ridcount++; 635 mtx_unlock(&pmap_ridmutex); 636 637 return rid; 638} 639 640static void 641pmap_free_rid(u_int32_t rid) 642{ 643 uint64_t bit; 644 int idx; 645 646 idx = rid / 64; 647 bit = ~(1UL << (rid & 63)); 648 649 mtx_lock(&pmap_ridmutex); 650 pmap_ridmap[idx] &= bit; 651 pmap_ridcount--; 652 mtx_unlock(&pmap_ridmutex); 653} 654 655/*************************************************** 656 * Low level helper routines..... 657 ***************************************************/ 658 659/* 660 * Install a pte into the VHPT 661 */ 662static PMAP_INLINE void 663pmap_install_pte(struct ia64_lpte *vhpte, struct ia64_lpte *pte) 664{ 665 u_int64_t *vhp, *p; 666 667 vhp = (u_int64_t *)vhpte; 668 p = (u_int64_t *)pte; 669 670 critical_enter(); 671 672 /* Invalidate the tag so the VHPT walker will not match this entry. */ 673 vhp[2] = 1UL << 63; 674 ia64_mf(); 675 676 vhp[0] = p[0]; 677 vhp[1] = p[1]; 678 ia64_mf(); 679 680 /* Install a proper tag now that we're done. */ 681 vhp[2] = p[2]; 682 ia64_mf(); 683 684 critical_exit(); 685} 686 687/* 688 * Compare essential parts of pte. 689 */ 690static PMAP_INLINE int 691pmap_equal_pte(struct ia64_lpte *pte1, struct ia64_lpte *pte2) 692{ 693 return *(u_int64_t *) pte1 == *(u_int64_t *) pte2; 694} 695 696/* 697 * this routine defines the region(s) of memory that should 698 * not be tested for the modified bit. 699 */ 700static PMAP_INLINE int 701pmap_track_modified(vm_offset_t va) 702{ 703 if ((va < kmi.clean_sva) || (va >= kmi.clean_eva)) 704 return 1; 705 else 706 return 0; 707} 708 709/*************************************************** 710 * Page table page management routines..... 711 ***************************************************/ 712 713void 714pmap_pinit0(struct pmap *pmap) 715{ 716 /* kernel_pmap is the same as any other pmap. */ 717 pmap_pinit(pmap); 718} 719 720/* 721 * Initialize a preallocated and zeroed pmap structure, 722 * such as one in a vmspace structure. 723 */ 724void 725pmap_pinit(struct pmap *pmap) 726{ 727 int i; 728 729 pmap->pm_flags = 0; 730 for (i = 0; i < 5; i++) 731 pmap->pm_rid[i] = pmap_allocate_rid(); 732 pmap->pm_ptphint = NULL; 733 pmap->pm_active = 0; 734 TAILQ_INIT(&pmap->pm_pvlist); 735 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 736} 737 738/* 739 * Wire in kernel global address entries. To avoid a race condition 740 * between pmap initialization and pmap_growkernel, this procedure 741 * should be called after the vmspace is attached to the process 742 * but before this pmap is activated. 743 */ 744void 745pmap_pinit2(struct pmap *pmap) 746{ 747} 748 749/*************************************************** 750 * Pmap allocation/deallocation routines. 751 ***************************************************/ 752 753/* 754 * Release any resources held by the given physical map. 755 * Called when a pmap initialized by pmap_pinit is being released. 756 * Should only be called if the map contains no valid mappings. 757 */ 758void 759pmap_release(pmap_t pmap) 760{ 761 int i; 762 763 for (i = 0; i < 5; i++) 764 if (pmap->pm_rid[i]) 765 pmap_free_rid(pmap->pm_rid[i]); 766} 767 768/* 769 * grow the number of kernel page table entries, if needed 770 */ 771void 772pmap_growkernel(vm_offset_t addr) 773{ 774 struct ia64_lpte *ptepage; 775 vm_page_t nkpg; 776 777 if (kernel_vm_end >= addr) 778 return; 779 780 critical_enter(); 781 782 while (kernel_vm_end < addr) { 783 /* We could handle more by increasing the size of kptdir. */ 784 if (nkpt == MAXKPT) 785 panic("pmap_growkernel: out of kernel address space"); 786 787 nkpg = vm_page_alloc(NULL, nkpt, 788 VM_ALLOC_NOOBJ | VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 789 if (!nkpg) 790 panic("pmap_growkernel: no memory to grow kernel"); 791 792 ptepage = (struct ia64_lpte *) 793 IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg)); 794 bzero(ptepage, PAGE_SIZE); 795 ia64_kptdir[KPTE_DIR_INDEX(kernel_vm_end)] = ptepage; 796 797 nkpt++; 798 kernel_vm_end += PAGE_SIZE * NKPTEPG; 799 } 800 801 critical_exit(); 802} 803 804/*************************************************** 805 * page management routines. 806 ***************************************************/ 807 808/* 809 * free the pv_entry back to the free list 810 */ 811static PMAP_INLINE void 812free_pv_entry(pv_entry_t pv) 813{ 814 pv_entry_count--; 815 uma_zfree(pvzone, pv); 816} 817 818/* 819 * get a new pv_entry, allocating a block from the system 820 * when needed. 821 * the memory allocation is performed bypassing the malloc code 822 * because of the possibility of allocations at interrupt time. 823 */ 824static pv_entry_t 825get_pv_entry(void) 826{ 827 pv_entry_count++; 828 if (pv_entry_high_water && 829 (pv_entry_count > pv_entry_high_water) && 830 (pmap_pagedaemon_waken == 0)) { 831 pmap_pagedaemon_waken = 1; 832 wakeup (&vm_pages_needed); 833 } 834 return uma_zalloc(pvzone, M_NOWAIT); 835} 836 837/* 838 * Add an ia64_lpte to the VHPT. 839 */ 840static void 841pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va) 842{ 843 struct ia64_lpte *vhpte; 844 845 pmap_vhpt_inserts++; 846 pmap_vhpt_resident++; 847 848 vhpte = (struct ia64_lpte *) ia64_thash(va); 849 850 if (vhpte->pte_chain) 851 pmap_vhpt_collisions++; 852 853 mtx_lock(&pmap_vhptmutex); 854 855 pte->pte_chain = vhpte->pte_chain; 856 ia64_mf(); 857 vhpte->pte_chain = ia64_tpa((vm_offset_t)pte); 858 ia64_mf(); 859 860 if (!vhpte->pte_p && pte->pte_p) 861 pmap_install_pte(vhpte, pte); 862 863 mtx_unlock(&pmap_vhptmutex); 864} 865 866/* 867 * Update VHPT after a pte has changed. 868 */ 869static void 870pmap_update_vhpt(struct ia64_lpte *pte, vm_offset_t va) 871{ 872 struct ia64_lpte *vhpte; 873 874 vhpte = (struct ia64_lpte *)ia64_thash(va); 875 876 mtx_lock(&pmap_vhptmutex); 877 878 if ((!vhpte->pte_p || vhpte->pte_tag == pte->pte_tag) && pte->pte_p) 879 pmap_install_pte(vhpte, pte); 880 881 mtx_unlock(&pmap_vhptmutex); 882} 883 884/* 885 * Remove the ia64_lpte matching va from the VHPT. Return zero if it 886 * worked or an appropriate error code otherwise. 887 */ 888static int 889pmap_remove_vhpt(vm_offset_t va) 890{ 891 struct ia64_lpte *pte; 892 struct ia64_lpte *lpte; 893 struct ia64_lpte *vhpte; 894 u_int64_t tag; 895 896 vhpte = (struct ia64_lpte *)ia64_thash(va); 897 898 /* 899 * If the VHPTE is invalid, there can't be a collision chain. 900 */ 901 if (!vhpte->pte_p) { 902 KASSERT(!vhpte->pte_chain, ("bad vhpte")); 903 return (ENOENT); 904 } 905 906 lpte = vhpte; 907 tag = ia64_ttag(va); 908 909 mtx_lock(&pmap_vhptmutex); 910 911 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(vhpte->pte_chain); 912 KASSERT(pte != NULL, ("foo")); 913 914 while (pte->pte_tag != tag) { 915 lpte = pte; 916 if (pte->pte_chain == 0) { 917 mtx_unlock(&pmap_vhptmutex); 918 return (ENOENT); 919 } 920 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->pte_chain); 921 } 922 923 /* Snip this pv_entry out of the collision chain. */ 924 lpte->pte_chain = pte->pte_chain; 925 ia64_mf(); 926 927 /* 928 * If the VHPTE matches as well, change it to map the first 929 * element from the chain if there is one. 930 */ 931 if (vhpte->pte_tag == tag) { 932 if (vhpte->pte_chain) { 933 pte = (void*)IA64_PHYS_TO_RR7(vhpte->pte_chain); 934 pmap_install_pte(vhpte, pte); 935 } else 936 vhpte->pte_p = 0; 937 } 938 939 mtx_unlock(&pmap_vhptmutex); 940 pmap_vhpt_resident--; 941 return (0); 942} 943 944/* 945 * Find the ia64_lpte for the given va, if any. 946 */ 947static struct ia64_lpte * 948pmap_find_vhpt(vm_offset_t va) 949{ 950 struct ia64_lpte *pte; 951 u_int64_t tag; 952 953 tag = ia64_ttag(va); 954 pte = (struct ia64_lpte *)ia64_thash(va); 955 if (pte->pte_chain == 0) 956 return (NULL); 957 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->pte_chain); 958 while (pte->pte_tag != tag) { 959 if (pte->pte_chain == 0) 960 return (NULL); 961 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(pte->pte_chain); 962 } 963 return (pte); 964} 965 966/* 967 * Remove an entry from the list of managed mappings. 968 */ 969static int 970pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pv_entry_t pv) 971{ 972 if (!pv) { 973 if (m->md.pv_list_count < pmap->pm_stats.resident_count) { 974 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 975 if (pmap == pv->pv_pmap && va == pv->pv_va) 976 break; 977 } 978 } else { 979 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 980 if (va == pv->pv_va) 981 break; 982 } 983 } 984 } 985 986 if (pv) { 987 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 988 m->md.pv_list_count--; 989 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 990 vm_page_flag_clear(m, PG_WRITEABLE); 991 992 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 993 free_pv_entry(pv); 994 return 0; 995 } else { 996 return ENOENT; 997 } 998} 999 1000/* 1001 * Create a pv entry for page at pa for 1002 * (pmap, va). 1003 */ 1004static void 1005pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) 1006{ 1007 pv_entry_t pv; 1008 1009 pv = get_pv_entry(); 1010 pv->pv_pmap = pmap; 1011 pv->pv_va = va; 1012 1013 vm_page_lock_queues(); 1014 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1015 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1016 m->md.pv_list_count++; 1017 vm_page_unlock_queues(); 1018} 1019 1020/* 1021 * Routine: pmap_extract 1022 * Function: 1023 * Extract the physical page address associated 1024 * with the given map/virtual_address pair. 1025 */ 1026vm_offset_t 1027pmap_extract(pmap, va) 1028 register pmap_t pmap; 1029 vm_offset_t va; 1030{ 1031 struct ia64_lpte *pte; 1032 pmap_t oldpmap; 1033 1034 if (!pmap) 1035 return 0; 1036 1037 oldpmap = pmap_install(pmap); 1038 pte = pmap_find_vhpt(va); 1039 pmap_install(oldpmap); 1040 1041 if (!pte) 1042 return 0; 1043 1044 return pmap_pte_pa(pte); 1045} 1046 1047/* 1048 * Routine: pmap_extract_and_hold 1049 * Function: 1050 * Atomically extract and hold the physical page 1051 * with the given pmap and virtual address pair 1052 * if that mapping permits the given protection. 1053 */ 1054vm_page_t 1055pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 1056{ 1057 vm_paddr_t pa; 1058 vm_page_t m; 1059 1060 m = NULL; 1061 mtx_lock(&Giant); 1062 if ((pa = pmap_extract(pmap, va)) != 0) { 1063 m = PHYS_TO_VM_PAGE(pa); 1064 vm_page_lock_queues(); 1065 vm_page_hold(m); 1066 vm_page_unlock_queues(); 1067 } 1068 mtx_unlock(&Giant); 1069 return (m); 1070} 1071 1072/*************************************************** 1073 * Low level mapping routines..... 1074 ***************************************************/ 1075 1076/* 1077 * Find the kernel lpte for mapping the given virtual address, which 1078 * must be in the part of region 5 which we can cover with our kernel 1079 * 'page tables'. 1080 */ 1081static struct ia64_lpte * 1082pmap_find_kpte(vm_offset_t va) 1083{ 1084 KASSERT((va >> 61) == 5, 1085 ("kernel mapping 0x%lx not in region 5", va)); 1086 KASSERT(IA64_RR_MASK(va) < (nkpt * PAGE_SIZE * NKPTEPG), 1087 ("kernel mapping 0x%lx out of range", va)); 1088 return (&ia64_kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)]); 1089} 1090 1091/* 1092 * Find a pte suitable for mapping a user-space address. If one exists 1093 * in the VHPT, that one will be returned, otherwise a new pte is 1094 * allocated. 1095 */ 1096static struct ia64_lpte * 1097pmap_find_pte(vm_offset_t va) 1098{ 1099 struct ia64_lpte *pte; 1100 1101 if (va >= VM_MAXUSER_ADDRESS) 1102 return pmap_find_kpte(va); 1103 1104 pte = pmap_find_vhpt(va); 1105 if (!pte) { 1106 pte = uma_zalloc(ptezone, M_WAITOK); 1107 pte->pte_p = 0; 1108 } 1109 return pte; 1110} 1111 1112/* 1113 * Free a pte which is now unused. This simply returns it to the zone 1114 * allocator if it is a user mapping. For kernel mappings, clear the 1115 * valid bit to make it clear that the mapping is not currently used. 1116 */ 1117static void 1118pmap_free_pte(struct ia64_lpte *pte, vm_offset_t va) 1119{ 1120 if (va < VM_MAXUSER_ADDRESS) 1121 uma_zfree(ptezone, pte); 1122 else 1123 pte->pte_p = 0; 1124} 1125 1126/* 1127 * Set a pte to contain a valid mapping and enter it in the VHPT. If 1128 * the pte was orginally valid, then its assumed to already be in the 1129 * VHPT. 1130 */ 1131static void 1132pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa, 1133 int ig, int pl, int ar) 1134{ 1135 int wasvalid = pte->pte_p; 1136 1137 pte->pte_p = 1; 1138 pte->pte_ma = PTE_MA_WB; 1139 if (ig & PTE_IG_MANAGED) { 1140 pte->pte_a = 0; 1141 pte->pte_d = 0; 1142 } else { 1143 pte->pte_a = 1; 1144 pte->pte_d = 1; 1145 } 1146 pte->pte_pl = pl; 1147 pte->pte_ar = ar; 1148 pte->pte_ppn = pa >> 12; 1149 pte->pte_ed = 0; 1150 pte->pte_ig = ig; 1151 1152 pte->pte_ps = PAGE_SHIFT; 1153 pte->pte_key = 0; 1154 1155 pte->pte_tag = ia64_ttag(va); 1156 1157 if (wasvalid) { 1158 pmap_update_vhpt(pte, va); 1159 } else { 1160 pmap_enter_vhpt(pte, va); 1161 } 1162} 1163 1164/* 1165 * If a pte contains a valid mapping, clear it and update the VHPT. 1166 */ 1167static void 1168pmap_clear_pte(struct ia64_lpte *pte, vm_offset_t va) 1169{ 1170 if (pte->pte_p) { 1171 pmap_remove_vhpt(va); 1172 ia64_ptc_g(va, PAGE_SHIFT << 2); 1173 pte->pte_p = 0; 1174 } 1175} 1176 1177/* 1178 * Remove the (possibly managed) mapping represented by pte from the 1179 * given pmap. 1180 */ 1181static int 1182pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vm_offset_t va, 1183 pv_entry_t pv, int freepte) 1184{ 1185 int error; 1186 vm_page_t m; 1187 1188 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 1189 ("removing pte for non-current pmap")); 1190 1191 /* 1192 * First remove from the VHPT. 1193 */ 1194 error = pmap_remove_vhpt(va); 1195 if (error) 1196 return error; 1197 1198 /* 1199 * Make sure pmap_set_pte() knows it isn't in the VHPT. 1200 */ 1201 pte->pte_p = 0; 1202 1203 if (pte->pte_ig & PTE_IG_WIRED) 1204 pmap->pm_stats.wired_count -= 1; 1205 1206 pmap->pm_stats.resident_count -= 1; 1207 if (pte->pte_ig & PTE_IG_MANAGED) { 1208 m = PHYS_TO_VM_PAGE(pmap_pte_pa(pte)); 1209 if (pte->pte_d) 1210 if (pmap_track_modified(va)) 1211 vm_page_dirty(m); 1212 if (pte->pte_a) 1213 vm_page_flag_set(m, PG_REFERENCED); 1214 1215 if (freepte) 1216 pmap_free_pte(pte, va); 1217 return pmap_remove_entry(pmap, m, va, pv); 1218 } else { 1219 if (freepte) 1220 pmap_free_pte(pte, va); 1221 return 0; 1222 } 1223} 1224 1225/* 1226 * Extract the physical page address associated with a kernel 1227 * virtual address. 1228 */ 1229vm_paddr_t 1230pmap_kextract(vm_offset_t va) 1231{ 1232 struct ia64_lpte *pte; 1233 vm_offset_t gwpage; 1234 1235 KASSERT(va >= IA64_RR_BASE(5), ("Must be kernel VA")); 1236 1237 /* Regions 6 and 7 are direct mapped. */ 1238 if (va >= IA64_RR_BASE(6)) 1239 return (IA64_RR_MASK(va)); 1240 1241 /* EPC gateway page? */ 1242 gwpage = (vm_offset_t)ia64_get_k5(); 1243 if (va >= gwpage && va < gwpage + VM_GATEWAY_SIZE) 1244 return (IA64_RR_MASK((vm_offset_t)ia64_gateway_page)); 1245 1246 /* Bail out if the virtual address is beyond our limits. */ 1247 if (IA64_RR_MASK(va) >= nkpt * PAGE_SIZE * NKPTEPG) 1248 return (0); 1249 1250 pte = pmap_find_kpte(va); 1251 if (!pte->pte_p) 1252 return (0); 1253 return ((pte->pte_ppn << 12) | (va & PAGE_MASK)); 1254} 1255 1256/* 1257 * Add a list of wired pages to the kva 1258 * this routine is only used for temporary 1259 * kernel mappings that do not need to have 1260 * page modification or references recorded. 1261 * Note that old mappings are simply written 1262 * over. The page *must* be wired. 1263 */ 1264void 1265pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 1266{ 1267 int i; 1268 struct ia64_lpte *pte; 1269 1270 for (i = 0; i < count; i++) { 1271 vm_offset_t tva = va + i * PAGE_SIZE; 1272 int wasvalid; 1273 pte = pmap_find_kpte(tva); 1274 wasvalid = pte->pte_p; 1275 pmap_set_pte(pte, tva, VM_PAGE_TO_PHYS(m[i]), 1276 0, PTE_PL_KERN, PTE_AR_RWX); 1277 if (wasvalid) 1278 ia64_ptc_g(tva, PAGE_SHIFT << 2); 1279 } 1280} 1281 1282/* 1283 * this routine jerks page mappings from the 1284 * kernel -- it is meant only for temporary mappings. 1285 */ 1286void 1287pmap_qremove(vm_offset_t va, int count) 1288{ 1289 int i; 1290 struct ia64_lpte *pte; 1291 1292 for (i = 0; i < count; i++) { 1293 pte = pmap_find_kpte(va); 1294 pmap_clear_pte(pte, va); 1295 va += PAGE_SIZE; 1296 } 1297} 1298 1299/* 1300 * Add a wired page to the kva. 1301 */ 1302void 1303pmap_kenter(vm_offset_t va, vm_offset_t pa) 1304{ 1305 struct ia64_lpte *pte; 1306 int wasvalid; 1307 1308 pte = pmap_find_kpte(va); 1309 wasvalid = pte->pte_p; 1310 pmap_set_pte(pte, va, pa, 0, PTE_PL_KERN, PTE_AR_RWX); 1311 if (wasvalid) 1312 ia64_ptc_g(va, PAGE_SHIFT << 2); 1313} 1314 1315/* 1316 * Remove a page from the kva 1317 */ 1318void 1319pmap_kremove(vm_offset_t va) 1320{ 1321 struct ia64_lpte *pte; 1322 1323 pte = pmap_find_kpte(va); 1324 pmap_clear_pte(pte, va); 1325} 1326 1327/* 1328 * Used to map a range of physical addresses into kernel 1329 * virtual address space. 1330 * 1331 * The value passed in '*virt' is a suggested virtual address for 1332 * the mapping. Architectures which can support a direct-mapped 1333 * physical to virtual region can return the appropriate address 1334 * within that region, leaving '*virt' unchanged. Other 1335 * architectures should map the pages starting at '*virt' and 1336 * update '*virt' with the first usable address after the mapped 1337 * region. 1338 */ 1339vm_offset_t 1340pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 1341{ 1342 return IA64_PHYS_TO_RR7(start); 1343} 1344 1345/* 1346 * Remove a single page from a process address space 1347 */ 1348static void 1349pmap_remove_page(pmap_t pmap, vm_offset_t va) 1350{ 1351 struct ia64_lpte *pte; 1352 1353 KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), 1354 ("removing page for non-current pmap")); 1355 1356 pte = pmap_find_vhpt(va); 1357 if (pte) { 1358 pmap_remove_pte(pmap, pte, va, 0, 1); 1359 pmap_invalidate_page(pmap, va); 1360 } 1361 return; 1362} 1363 1364/* 1365 * Remove the given range of addresses from the specified map. 1366 * 1367 * It is assumed that the start and end are properly 1368 * rounded to the page size. 1369 */ 1370void 1371pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1372{ 1373 pmap_t oldpmap; 1374 vm_offset_t va; 1375 pv_entry_t pv; 1376 struct ia64_lpte *pte; 1377 1378 if (pmap == NULL) 1379 return; 1380 1381 if (pmap->pm_stats.resident_count == 0) 1382 return; 1383 1384 oldpmap = pmap_install(pmap); 1385 1386 /* 1387 * special handling of removing one page. a very 1388 * common operation and easy to short circuit some 1389 * code. 1390 */ 1391 if (sva + PAGE_SIZE == eva) { 1392 pmap_remove_page(pmap, sva); 1393 pmap_install(oldpmap); 1394 return; 1395 } 1396 1397 if (pmap->pm_stats.resident_count < ((eva - sva) >> PAGE_SHIFT)) { 1398 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1399 va = pv->pv_va; 1400 if (va >= sva && va < eva) { 1401 pte = pmap_find_vhpt(va); 1402 KASSERT(pte != NULL, ("pte")); 1403 pmap_remove_pte(pmap, pte, va, pv, 1); 1404 pmap_invalidate_page(pmap, va); 1405 } 1406 } 1407 1408 } else { 1409 for (va = sva; va < eva; va = va += PAGE_SIZE) { 1410 pte = pmap_find_vhpt(va); 1411 if (pte) { 1412 pmap_remove_pte(pmap, pte, va, 0, 1); 1413 pmap_invalidate_page(pmap, va); 1414 } 1415 } 1416 } 1417 1418 pmap_install(oldpmap); 1419} 1420 1421/* 1422 * Routine: pmap_remove_all 1423 * Function: 1424 * Removes this physical page from 1425 * all physical maps in which it resides. 1426 * Reflects back modify bits to the pager. 1427 * 1428 * Notes: 1429 * Original versions of this routine were very 1430 * inefficient because they iteratively called 1431 * pmap_remove (slow...) 1432 */ 1433 1434void 1435pmap_remove_all(vm_page_t m) 1436{ 1437 pmap_t oldpmap; 1438 pv_entry_t pv; 1439 int s; 1440 1441#if defined(PMAP_DIAGNOSTIC) 1442 /* 1443 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 1444 * pages! 1445 */ 1446 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) { 1447 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", VM_PAGE_TO_PHYS(m)); 1448 } 1449#endif 1450 1451 s = splvm(); 1452 1453 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1454 struct ia64_lpte *pte; 1455 pmap_t pmap = pv->pv_pmap; 1456 vm_offset_t va = pv->pv_va; 1457 1458 oldpmap = pmap_install(pmap); 1459 pte = pmap_find_vhpt(va); 1460 KASSERT(pte != NULL, ("pte")); 1461 if (pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(m)) 1462 panic("pmap_remove_all: pv_table for %lx is inconsistent", VM_PAGE_TO_PHYS(m)); 1463 pmap_remove_pte(pmap, pte, va, pv, 1); 1464 pmap_invalidate_page(pmap, va); 1465 pmap_install(oldpmap); 1466 } 1467 1468 vm_page_flag_clear(m, PG_WRITEABLE); 1469 1470 splx(s); 1471 return; 1472} 1473 1474/* 1475 * Set the physical protection on the 1476 * specified range of this map as requested. 1477 */ 1478void 1479pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1480{ 1481 pmap_t oldpmap; 1482 struct ia64_lpte *pte; 1483 int newprot; 1484 1485 if (pmap == NULL) 1486 return; 1487 1488 oldpmap = pmap_install(pmap); 1489 1490 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1491 pmap_remove(pmap, sva, eva); 1492 pmap_install(oldpmap); 1493 return; 1494 } 1495 1496 if (prot & VM_PROT_WRITE) { 1497 pmap_install(oldpmap); 1498 return; 1499 } 1500 1501 newprot = pte_prot(pmap, prot); 1502 1503 if ((sva & PAGE_MASK) || (eva & PAGE_MASK)) 1504 panic("pmap_protect: unaligned addresses"); 1505 1506 while (sva < eva) { 1507 /* 1508 * If page is invalid, skip this page 1509 */ 1510 pte = pmap_find_vhpt(sva); 1511 if (!pte) { 1512 sva += PAGE_SIZE; 1513 continue; 1514 } 1515 1516 if (pmap_pte_prot(pte) != newprot) { 1517 if (pte->pte_ig & PTE_IG_MANAGED) { 1518 vm_offset_t pa = pmap_pte_pa(pte); 1519 vm_page_t m = PHYS_TO_VM_PAGE(pa); 1520 if (pte->pte_d) { 1521 if (pmap_track_modified(sva)) 1522 vm_page_dirty(m); 1523 pte->pte_d = 0; 1524 } 1525 if (pte->pte_a) { 1526 vm_page_flag_set(m, PG_REFERENCED); 1527 pte->pte_a = 0; 1528 } 1529 } 1530 pmap_pte_set_prot(pte, newprot); 1531 pmap_update_vhpt(pte, sva); 1532 pmap_invalidate_page(pmap, sva); 1533 } 1534 1535 sva += PAGE_SIZE; 1536 } 1537 pmap_install(oldpmap); 1538} 1539 1540/* 1541 * Insert the given physical page (p) at 1542 * the specified virtual address (v) in the 1543 * target physical map with the protection requested. 1544 * 1545 * If specified, the page will be wired down, meaning 1546 * that the related pte can not be reclaimed. 1547 * 1548 * NB: This is the only routine which MAY NOT lazy-evaluate 1549 * or lose information. That is, this routine must actually 1550 * insert this page into the given map NOW. 1551 */ 1552void 1553pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1554 boolean_t wired) 1555{ 1556 pmap_t oldpmap; 1557 vm_offset_t pa; 1558 vm_offset_t opa; 1559 struct ia64_lpte origpte; 1560 struct ia64_lpte *pte; 1561 int managed; 1562 1563 if (pmap == NULL) 1564 return; 1565 1566 oldpmap = pmap_install(pmap); 1567 1568 va &= ~PAGE_MASK; 1569#ifdef PMAP_DIAGNOSTIC 1570 if (va > VM_MAX_KERNEL_ADDRESS) 1571 panic("pmap_enter: toobig"); 1572#endif 1573 1574 /* 1575 * Find (or create) a pte for the given mapping. 1576 */ 1577 pte = pmap_find_pte(va); 1578 origpte = *pte; 1579 1580 if (origpte.pte_p) 1581 opa = pmap_pte_pa(&origpte); 1582 else 1583 opa = 0; 1584 managed = 0; 1585 1586 pa = VM_PAGE_TO_PHYS(m) & ~PAGE_MASK; 1587 1588 /* 1589 * Mapping has not changed, must be protection or wiring change. 1590 */ 1591 if (origpte.pte_p && (opa == pa)) { 1592 /* 1593 * Wiring change, just update stats. We don't worry about 1594 * wiring PT pages as they remain resident as long as there 1595 * are valid mappings in them. Hence, if a user page is wired, 1596 * the PT page will be also. 1597 */ 1598 if (wired && ((origpte.pte_ig & PTE_IG_WIRED) == 0)) 1599 pmap->pm_stats.wired_count++; 1600 else if (!wired && (origpte.pte_ig & PTE_IG_WIRED)) 1601 pmap->pm_stats.wired_count--; 1602 1603 /* 1604 * We might be turning off write access to the page, 1605 * so we go ahead and sense modify status. 1606 */ 1607 if (origpte.pte_ig & PTE_IG_MANAGED) { 1608 if (origpte.pte_d && pmap_track_modified(va)) { 1609 vm_page_t om; 1610 om = PHYS_TO_VM_PAGE(opa); 1611 vm_page_dirty(om); 1612 } 1613 } 1614 1615 managed = origpte.pte_ig & PTE_IG_MANAGED; 1616 goto validate; 1617 } 1618 /* 1619 * Mapping has changed, invalidate old range and fall 1620 * through to handle validating new mapping. 1621 */ 1622 if (opa) { 1623 int error; 1624 vm_page_lock_queues(); 1625 error = pmap_remove_pte(pmap, pte, va, 0, 0); 1626 vm_page_unlock_queues(); 1627 if (error) 1628 panic("pmap_enter: pte vanished, va: 0x%lx", va); 1629 } 1630 1631 /* 1632 * Enter on the PV list if part of our managed memory. 1633 */ 1634 if (pmap_initialized && (m->flags & PG_FICTITIOUS) == 0) { 1635 pmap_insert_entry(pmap, va, m); 1636 managed |= PTE_IG_MANAGED; 1637 } 1638 1639 /* 1640 * Increment counters 1641 */ 1642 pmap->pm_stats.resident_count++; 1643 if (wired) 1644 pmap->pm_stats.wired_count++; 1645 1646validate: 1647 1648 /* 1649 * Now validate mapping with desired protection/wiring. This 1650 * adds the pte to the VHPT if necessary. 1651 */ 1652 pmap_set_pte(pte, va, pa, managed | (wired ? PTE_IG_WIRED : 0), 1653 pte_prot_pl(pmap, prot), pte_prot_ar(pmap, prot)); 1654 1655 /* 1656 * if the mapping or permission bits are different, we need 1657 * to invalidate the page. 1658 */ 1659 if (!pmap_equal_pte(&origpte, pte)) 1660 pmap_invalidate_page(pmap, va); 1661 1662 pmap_install(oldpmap); 1663} 1664 1665/* 1666 * this code makes some *MAJOR* assumptions: 1667 * 1. Current pmap & pmap exists. 1668 * 2. Not wired. 1669 * 3. Read access. 1670 * 4. No page table pages. 1671 * 5. Tlbflush is deferred to calling procedure. 1672 * 6. Page IS managed. 1673 * but is *MUCH* faster than pmap_enter... 1674 */ 1675 1676vm_page_t 1677pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) 1678{ 1679 struct ia64_lpte *pte; 1680 pmap_t oldpmap; 1681 1682 oldpmap = pmap_install(pmap); 1683 1684 pte = pmap_find_pte(va); 1685 if (pte->pte_p) 1686 goto reinstall; 1687 1688 /* 1689 * Enter on the PV list since its part of our managed memory. 1690 */ 1691 pmap_insert_entry(pmap, va, m); 1692 1693 /* 1694 * Increment counters 1695 */ 1696 pmap->pm_stats.resident_count++; 1697 1698 /* 1699 * Initialise PTE with read-only protection and enter into VHPT. 1700 */ 1701 pmap_set_pte(pte, va, VM_PAGE_TO_PHYS(m), 1702 PTE_IG_MANAGED, 1703 PTE_PL_USER, PTE_AR_R); 1704reinstall: 1705 pmap_install(oldpmap); 1706 return (NULL); 1707} 1708 1709/* 1710 * Make temporary mapping for a physical address. This is called 1711 * during dump. 1712 */ 1713void * 1714pmap_kenter_temporary(vm_offset_t pa, int i) 1715{ 1716 return (void *) IA64_PHYS_TO_RR7(pa - (i * PAGE_SIZE)); 1717} 1718 1719/* 1720 * pmap_object_init_pt preloads the ptes for a given object 1721 * into the specified pmap. This eliminates the blast of soft 1722 * faults on process startup and immediately after an mmap. 1723 */ 1724void 1725pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 1726 vm_object_t object, vm_pindex_t pindex, 1727 vm_size_t size) 1728{ 1729 1730 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1731 KASSERT(object->type == OBJT_DEVICE, 1732 ("pmap_object_init_pt: non-device object")); 1733} 1734 1735/* 1736 * Routine: pmap_change_wiring 1737 * Function: Change the wiring attribute for a map/virtual-address 1738 * pair. 1739 * In/out conditions: 1740 * The mapping must already exist in the pmap. 1741 */ 1742void 1743pmap_change_wiring(pmap, va, wired) 1744 register pmap_t pmap; 1745 vm_offset_t va; 1746 boolean_t wired; 1747{ 1748 pmap_t oldpmap; 1749 struct ia64_lpte *pte; 1750 1751 if (pmap == NULL) 1752 return; 1753 1754 oldpmap = pmap_install(pmap); 1755 1756 pte = pmap_find_vhpt(va); 1757 KASSERT(pte != NULL, ("pte")); 1758 if (wired && !pmap_pte_w(pte)) 1759 pmap->pm_stats.wired_count++; 1760 else if (!wired && pmap_pte_w(pte)) 1761 pmap->pm_stats.wired_count--; 1762 1763 /* 1764 * Wiring is not a hardware characteristic so there is no need to 1765 * invalidate TLB. 1766 */ 1767 pmap_pte_set_w(pte, wired); 1768 1769 pmap_install(oldpmap); 1770} 1771 1772 1773 1774/* 1775 * Copy the range specified by src_addr/len 1776 * from the source map to the range dst_addr/len 1777 * in the destination map. 1778 * 1779 * This routine is only advisory and need not do anything. 1780 */ 1781 1782void 1783pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 1784 vm_offset_t src_addr) 1785{ 1786} 1787 1788 1789/* 1790 * pmap_zero_page zeros the specified hardware page by 1791 * mapping it into virtual memory and using bzero to clear 1792 * its contents. 1793 */ 1794 1795void 1796pmap_zero_page(vm_page_t m) 1797{ 1798 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 1799 bzero((caddr_t) va, PAGE_SIZE); 1800} 1801 1802 1803/* 1804 * pmap_zero_page_area zeros the specified hardware page by 1805 * mapping it into virtual memory and using bzero to clear 1806 * its contents. 1807 * 1808 * off and size must reside within a single page. 1809 */ 1810 1811void 1812pmap_zero_page_area(vm_page_t m, int off, int size) 1813{ 1814 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 1815 bzero((char *)(caddr_t)va + off, size); 1816} 1817 1818 1819/* 1820 * pmap_zero_page_idle zeros the specified hardware page by 1821 * mapping it into virtual memory and using bzero to clear 1822 * its contents. This is for the vm_idlezero process. 1823 */ 1824 1825void 1826pmap_zero_page_idle(vm_page_t m) 1827{ 1828 vm_offset_t va = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 1829 bzero((caddr_t) va, PAGE_SIZE); 1830} 1831 1832 1833/* 1834 * pmap_copy_page copies the specified (machine independent) 1835 * page by mapping the page into virtual memory and using 1836 * bcopy to copy the page, one machine dependent page at a 1837 * time. 1838 */ 1839void 1840pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1841{ 1842 vm_offset_t src = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(msrc)); 1843 vm_offset_t dst = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(mdst)); 1844 bcopy((caddr_t) src, (caddr_t) dst, PAGE_SIZE); 1845} 1846 1847/* 1848 * Returns true if the pmap's pv is one of the first 1849 * 16 pvs linked to from this page. This count may 1850 * be changed upwards or downwards in the future; it 1851 * is only necessary that true be returned for a small 1852 * subset of pmaps for proper page aging. 1853 */ 1854boolean_t 1855pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 1856{ 1857 pv_entry_t pv; 1858 int loops = 0; 1859 int s; 1860 1861 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 1862 return FALSE; 1863 1864 s = splvm(); 1865 1866 /* 1867 * Not found, check current mappings returning immediately if found. 1868 */ 1869 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1870 if (pv->pv_pmap == pmap) { 1871 splx(s); 1872 return TRUE; 1873 } 1874 loops++; 1875 if (loops >= 16) 1876 break; 1877 } 1878 splx(s); 1879 return (FALSE); 1880} 1881 1882#define PMAP_REMOVE_PAGES_CURPROC_ONLY 1883/* 1884 * Remove all pages from specified address space 1885 * this aids process exit speeds. Also, this code 1886 * is special cased for current process only, but 1887 * can have the more generic (and slightly slower) 1888 * mode enabled. This is much faster than pmap_remove 1889 * in the case of running down an entire address space. 1890 */ 1891void 1892pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1893{ 1894 pv_entry_t pv, npv; 1895 int s; 1896 1897#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY 1898 if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))) { 1899 printf("warning: pmap_remove_pages called with non-current pmap\n"); 1900 return; 1901 } 1902#endif 1903 1904 s = splvm(); 1905 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); 1906 pv; 1907 pv = npv) { 1908 struct ia64_lpte *pte; 1909 1910 npv = TAILQ_NEXT(pv, pv_plist); 1911 1912 if (pv->pv_va >= eva || pv->pv_va < sva) { 1913 continue; 1914 } 1915 1916 pte = pmap_find_vhpt(pv->pv_va); 1917 KASSERT(pte != NULL, ("pte")); 1918 if (pte->pte_ig & PTE_IG_WIRED) 1919 continue; 1920 1921 pmap_remove_pte(pmap, pte, pv->pv_va, pv, 1); 1922 } 1923 splx(s); 1924 1925 pmap_invalidate_all(pmap); 1926} 1927 1928/* 1929 * pmap_page_protect: 1930 * 1931 * Lower the permission for all mappings to a given page. 1932 */ 1933void 1934pmap_page_protect(vm_page_t m, vm_prot_t prot) 1935{ 1936 pv_entry_t pv; 1937 1938 if ((prot & VM_PROT_WRITE) != 0) 1939 return; 1940 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 1941 if ((m->flags & PG_WRITEABLE) == 0) 1942 return; 1943 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1944 int newprot = pte_prot(pv->pv_pmap, prot); 1945 pmap_t oldpmap = pmap_install(pv->pv_pmap); 1946 struct ia64_lpte *pte; 1947 pte = pmap_find_vhpt(pv->pv_va); 1948 KASSERT(pte != NULL, ("pte")); 1949 pmap_pte_set_prot(pte, newprot); 1950 pmap_update_vhpt(pte, pv->pv_va); 1951 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1952 pmap_install(oldpmap); 1953 } 1954 vm_page_flag_clear(m, PG_WRITEABLE); 1955 } else { 1956 pmap_remove_all(m); 1957 } 1958} 1959 1960/* 1961 * pmap_ts_referenced: 1962 * 1963 * Return a count of reference bits for a page, clearing those bits. 1964 * It is not necessary for every reference bit to be cleared, but it 1965 * is necessary that 0 only be returned when there are truly no 1966 * reference bits set. 1967 * 1968 * XXX: The exact number of bits to check and clear is a matter that 1969 * should be tested and standardized at some point in the future for 1970 * optimal aging of shared pages. 1971 */ 1972int 1973pmap_ts_referenced(vm_page_t m) 1974{ 1975 pv_entry_t pv; 1976 int count = 0; 1977 1978 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 1979 return 0; 1980 1981 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1982 pmap_t oldpmap = pmap_install(pv->pv_pmap); 1983 struct ia64_lpte *pte; 1984 pte = pmap_find_vhpt(pv->pv_va); 1985 KASSERT(pte != NULL, ("pte")); 1986 if (pte->pte_a) { 1987 count++; 1988 pte->pte_a = 0; 1989 pmap_update_vhpt(pte, pv->pv_va); 1990 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1991 } 1992 pmap_install(oldpmap); 1993 } 1994 1995 return count; 1996} 1997 1998#if 0 1999/* 2000 * pmap_is_referenced: 2001 * 2002 * Return whether or not the specified physical page was referenced 2003 * in any physical maps. 2004 */ 2005static boolean_t 2006pmap_is_referenced(vm_page_t m) 2007{ 2008 pv_entry_t pv; 2009 2010 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2011 return FALSE; 2012 2013 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2014 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2015 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2016 pmap_install(oldpmap); 2017 KASSERT(pte != NULL, ("pte")); 2018 if (pte->pte_a) 2019 return 1; 2020 } 2021 2022 return 0; 2023} 2024#endif 2025 2026/* 2027 * pmap_is_modified: 2028 * 2029 * Return whether or not the specified physical page was modified 2030 * in any physical maps. 2031 */ 2032boolean_t 2033pmap_is_modified(vm_page_t m) 2034{ 2035 pv_entry_t pv; 2036 2037 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2038 return FALSE; 2039 2040 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2041 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2042 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2043 pmap_install(oldpmap); 2044 KASSERT(pte != NULL, ("pte")); 2045 if (pte->pte_d) 2046 return 1; 2047 } 2048 2049 return 0; 2050} 2051 2052/* 2053 * pmap_is_prefaultable: 2054 * 2055 * Return whether or not the specified virtual address is elgible 2056 * for prefault. 2057 */ 2058boolean_t 2059pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2060{ 2061 struct ia64_lpte *pte; 2062 2063 pte = pmap_find_vhpt(addr); 2064 if (pte && pte->pte_p) 2065 return (FALSE); 2066 return (TRUE); 2067} 2068 2069/* 2070 * Clear the modify bits on the specified physical page. 2071 */ 2072void 2073pmap_clear_modify(vm_page_t m) 2074{ 2075 pv_entry_t pv; 2076 2077 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2078 return; 2079 2080 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2081 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2082 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2083 KASSERT(pte != NULL, ("pte")); 2084 if (pte->pte_d) { 2085 pte->pte_d = 0; 2086 pmap_update_vhpt(pte, pv->pv_va); 2087 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2088 } 2089 pmap_install(oldpmap); 2090 } 2091} 2092 2093/* 2094 * pmap_clear_reference: 2095 * 2096 * Clear the reference bit on the specified physical page. 2097 */ 2098void 2099pmap_clear_reference(vm_page_t m) 2100{ 2101 pv_entry_t pv; 2102 2103 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2104 return; 2105 2106 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2107 pmap_t oldpmap = pmap_install(pv->pv_pmap); 2108 struct ia64_lpte *pte = pmap_find_vhpt(pv->pv_va); 2109 KASSERT(pte != NULL, ("pte")); 2110 if (pte->pte_a) { 2111 pte->pte_a = 0; 2112 pmap_update_vhpt(pte, pv->pv_va); 2113 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2114 } 2115 pmap_install(oldpmap); 2116 } 2117} 2118 2119/* 2120 * Miscellaneous support routines follow 2121 */ 2122 2123static void 2124ia64_protection_init() 2125{ 2126 int prot, *kp, *up; 2127 2128 kp = protection_codes[0]; 2129 up = protection_codes[1]; 2130 2131 for (prot = 0; prot < 8; prot++) { 2132 switch (prot) { 2133 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 2134 *kp++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2135 *up++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2136 break; 2137 2138 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 2139 *kp++ = (PTE_AR_X_RX << 2) | PTE_PL_KERN; 2140 *up++ = (PTE_AR_X_RX << 2) | PTE_PL_USER; 2141 break; 2142 2143 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 2144 *kp++ = (PTE_AR_RW << 2) | PTE_PL_KERN; 2145 *up++ = (PTE_AR_RW << 2) | PTE_PL_USER; 2146 break; 2147 2148 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 2149 *kp++ = (PTE_AR_RWX << 2) | PTE_PL_KERN; 2150 *up++ = (PTE_AR_RWX << 2) | PTE_PL_USER; 2151 break; 2152 2153 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 2154 *kp++ = (PTE_AR_R << 2) | PTE_PL_KERN; 2155 *up++ = (PTE_AR_R << 2) | PTE_PL_USER; 2156 break; 2157 2158 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 2159 *kp++ = (PTE_AR_RX << 2) | PTE_PL_KERN; 2160 *up++ = (PTE_AR_RX << 2) | PTE_PL_USER; 2161 break; 2162 2163 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 2164 *kp++ = (PTE_AR_RW << 2) | PTE_PL_KERN; 2165 *up++ = (PTE_AR_RW << 2) | PTE_PL_USER; 2166 break; 2167 2168 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 2169 *kp++ = (PTE_AR_RWX << 2) | PTE_PL_KERN; 2170 *up++ = (PTE_AR_RWX << 2) | PTE_PL_USER; 2171 break; 2172 } 2173 } 2174} 2175 2176/* 2177 * Map a set of physical memory pages into the kernel virtual 2178 * address space. Return a pointer to where it is mapped. This 2179 * routine is intended to be used for mapping device memory, 2180 * NOT real memory. 2181 */ 2182void * 2183pmap_mapdev(vm_offset_t pa, vm_size_t size) 2184{ 2185 return (void*) IA64_PHYS_TO_RR6(pa); 2186} 2187 2188/* 2189 * 'Unmap' a range mapped by pmap_mapdev(). 2190 */ 2191void 2192pmap_unmapdev(vm_offset_t va, vm_size_t size) 2193{ 2194 return; 2195} 2196 2197/* 2198 * perform the pmap work for mincore 2199 */ 2200int 2201pmap_mincore(pmap_t pmap, vm_offset_t addr) 2202{ 2203 pmap_t oldpmap; 2204 struct ia64_lpte *pte; 2205 int val = 0; 2206 2207 oldpmap = pmap_install(pmap); 2208 pte = pmap_find_vhpt(addr); 2209 pmap_install(oldpmap); 2210 2211 if (!pte) 2212 return 0; 2213 2214 if (pmap_pte_v(pte)) { 2215 vm_page_t m; 2216 vm_offset_t pa; 2217 2218 val = MINCORE_INCORE; 2219 if ((pte->pte_ig & PTE_IG_MANAGED) == 0) 2220 return val; 2221 2222 pa = pmap_pte_pa(pte); 2223 2224 m = PHYS_TO_VM_PAGE(pa); 2225 2226 /* 2227 * Modified by us 2228 */ 2229 if (pte->pte_d) 2230 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER; 2231 else { 2232 /* 2233 * Modified by someone 2234 */ 2235 vm_page_lock_queues(); 2236 if (pmap_is_modified(m)) 2237 val |= MINCORE_MODIFIED_OTHER; 2238 vm_page_unlock_queues(); 2239 } 2240 /* 2241 * Referenced by us 2242 */ 2243 if (pte->pte_a) 2244 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER; 2245 else { 2246 /* 2247 * Referenced by someone 2248 */ 2249 vm_page_lock_queues(); 2250 if (pmap_ts_referenced(m)) { 2251 val |= MINCORE_REFERENCED_OTHER; 2252 vm_page_flag_set(m, PG_REFERENCED); 2253 } 2254 vm_page_unlock_queues(); 2255 } 2256 } 2257 return val; 2258} 2259 2260void 2261pmap_activate(struct thread *td) 2262{ 2263 pmap_install(vmspace_pmap(td->td_proc->p_vmspace)); 2264} 2265 2266pmap_t 2267pmap_switch(pmap_t pm) 2268{ 2269 pmap_t prevpm; 2270 int i; 2271 2272 mtx_assert(&sched_lock, MA_OWNED); 2273 2274 prevpm = PCPU_GET(current_pmap); 2275 if (prevpm == pm) 2276 return (prevpm); 2277 if (prevpm != NULL) 2278 atomic_clear_32(&prevpm->pm_active, PCPU_GET(cpumask)); 2279 if (pm == NULL) { 2280 for (i = 0; i < 5; i++) { 2281 ia64_set_rr(IA64_RR_BASE(i), 2282 (i << 8)|(PAGE_SHIFT << 2)|1); 2283 } 2284 } else { 2285 for (i = 0; i < 5; i++) { 2286 ia64_set_rr(IA64_RR_BASE(i), 2287 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1); 2288 } 2289 atomic_set_32(&pm->pm_active, PCPU_GET(cpumask)); 2290 } 2291 PCPU_SET(current_pmap, pm); 2292 __asm __volatile("srlz.d"); 2293 return (prevpm); 2294} 2295 2296static pmap_t 2297pmap_install(pmap_t pm) 2298{ 2299 pmap_t prevpm; 2300 2301 mtx_lock_spin(&sched_lock); 2302 prevpm = pmap_switch(pm); 2303 mtx_unlock_spin(&sched_lock); 2304 return (prevpm); 2305} 2306 2307vm_offset_t 2308pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) 2309{ 2310 2311 return addr; 2312} 2313 2314#include "opt_ddb.h" 2315 2316#ifdef DDB 2317 2318#include <ddb/ddb.h> 2319 2320static const char* psnames[] = { 2321 "1B", "2B", "4B", "8B", 2322 "16B", "32B", "64B", "128B", 2323 "256B", "512B", "1K", "2K", 2324 "4K", "8K", "16K", "32K", 2325 "64K", "128K", "256K", "512K", 2326 "1M", "2M", "4M", "8M", 2327 "16M", "32M", "64M", "128M", 2328 "256M", "512M", "1G", "2G" 2329}; 2330 2331static void 2332print_trs(int type) 2333{ 2334 struct ia64_pal_result res; 2335 int i, maxtr; 2336 struct { 2337 struct ia64_pte pte; 2338 struct ia64_itir itir; 2339 struct ia64_ifa ifa; 2340 struct ia64_rr rr; 2341 } buf; 2342 static const char* manames[] = { 2343 "WB", "bad", "bad", "bad", 2344 "UC", "UCE", "WC", "NaT", 2345 2346 }; 2347 2348 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 2349 if (res.pal_status != 0) { 2350 db_printf("Can't get VM summary\n"); 2351 return; 2352 } 2353 2354 if (type == 0) 2355 maxtr = (res.pal_result[0] >> 40) & 0xff; 2356 else 2357 maxtr = (res.pal_result[0] >> 32) & 0xff; 2358 2359 db_printf("V RID Virtual Page Physical Page PgSz ED AR PL D A MA P KEY\n"); 2360 for (i = 0; i <= maxtr; i++) { 2361 bzero(&buf, sizeof(buf)); 2362 res = ia64_call_pal_stacked_physical 2363 (PAL_VM_TR_READ, i, type, ia64_tpa((u_int64_t) &buf)); 2364 if (!(res.pal_result[0] & 1)) 2365 buf.pte.pte_ar = 0; 2366 if (!(res.pal_result[0] & 2)) 2367 buf.pte.pte_pl = 0; 2368 if (!(res.pal_result[0] & 4)) 2369 buf.pte.pte_d = 0; 2370 if (!(res.pal_result[0] & 8)) 2371 buf.pte.pte_ma = 0; 2372 db_printf( 2373 "%d %06x %013lx %013lx %4s %d %d %d %d %d %-3s %d %06x\n", 2374 buf.ifa.ifa_ig & 1, 2375 buf.rr.rr_rid, 2376 buf.ifa.ifa_vpn, 2377 buf.pte.pte_ppn, 2378 psnames[buf.itir.itir_ps], 2379 buf.pte.pte_ed, 2380 buf.pte.pte_ar, 2381 buf.pte.pte_pl, 2382 buf.pte.pte_d, 2383 buf.pte.pte_a, 2384 manames[buf.pte.pte_ma], 2385 buf.pte.pte_p, 2386 buf.itir.itir_key); 2387 } 2388} 2389 2390DB_COMMAND(itr, db_itr) 2391{ 2392 print_trs(0); 2393} 2394 2395DB_COMMAND(dtr, db_dtr) 2396{ 2397 print_trs(1); 2398} 2399 2400DB_COMMAND(rr, db_rr) 2401{ 2402 int i; 2403 u_int64_t t; 2404 struct ia64_rr rr; 2405 2406 printf("RR RID PgSz VE\n"); 2407 for (i = 0; i < 8; i++) { 2408 __asm __volatile ("mov %0=rr[%1]" 2409 : "=r"(t) 2410 : "r"(IA64_RR_BASE(i))); 2411 *(u_int64_t *) &rr = t; 2412 printf("%d %06x %4s %d\n", 2413 i, rr.rr_rid, psnames[rr.rr_ps], rr.rr_ve); 2414 } 2415} 2416 2417DB_COMMAND(thash, db_thash) 2418{ 2419 if (!have_addr) 2420 return; 2421 2422 db_printf("%p\n", (void *) ia64_thash(addr)); 2423} 2424 2425DB_COMMAND(ttag, db_ttag) 2426{ 2427 if (!have_addr) 2428 return; 2429 2430 db_printf("0x%lx\n", ia64_ttag(addr)); 2431} 2432 2433#endif 2434