95 96/* 97 * Manages physical address maps. 98 * 99 * In addition to hardware address maps, this module is called upon to 100 * provide software-use-only maps which may or may not be stored in the 101 * same form as hardware maps. These pseudo-maps are used to store 102 * intermediate results from copy operations to and from address spaces. 103 * 104 * Since the information managed by this module is also stored by the 105 * logical address mapping module, this module may throw away valid virtual 106 * to physical mappings at almost any time. However, invalidations of 107 * mappings must be done as requested. 108 * 109 * In order to cope with hardware architectures which make virtual to 110 * physical map invalidates expensive, this module may delay invalidate 111 * reduced protection operations until such time as they are actually 112 * necessary. This module is given full information as to which processors 113 * are currently using which maps, and to when physical maps must be made 114 * correct. 115 */ 116 117#include <sys/param.h> 118#include <sys/kernel.h> 119#include <sys/ktr.h> 120#include <sys/lock.h> 121#include <sys/msgbuf.h> 122#include <sys/mutex.h> 123#include <sys/proc.h> 124#include <sys/sysctl.h> 125#include <sys/systm.h> 126#include <sys/vmmeter.h> 127 128#include <dev/ofw/openfirm.h> 129 130#include <vm/vm.h> 131#include <vm/vm_param.h> 132#include <vm/vm_kern.h> 133#include <vm/vm_page.h> 134#include <vm/vm_map.h> 135#include <vm/vm_object.h> 136#include <vm/vm_extern.h> 137#include <vm/vm_pageout.h> 138#include <vm/vm_pager.h> 139#include <vm/uma.h> 140 141#include <machine/powerpc.h> 142#include <machine/bat.h> 143#include <machine/frame.h> 144#include <machine/md_var.h> 145#include <machine/psl.h> 146#include <machine/pte.h> 147#include <machine/sr.h> 148 149#define PMAP_DEBUG 150 151#define TODO panic("%s: not implemented", __func__); 152 153#define PMAP_LOCK(pm) 154#define PMAP_UNLOCK(pm) 155 156#define TLBIE(va) __asm __volatile("tlbie %0" :: "r"(va)) 157#define TLBSYNC() __asm __volatile("tlbsync"); 158#define SYNC() __asm __volatile("sync"); 159#define EIEIO() __asm __volatile("eieio"); 160 161#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4)) 162#define VSID_TO_SR(vsid) ((vsid) & 0xf) 163#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff) 164 165#define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */ 166#define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */ 167#define PVO_WIRED 0x0010 /* PVO entry is wired */ 168#define PVO_MANAGED 0x0020 /* PVO entry is managed */ 169#define PVO_EXECUTABLE 0x0040 /* PVO entry is executable */ 170#define PVO_BOOTSTRAP 0x0080 /* PVO entry allocated during 171 bootstrap */ 172#define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF) 173#define PVO_ISEXECUTABLE(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE) 174#define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK) 175#define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID) 176#define PVO_PTEGIDX_CLR(pvo) \ 177 ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) 178#define PVO_PTEGIDX_SET(pvo, i) \ 179 ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) 180 181#define PMAP_PVO_CHECK(pvo) 182 183struct ofw_map { 184 vm_offset_t om_va; 185 vm_size_t om_len; 186 vm_offset_t om_pa; 187 u_int om_mode; 188}; 189 190int pmap_bootstrapped = 0; 191 192/* 193 * Virtual and physical address of message buffer. 194 */ 195struct msgbuf *msgbufp; 196vm_offset_t msgbuf_phys; 197 198/* 199 * Physical addresses of first and last available physical page. 200 */ 201vm_offset_t avail_start; 202vm_offset_t avail_end; 203 204int pmap_pagedaemon_waken; 205 206/* 207 * Map of physical memory regions. 208 */ 209vm_offset_t phys_avail[128]; 210u_int phys_avail_count; 211static struct mem_region *regions; 212static struct mem_region *pregions; 213int regions_sz, pregions_sz; 214static struct ofw_map *translations; 215 216/* 217 * First and last available kernel virtual addresses. 218 */ 219vm_offset_t virtual_avail; 220vm_offset_t virtual_end; 221vm_offset_t kernel_vm_end; 222 223/* 224 * Kernel pmap. 225 */ 226struct pmap kernel_pmap_store; 227extern struct pmap ofw_pmap; 228 229/* 230 * PTEG data. 231 */ 232static struct pteg *pmap_pteg_table; 233u_int pmap_pteg_count; 234u_int pmap_pteg_mask; 235 236/* 237 * PVO data. 238 */ 239struct pvo_head *pmap_pvo_table; /* pvo entries by pteg index */ 240struct pvo_head pmap_pvo_kunmanaged = 241 LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */ 242struct pvo_head pmap_pvo_unmanaged = 243 LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */ 244 245uma_zone_t pmap_upvo_zone; /* zone for pvo entries for unmanaged pages */ 246uma_zone_t pmap_mpvo_zone; /* zone for pvo entries for managed pages */ 247struct vm_object pmap_upvo_zone_obj; 248struct vm_object pmap_mpvo_zone_obj; 249static vm_object_t pmap_pvo_obj; 250static u_int pmap_pvo_count; 251 252#define BPVO_POOL_SIZE 32768 253static struct pvo_entry *pmap_bpvo_pool; 254static int pmap_bpvo_pool_index = 0; 255 256#define VSID_NBPW (sizeof(u_int32_t) * 8) 257static u_int pmap_vsid_bitmap[NPMAPS / VSID_NBPW]; 258 259static boolean_t pmap_initialized = FALSE; 260 261/* 262 * Statistics. 263 */ 264u_int pmap_pte_valid = 0; 265u_int pmap_pte_overflow = 0; 266u_int pmap_pte_replacements = 0; 267u_int pmap_pvo_entries = 0; 268u_int pmap_pvo_enter_calls = 0; 269u_int pmap_pvo_remove_calls = 0; 270u_int pmap_pte_spills = 0; 271SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_valid, CTLFLAG_RD, &pmap_pte_valid, 272 0, ""); 273SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_overflow, CTLFLAG_RD, 274 &pmap_pte_overflow, 0, ""); 275SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_replacements, CTLFLAG_RD, 276 &pmap_pte_replacements, 0, ""); 277SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_entries, CTLFLAG_RD, &pmap_pvo_entries, 278 0, ""); 279SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_enter_calls, CTLFLAG_RD, 280 &pmap_pvo_enter_calls, 0, ""); 281SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_remove_calls, CTLFLAG_RD, 282 &pmap_pvo_remove_calls, 0, ""); 283SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_spills, CTLFLAG_RD, 284 &pmap_pte_spills, 0, ""); 285 286struct pvo_entry *pmap_pvo_zeropage; 287 288vm_offset_t pmap_rkva_start = VM_MIN_KERNEL_ADDRESS; 289u_int pmap_rkva_count = 4; 290 291/* 292 * Allocate physical memory for use in pmap_bootstrap. 293 */ 294static vm_offset_t pmap_bootstrap_alloc(vm_size_t, u_int); 295 296/* 297 * PTE calls. 298 */ 299static int pmap_pte_insert(u_int, struct pte *); 300 301/* 302 * PVO calls. 303 */ 304static int pmap_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *, 305 vm_offset_t, vm_offset_t, u_int, int); 306static void pmap_pvo_remove(struct pvo_entry *, int); 307static struct pvo_entry *pmap_pvo_find_va(pmap_t, vm_offset_t, int *); 308static struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int); 309 310/* 311 * Utility routines. 312 */ 313static void * pmap_pvo_allocf(uma_zone_t, int, u_int8_t *, int); 314static struct pvo_entry *pmap_rkva_alloc(void); 315static void pmap_pa_map(struct pvo_entry *, vm_offset_t, 316 struct pte *, int *); 317static void pmap_pa_unmap(struct pvo_entry *, struct pte *, int *); 318static void pmap_syncicache(vm_offset_t, vm_size_t); 319static boolean_t pmap_query_bit(vm_page_t, int); 320static u_int pmap_clear_bit(vm_page_t, int, int *); 321static void tlbia(void); 322 323static __inline int 324va_to_sr(u_int *sr, vm_offset_t va) 325{ 326 return (sr[(uintptr_t)va >> ADDR_SR_SHFT]); 327} 328 329static __inline u_int 330va_to_pteg(u_int sr, vm_offset_t addr) 331{ 332 u_int hash; 333 334 hash = (sr & SR_VSID_MASK) ^ (((u_int)addr & ADDR_PIDX) >> 335 ADDR_PIDX_SHFT); 336 return (hash & pmap_pteg_mask); 337} 338 339static __inline struct pvo_head * 340pa_to_pvoh(vm_offset_t pa, vm_page_t *pg_p) 341{ 342 struct vm_page *pg; 343 344 pg = PHYS_TO_VM_PAGE(pa); 345 346 if (pg_p != NULL) 347 *pg_p = pg; 348 349 if (pg == NULL) 350 return (&pmap_pvo_unmanaged); 351 352 return (&pg->md.mdpg_pvoh); 353} 354 355static __inline struct pvo_head * 356vm_page_to_pvoh(vm_page_t m) 357{ 358 359 return (&m->md.mdpg_pvoh); 360} 361 362static __inline void 363pmap_attr_clear(vm_page_t m, int ptebit) 364{ 365 366 m->md.mdpg_attrs &= ~ptebit; 367} 368 369static __inline int 370pmap_attr_fetch(vm_page_t m) 371{ 372 373 return (m->md.mdpg_attrs); 374} 375 376static __inline void 377pmap_attr_save(vm_page_t m, int ptebit) 378{ 379 380 m->md.mdpg_attrs |= ptebit; 381} 382 383static __inline int 384pmap_pte_compare(const struct pte *pt, const struct pte *pvo_pt) 385{ 386 if (pt->pte_hi == pvo_pt->pte_hi) 387 return (1); 388 389 return (0); 390} 391 392static __inline int 393pmap_pte_match(struct pte *pt, u_int sr, vm_offset_t va, int which) 394{ 395 return (pt->pte_hi & ~PTE_VALID) == 396 (((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 397 ((va >> ADDR_API_SHFT) & PTE_API) | which); 398} 399 400static __inline void 401pmap_pte_create(struct pte *pt, u_int sr, vm_offset_t va, u_int pte_lo) 402{ 403 /* 404 * Construct a PTE. Default to IMB initially. Valid bit only gets 405 * set when the real pte is set in memory. 406 * 407 * Note: Don't set the valid bit for correct operation of tlb update. 408 */ 409 pt->pte_hi = ((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 410 (((va & ADDR_PIDX) >> ADDR_API_SHFT) & PTE_API); 411 pt->pte_lo = pte_lo; 412} 413 414static __inline void 415pmap_pte_synch(struct pte *pt, struct pte *pvo_pt) 416{ 417 418 pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF | PTE_CHG); 419} 420 421static __inline void 422pmap_pte_clear(struct pte *pt, vm_offset_t va, int ptebit) 423{ 424 425 /* 426 * As shown in Section 7.6.3.2.3 427 */ 428 pt->pte_lo &= ~ptebit; 429 TLBIE(va); 430 EIEIO(); 431 TLBSYNC(); 432 SYNC(); 433} 434 435static __inline void 436pmap_pte_set(struct pte *pt, struct pte *pvo_pt) 437{ 438 439 pvo_pt->pte_hi |= PTE_VALID; 440 441 /* 442 * Update the PTE as defined in section 7.6.3.1. 443 * Note that the REF/CHG bits are from pvo_pt and thus should havce 444 * been saved so this routine can restore them (if desired). 445 */ 446 pt->pte_lo = pvo_pt->pte_lo; 447 EIEIO(); 448 pt->pte_hi = pvo_pt->pte_hi; 449 SYNC(); 450 pmap_pte_valid++; 451} 452 453static __inline void 454pmap_pte_unset(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 455{ 456 457 pvo_pt->pte_hi &= ~PTE_VALID; 458 459 /* 460 * Force the reg & chg bits back into the PTEs. 461 */ 462 SYNC(); 463 464 /* 465 * Invalidate the pte. 466 */ 467 pt->pte_hi &= ~PTE_VALID; 468 469 SYNC(); 470 TLBIE(va); 471 EIEIO(); 472 TLBSYNC(); 473 SYNC(); 474 475 /* 476 * Save the reg & chg bits. 477 */ 478 pmap_pte_synch(pt, pvo_pt); 479 pmap_pte_valid--; 480} 481 482static __inline void 483pmap_pte_change(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 484{ 485 486 /* 487 * Invalidate the PTE 488 */ 489 pmap_pte_unset(pt, pvo_pt, va); 490 pmap_pte_set(pt, pvo_pt); 491} 492 493/* 494 * Quick sort callout for comparing memory regions. 495 */ 496static int mr_cmp(const void *a, const void *b); 497static int om_cmp(const void *a, const void *b); 498 499static int 500mr_cmp(const void *a, const void *b) 501{ 502 const struct mem_region *regiona; 503 const struct mem_region *regionb; 504 505 regiona = a; 506 regionb = b; 507 if (regiona->mr_start < regionb->mr_start) 508 return (-1); 509 else if (regiona->mr_start > regionb->mr_start) 510 return (1); 511 else 512 return (0); 513} 514 515static int 516om_cmp(const void *a, const void *b) 517{ 518 const struct ofw_map *mapa; 519 const struct ofw_map *mapb; 520 521 mapa = a; 522 mapb = b; 523 if (mapa->om_pa < mapb->om_pa) 524 return (-1); 525 else if (mapa->om_pa > mapb->om_pa) 526 return (1); 527 else 528 return (0); 529} 530 531void 532pmap_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) 533{ 534 ihandle_t mmui; 535 phandle_t chosen, mmu; 536 int sz; 537 int i, j; 538 int ofw_mappings; 539 vm_size_t size, physsz; 540 vm_offset_t pa, va, off; 541 u_int batl, batu; 542 543 /* 544 * Set up BAT0 to map the lowest 256 MB area 545 */ 546 battable[0x0].batl = BATL(0x00000000, BAT_M, BAT_PP_RW); 547 battable[0x0].batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs); 548 549 /* 550 * Map PCI memory space. 551 */ 552 battable[0x8].batl = BATL(0x80000000, BAT_I|BAT_G, BAT_PP_RW); 553 battable[0x8].batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs); 554 555 battable[0x9].batl = BATL(0x90000000, BAT_I|BAT_G, BAT_PP_RW); 556 battable[0x9].batu = BATU(0x90000000, BAT_BL_256M, BAT_Vs); 557 558 battable[0xa].batl = BATL(0xa0000000, BAT_I|BAT_G, BAT_PP_RW); 559 battable[0xa].batu = BATU(0xa0000000, BAT_BL_256M, BAT_Vs); 560 561 battable[0xb].batl = BATL(0xb0000000, BAT_I|BAT_G, BAT_PP_RW); 562 battable[0xb].batu = BATU(0xb0000000, BAT_BL_256M, BAT_Vs); 563 564 /* 565 * Map obio devices. 566 */ 567 battable[0xf].batl = BATL(0xf0000000, BAT_I|BAT_G, BAT_PP_RW); 568 battable[0xf].batu = BATU(0xf0000000, BAT_BL_256M, BAT_Vs); 569 570 /* 571 * Use an IBAT and a DBAT to map the bottom segment of memory 572 * where we are. 573 */ 574 batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs); 575 batl = BATL(0x00000000, BAT_M, BAT_PP_RW); 576 __asm ("mtibatu 0,%0; mtibatl 0,%1; mtdbatu 0,%0; mtdbatl 0,%1" 577 :: "r"(batu), "r"(batl)); 578 579#if 0 580 /* map frame buffer */ 581 batu = BATU(0x90000000, BAT_BL_256M, BAT_Vs); 582 batl = BATL(0x90000000, BAT_I|BAT_G, BAT_PP_RW); 583 __asm ("mtdbatu 1,%0; mtdbatl 1,%1" 584 :: "r"(batu), "r"(batl)); 585#endif 586 587#if 1 588 /* map pci space */ 589 batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs); 590 batl = BATL(0x80000000, BAT_I|BAT_G, BAT_PP_RW); 591 __asm ("mtdbatu 1,%0; mtdbatl 1,%1" 592 :: "r"(batu), "r"(batl)); 593#endif 594 595 /* 596 * Set the start and end of kva. 597 */ 598 virtual_avail = VM_MIN_KERNEL_ADDRESS; 599 virtual_end = VM_MAX_KERNEL_ADDRESS; 600 601 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 602 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 603 604 qsort(pregions, pregions_sz, sizeof(*pregions), mr_cmp); 605 for (i = 0; i < pregions_sz; i++) { 606 vm_offset_t pa; 607 vm_offset_t end; 608 609 CTR3(KTR_PMAP, "physregion: %#x - %#x (%#x)", 610 pregions[i].mr_start, 611 pregions[i].mr_start + pregions[i].mr_size, 612 pregions[i].mr_size); 613 /* 614 * Install entries into the BAT table to allow all 615 * of physmem to be convered by on-demand BAT entries. 616 * The loop will sometimes set the same battable element 617 * twice, but that's fine since they won't be used for 618 * a while yet. 619 */ 620 pa = pregions[i].mr_start & 0xf0000000; 621 end = pregions[i].mr_start + pregions[i].mr_size; 622 do { 623 u_int n = pa >> ADDR_SR_SHFT; 624 625 battable[n].batl = BATL(pa, BAT_M, BAT_PP_RW); 626 battable[n].batu = BATU(pa, BAT_BL_256M, BAT_Vs); 627 pa += SEGMENT_LENGTH; 628 } while (pa < end); 629 } 630 631 if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz) 632 panic("pmap_bootstrap: phys_avail too small"); 633 qsort(regions, regions_sz, sizeof(*regions), mr_cmp); 634 phys_avail_count = 0; 635 physsz = 0; 636 for (i = 0, j = 0; i < regions_sz; i++, j += 2) { 637 CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start, 638 regions[i].mr_start + regions[i].mr_size, 639 regions[i].mr_size); 640 phys_avail[j] = regions[i].mr_start; 641 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 642 phys_avail_count++; 643 physsz += regions[i].mr_size; 644 } 645 physmem = btoc(physsz); 646 647 /* 648 * Allocate PTEG table. 649 */ 650#ifdef PTEGCOUNT 651 pmap_pteg_count = PTEGCOUNT; 652#else 653 pmap_pteg_count = 0x1000; 654 655 while (pmap_pteg_count < physmem) 656 pmap_pteg_count <<= 1; 657 658 pmap_pteg_count >>= 1; 659#endif /* PTEGCOUNT */ 660 661 size = pmap_pteg_count * sizeof(struct pteg); 662 CTR2(KTR_PMAP, "pmap_bootstrap: %d PTEGs, %d bytes", pmap_pteg_count, 663 size); 664 pmap_pteg_table = (struct pteg *)pmap_bootstrap_alloc(size, size); 665 CTR1(KTR_PMAP, "pmap_bootstrap: PTEG table at %p", pmap_pteg_table); 666 bzero((void *)pmap_pteg_table, pmap_pteg_count * sizeof(struct pteg)); 667 pmap_pteg_mask = pmap_pteg_count - 1; 668 669 /* 670 * Allocate pv/overflow lists. 671 */ 672 size = sizeof(struct pvo_head) * pmap_pteg_count; 673 pmap_pvo_table = (struct pvo_head *)pmap_bootstrap_alloc(size, 674 PAGE_SIZE); 675 CTR1(KTR_PMAP, "pmap_bootstrap: PVO table at %p", pmap_pvo_table); 676 for (i = 0; i < pmap_pteg_count; i++) 677 LIST_INIT(&pmap_pvo_table[i]); 678 679 /* 680 * Allocate the message buffer. 681 */ 682 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE, 0); 683 684 /* 685 * Initialise the unmanaged pvo pool. 686 */ 687 pmap_bpvo_pool = (struct pvo_entry *)pmap_bootstrap_alloc( 688 BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0); 689 pmap_bpvo_pool_index = 0; 690 691 /* 692 * Make sure kernel vsid is allocated as well as VSID 0. 693 */ 694 pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW] 695 |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 696 pmap_vsid_bitmap[0] |= 1; 697 698 /* 699 * Set up the OpenFirmware pmap and add it's mappings. 700 */ 701 pmap_pinit(&ofw_pmap); 702 ofw_pmap.pm_sr[KERNEL_SR] = KERNEL_SEGMENT; 703 if ((chosen = OF_finddevice("/chosen")) == -1) 704 panic("pmap_bootstrap: can't find /chosen"); 705 OF_getprop(chosen, "mmu", &mmui, 4); 706 if ((mmu = OF_instance_to_package(mmui)) == -1) 707 panic("pmap_bootstrap: can't get mmu package"); 708 if ((sz = OF_getproplen(mmu, "translations")) == -1) 709 panic("pmap_bootstrap: can't get ofw translation count"); 710 translations = NULL; 711 for (i = 0; phys_avail[i + 2] != 0; i += 2) { 712 if (phys_avail[i + 1] >= sz) 713 translations = (struct ofw_map *)phys_avail[i]; 714 } 715 if (translations == NULL) 716 panic("pmap_bootstrap: no space to copy translations"); 717 bzero(translations, sz); 718 if (OF_getprop(mmu, "translations", translations, sz) == -1) 719 panic("pmap_bootstrap: can't get ofw translations"); 720 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 721 sz /= sizeof(*translations); 722 qsort(translations, sz, sizeof (*translations), om_cmp); 723 for (i = 0, ofw_mappings = 0; i < sz; i++) { 724 CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x", 725 translations[i].om_pa, translations[i].om_va, 726 translations[i].om_len); 727 728 /* 729 * If the mapping is 1:1, let the RAM and device on-demand 730 * BAT tables take care of the translation. 731 */ 732 if (translations[i].om_va == translations[i].om_pa) 733 continue; 734 735 /* Enter the pages */ 736 for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) { 737 struct vm_page m; 738 739 m.phys_addr = translations[i].om_pa + off; 740 pmap_enter(&ofw_pmap, translations[i].om_va + off, &m, 741 VM_PROT_ALL, 1); 742 ofw_mappings++; 743 } 744 } 745#ifdef SMP 746 TLBSYNC(); 747#endif 748 749 /* 750 * Initialize the kernel pmap (which is statically allocated). 751 */ 752 for (i = 0; i < 16; i++) { 753 kernel_pmap->pm_sr[i] = EMPTY_SEGMENT; 754 } 755 kernel_pmap->pm_sr[KERNEL_SR] = KERNEL_SEGMENT; 756 kernel_pmap->pm_active = ~0; 757 758 /* 759 * Allocate a kernel stack with a guard page for thread0 and map it 760 * into the kernel page map. 761 */ 762 pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, 0); 763 kstack0_phys = pa; 764 kstack0 = virtual_avail + (KSTACK_GUARD_PAGES * PAGE_SIZE); 765 CTR2(KTR_PMAP, "pmap_bootstrap: kstack0 at %#x (%#x)", kstack0_phys, 766 kstack0); 767 virtual_avail += (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE; 768 for (i = 0; i < KSTACK_PAGES; i++) { 769 pa = kstack0_phys + i * PAGE_SIZE; 770 va = kstack0 + i * PAGE_SIZE; 771 pmap_kenter(va, pa); 772 TLBIE(va); 773 } 774 775 /* 776 * Calculate the first and last available physical addresses. 777 */ 778 avail_start = phys_avail[0]; 779 for (i = 0; phys_avail[i + 2] != 0; i += 2) 780 ; 781 avail_end = phys_avail[i + 1]; 782 Maxmem = powerpc_btop(avail_end); 783 784 /* 785 * Allocate virtual address space for the message buffer. 786 */ 787 msgbufp = (struct msgbuf *)virtual_avail; 788 virtual_avail += round_page(MSGBUF_SIZE); 789 790 /* 791 * Initialize hardware. 792 */ 793 for (i = 0; i < 16; i++) { 794 mtsrin(i << ADDR_SR_SHFT, EMPTY_SEGMENT); 795 } 796 __asm __volatile ("mtsr %0,%1" 797 :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); 798 __asm __volatile ("sync; mtsdr1 %0; isync" 799 :: "r"((u_int)pmap_pteg_table | (pmap_pteg_mask >> 10))); 800 tlbia(); 801 802 pmap_bootstrapped++; 803} 804 805/* 806 * Activate a user pmap. The pmap must be activated before it's address 807 * space can be accessed in any way. 808 */ 809void 810pmap_activate(struct thread *td) 811{ 812 pmap_t pm, pmr; 813 814 /* 815 * Load all the data we need up front to encourage the compiler to 816 * not issue any loads while we have interrupts disabled below. 817 */ 818 pm = &td->td_proc->p_vmspace->vm_pmap; 819 820 if ((pmr = (pmap_t)pmap_kextract((vm_offset_t)pm)) == NULL) 821 pmr = pm; 822 823 pm->pm_active |= PCPU_GET(cpumask); 824 PCPU_SET(curpmap, pmr); 825} 826 827void 828pmap_deactivate(struct thread *td) 829{ 830 pmap_t pm; 831 832 pm = &td->td_proc->p_vmspace->vm_pmap; 833 pm->pm_active &= ~(PCPU_GET(cpumask)); 834 PCPU_SET(curpmap, NULL); 835} 836 837vm_offset_t 838pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 839{ 840 841 return (va); 842} 843 844void 845pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 846{ 847 struct pvo_entry *pvo; 848 849 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 850 851 if (pvo != NULL) { 852 if (wired) { 853 if ((pvo->pvo_vaddr & PVO_WIRED) == 0) 854 pm->pm_stats.wired_count++; 855 pvo->pvo_vaddr |= PVO_WIRED; 856 } else { 857 if ((pvo->pvo_vaddr & PVO_WIRED) != 0) 858 pm->pm_stats.wired_count--; 859 pvo->pvo_vaddr &= ~PVO_WIRED; 860 } 861 } 862} 863 864void 865pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 866 vm_size_t len, vm_offset_t src_addr) 867{ 868 869 /* 870 * This is not needed as it's mainly an optimisation. 871 * It may want to be implemented later though. 872 */ 873} 874 875void 876pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 877{ 878 vm_offset_t dst; 879 vm_offset_t src; 880 881 dst = VM_PAGE_TO_PHYS(mdst); 882 src = VM_PAGE_TO_PHYS(msrc); 883 884 kcopy((void *)src, (void *)dst, PAGE_SIZE); 885} 886 887/* 888 * Zero a page of physical memory by temporarily mapping it into the tlb. 889 */ 890void 891pmap_zero_page(vm_page_t m) 892{ 893 vm_offset_t pa = VM_PAGE_TO_PHYS(m); 894 caddr_t va; 895 896 if (pa < SEGMENT_LENGTH) { 897 va = (caddr_t) pa; 898 } else if (pmap_initialized) { 899 if (pmap_pvo_zeropage == NULL) 900 pmap_pvo_zeropage = pmap_rkva_alloc(); 901 pmap_pa_map(pmap_pvo_zeropage, pa, NULL, NULL); 902 va = (caddr_t)PVO_VADDR(pmap_pvo_zeropage); 903 } else { 904 panic("pmap_zero_page: can't zero pa %#x", pa); 905 } 906 907 bzero(va, PAGE_SIZE); 908 909 if (pa >= SEGMENT_LENGTH) 910 pmap_pa_unmap(pmap_pvo_zeropage, NULL, NULL); 911} 912 913void 914pmap_zero_page_area(vm_page_t m, int off, int size) 915{ 916 vm_offset_t pa = VM_PAGE_TO_PHYS(m); 917 caddr_t va; 918 919 if (pa < SEGMENT_LENGTH) { 920 va = (caddr_t) pa; 921 } else if (pmap_initialized) { 922 if (pmap_pvo_zeropage == NULL) 923 pmap_pvo_zeropage = pmap_rkva_alloc(); 924 pmap_pa_map(pmap_pvo_zeropage, pa, NULL, NULL); 925 va = (caddr_t)PVO_VADDR(pmap_pvo_zeropage); 926 } else { 927 panic("pmap_zero_page: can't zero pa %#x", pa); 928 } 929 930 bzero(va + off, size); 931 932 if (pa >= SEGMENT_LENGTH) 933 pmap_pa_unmap(pmap_pvo_zeropage, NULL, NULL); 934} 935 936void 937pmap_zero_page_idle(vm_page_t m) 938{ 939 940 /* XXX this is called outside of Giant, is pmap_zero_page safe? */ 941 /* XXX maybe have a dedicated mapping for this to avoid the problem? */ 942 mtx_lock(&Giant); 943 pmap_zero_page(m); 944 mtx_unlock(&Giant); 945} 946 947/* 948 * Map the given physical page at the specified virtual address in the 949 * target pmap with the protection requested. If specified the page 950 * will be wired down. 951 */ 952void 953pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 954 boolean_t wired) 955{ 956 struct pvo_head *pvo_head; 957 uma_zone_t zone; 958 vm_page_t pg; 959 u_int pte_lo, pvo_flags, was_exec, i; 960 int error; 961 962 if (!pmap_initialized) { 963 pvo_head = &pmap_pvo_kunmanaged; 964 zone = pmap_upvo_zone; 965 pvo_flags = 0; 966 pg = NULL; 967 was_exec = PTE_EXEC; 968 } else { 969 pvo_head = vm_page_to_pvoh(m); 970 pg = m; 971 zone = pmap_mpvo_zone; 972 pvo_flags = PVO_MANAGED; 973 was_exec = 0; 974 } 975 976 /* 977 * If this is a managed page, and it's the first reference to the page, 978 * clear the execness of the page. Otherwise fetch the execness. 979 */ 980 if (pg != NULL) { 981 if (LIST_EMPTY(pvo_head)) { 982 pmap_attr_clear(pg, PTE_EXEC); 983 } else { 984 was_exec = pmap_attr_fetch(pg) & PTE_EXEC; 985 } 986 } 987 988 989 /* 990 * Assume the page is cache inhibited and access is guarded unless 991 * it's in our available memory array. 992 */ 993 pte_lo = PTE_I | PTE_G; 994 for (i = 0; i < pregions_sz; i++) { 995 if ((VM_PAGE_TO_PHYS(m) >= pregions[i].mr_start) && 996 (VM_PAGE_TO_PHYS(m) < 997 (pregions[i].mr_start + pregions[i].mr_size))) { 998 pte_lo &= ~(PTE_I | PTE_G); 999 break; 1000 } 1001 } 1002 1003 if (prot & VM_PROT_WRITE) 1004 pte_lo |= PTE_BW; 1005 else 1006 pte_lo |= PTE_BR; 1007 1008 pvo_flags |= (prot & VM_PROT_EXECUTE); 1009 1010 if (wired) 1011 pvo_flags |= PVO_WIRED; 1012 1013 error = pmap_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), 1014 pte_lo, pvo_flags); 1015 1016 /* 1017 * Flush the real page from the instruction cache if this page is 1018 * mapped executable and cacheable and was not previously mapped (or 1019 * was not mapped executable). 1020 */ 1021 if (error == 0 && (pvo_flags & PVO_EXECUTABLE) && 1022 (pte_lo & PTE_I) == 0 && was_exec == 0) { 1023 /* 1024 * Flush the real memory from the cache. 1025 */ 1026 pmap_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); 1027 if (pg != NULL) 1028 pmap_attr_save(pg, PTE_EXEC); 1029 } 1030 1031 /* XXX syncicache always until problems are sorted */ 1032 pmap_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); 1033} 1034 1035vm_offset_t 1036pmap_extract(pmap_t pm, vm_offset_t va) 1037{ 1038 struct pvo_entry *pvo; 1039 1040 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 1041 1042 if (pvo != NULL) { 1043 return ((pvo->pvo_pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF)); 1044 } 1045 1046 return (0); 1047} 1048 1049/* 1050 * Grow the number of kernel page table entries. Unneeded. 1051 */ 1052void 1053pmap_growkernel(vm_offset_t addr) 1054{ 1055} 1056 1057void 1058pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) 1059{ 1060 1061 CTR0(KTR_PMAP, "pmap_init"); 1062 1063 pmap_pvo_obj = vm_object_allocate(OBJT_PHYS, 16); 1064 pmap_pvo_count = 0; 1065 pmap_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), 1066 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); 1067 uma_zone_set_allocf(pmap_upvo_zone, pmap_pvo_allocf); 1068 pmap_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry), 1069 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); 1070 uma_zone_set_allocf(pmap_mpvo_zone, pmap_pvo_allocf); 1071 pmap_initialized = TRUE; 1072} 1073 1074void 1075pmap_init2(void) 1076{ 1077 1078 CTR0(KTR_PMAP, "pmap_init2"); 1079} 1080 1081boolean_t 1082pmap_is_modified(vm_page_t m) 1083{ 1084 1085 if ((m->flags & (PG_FICTITIOUS |PG_UNMANAGED)) != 0) 1086 return (FALSE); 1087 1088 return (pmap_query_bit(m, PTE_CHG)); 1089} 1090 1091void 1092pmap_clear_reference(vm_page_t m) 1093{ 1094 1095 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1096 return; 1097 pmap_clear_bit(m, PTE_REF, NULL); 1098} 1099 1100void 1101pmap_clear_modify(vm_page_t m) 1102{ 1103 1104 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1105 return; 1106 pmap_clear_bit(m, PTE_CHG, NULL); 1107} 1108 1109/* 1110 * pmap_ts_referenced: 1111 * 1112 * Return a count of reference bits for a page, clearing those bits. 1113 * It is not necessary for every reference bit to be cleared, but it 1114 * is necessary that 0 only be returned when there are truly no 1115 * reference bits set. 1116 * 1117 * XXX: The exact number of bits to check and clear is a matter that 1118 * should be tested and standardized at some point in the future for 1119 * optimal aging of shared pages. 1120 */ 1121int 1122pmap_ts_referenced(vm_page_t m) 1123{ 1124 int count; 1125 1126 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1127 return (0); 1128 1129 count = pmap_clear_bit(m, PTE_REF, NULL); 1130 1131 return (count); 1132} 1133 1134/* 1135 * Map a wired page into kernel virtual address space. 1136 */ 1137void 1138pmap_kenter(vm_offset_t va, vm_offset_t pa) 1139{ 1140 u_int pte_lo; 1141 int error; 1142 int i; 1143 1144#if 0 1145 if (va < VM_MIN_KERNEL_ADDRESS) 1146 panic("pmap_kenter: attempt to enter non-kernel address %#x", 1147 va); 1148#endif 1149 1150 pte_lo = PTE_I | PTE_G; 1151 for (i = 0; i < pregions_sz; i++) { 1152 if ((pa >= pregions[i].mr_start) && 1153 (pa < (pregions[i].mr_start + pregions[i].mr_size))) { 1154 pte_lo &= ~(PTE_I | PTE_G); 1155 break; 1156 } 1157 } 1158 1159 error = pmap_pvo_enter(kernel_pmap, pmap_upvo_zone, 1160 &pmap_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED); 1161 1162 if (error != 0 && error != ENOENT) 1163 panic("pmap_kenter: failed to enter va %#x pa %#x: %d", va, 1164 pa, error); 1165 1166 /* 1167 * Flush the real memory from the instruction cache. 1168 */ 1169 if ((pte_lo & (PTE_I | PTE_G)) == 0) { 1170 pmap_syncicache(pa, PAGE_SIZE); 1171 } 1172} 1173 1174/* 1175 * Extract the physical page address associated with the given kernel virtual 1176 * address. 1177 */ 1178vm_offset_t 1179pmap_kextract(vm_offset_t va) 1180{ 1181 struct pvo_entry *pvo; 1182 1183 pvo = pmap_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL); 1184 if (pvo == NULL) { 1185 return (0); 1186 } 1187 1188 return ((pvo->pvo_pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF)); 1189} 1190 1191/* 1192 * Remove a wired page from kernel virtual address space. 1193 */ 1194void 1195pmap_kremove(vm_offset_t va) 1196{ 1197 1198 pmap_remove(kernel_pmap, va, va + PAGE_SIZE); 1199} 1200 1201/* 1202 * Map a range of physical addresses into kernel virtual address space. 1203 * 1204 * The value passed in *virt is a suggested virtual address for the mapping. 1205 * Architectures which can support a direct-mapped physical to virtual region 1206 * can return the appropriate address within that region, leaving '*virt' 1207 * unchanged. We cannot and therefore do not; *virt is updated with the 1208 * first usable address after the mapped region. 1209 */ 1210vm_offset_t 1211pmap_map(vm_offset_t *virt, vm_offset_t pa_start, vm_offset_t pa_end, int prot) 1212{ 1213 vm_offset_t sva, va; 1214 1215 sva = *virt; 1216 va = sva; 1217 for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE) 1218 pmap_kenter(va, pa_start); 1219 *virt = va; 1220 return (sva); 1221} 1222 1223int 1224pmap_mincore(pmap_t pmap, vm_offset_t addr) 1225{ 1226 TODO; 1227 return (0); 1228} 1229 1230void 1231pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1232 vm_pindex_t pindex, vm_size_t size, int limit) 1233{ 1234 1235 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1236 ("pmap_remove_pages: non current pmap")); 1237 /* XXX */ 1238} 1239 1240/* 1241 * Lower the permission for all mappings to a given page. 1242 */ 1243void 1244pmap_page_protect(vm_page_t m, vm_prot_t prot) 1245{ 1246 struct pvo_head *pvo_head; 1247 struct pvo_entry *pvo, *next_pvo; 1248 struct pte *pt; 1249 1250 /* 1251 * Since the routine only downgrades protection, if the 1252 * maximal protection is desired, there isn't any change 1253 * to be made. 1254 */ 1255 if ((prot & (VM_PROT_READ|VM_PROT_WRITE)) == 1256 (VM_PROT_READ|VM_PROT_WRITE)) 1257 return; 1258 1259 pvo_head = vm_page_to_pvoh(m); 1260 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 1261 next_pvo = LIST_NEXT(pvo, pvo_vlink); 1262 PMAP_PVO_CHECK(pvo); /* sanity check */ 1263 1264 /* 1265 * Downgrading to no mapping at all, we just remove the entry. 1266 */ 1267 if ((prot & VM_PROT_READ) == 0) { 1268 pmap_pvo_remove(pvo, -1); 1269 continue; 1270 } 1271 1272 /* 1273 * If EXEC permission is being revoked, just clear the flag 1274 * in the PVO. 1275 */ 1276 if ((prot & VM_PROT_EXECUTE) == 0) 1277 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1278 1279 /* 1280 * If this entry is already RO, don't diddle with the page 1281 * table. 1282 */ 1283 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) { 1284 PMAP_PVO_CHECK(pvo); 1285 continue; 1286 } 1287 1288 /* 1289 * Grab the PTE before we diddle the bits so pvo_to_pte can 1290 * verify the pte contents are as expected. 1291 */ 1292 pt = pmap_pvo_to_pte(pvo, -1); 1293 pvo->pvo_pte.pte_lo &= ~PTE_PP; 1294 pvo->pvo_pte.pte_lo |= PTE_BR; 1295 if (pt != NULL) 1296 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1297 PMAP_PVO_CHECK(pvo); /* sanity check */ 1298 } 1299} 1300 1301/* 1302 * Returns true if the pmap's pv is one of the first 1303 * 16 pvs linked to from this page. This count may 1304 * be changed upwards or downwards in the future; it 1305 * is only necessary that true be returned for a small 1306 * subset of pmaps for proper page aging. 1307 */ 1308boolean_t 1309pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 1310{ 1311 int loops; 1312 struct pvo_entry *pvo; 1313 1314 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 1315 return FALSE; 1316 1317 loops = 0; 1318 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 1319 if (pvo->pvo_pmap == pmap) 1320 return (TRUE); 1321 if (++loops >= 16) 1322 break; 1323 } 1324 1325 return (FALSE); 1326} 1327 1328static u_int pmap_vsidcontext; 1329 1330void 1331pmap_pinit(pmap_t pmap) 1332{ 1333 int i, mask; 1334 u_int entropy; 1335 1336 entropy = 0; 1337 __asm __volatile("mftb %0" : "=r"(entropy)); 1338 1339 /* 1340 * Allocate some segment registers for this pmap. 1341 */ 1342 for (i = 0; i < NPMAPS; i += VSID_NBPW) { 1343 u_int hash, n; 1344 1345 /* 1346 * Create a new value by mutiplying by a prime and adding in 1347 * entropy from the timebase register. This is to make the 1348 * VSID more random so that the PT hash function collides 1349 * less often. (Note that the prime casues gcc to do shifts 1350 * instead of a multiply.) 1351 */ 1352 pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy; 1353 hash = pmap_vsidcontext & (NPMAPS - 1); 1354 if (hash == 0) /* 0 is special, avoid it */ 1355 continue; 1356 n = hash >> 5; 1357 mask = 1 << (hash & (VSID_NBPW - 1)); 1358 hash = (pmap_vsidcontext & 0xfffff); 1359 if (pmap_vsid_bitmap[n] & mask) { /* collision? */ 1360 /* anything free in this bucket? */ 1361 if (pmap_vsid_bitmap[n] == 0xffffffff) { 1362 entropy = (pmap_vsidcontext >> 20); 1363 continue; 1364 } 1365 i = ffs(~pmap_vsid_bitmap[i]) - 1; 1366 mask = 1 << i; 1367 hash &= 0xfffff & ~(VSID_NBPW - 1); 1368 hash |= i; 1369 } 1370 pmap_vsid_bitmap[n] |= mask; 1371 for (i = 0; i < 16; i++) 1372 pmap->pm_sr[i] = VSID_MAKE(i, hash); 1373 return; 1374 } 1375 1376 panic("pmap_pinit: out of segments"); 1377} 1378 1379/* 1380 * Initialize the pmap associated with process 0. 1381 */ 1382void 1383pmap_pinit0(pmap_t pm) 1384{ 1385 1386 pmap_pinit(pm); 1387 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1388} 1389 1390void 1391pmap_pinit2(pmap_t pmap) 1392{ 1393 /* XXX: Remove this stub when no longer called */ 1394} 1395 1396void 1397pmap_prefault(pmap_t pm, vm_offset_t va, vm_map_entry_t entry) 1398{ 1399 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1400 ("pmap_prefault: non current pmap")); 1401 /* XXX */ 1402} 1403 1404/* 1405 * Set the physical protection on the specified range of this map as requested. 1406 */ 1407void 1408pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1409{ 1410 struct pvo_entry *pvo; 1411 struct pte *pt; 1412 int pteidx; 1413 1414 CTR4(KTR_PMAP, "pmap_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva, 1415 eva, prot); 1416 1417 1418 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1419 ("pmap_protect: non current pmap")); 1420 1421 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1422 pmap_remove(pm, sva, eva); 1423 return; 1424 } 1425 1426 for (; sva < eva; sva += PAGE_SIZE) { 1427 pvo = pmap_pvo_find_va(pm, sva, &pteidx); 1428 if (pvo == NULL) 1429 continue; 1430 1431 if ((prot & VM_PROT_EXECUTE) == 0) 1432 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1433 1434 /* 1435 * Grab the PTE pointer before we diddle with the cached PTE 1436 * copy. 1437 */ 1438 pt = pmap_pvo_to_pte(pvo, pteidx); 1439 /* 1440 * Change the protection of the page. 1441 */ 1442 pvo->pvo_pte.pte_lo &= ~PTE_PP; 1443 pvo->pvo_pte.pte_lo |= PTE_BR; 1444 1445 /* 1446 * If the PVO is in the page table, update that pte as well. 1447 */ 1448 if (pt != NULL) 1449 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1450 } 1451} 1452 1453/* 1454 * Map a list of wired pages into kernel virtual address space. This is 1455 * intended for temporary mappings which do not need page modification or 1456 * references recorded. Existing mappings in the region are overwritten. 1457 */ 1458void 1459pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 1460{ 1461 vm_offset_t va; 1462 1463 va = sva; 1464 while (count-- > 0) { 1465 pmap_kenter(va, VM_PAGE_TO_PHYS(*m)); 1466 va += PAGE_SIZE; 1467 m++; 1468 } 1469} 1470 1471/* 1472 * Remove page mappings from kernel virtual address space. Intended for 1473 * temporary mappings entered by pmap_qenter. 1474 */ 1475void 1476pmap_qremove(vm_offset_t sva, int count) 1477{ 1478 vm_offset_t va; 1479 1480 va = sva; 1481 while (count-- > 0) { 1482 pmap_kremove(va); 1483 va += PAGE_SIZE; 1484 } 1485} 1486 1487void 1488pmap_release(pmap_t pmap) 1489{ 1490 int idx, mask; 1491 1492 /* 1493 * Free segment register's VSID 1494 */ 1495 if (pmap->pm_sr[0] == 0) 1496 panic("pmap_release"); 1497 1498 idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1); 1499 mask = 1 << (idx % VSID_NBPW); 1500 idx /= VSID_NBPW; 1501 pmap_vsid_bitmap[idx] &= ~mask; 1502} 1503 1504/* 1505 * Remove the given range of addresses from the specified map. 1506 */ 1507void 1508pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1509{ 1510 struct pvo_entry *pvo; 1511 int pteidx; 1512 1513 for (; sva < eva; sva += PAGE_SIZE) { 1514 pvo = pmap_pvo_find_va(pm, sva, &pteidx); 1515 if (pvo != NULL) { 1516 pmap_pvo_remove(pvo, pteidx); 1517 } 1518 } 1519} 1520 1521/* 1522 * Remove physical page from all pmaps in which it resides. pmap_pvo_remove() 1523 * will reflect changes in pte's back to the vm_page. 1524 */ 1525void 1526pmap_remove_all(vm_page_t m) 1527{ 1528 struct pvo_head *pvo_head; 1529 struct pvo_entry *pvo, *next_pvo; 1530 1531 KASSERT((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0, 1532 ("pv_remove_all: illegal for unmanaged page %#x", 1533 VM_PAGE_TO_PHYS(m))); 1534 1535 pvo_head = vm_page_to_pvoh(m); 1536 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 1537 next_pvo = LIST_NEXT(pvo, pvo_vlink); 1538 1539 PMAP_PVO_CHECK(pvo); /* sanity check */ 1540 pmap_pvo_remove(pvo, -1); 1541 } 1542 vm_page_flag_clear(m, PG_WRITEABLE); 1543} 1544 1545/* 1546 * Remove all pages from specified address space, this aids process exit 1547 * speeds. This is much faster than pmap_remove in the case of running down 1548 * an entire address space. Only works for the current pmap. 1549 */ 1550void 1551pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1552{ 1553 1554 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1555 ("pmap_remove_pages: non current pmap")); 1556 pmap_remove(pm, sva, eva); 1557} 1558
| 95 96/* 97 * Manages physical address maps. 98 * 99 * In addition to hardware address maps, this module is called upon to 100 * provide software-use-only maps which may or may not be stored in the 101 * same form as hardware maps. These pseudo-maps are used to store 102 * intermediate results from copy operations to and from address spaces. 103 * 104 * Since the information managed by this module is also stored by the 105 * logical address mapping module, this module may throw away valid virtual 106 * to physical mappings at almost any time. However, invalidations of 107 * mappings must be done as requested. 108 * 109 * In order to cope with hardware architectures which make virtual to 110 * physical map invalidates expensive, this module may delay invalidate 111 * reduced protection operations until such time as they are actually 112 * necessary. This module is given full information as to which processors 113 * are currently using which maps, and to when physical maps must be made 114 * correct. 115 */ 116 117#include <sys/param.h> 118#include <sys/kernel.h> 119#include <sys/ktr.h> 120#include <sys/lock.h> 121#include <sys/msgbuf.h> 122#include <sys/mutex.h> 123#include <sys/proc.h> 124#include <sys/sysctl.h> 125#include <sys/systm.h> 126#include <sys/vmmeter.h> 127 128#include <dev/ofw/openfirm.h> 129 130#include <vm/vm.h> 131#include <vm/vm_param.h> 132#include <vm/vm_kern.h> 133#include <vm/vm_page.h> 134#include <vm/vm_map.h> 135#include <vm/vm_object.h> 136#include <vm/vm_extern.h> 137#include <vm/vm_pageout.h> 138#include <vm/vm_pager.h> 139#include <vm/uma.h> 140 141#include <machine/powerpc.h> 142#include <machine/bat.h> 143#include <machine/frame.h> 144#include <machine/md_var.h> 145#include <machine/psl.h> 146#include <machine/pte.h> 147#include <machine/sr.h> 148 149#define PMAP_DEBUG 150 151#define TODO panic("%s: not implemented", __func__); 152 153#define PMAP_LOCK(pm) 154#define PMAP_UNLOCK(pm) 155 156#define TLBIE(va) __asm __volatile("tlbie %0" :: "r"(va)) 157#define TLBSYNC() __asm __volatile("tlbsync"); 158#define SYNC() __asm __volatile("sync"); 159#define EIEIO() __asm __volatile("eieio"); 160 161#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4)) 162#define VSID_TO_SR(vsid) ((vsid) & 0xf) 163#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff) 164 165#define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */ 166#define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */ 167#define PVO_WIRED 0x0010 /* PVO entry is wired */ 168#define PVO_MANAGED 0x0020 /* PVO entry is managed */ 169#define PVO_EXECUTABLE 0x0040 /* PVO entry is executable */ 170#define PVO_BOOTSTRAP 0x0080 /* PVO entry allocated during 171 bootstrap */ 172#define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF) 173#define PVO_ISEXECUTABLE(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE) 174#define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK) 175#define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID) 176#define PVO_PTEGIDX_CLR(pvo) \ 177 ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) 178#define PVO_PTEGIDX_SET(pvo, i) \ 179 ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) 180 181#define PMAP_PVO_CHECK(pvo) 182 183struct ofw_map { 184 vm_offset_t om_va; 185 vm_size_t om_len; 186 vm_offset_t om_pa; 187 u_int om_mode; 188}; 189 190int pmap_bootstrapped = 0; 191 192/* 193 * Virtual and physical address of message buffer. 194 */ 195struct msgbuf *msgbufp; 196vm_offset_t msgbuf_phys; 197 198/* 199 * Physical addresses of first and last available physical page. 200 */ 201vm_offset_t avail_start; 202vm_offset_t avail_end; 203 204int pmap_pagedaemon_waken; 205 206/* 207 * Map of physical memory regions. 208 */ 209vm_offset_t phys_avail[128]; 210u_int phys_avail_count; 211static struct mem_region *regions; 212static struct mem_region *pregions; 213int regions_sz, pregions_sz; 214static struct ofw_map *translations; 215 216/* 217 * First and last available kernel virtual addresses. 218 */ 219vm_offset_t virtual_avail; 220vm_offset_t virtual_end; 221vm_offset_t kernel_vm_end; 222 223/* 224 * Kernel pmap. 225 */ 226struct pmap kernel_pmap_store; 227extern struct pmap ofw_pmap; 228 229/* 230 * PTEG data. 231 */ 232static struct pteg *pmap_pteg_table; 233u_int pmap_pteg_count; 234u_int pmap_pteg_mask; 235 236/* 237 * PVO data. 238 */ 239struct pvo_head *pmap_pvo_table; /* pvo entries by pteg index */ 240struct pvo_head pmap_pvo_kunmanaged = 241 LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */ 242struct pvo_head pmap_pvo_unmanaged = 243 LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */ 244 245uma_zone_t pmap_upvo_zone; /* zone for pvo entries for unmanaged pages */ 246uma_zone_t pmap_mpvo_zone; /* zone for pvo entries for managed pages */ 247struct vm_object pmap_upvo_zone_obj; 248struct vm_object pmap_mpvo_zone_obj; 249static vm_object_t pmap_pvo_obj; 250static u_int pmap_pvo_count; 251 252#define BPVO_POOL_SIZE 32768 253static struct pvo_entry *pmap_bpvo_pool; 254static int pmap_bpvo_pool_index = 0; 255 256#define VSID_NBPW (sizeof(u_int32_t) * 8) 257static u_int pmap_vsid_bitmap[NPMAPS / VSID_NBPW]; 258 259static boolean_t pmap_initialized = FALSE; 260 261/* 262 * Statistics. 263 */ 264u_int pmap_pte_valid = 0; 265u_int pmap_pte_overflow = 0; 266u_int pmap_pte_replacements = 0; 267u_int pmap_pvo_entries = 0; 268u_int pmap_pvo_enter_calls = 0; 269u_int pmap_pvo_remove_calls = 0; 270u_int pmap_pte_spills = 0; 271SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_valid, CTLFLAG_RD, &pmap_pte_valid, 272 0, ""); 273SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_overflow, CTLFLAG_RD, 274 &pmap_pte_overflow, 0, ""); 275SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_replacements, CTLFLAG_RD, 276 &pmap_pte_replacements, 0, ""); 277SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_entries, CTLFLAG_RD, &pmap_pvo_entries, 278 0, ""); 279SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_enter_calls, CTLFLAG_RD, 280 &pmap_pvo_enter_calls, 0, ""); 281SYSCTL_INT(_machdep, OID_AUTO, pmap_pvo_remove_calls, CTLFLAG_RD, 282 &pmap_pvo_remove_calls, 0, ""); 283SYSCTL_INT(_machdep, OID_AUTO, pmap_pte_spills, CTLFLAG_RD, 284 &pmap_pte_spills, 0, ""); 285 286struct pvo_entry *pmap_pvo_zeropage; 287 288vm_offset_t pmap_rkva_start = VM_MIN_KERNEL_ADDRESS; 289u_int pmap_rkva_count = 4; 290 291/* 292 * Allocate physical memory for use in pmap_bootstrap. 293 */ 294static vm_offset_t pmap_bootstrap_alloc(vm_size_t, u_int); 295 296/* 297 * PTE calls. 298 */ 299static int pmap_pte_insert(u_int, struct pte *); 300 301/* 302 * PVO calls. 303 */ 304static int pmap_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *, 305 vm_offset_t, vm_offset_t, u_int, int); 306static void pmap_pvo_remove(struct pvo_entry *, int); 307static struct pvo_entry *pmap_pvo_find_va(pmap_t, vm_offset_t, int *); 308static struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int); 309 310/* 311 * Utility routines. 312 */ 313static void * pmap_pvo_allocf(uma_zone_t, int, u_int8_t *, int); 314static struct pvo_entry *pmap_rkva_alloc(void); 315static void pmap_pa_map(struct pvo_entry *, vm_offset_t, 316 struct pte *, int *); 317static void pmap_pa_unmap(struct pvo_entry *, struct pte *, int *); 318static void pmap_syncicache(vm_offset_t, vm_size_t); 319static boolean_t pmap_query_bit(vm_page_t, int); 320static u_int pmap_clear_bit(vm_page_t, int, int *); 321static void tlbia(void); 322 323static __inline int 324va_to_sr(u_int *sr, vm_offset_t va) 325{ 326 return (sr[(uintptr_t)va >> ADDR_SR_SHFT]); 327} 328 329static __inline u_int 330va_to_pteg(u_int sr, vm_offset_t addr) 331{ 332 u_int hash; 333 334 hash = (sr & SR_VSID_MASK) ^ (((u_int)addr & ADDR_PIDX) >> 335 ADDR_PIDX_SHFT); 336 return (hash & pmap_pteg_mask); 337} 338 339static __inline struct pvo_head * 340pa_to_pvoh(vm_offset_t pa, vm_page_t *pg_p) 341{ 342 struct vm_page *pg; 343 344 pg = PHYS_TO_VM_PAGE(pa); 345 346 if (pg_p != NULL) 347 *pg_p = pg; 348 349 if (pg == NULL) 350 return (&pmap_pvo_unmanaged); 351 352 return (&pg->md.mdpg_pvoh); 353} 354 355static __inline struct pvo_head * 356vm_page_to_pvoh(vm_page_t m) 357{ 358 359 return (&m->md.mdpg_pvoh); 360} 361 362static __inline void 363pmap_attr_clear(vm_page_t m, int ptebit) 364{ 365 366 m->md.mdpg_attrs &= ~ptebit; 367} 368 369static __inline int 370pmap_attr_fetch(vm_page_t m) 371{ 372 373 return (m->md.mdpg_attrs); 374} 375 376static __inline void 377pmap_attr_save(vm_page_t m, int ptebit) 378{ 379 380 m->md.mdpg_attrs |= ptebit; 381} 382 383static __inline int 384pmap_pte_compare(const struct pte *pt, const struct pte *pvo_pt) 385{ 386 if (pt->pte_hi == pvo_pt->pte_hi) 387 return (1); 388 389 return (0); 390} 391 392static __inline int 393pmap_pte_match(struct pte *pt, u_int sr, vm_offset_t va, int which) 394{ 395 return (pt->pte_hi & ~PTE_VALID) == 396 (((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 397 ((va >> ADDR_API_SHFT) & PTE_API) | which); 398} 399 400static __inline void 401pmap_pte_create(struct pte *pt, u_int sr, vm_offset_t va, u_int pte_lo) 402{ 403 /* 404 * Construct a PTE. Default to IMB initially. Valid bit only gets 405 * set when the real pte is set in memory. 406 * 407 * Note: Don't set the valid bit for correct operation of tlb update. 408 */ 409 pt->pte_hi = ((sr & SR_VSID_MASK) << PTE_VSID_SHFT) | 410 (((va & ADDR_PIDX) >> ADDR_API_SHFT) & PTE_API); 411 pt->pte_lo = pte_lo; 412} 413 414static __inline void 415pmap_pte_synch(struct pte *pt, struct pte *pvo_pt) 416{ 417 418 pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF | PTE_CHG); 419} 420 421static __inline void 422pmap_pte_clear(struct pte *pt, vm_offset_t va, int ptebit) 423{ 424 425 /* 426 * As shown in Section 7.6.3.2.3 427 */ 428 pt->pte_lo &= ~ptebit; 429 TLBIE(va); 430 EIEIO(); 431 TLBSYNC(); 432 SYNC(); 433} 434 435static __inline void 436pmap_pte_set(struct pte *pt, struct pte *pvo_pt) 437{ 438 439 pvo_pt->pte_hi |= PTE_VALID; 440 441 /* 442 * Update the PTE as defined in section 7.6.3.1. 443 * Note that the REF/CHG bits are from pvo_pt and thus should havce 444 * been saved so this routine can restore them (if desired). 445 */ 446 pt->pte_lo = pvo_pt->pte_lo; 447 EIEIO(); 448 pt->pte_hi = pvo_pt->pte_hi; 449 SYNC(); 450 pmap_pte_valid++; 451} 452 453static __inline void 454pmap_pte_unset(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 455{ 456 457 pvo_pt->pte_hi &= ~PTE_VALID; 458 459 /* 460 * Force the reg & chg bits back into the PTEs. 461 */ 462 SYNC(); 463 464 /* 465 * Invalidate the pte. 466 */ 467 pt->pte_hi &= ~PTE_VALID; 468 469 SYNC(); 470 TLBIE(va); 471 EIEIO(); 472 TLBSYNC(); 473 SYNC(); 474 475 /* 476 * Save the reg & chg bits. 477 */ 478 pmap_pte_synch(pt, pvo_pt); 479 pmap_pte_valid--; 480} 481 482static __inline void 483pmap_pte_change(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) 484{ 485 486 /* 487 * Invalidate the PTE 488 */ 489 pmap_pte_unset(pt, pvo_pt, va); 490 pmap_pte_set(pt, pvo_pt); 491} 492 493/* 494 * Quick sort callout for comparing memory regions. 495 */ 496static int mr_cmp(const void *a, const void *b); 497static int om_cmp(const void *a, const void *b); 498 499static int 500mr_cmp(const void *a, const void *b) 501{ 502 const struct mem_region *regiona; 503 const struct mem_region *regionb; 504 505 regiona = a; 506 regionb = b; 507 if (regiona->mr_start < regionb->mr_start) 508 return (-1); 509 else if (regiona->mr_start > regionb->mr_start) 510 return (1); 511 else 512 return (0); 513} 514 515static int 516om_cmp(const void *a, const void *b) 517{ 518 const struct ofw_map *mapa; 519 const struct ofw_map *mapb; 520 521 mapa = a; 522 mapb = b; 523 if (mapa->om_pa < mapb->om_pa) 524 return (-1); 525 else if (mapa->om_pa > mapb->om_pa) 526 return (1); 527 else 528 return (0); 529} 530 531void 532pmap_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) 533{ 534 ihandle_t mmui; 535 phandle_t chosen, mmu; 536 int sz; 537 int i, j; 538 int ofw_mappings; 539 vm_size_t size, physsz; 540 vm_offset_t pa, va, off; 541 u_int batl, batu; 542 543 /* 544 * Set up BAT0 to map the lowest 256 MB area 545 */ 546 battable[0x0].batl = BATL(0x00000000, BAT_M, BAT_PP_RW); 547 battable[0x0].batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs); 548 549 /* 550 * Map PCI memory space. 551 */ 552 battable[0x8].batl = BATL(0x80000000, BAT_I|BAT_G, BAT_PP_RW); 553 battable[0x8].batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs); 554 555 battable[0x9].batl = BATL(0x90000000, BAT_I|BAT_G, BAT_PP_RW); 556 battable[0x9].batu = BATU(0x90000000, BAT_BL_256M, BAT_Vs); 557 558 battable[0xa].batl = BATL(0xa0000000, BAT_I|BAT_G, BAT_PP_RW); 559 battable[0xa].batu = BATU(0xa0000000, BAT_BL_256M, BAT_Vs); 560 561 battable[0xb].batl = BATL(0xb0000000, BAT_I|BAT_G, BAT_PP_RW); 562 battable[0xb].batu = BATU(0xb0000000, BAT_BL_256M, BAT_Vs); 563 564 /* 565 * Map obio devices. 566 */ 567 battable[0xf].batl = BATL(0xf0000000, BAT_I|BAT_G, BAT_PP_RW); 568 battable[0xf].batu = BATU(0xf0000000, BAT_BL_256M, BAT_Vs); 569 570 /* 571 * Use an IBAT and a DBAT to map the bottom segment of memory 572 * where we are. 573 */ 574 batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs); 575 batl = BATL(0x00000000, BAT_M, BAT_PP_RW); 576 __asm ("mtibatu 0,%0; mtibatl 0,%1; mtdbatu 0,%0; mtdbatl 0,%1" 577 :: "r"(batu), "r"(batl)); 578 579#if 0 580 /* map frame buffer */ 581 batu = BATU(0x90000000, BAT_BL_256M, BAT_Vs); 582 batl = BATL(0x90000000, BAT_I|BAT_G, BAT_PP_RW); 583 __asm ("mtdbatu 1,%0; mtdbatl 1,%1" 584 :: "r"(batu), "r"(batl)); 585#endif 586 587#if 1 588 /* map pci space */ 589 batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs); 590 batl = BATL(0x80000000, BAT_I|BAT_G, BAT_PP_RW); 591 __asm ("mtdbatu 1,%0; mtdbatl 1,%1" 592 :: "r"(batu), "r"(batl)); 593#endif 594 595 /* 596 * Set the start and end of kva. 597 */ 598 virtual_avail = VM_MIN_KERNEL_ADDRESS; 599 virtual_end = VM_MAX_KERNEL_ADDRESS; 600 601 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 602 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 603 604 qsort(pregions, pregions_sz, sizeof(*pregions), mr_cmp); 605 for (i = 0; i < pregions_sz; i++) { 606 vm_offset_t pa; 607 vm_offset_t end; 608 609 CTR3(KTR_PMAP, "physregion: %#x - %#x (%#x)", 610 pregions[i].mr_start, 611 pregions[i].mr_start + pregions[i].mr_size, 612 pregions[i].mr_size); 613 /* 614 * Install entries into the BAT table to allow all 615 * of physmem to be convered by on-demand BAT entries. 616 * The loop will sometimes set the same battable element 617 * twice, but that's fine since they won't be used for 618 * a while yet. 619 */ 620 pa = pregions[i].mr_start & 0xf0000000; 621 end = pregions[i].mr_start + pregions[i].mr_size; 622 do { 623 u_int n = pa >> ADDR_SR_SHFT; 624 625 battable[n].batl = BATL(pa, BAT_M, BAT_PP_RW); 626 battable[n].batu = BATU(pa, BAT_BL_256M, BAT_Vs); 627 pa += SEGMENT_LENGTH; 628 } while (pa < end); 629 } 630 631 if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz) 632 panic("pmap_bootstrap: phys_avail too small"); 633 qsort(regions, regions_sz, sizeof(*regions), mr_cmp); 634 phys_avail_count = 0; 635 physsz = 0; 636 for (i = 0, j = 0; i < regions_sz; i++, j += 2) { 637 CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start, 638 regions[i].mr_start + regions[i].mr_size, 639 regions[i].mr_size); 640 phys_avail[j] = regions[i].mr_start; 641 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 642 phys_avail_count++; 643 physsz += regions[i].mr_size; 644 } 645 physmem = btoc(physsz); 646 647 /* 648 * Allocate PTEG table. 649 */ 650#ifdef PTEGCOUNT 651 pmap_pteg_count = PTEGCOUNT; 652#else 653 pmap_pteg_count = 0x1000; 654 655 while (pmap_pteg_count < physmem) 656 pmap_pteg_count <<= 1; 657 658 pmap_pteg_count >>= 1; 659#endif /* PTEGCOUNT */ 660 661 size = pmap_pteg_count * sizeof(struct pteg); 662 CTR2(KTR_PMAP, "pmap_bootstrap: %d PTEGs, %d bytes", pmap_pteg_count, 663 size); 664 pmap_pteg_table = (struct pteg *)pmap_bootstrap_alloc(size, size); 665 CTR1(KTR_PMAP, "pmap_bootstrap: PTEG table at %p", pmap_pteg_table); 666 bzero((void *)pmap_pteg_table, pmap_pteg_count * sizeof(struct pteg)); 667 pmap_pteg_mask = pmap_pteg_count - 1; 668 669 /* 670 * Allocate pv/overflow lists. 671 */ 672 size = sizeof(struct pvo_head) * pmap_pteg_count; 673 pmap_pvo_table = (struct pvo_head *)pmap_bootstrap_alloc(size, 674 PAGE_SIZE); 675 CTR1(KTR_PMAP, "pmap_bootstrap: PVO table at %p", pmap_pvo_table); 676 for (i = 0; i < pmap_pteg_count; i++) 677 LIST_INIT(&pmap_pvo_table[i]); 678 679 /* 680 * Allocate the message buffer. 681 */ 682 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE, 0); 683 684 /* 685 * Initialise the unmanaged pvo pool. 686 */ 687 pmap_bpvo_pool = (struct pvo_entry *)pmap_bootstrap_alloc( 688 BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0); 689 pmap_bpvo_pool_index = 0; 690 691 /* 692 * Make sure kernel vsid is allocated as well as VSID 0. 693 */ 694 pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW] 695 |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 696 pmap_vsid_bitmap[0] |= 1; 697 698 /* 699 * Set up the OpenFirmware pmap and add it's mappings. 700 */ 701 pmap_pinit(&ofw_pmap); 702 ofw_pmap.pm_sr[KERNEL_SR] = KERNEL_SEGMENT; 703 if ((chosen = OF_finddevice("/chosen")) == -1) 704 panic("pmap_bootstrap: can't find /chosen"); 705 OF_getprop(chosen, "mmu", &mmui, 4); 706 if ((mmu = OF_instance_to_package(mmui)) == -1) 707 panic("pmap_bootstrap: can't get mmu package"); 708 if ((sz = OF_getproplen(mmu, "translations")) == -1) 709 panic("pmap_bootstrap: can't get ofw translation count"); 710 translations = NULL; 711 for (i = 0; phys_avail[i + 2] != 0; i += 2) { 712 if (phys_avail[i + 1] >= sz) 713 translations = (struct ofw_map *)phys_avail[i]; 714 } 715 if (translations == NULL) 716 panic("pmap_bootstrap: no space to copy translations"); 717 bzero(translations, sz); 718 if (OF_getprop(mmu, "translations", translations, sz) == -1) 719 panic("pmap_bootstrap: can't get ofw translations"); 720 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 721 sz /= sizeof(*translations); 722 qsort(translations, sz, sizeof (*translations), om_cmp); 723 for (i = 0, ofw_mappings = 0; i < sz; i++) { 724 CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x", 725 translations[i].om_pa, translations[i].om_va, 726 translations[i].om_len); 727 728 /* 729 * If the mapping is 1:1, let the RAM and device on-demand 730 * BAT tables take care of the translation. 731 */ 732 if (translations[i].om_va == translations[i].om_pa) 733 continue; 734 735 /* Enter the pages */ 736 for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) { 737 struct vm_page m; 738 739 m.phys_addr = translations[i].om_pa + off; 740 pmap_enter(&ofw_pmap, translations[i].om_va + off, &m, 741 VM_PROT_ALL, 1); 742 ofw_mappings++; 743 } 744 } 745#ifdef SMP 746 TLBSYNC(); 747#endif 748 749 /* 750 * Initialize the kernel pmap (which is statically allocated). 751 */ 752 for (i = 0; i < 16; i++) { 753 kernel_pmap->pm_sr[i] = EMPTY_SEGMENT; 754 } 755 kernel_pmap->pm_sr[KERNEL_SR] = KERNEL_SEGMENT; 756 kernel_pmap->pm_active = ~0; 757 758 /* 759 * Allocate a kernel stack with a guard page for thread0 and map it 760 * into the kernel page map. 761 */ 762 pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, 0); 763 kstack0_phys = pa; 764 kstack0 = virtual_avail + (KSTACK_GUARD_PAGES * PAGE_SIZE); 765 CTR2(KTR_PMAP, "pmap_bootstrap: kstack0 at %#x (%#x)", kstack0_phys, 766 kstack0); 767 virtual_avail += (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE; 768 for (i = 0; i < KSTACK_PAGES; i++) { 769 pa = kstack0_phys + i * PAGE_SIZE; 770 va = kstack0 + i * PAGE_SIZE; 771 pmap_kenter(va, pa); 772 TLBIE(va); 773 } 774 775 /* 776 * Calculate the first and last available physical addresses. 777 */ 778 avail_start = phys_avail[0]; 779 for (i = 0; phys_avail[i + 2] != 0; i += 2) 780 ; 781 avail_end = phys_avail[i + 1]; 782 Maxmem = powerpc_btop(avail_end); 783 784 /* 785 * Allocate virtual address space for the message buffer. 786 */ 787 msgbufp = (struct msgbuf *)virtual_avail; 788 virtual_avail += round_page(MSGBUF_SIZE); 789 790 /* 791 * Initialize hardware. 792 */ 793 for (i = 0; i < 16; i++) { 794 mtsrin(i << ADDR_SR_SHFT, EMPTY_SEGMENT); 795 } 796 __asm __volatile ("mtsr %0,%1" 797 :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); 798 __asm __volatile ("sync; mtsdr1 %0; isync" 799 :: "r"((u_int)pmap_pteg_table | (pmap_pteg_mask >> 10))); 800 tlbia(); 801 802 pmap_bootstrapped++; 803} 804 805/* 806 * Activate a user pmap. The pmap must be activated before it's address 807 * space can be accessed in any way. 808 */ 809void 810pmap_activate(struct thread *td) 811{ 812 pmap_t pm, pmr; 813 814 /* 815 * Load all the data we need up front to encourage the compiler to 816 * not issue any loads while we have interrupts disabled below. 817 */ 818 pm = &td->td_proc->p_vmspace->vm_pmap; 819 820 if ((pmr = (pmap_t)pmap_kextract((vm_offset_t)pm)) == NULL) 821 pmr = pm; 822 823 pm->pm_active |= PCPU_GET(cpumask); 824 PCPU_SET(curpmap, pmr); 825} 826 827void 828pmap_deactivate(struct thread *td) 829{ 830 pmap_t pm; 831 832 pm = &td->td_proc->p_vmspace->vm_pmap; 833 pm->pm_active &= ~(PCPU_GET(cpumask)); 834 PCPU_SET(curpmap, NULL); 835} 836 837vm_offset_t 838pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 839{ 840 841 return (va); 842} 843 844void 845pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 846{ 847 struct pvo_entry *pvo; 848 849 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 850 851 if (pvo != NULL) { 852 if (wired) { 853 if ((pvo->pvo_vaddr & PVO_WIRED) == 0) 854 pm->pm_stats.wired_count++; 855 pvo->pvo_vaddr |= PVO_WIRED; 856 } else { 857 if ((pvo->pvo_vaddr & PVO_WIRED) != 0) 858 pm->pm_stats.wired_count--; 859 pvo->pvo_vaddr &= ~PVO_WIRED; 860 } 861 } 862} 863 864void 865pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 866 vm_size_t len, vm_offset_t src_addr) 867{ 868 869 /* 870 * This is not needed as it's mainly an optimisation. 871 * It may want to be implemented later though. 872 */ 873} 874 875void 876pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 877{ 878 vm_offset_t dst; 879 vm_offset_t src; 880 881 dst = VM_PAGE_TO_PHYS(mdst); 882 src = VM_PAGE_TO_PHYS(msrc); 883 884 kcopy((void *)src, (void *)dst, PAGE_SIZE); 885} 886 887/* 888 * Zero a page of physical memory by temporarily mapping it into the tlb. 889 */ 890void 891pmap_zero_page(vm_page_t m) 892{ 893 vm_offset_t pa = VM_PAGE_TO_PHYS(m); 894 caddr_t va; 895 896 if (pa < SEGMENT_LENGTH) { 897 va = (caddr_t) pa; 898 } else if (pmap_initialized) { 899 if (pmap_pvo_zeropage == NULL) 900 pmap_pvo_zeropage = pmap_rkva_alloc(); 901 pmap_pa_map(pmap_pvo_zeropage, pa, NULL, NULL); 902 va = (caddr_t)PVO_VADDR(pmap_pvo_zeropage); 903 } else { 904 panic("pmap_zero_page: can't zero pa %#x", pa); 905 } 906 907 bzero(va, PAGE_SIZE); 908 909 if (pa >= SEGMENT_LENGTH) 910 pmap_pa_unmap(pmap_pvo_zeropage, NULL, NULL); 911} 912 913void 914pmap_zero_page_area(vm_page_t m, int off, int size) 915{ 916 vm_offset_t pa = VM_PAGE_TO_PHYS(m); 917 caddr_t va; 918 919 if (pa < SEGMENT_LENGTH) { 920 va = (caddr_t) pa; 921 } else if (pmap_initialized) { 922 if (pmap_pvo_zeropage == NULL) 923 pmap_pvo_zeropage = pmap_rkva_alloc(); 924 pmap_pa_map(pmap_pvo_zeropage, pa, NULL, NULL); 925 va = (caddr_t)PVO_VADDR(pmap_pvo_zeropage); 926 } else { 927 panic("pmap_zero_page: can't zero pa %#x", pa); 928 } 929 930 bzero(va + off, size); 931 932 if (pa >= SEGMENT_LENGTH) 933 pmap_pa_unmap(pmap_pvo_zeropage, NULL, NULL); 934} 935 936void 937pmap_zero_page_idle(vm_page_t m) 938{ 939 940 /* XXX this is called outside of Giant, is pmap_zero_page safe? */ 941 /* XXX maybe have a dedicated mapping for this to avoid the problem? */ 942 mtx_lock(&Giant); 943 pmap_zero_page(m); 944 mtx_unlock(&Giant); 945} 946 947/* 948 * Map the given physical page at the specified virtual address in the 949 * target pmap with the protection requested. If specified the page 950 * will be wired down. 951 */ 952void 953pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 954 boolean_t wired) 955{ 956 struct pvo_head *pvo_head; 957 uma_zone_t zone; 958 vm_page_t pg; 959 u_int pte_lo, pvo_flags, was_exec, i; 960 int error; 961 962 if (!pmap_initialized) { 963 pvo_head = &pmap_pvo_kunmanaged; 964 zone = pmap_upvo_zone; 965 pvo_flags = 0; 966 pg = NULL; 967 was_exec = PTE_EXEC; 968 } else { 969 pvo_head = vm_page_to_pvoh(m); 970 pg = m; 971 zone = pmap_mpvo_zone; 972 pvo_flags = PVO_MANAGED; 973 was_exec = 0; 974 } 975 976 /* 977 * If this is a managed page, and it's the first reference to the page, 978 * clear the execness of the page. Otherwise fetch the execness. 979 */ 980 if (pg != NULL) { 981 if (LIST_EMPTY(pvo_head)) { 982 pmap_attr_clear(pg, PTE_EXEC); 983 } else { 984 was_exec = pmap_attr_fetch(pg) & PTE_EXEC; 985 } 986 } 987 988 989 /* 990 * Assume the page is cache inhibited and access is guarded unless 991 * it's in our available memory array. 992 */ 993 pte_lo = PTE_I | PTE_G; 994 for (i = 0; i < pregions_sz; i++) { 995 if ((VM_PAGE_TO_PHYS(m) >= pregions[i].mr_start) && 996 (VM_PAGE_TO_PHYS(m) < 997 (pregions[i].mr_start + pregions[i].mr_size))) { 998 pte_lo &= ~(PTE_I | PTE_G); 999 break; 1000 } 1001 } 1002 1003 if (prot & VM_PROT_WRITE) 1004 pte_lo |= PTE_BW; 1005 else 1006 pte_lo |= PTE_BR; 1007 1008 pvo_flags |= (prot & VM_PROT_EXECUTE); 1009 1010 if (wired) 1011 pvo_flags |= PVO_WIRED; 1012 1013 error = pmap_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), 1014 pte_lo, pvo_flags); 1015 1016 /* 1017 * Flush the real page from the instruction cache if this page is 1018 * mapped executable and cacheable and was not previously mapped (or 1019 * was not mapped executable). 1020 */ 1021 if (error == 0 && (pvo_flags & PVO_EXECUTABLE) && 1022 (pte_lo & PTE_I) == 0 && was_exec == 0) { 1023 /* 1024 * Flush the real memory from the cache. 1025 */ 1026 pmap_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); 1027 if (pg != NULL) 1028 pmap_attr_save(pg, PTE_EXEC); 1029 } 1030 1031 /* XXX syncicache always until problems are sorted */ 1032 pmap_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE); 1033} 1034 1035vm_offset_t 1036pmap_extract(pmap_t pm, vm_offset_t va) 1037{ 1038 struct pvo_entry *pvo; 1039 1040 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 1041 1042 if (pvo != NULL) { 1043 return ((pvo->pvo_pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF)); 1044 } 1045 1046 return (0); 1047} 1048 1049/* 1050 * Grow the number of kernel page table entries. Unneeded. 1051 */ 1052void 1053pmap_growkernel(vm_offset_t addr) 1054{ 1055} 1056 1057void 1058pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) 1059{ 1060 1061 CTR0(KTR_PMAP, "pmap_init"); 1062 1063 pmap_pvo_obj = vm_object_allocate(OBJT_PHYS, 16); 1064 pmap_pvo_count = 0; 1065 pmap_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), 1066 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); 1067 uma_zone_set_allocf(pmap_upvo_zone, pmap_pvo_allocf); 1068 pmap_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry), 1069 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); 1070 uma_zone_set_allocf(pmap_mpvo_zone, pmap_pvo_allocf); 1071 pmap_initialized = TRUE; 1072} 1073 1074void 1075pmap_init2(void) 1076{ 1077 1078 CTR0(KTR_PMAP, "pmap_init2"); 1079} 1080 1081boolean_t 1082pmap_is_modified(vm_page_t m) 1083{ 1084 1085 if ((m->flags & (PG_FICTITIOUS |PG_UNMANAGED)) != 0) 1086 return (FALSE); 1087 1088 return (pmap_query_bit(m, PTE_CHG)); 1089} 1090 1091void 1092pmap_clear_reference(vm_page_t m) 1093{ 1094 1095 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1096 return; 1097 pmap_clear_bit(m, PTE_REF, NULL); 1098} 1099 1100void 1101pmap_clear_modify(vm_page_t m) 1102{ 1103 1104 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1105 return; 1106 pmap_clear_bit(m, PTE_CHG, NULL); 1107} 1108 1109/* 1110 * pmap_ts_referenced: 1111 * 1112 * Return a count of reference bits for a page, clearing those bits. 1113 * It is not necessary for every reference bit to be cleared, but it 1114 * is necessary that 0 only be returned when there are truly no 1115 * reference bits set. 1116 * 1117 * XXX: The exact number of bits to check and clear is a matter that 1118 * should be tested and standardized at some point in the future for 1119 * optimal aging of shared pages. 1120 */ 1121int 1122pmap_ts_referenced(vm_page_t m) 1123{ 1124 int count; 1125 1126 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1127 return (0); 1128 1129 count = pmap_clear_bit(m, PTE_REF, NULL); 1130 1131 return (count); 1132} 1133 1134/* 1135 * Map a wired page into kernel virtual address space. 1136 */ 1137void 1138pmap_kenter(vm_offset_t va, vm_offset_t pa) 1139{ 1140 u_int pte_lo; 1141 int error; 1142 int i; 1143 1144#if 0 1145 if (va < VM_MIN_KERNEL_ADDRESS) 1146 panic("pmap_kenter: attempt to enter non-kernel address %#x", 1147 va); 1148#endif 1149 1150 pte_lo = PTE_I | PTE_G; 1151 for (i = 0; i < pregions_sz; i++) { 1152 if ((pa >= pregions[i].mr_start) && 1153 (pa < (pregions[i].mr_start + pregions[i].mr_size))) { 1154 pte_lo &= ~(PTE_I | PTE_G); 1155 break; 1156 } 1157 } 1158 1159 error = pmap_pvo_enter(kernel_pmap, pmap_upvo_zone, 1160 &pmap_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED); 1161 1162 if (error != 0 && error != ENOENT) 1163 panic("pmap_kenter: failed to enter va %#x pa %#x: %d", va, 1164 pa, error); 1165 1166 /* 1167 * Flush the real memory from the instruction cache. 1168 */ 1169 if ((pte_lo & (PTE_I | PTE_G)) == 0) { 1170 pmap_syncicache(pa, PAGE_SIZE); 1171 } 1172} 1173 1174/* 1175 * Extract the physical page address associated with the given kernel virtual 1176 * address. 1177 */ 1178vm_offset_t 1179pmap_kextract(vm_offset_t va) 1180{ 1181 struct pvo_entry *pvo; 1182 1183 pvo = pmap_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL); 1184 if (pvo == NULL) { 1185 return (0); 1186 } 1187 1188 return ((pvo->pvo_pte.pte_lo & PTE_RPGN) | (va & ADDR_POFF)); 1189} 1190 1191/* 1192 * Remove a wired page from kernel virtual address space. 1193 */ 1194void 1195pmap_kremove(vm_offset_t va) 1196{ 1197 1198 pmap_remove(kernel_pmap, va, va + PAGE_SIZE); 1199} 1200 1201/* 1202 * Map a range of physical addresses into kernel virtual address space. 1203 * 1204 * The value passed in *virt is a suggested virtual address for the mapping. 1205 * Architectures which can support a direct-mapped physical to virtual region 1206 * can return the appropriate address within that region, leaving '*virt' 1207 * unchanged. We cannot and therefore do not; *virt is updated with the 1208 * first usable address after the mapped region. 1209 */ 1210vm_offset_t 1211pmap_map(vm_offset_t *virt, vm_offset_t pa_start, vm_offset_t pa_end, int prot) 1212{ 1213 vm_offset_t sva, va; 1214 1215 sva = *virt; 1216 va = sva; 1217 for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE) 1218 pmap_kenter(va, pa_start); 1219 *virt = va; 1220 return (sva); 1221} 1222 1223int 1224pmap_mincore(pmap_t pmap, vm_offset_t addr) 1225{ 1226 TODO; 1227 return (0); 1228} 1229 1230void 1231pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1232 vm_pindex_t pindex, vm_size_t size, int limit) 1233{ 1234 1235 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1236 ("pmap_remove_pages: non current pmap")); 1237 /* XXX */ 1238} 1239 1240/* 1241 * Lower the permission for all mappings to a given page. 1242 */ 1243void 1244pmap_page_protect(vm_page_t m, vm_prot_t prot) 1245{ 1246 struct pvo_head *pvo_head; 1247 struct pvo_entry *pvo, *next_pvo; 1248 struct pte *pt; 1249 1250 /* 1251 * Since the routine only downgrades protection, if the 1252 * maximal protection is desired, there isn't any change 1253 * to be made. 1254 */ 1255 if ((prot & (VM_PROT_READ|VM_PROT_WRITE)) == 1256 (VM_PROT_READ|VM_PROT_WRITE)) 1257 return; 1258 1259 pvo_head = vm_page_to_pvoh(m); 1260 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 1261 next_pvo = LIST_NEXT(pvo, pvo_vlink); 1262 PMAP_PVO_CHECK(pvo); /* sanity check */ 1263 1264 /* 1265 * Downgrading to no mapping at all, we just remove the entry. 1266 */ 1267 if ((prot & VM_PROT_READ) == 0) { 1268 pmap_pvo_remove(pvo, -1); 1269 continue; 1270 } 1271 1272 /* 1273 * If EXEC permission is being revoked, just clear the flag 1274 * in the PVO. 1275 */ 1276 if ((prot & VM_PROT_EXECUTE) == 0) 1277 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1278 1279 /* 1280 * If this entry is already RO, don't diddle with the page 1281 * table. 1282 */ 1283 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) { 1284 PMAP_PVO_CHECK(pvo); 1285 continue; 1286 } 1287 1288 /* 1289 * Grab the PTE before we diddle the bits so pvo_to_pte can 1290 * verify the pte contents are as expected. 1291 */ 1292 pt = pmap_pvo_to_pte(pvo, -1); 1293 pvo->pvo_pte.pte_lo &= ~PTE_PP; 1294 pvo->pvo_pte.pte_lo |= PTE_BR; 1295 if (pt != NULL) 1296 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1297 PMAP_PVO_CHECK(pvo); /* sanity check */ 1298 } 1299} 1300 1301/* 1302 * Returns true if the pmap's pv is one of the first 1303 * 16 pvs linked to from this page. This count may 1304 * be changed upwards or downwards in the future; it 1305 * is only necessary that true be returned for a small 1306 * subset of pmaps for proper page aging. 1307 */ 1308boolean_t 1309pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 1310{ 1311 int loops; 1312 struct pvo_entry *pvo; 1313 1314 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 1315 return FALSE; 1316 1317 loops = 0; 1318 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 1319 if (pvo->pvo_pmap == pmap) 1320 return (TRUE); 1321 if (++loops >= 16) 1322 break; 1323 } 1324 1325 return (FALSE); 1326} 1327 1328static u_int pmap_vsidcontext; 1329 1330void 1331pmap_pinit(pmap_t pmap) 1332{ 1333 int i, mask; 1334 u_int entropy; 1335 1336 entropy = 0; 1337 __asm __volatile("mftb %0" : "=r"(entropy)); 1338 1339 /* 1340 * Allocate some segment registers for this pmap. 1341 */ 1342 for (i = 0; i < NPMAPS; i += VSID_NBPW) { 1343 u_int hash, n; 1344 1345 /* 1346 * Create a new value by mutiplying by a prime and adding in 1347 * entropy from the timebase register. This is to make the 1348 * VSID more random so that the PT hash function collides 1349 * less often. (Note that the prime casues gcc to do shifts 1350 * instead of a multiply.) 1351 */ 1352 pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy; 1353 hash = pmap_vsidcontext & (NPMAPS - 1); 1354 if (hash == 0) /* 0 is special, avoid it */ 1355 continue; 1356 n = hash >> 5; 1357 mask = 1 << (hash & (VSID_NBPW - 1)); 1358 hash = (pmap_vsidcontext & 0xfffff); 1359 if (pmap_vsid_bitmap[n] & mask) { /* collision? */ 1360 /* anything free in this bucket? */ 1361 if (pmap_vsid_bitmap[n] == 0xffffffff) { 1362 entropy = (pmap_vsidcontext >> 20); 1363 continue; 1364 } 1365 i = ffs(~pmap_vsid_bitmap[i]) - 1; 1366 mask = 1 << i; 1367 hash &= 0xfffff & ~(VSID_NBPW - 1); 1368 hash |= i; 1369 } 1370 pmap_vsid_bitmap[n] |= mask; 1371 for (i = 0; i < 16; i++) 1372 pmap->pm_sr[i] = VSID_MAKE(i, hash); 1373 return; 1374 } 1375 1376 panic("pmap_pinit: out of segments"); 1377} 1378 1379/* 1380 * Initialize the pmap associated with process 0. 1381 */ 1382void 1383pmap_pinit0(pmap_t pm) 1384{ 1385 1386 pmap_pinit(pm); 1387 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1388} 1389 1390void 1391pmap_pinit2(pmap_t pmap) 1392{ 1393 /* XXX: Remove this stub when no longer called */ 1394} 1395 1396void 1397pmap_prefault(pmap_t pm, vm_offset_t va, vm_map_entry_t entry) 1398{ 1399 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1400 ("pmap_prefault: non current pmap")); 1401 /* XXX */ 1402} 1403 1404/* 1405 * Set the physical protection on the specified range of this map as requested. 1406 */ 1407void 1408pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1409{ 1410 struct pvo_entry *pvo; 1411 struct pte *pt; 1412 int pteidx; 1413 1414 CTR4(KTR_PMAP, "pmap_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva, 1415 eva, prot); 1416 1417 1418 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1419 ("pmap_protect: non current pmap")); 1420 1421 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1422 pmap_remove(pm, sva, eva); 1423 return; 1424 } 1425 1426 for (; sva < eva; sva += PAGE_SIZE) { 1427 pvo = pmap_pvo_find_va(pm, sva, &pteidx); 1428 if (pvo == NULL) 1429 continue; 1430 1431 if ((prot & VM_PROT_EXECUTE) == 0) 1432 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1433 1434 /* 1435 * Grab the PTE pointer before we diddle with the cached PTE 1436 * copy. 1437 */ 1438 pt = pmap_pvo_to_pte(pvo, pteidx); 1439 /* 1440 * Change the protection of the page. 1441 */ 1442 pvo->pvo_pte.pte_lo &= ~PTE_PP; 1443 pvo->pvo_pte.pte_lo |= PTE_BR; 1444 1445 /* 1446 * If the PVO is in the page table, update that pte as well. 1447 */ 1448 if (pt != NULL) 1449 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1450 } 1451} 1452 1453/* 1454 * Map a list of wired pages into kernel virtual address space. This is 1455 * intended for temporary mappings which do not need page modification or 1456 * references recorded. Existing mappings in the region are overwritten. 1457 */ 1458void 1459pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 1460{ 1461 vm_offset_t va; 1462 1463 va = sva; 1464 while (count-- > 0) { 1465 pmap_kenter(va, VM_PAGE_TO_PHYS(*m)); 1466 va += PAGE_SIZE; 1467 m++; 1468 } 1469} 1470 1471/* 1472 * Remove page mappings from kernel virtual address space. Intended for 1473 * temporary mappings entered by pmap_qenter. 1474 */ 1475void 1476pmap_qremove(vm_offset_t sva, int count) 1477{ 1478 vm_offset_t va; 1479 1480 va = sva; 1481 while (count-- > 0) { 1482 pmap_kremove(va); 1483 va += PAGE_SIZE; 1484 } 1485} 1486 1487void 1488pmap_release(pmap_t pmap) 1489{ 1490 int idx, mask; 1491 1492 /* 1493 * Free segment register's VSID 1494 */ 1495 if (pmap->pm_sr[0] == 0) 1496 panic("pmap_release"); 1497 1498 idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1); 1499 mask = 1 << (idx % VSID_NBPW); 1500 idx /= VSID_NBPW; 1501 pmap_vsid_bitmap[idx] &= ~mask; 1502} 1503 1504/* 1505 * Remove the given range of addresses from the specified map. 1506 */ 1507void 1508pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1509{ 1510 struct pvo_entry *pvo; 1511 int pteidx; 1512 1513 for (; sva < eva; sva += PAGE_SIZE) { 1514 pvo = pmap_pvo_find_va(pm, sva, &pteidx); 1515 if (pvo != NULL) { 1516 pmap_pvo_remove(pvo, pteidx); 1517 } 1518 } 1519} 1520 1521/* 1522 * Remove physical page from all pmaps in which it resides. pmap_pvo_remove() 1523 * will reflect changes in pte's back to the vm_page. 1524 */ 1525void 1526pmap_remove_all(vm_page_t m) 1527{ 1528 struct pvo_head *pvo_head; 1529 struct pvo_entry *pvo, *next_pvo; 1530 1531 KASSERT((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0, 1532 ("pv_remove_all: illegal for unmanaged page %#x", 1533 VM_PAGE_TO_PHYS(m))); 1534 1535 pvo_head = vm_page_to_pvoh(m); 1536 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 1537 next_pvo = LIST_NEXT(pvo, pvo_vlink); 1538 1539 PMAP_PVO_CHECK(pvo); /* sanity check */ 1540 pmap_pvo_remove(pvo, -1); 1541 } 1542 vm_page_flag_clear(m, PG_WRITEABLE); 1543} 1544 1545/* 1546 * Remove all pages from specified address space, this aids process exit 1547 * speeds. This is much faster than pmap_remove in the case of running down 1548 * an entire address space. Only works for the current pmap. 1549 */ 1550void 1551pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1552{ 1553 1554 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, 1555 ("pmap_remove_pages: non current pmap")); 1556 pmap_remove(pm, sva, eva); 1557} 1558
|
1713 * Allocate a physical page of memory directly from the phys_avail map. 1714 * Can only be called from pmap_bootstrap before avail start and end are 1715 * calculated. 1716 */ 1717static vm_offset_t 1718pmap_bootstrap_alloc(vm_size_t size, u_int align) 1719{ 1720 vm_offset_t s, e; 1721 int i, j; 1722 1723 size = round_page(size); 1724 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1725 if (align != 0) 1726 s = (phys_avail[i] + align - 1) & ~(align - 1); 1727 else 1728 s = phys_avail[i]; 1729 e = s + size; 1730 1731 if (s < phys_avail[i] || e > phys_avail[i + 1]) 1732 continue; 1733 1734 if (s == phys_avail[i]) { 1735 phys_avail[i] += size; 1736 } else if (e == phys_avail[i + 1]) { 1737 phys_avail[i + 1] -= size; 1738 } else { 1739 for (j = phys_avail_count * 2; j > i; j -= 2) { 1740 phys_avail[j] = phys_avail[j - 2]; 1741 phys_avail[j + 1] = phys_avail[j - 1]; 1742 } 1743 1744 phys_avail[i + 3] = phys_avail[i + 1]; 1745 phys_avail[i + 1] = s; 1746 phys_avail[i + 2] = e; 1747 phys_avail_count++; 1748 } 1749 1750 return (s); 1751 } 1752 panic("pmap_bootstrap_alloc: could not allocate memory"); 1753} 1754 1755/* 1756 * Return an unmapped pvo for a kernel virtual address. 1757 * Used by pmap functions that operate on physical pages. 1758 */ 1759static struct pvo_entry * 1760pmap_rkva_alloc(void) 1761{ 1762 struct pvo_entry *pvo; 1763 struct pte *pt; 1764 vm_offset_t kva; 1765 int pteidx; 1766 1767 if (pmap_rkva_count == 0) 1768 panic("pmap_rkva_alloc: no more reserved KVAs"); 1769 1770 kva = pmap_rkva_start + (PAGE_SIZE * --pmap_rkva_count); 1771 pmap_kenter(kva, 0); 1772 1773 pvo = pmap_pvo_find_va(kernel_pmap, kva, &pteidx); 1774 1775 if (pvo == NULL) 1776 panic("pmap_kva_alloc: pmap_pvo_find_va failed"); 1777 1778 pt = pmap_pvo_to_pte(pvo, pteidx); 1779 1780 if (pt == NULL) 1781 panic("pmap_kva_alloc: pmap_pvo_to_pte failed"); 1782 1783 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1784 PVO_PTEGIDX_CLR(pvo); 1785 1786 pmap_pte_overflow++; 1787 1788 return (pvo); 1789} 1790 1791static void 1792pmap_pa_map(struct pvo_entry *pvo, vm_offset_t pa, struct pte *saved_pt, 1793 int *depth_p) 1794{ 1795 struct pte *pt; 1796 1797 /* 1798 * If this pvo already has a valid pte, we need to save it so it can 1799 * be restored later. We then just reload the new PTE over the old 1800 * slot. 1801 */ 1802 if (saved_pt != NULL) { 1803 pt = pmap_pvo_to_pte(pvo, -1); 1804 1805 if (pt != NULL) { 1806 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1807 PVO_PTEGIDX_CLR(pvo); 1808 pmap_pte_overflow++; 1809 } 1810 1811 *saved_pt = pvo->pvo_pte; 1812 1813 pvo->pvo_pte.pte_lo &= ~PTE_RPGN; 1814 } 1815 1816 pvo->pvo_pte.pte_lo |= pa; 1817 1818 if (!pmap_pte_spill(pvo->pvo_vaddr)) 1819 panic("pmap_pa_map: could not spill pvo %p", pvo); 1820 1821 if (depth_p != NULL) 1822 (*depth_p)++; 1823} 1824 1825static void 1826pmap_pa_unmap(struct pvo_entry *pvo, struct pte *saved_pt, int *depth_p) 1827{ 1828 struct pte *pt; 1829 1830 pt = pmap_pvo_to_pte(pvo, -1); 1831 1832 if (pt != NULL) { 1833 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1834 PVO_PTEGIDX_CLR(pvo); 1835 pmap_pte_overflow++; 1836 } 1837 1838 pvo->pvo_pte.pte_lo &= ~PTE_RPGN; 1839 1840 /* 1841 * If there is a saved PTE and it's valid, restore it and return. 1842 */ 1843 if (saved_pt != NULL && (saved_pt->pte_lo & PTE_RPGN) != 0) { 1844 if (depth_p != NULL && --(*depth_p) == 0) 1845 panic("pmap_pa_unmap: restoring but depth == 0"); 1846 1847 pvo->pvo_pte = *saved_pt; 1848 1849 if (!pmap_pte_spill(pvo->pvo_vaddr)) 1850 panic("pmap_pa_unmap: could not spill pvo %p", pvo); 1851 } 1852} 1853 1854static void 1855pmap_syncicache(vm_offset_t pa, vm_size_t len) 1856{ 1857 __syncicache((void *)pa, len); 1858} 1859 1860static void 1861tlbia(void) 1862{ 1863 caddr_t i; 1864 1865 SYNC(); 1866 for (i = 0; i < (caddr_t)0x00040000; i += 0x00001000) { 1867 TLBIE(i); 1868 EIEIO(); 1869 } 1870 TLBSYNC(); 1871 SYNC(); 1872} 1873 1874static int 1875pmap_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head, 1876 vm_offset_t va, vm_offset_t pa, u_int pte_lo, int flags) 1877{ 1878 struct pvo_entry *pvo; 1879 u_int sr; 1880 int first; 1881 u_int ptegidx; 1882 int i; 1883 int bootstrap; 1884 1885 pmap_pvo_enter_calls++; 1886 first = 0; 1887 1888 bootstrap = 0; 1889 1890 /* 1891 * Compute the PTE Group index. 1892 */ 1893 va &= ~ADDR_POFF; 1894 sr = va_to_sr(pm->pm_sr, va); 1895 ptegidx = va_to_pteg(sr, va); 1896 1897 /* 1898 * Remove any existing mapping for this page. Reuse the pvo entry if 1899 * there is a mapping. 1900 */ 1901 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1902 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1903 if ((pvo->pvo_pte.pte_lo & PTE_RPGN) == pa && 1904 (pvo->pvo_pte.pte_lo & PTE_PP) == 1905 (pte_lo & PTE_PP)) { 1906 return (0); 1907 } 1908 pmap_pvo_remove(pvo, -1); 1909 break; 1910 } 1911 } 1912 1913 /* 1914 * If we aren't overwriting a mapping, try to allocate. 1915 */ 1916 if (pmap_initialized) { 1917 pvo = uma_zalloc(zone, M_NOWAIT); 1918 } else { 1919 if (pmap_bpvo_pool_index >= BPVO_POOL_SIZE) { 1920 panic("pmap_enter: bpvo pool exhausted, %d, %d, %d", 1921 pmap_bpvo_pool_index, BPVO_POOL_SIZE, 1922 BPVO_POOL_SIZE * sizeof(struct pvo_entry)); 1923 } 1924 pvo = &pmap_bpvo_pool[pmap_bpvo_pool_index]; 1925 pmap_bpvo_pool_index++; 1926 bootstrap = 1; 1927 } 1928 1929 if (pvo == NULL) { 1930 return (ENOMEM); 1931 } 1932 1933 pmap_pvo_entries++; 1934 pvo->pvo_vaddr = va; 1935 pvo->pvo_pmap = pm; 1936 LIST_INSERT_HEAD(&pmap_pvo_table[ptegidx], pvo, pvo_olink); 1937 pvo->pvo_vaddr &= ~ADDR_POFF; 1938 if (flags & VM_PROT_EXECUTE) 1939 pvo->pvo_vaddr |= PVO_EXECUTABLE; 1940 if (flags & PVO_WIRED) 1941 pvo->pvo_vaddr |= PVO_WIRED; 1942 if (pvo_head != &pmap_pvo_kunmanaged) 1943 pvo->pvo_vaddr |= PVO_MANAGED; 1944 if (bootstrap) 1945 pvo->pvo_vaddr |= PVO_BOOTSTRAP; 1946 pmap_pte_create(&pvo->pvo_pte, sr, va, pa | pte_lo); 1947 1948 /* 1949 * Remember if the list was empty and therefore will be the first 1950 * item. 1951 */ 1952 if (LIST_FIRST(pvo_head) == NULL) 1953 first = 1; 1954 1955 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 1956 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1957 pvo->pvo_pmap->pm_stats.wired_count++; 1958 pvo->pvo_pmap->pm_stats.resident_count++; 1959 1960 /* 1961 * We hope this succeeds but it isn't required. 1962 */ 1963 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 1964 if (i >= 0) { 1965 PVO_PTEGIDX_SET(pvo, i); 1966 } else { 1967 panic("pmap_pvo_enter: overflow"); 1968 pmap_pte_overflow++; 1969 } 1970 1971 return (first ? ENOENT : 0); 1972} 1973 1974static void 1975pmap_pvo_remove(struct pvo_entry *pvo, int pteidx) 1976{ 1977 struct pte *pt; 1978 1979 /* 1980 * If there is an active pte entry, we need to deactivate it (and 1981 * save the ref & cfg bits). 1982 */ 1983 pt = pmap_pvo_to_pte(pvo, pteidx); 1984 if (pt != NULL) { 1985 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1986 PVO_PTEGIDX_CLR(pvo); 1987 } else { 1988 pmap_pte_overflow--; 1989 } 1990 1991 /* 1992 * Update our statistics. 1993 */ 1994 pvo->pvo_pmap->pm_stats.resident_count--; 1995 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1996 pvo->pvo_pmap->pm_stats.wired_count--; 1997 1998 /* 1999 * Save the REF/CHG bits into their cache if the page is managed. 2000 */ 2001 if (pvo->pvo_vaddr & PVO_MANAGED) { 2002 struct vm_page *pg; 2003 2004 pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte_lo & PTE_RPGN); 2005 if (pg != NULL) { 2006 pmap_attr_save(pg, pvo->pvo_pte.pte_lo & 2007 (PTE_REF | PTE_CHG)); 2008 } 2009 } 2010 2011 /* 2012 * Remove this PVO from the PV list. 2013 */ 2014 LIST_REMOVE(pvo, pvo_vlink); 2015 2016 /* 2017 * Remove this from the overflow list and return it to the pool 2018 * if we aren't going to reuse it. 2019 */ 2020 LIST_REMOVE(pvo, pvo_olink); 2021 if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP)) 2022 uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? pmap_mpvo_zone : 2023 pmap_upvo_zone, pvo); 2024 pmap_pvo_entries--; 2025 pmap_pvo_remove_calls++; 2026} 2027 2028static __inline int 2029pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 2030{ 2031 int pteidx; 2032 2033 /* 2034 * We can find the actual pte entry without searching by grabbing 2035 * the PTEG index from 3 unused bits in pte_lo[11:9] and by 2036 * noticing the HID bit. 2037 */ 2038 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 2039 if (pvo->pvo_pte.pte_hi & PTE_HID) 2040 pteidx ^= pmap_pteg_mask * 8; 2041 2042 return (pteidx); 2043} 2044 2045static struct pvo_entry * 2046pmap_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p) 2047{ 2048 struct pvo_entry *pvo; 2049 int ptegidx; 2050 u_int sr; 2051 2052 va &= ~ADDR_POFF; 2053 sr = va_to_sr(pm->pm_sr, va); 2054 ptegidx = va_to_pteg(sr, va); 2055 2056 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2057 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 2058 if (pteidx_p) 2059 *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx); 2060 return (pvo); 2061 } 2062 } 2063 2064 return (NULL); 2065} 2066 2067static struct pte * 2068pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 2069{ 2070 struct pte *pt; 2071 2072 /* 2073 * If we haven't been supplied the ptegidx, calculate it. 2074 */ 2075 if (pteidx == -1) { 2076 int ptegidx; 2077 u_int sr; 2078 2079 sr = va_to_sr(pvo->pvo_pmap->pm_sr, pvo->pvo_vaddr); 2080 ptegidx = va_to_pteg(sr, pvo->pvo_vaddr); 2081 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 2082 } 2083 2084 pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7]; 2085 2086 if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 2087 panic("pmap_pvo_to_pte: pvo %p has valid pte in pvo but no " 2088 "valid pte index", pvo); 2089 } 2090 2091 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 2092 panic("pmap_pvo_to_pte: pvo %p has valid pte index in pvo " 2093 "pvo but no valid pte", pvo); 2094 } 2095 2096 if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 2097 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) { 2098 panic("pmap_pvo_to_pte: pvo %p has valid pte in " 2099 "pmap_pteg_table %p but invalid in pvo", pvo, pt); 2100 } 2101 2102 if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) 2103 != 0) { 2104 panic("pmap_pvo_to_pte: pvo %p pte does not match " 2105 "pte %p in pmap_pteg_table", pvo, pt); 2106 } 2107 2108 return (pt); 2109 } 2110 2111 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 2112 panic("pmap_pvo_to_pte: pvo %p has invalid pte %p in " 2113 "pmap_pteg_table but valid in pvo", pvo, pt); 2114 } 2115 2116 return (NULL); 2117} 2118 2119static void * 2120pmap_pvo_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 2121{ 2122 vm_page_t m; 2123 2124 if (bytes != PAGE_SIZE) 2125 panic("pmap_pvo_allocf: benno was shortsighted. hit him."); 2126 2127 *flags = UMA_SLAB_PRIV; 2128 m = vm_page_alloc(pmap_pvo_obj, pmap_pvo_count, VM_ALLOC_SYSTEM); 2129 if (m == NULL) 2130 return (NULL); 2131 pmap_pvo_count++; 2132 return ((void *)VM_PAGE_TO_PHYS(m)); 2133} 2134 2135/* 2136 * XXX: THIS STUFF SHOULD BE IN pte.c? 2137 */ 2138int 2139pmap_pte_spill(vm_offset_t addr) 2140{ 2141 struct pvo_entry *source_pvo, *victim_pvo; 2142 struct pvo_entry *pvo; 2143 int ptegidx, i, j; 2144 u_int sr; 2145 struct pteg *pteg; 2146 struct pte *pt; 2147 2148 pmap_pte_spills++; 2149 2150 sr = mfsrin(addr); 2151 ptegidx = va_to_pteg(sr, addr); 2152 2153 /* 2154 * Have to substitute some entry. Use the primary hash for this. 2155 * Use low bits of timebase as random generator. 2156 */ 2157 pteg = &pmap_pteg_table[ptegidx]; 2158 __asm __volatile("mftb %0" : "=r"(i)); 2159 i &= 7; 2160 pt = &pteg->pt[i]; 2161 2162 source_pvo = NULL; 2163 victim_pvo = NULL; 2164 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2165 /* 2166 * We need to find a pvo entry for this address. 2167 */ 2168 PMAP_PVO_CHECK(pvo); 2169 if (source_pvo == NULL && 2170 pmap_pte_match(&pvo->pvo_pte, sr, addr, 2171 pvo->pvo_pte.pte_hi & PTE_HID)) { 2172 /* 2173 * Now found an entry to be spilled into the pteg. 2174 * The PTE is now valid, so we know it's active. 2175 */ 2176 j = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 2177 2178 if (j >= 0) { 2179 PVO_PTEGIDX_SET(pvo, j); 2180 pmap_pte_overflow--; 2181 PMAP_PVO_CHECK(pvo); 2182 return (1); 2183 } 2184 2185 source_pvo = pvo; 2186 2187 if (victim_pvo != NULL) 2188 break; 2189 } 2190 2191 /* 2192 * We also need the pvo entry of the victim we are replacing 2193 * so save the R & C bits of the PTE. 2194 */ 2195 if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 2196 pmap_pte_compare(pt, &pvo->pvo_pte)) { 2197 victim_pvo = pvo; 2198 if (source_pvo != NULL) 2199 break; 2200 } 2201 } 2202 2203 if (source_pvo == NULL) 2204 return (0); 2205 2206 if (victim_pvo == NULL) { 2207 if ((pt->pte_hi & PTE_HID) == 0) 2208 panic("pmap_pte_spill: victim p-pte (%p) has no pvo" 2209 "entry", pt); 2210 2211 /* 2212 * If this is a secondary PTE, we need to search it's primary 2213 * pvo bucket for the matching PVO. 2214 */ 2215 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx ^ pmap_pteg_mask], 2216 pvo_olink) { 2217 PMAP_PVO_CHECK(pvo); 2218 /* 2219 * We also need the pvo entry of the victim we are 2220 * replacing so save the R & C bits of the PTE. 2221 */ 2222 if (pmap_pte_compare(pt, &pvo->pvo_pte)) { 2223 victim_pvo = pvo; 2224 break; 2225 } 2226 } 2227 2228 if (victim_pvo == NULL) 2229 panic("pmap_pte_spill: victim s-pte (%p) has no pvo" 2230 "entry", pt); 2231 } 2232 2233 /* 2234 * We are invalidating the TLB entry for the EA we are replacing even 2235 * though it's valid. If we don't, we lose any ref/chg bit changes 2236 * contained in the TLB entry. 2237 */ 2238 source_pvo->pvo_pte.pte_hi &= ~PTE_HID; 2239 2240 pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr); 2241 pmap_pte_set(pt, &source_pvo->pvo_pte); 2242 2243 PVO_PTEGIDX_CLR(victim_pvo); 2244 PVO_PTEGIDX_SET(source_pvo, i); 2245 pmap_pte_replacements++; 2246 2247 PMAP_PVO_CHECK(victim_pvo); 2248 PMAP_PVO_CHECK(source_pvo); 2249 2250 return (1); 2251} 2252 2253static int 2254pmap_pte_insert(u_int ptegidx, struct pte *pvo_pt) 2255{ 2256 struct pte *pt; 2257 int i; 2258 2259 /* 2260 * First try primary hash. 2261 */ 2262 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 2263 if ((pt->pte_hi & PTE_VALID) == 0) { 2264 pvo_pt->pte_hi &= ~PTE_HID; 2265 pmap_pte_set(pt, pvo_pt); 2266 return (i); 2267 } 2268 } 2269 2270 /* 2271 * Now try secondary hash. 2272 */ 2273 ptegidx ^= pmap_pteg_mask; 2274 ptegidx++; 2275 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 2276 if ((pt->pte_hi & PTE_VALID) == 0) { 2277 pvo_pt->pte_hi |= PTE_HID; 2278 pmap_pte_set(pt, pvo_pt); 2279 return (i); 2280 } 2281 } 2282 2283 panic("pmap_pte_insert: overflow"); 2284 return (-1); 2285} 2286 2287static boolean_t 2288pmap_query_bit(vm_page_t m, int ptebit) 2289{ 2290 struct pvo_entry *pvo; 2291 struct pte *pt; 2292 2293 if (pmap_attr_fetch(m) & ptebit) 2294 return (TRUE); 2295 2296 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2297 PMAP_PVO_CHECK(pvo); /* sanity check */ 2298 2299 /* 2300 * See if we saved the bit off. If so, cache it and return 2301 * success. 2302 */ 2303 if (pvo->pvo_pte.pte_lo & ptebit) { 2304 pmap_attr_save(m, ptebit); 2305 PMAP_PVO_CHECK(pvo); /* sanity check */ 2306 return (TRUE); 2307 } 2308 } 2309 2310 /* 2311 * No luck, now go through the hard part of looking at the PTEs 2312 * themselves. Sync so that any pending REF/CHG bits are flushed to 2313 * the PTEs. 2314 */ 2315 SYNC(); 2316 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2317 PMAP_PVO_CHECK(pvo); /* sanity check */ 2318 2319 /* 2320 * See if this pvo has a valid PTE. if so, fetch the 2321 * REF/CHG bits from the valid PTE. If the appropriate 2322 * ptebit is set, cache it and return success. 2323 */ 2324 pt = pmap_pvo_to_pte(pvo, -1); 2325 if (pt != NULL) { 2326 pmap_pte_synch(pt, &pvo->pvo_pte); 2327 if (pvo->pvo_pte.pte_lo & ptebit) { 2328 pmap_attr_save(m, ptebit); 2329 PMAP_PVO_CHECK(pvo); /* sanity check */ 2330 return (TRUE); 2331 } 2332 } 2333 } 2334 2335 return (TRUE); 2336} 2337 2338static u_int 2339pmap_clear_bit(vm_page_t m, int ptebit, int *origbit) 2340{ 2341 u_int count; 2342 struct pvo_entry *pvo; 2343 struct pte *pt; 2344 int rv; 2345 2346 /* 2347 * Clear the cached value. 2348 */ 2349 rv = pmap_attr_fetch(m); 2350 pmap_attr_clear(m, ptebit); 2351 2352 /* 2353 * Sync so that any pending REF/CHG bits are flushed to the PTEs (so 2354 * we can reset the right ones). note that since the pvo entries and 2355 * list heads are accessed via BAT0 and are never placed in the page 2356 * table, we don't have to worry about further accesses setting the 2357 * REF/CHG bits. 2358 */ 2359 SYNC(); 2360 2361 /* 2362 * For each pvo entry, clear the pvo's ptebit. If this pvo has a 2363 * valid pte clear the ptebit from the valid pte. 2364 */ 2365 count = 0; 2366 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2367 PMAP_PVO_CHECK(pvo); /* sanity check */ 2368 pt = pmap_pvo_to_pte(pvo, -1); 2369 if (pt != NULL) { 2370 pmap_pte_synch(pt, &pvo->pvo_pte); 2371 if (pvo->pvo_pte.pte_lo & ptebit) { 2372 count++; 2373 pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit); 2374 } 2375 } 2376 rv |= pvo->pvo_pte.pte_lo; 2377 pvo->pvo_pte.pte_lo &= ~ptebit; 2378 PMAP_PVO_CHECK(pvo); /* sanity check */ 2379 } 2380 2381 if (origbit != NULL) { 2382 *origbit = rv; 2383 } 2384 2385 return (count); 2386} 2387 2388/* 2389 * Return true if the physical range is encompassed by the battable[idx] 2390 */ 2391static int 2392pmap_bat_mapped(int idx, vm_offset_t pa, vm_size_t size) 2393{ 2394 u_int prot; 2395 u_int32_t start; 2396 u_int32_t end; 2397 u_int32_t bat_ble; 2398 2399 /* 2400 * Return immediately if not a valid mapping 2401 */ 2402 if (!battable[idx].batu & BAT_Vs) 2403 return (EINVAL); 2404 2405 /* 2406 * The BAT entry must be cache-inhibited, guarded, and r/w 2407 * so it can function as an i/o page 2408 */ 2409 prot = battable[idx].batl & (BAT_I|BAT_G|BAT_PP_RW); 2410 if (prot != (BAT_I|BAT_G|BAT_PP_RW)) 2411 return (EPERM); 2412 2413 /* 2414 * The address should be within the BAT range. Assume that the 2415 * start address in the BAT has the correct alignment (thus 2416 * not requiring masking) 2417 */ 2418 start = battable[idx].batl & BAT_PBS; 2419 bat_ble = (battable[idx].batu & ~(BAT_EBS)) | 0x03; 2420 end = start | (bat_ble << 15) | 0x7fff; 2421 2422 if ((pa < start) || ((pa + size) > end)) 2423 return (ERANGE); 2424 2425 return (0); 2426} 2427 2428 2429/* 2430 * Map a set of physical memory pages into the kernel virtual 2431 * address space. Return a pointer to where it is mapped. This 2432 * routine is intended to be used for mapping device memory, 2433 * NOT real memory. 2434 */ 2435void * 2436pmap_mapdev(vm_offset_t pa, vm_size_t size) 2437{ 2438 vm_offset_t va, tmpva, ppa, offset; 2439 int i; 2440 2441 ppa = trunc_page(pa); 2442 offset = pa & PAGE_MASK; 2443 size = roundup(offset + size, PAGE_SIZE); 2444 2445 GIANT_REQUIRED; 2446 2447 /* 2448 * If the physical address lies within a valid BAT table entry, 2449 * return the 1:1 mapping. This currently doesn't work 2450 * for regions that overlap 256M BAT segments. 2451 */ 2452 for (i = 0; i < 16; i++) { 2453 if (pmap_bat_mapped(i, pa, size) == 0) 2454 return ((void *) pa); 2455 } 2456 2457 va = kmem_alloc_pageable(kernel_map, size); 2458 if (!va) 2459 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2460 2461 for (tmpva = va; size > 0;) { 2462 pmap_kenter(tmpva, ppa); 2463 TLBIE(tmpva); /* XXX or should it be invalidate-all ? */ 2464 size -= PAGE_SIZE; 2465 tmpva += PAGE_SIZE; 2466 ppa += PAGE_SIZE; 2467 } 2468 2469 return ((void *)(va + offset)); 2470} 2471 2472void 2473pmap_unmapdev(vm_offset_t va, vm_size_t size) 2474{ 2475 vm_offset_t base, offset; 2476 2477 /* 2478 * If this is outside kernel virtual space, then it's a 2479 * battable entry and doesn't require unmapping 2480 */ 2481 if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)) { 2482 base = trunc_page(va); 2483 offset = va & PAGE_MASK; 2484 size = roundup(offset + size, PAGE_SIZE); 2485 kmem_free(kernel_map, base, size); 2486 } 2487}
| 1560 * Allocate a physical page of memory directly from the phys_avail map. 1561 * Can only be called from pmap_bootstrap before avail start and end are 1562 * calculated. 1563 */ 1564static vm_offset_t 1565pmap_bootstrap_alloc(vm_size_t size, u_int align) 1566{ 1567 vm_offset_t s, e; 1568 int i, j; 1569 1570 size = round_page(size); 1571 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1572 if (align != 0) 1573 s = (phys_avail[i] + align - 1) & ~(align - 1); 1574 else 1575 s = phys_avail[i]; 1576 e = s + size; 1577 1578 if (s < phys_avail[i] || e > phys_avail[i + 1]) 1579 continue; 1580 1581 if (s == phys_avail[i]) { 1582 phys_avail[i] += size; 1583 } else if (e == phys_avail[i + 1]) { 1584 phys_avail[i + 1] -= size; 1585 } else { 1586 for (j = phys_avail_count * 2; j > i; j -= 2) { 1587 phys_avail[j] = phys_avail[j - 2]; 1588 phys_avail[j + 1] = phys_avail[j - 1]; 1589 } 1590 1591 phys_avail[i + 3] = phys_avail[i + 1]; 1592 phys_avail[i + 1] = s; 1593 phys_avail[i + 2] = e; 1594 phys_avail_count++; 1595 } 1596 1597 return (s); 1598 } 1599 panic("pmap_bootstrap_alloc: could not allocate memory"); 1600} 1601 1602/* 1603 * Return an unmapped pvo for a kernel virtual address. 1604 * Used by pmap functions that operate on physical pages. 1605 */ 1606static struct pvo_entry * 1607pmap_rkva_alloc(void) 1608{ 1609 struct pvo_entry *pvo; 1610 struct pte *pt; 1611 vm_offset_t kva; 1612 int pteidx; 1613 1614 if (pmap_rkva_count == 0) 1615 panic("pmap_rkva_alloc: no more reserved KVAs"); 1616 1617 kva = pmap_rkva_start + (PAGE_SIZE * --pmap_rkva_count); 1618 pmap_kenter(kva, 0); 1619 1620 pvo = pmap_pvo_find_va(kernel_pmap, kva, &pteidx); 1621 1622 if (pvo == NULL) 1623 panic("pmap_kva_alloc: pmap_pvo_find_va failed"); 1624 1625 pt = pmap_pvo_to_pte(pvo, pteidx); 1626 1627 if (pt == NULL) 1628 panic("pmap_kva_alloc: pmap_pvo_to_pte failed"); 1629 1630 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1631 PVO_PTEGIDX_CLR(pvo); 1632 1633 pmap_pte_overflow++; 1634 1635 return (pvo); 1636} 1637 1638static void 1639pmap_pa_map(struct pvo_entry *pvo, vm_offset_t pa, struct pte *saved_pt, 1640 int *depth_p) 1641{ 1642 struct pte *pt; 1643 1644 /* 1645 * If this pvo already has a valid pte, we need to save it so it can 1646 * be restored later. We then just reload the new PTE over the old 1647 * slot. 1648 */ 1649 if (saved_pt != NULL) { 1650 pt = pmap_pvo_to_pte(pvo, -1); 1651 1652 if (pt != NULL) { 1653 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1654 PVO_PTEGIDX_CLR(pvo); 1655 pmap_pte_overflow++; 1656 } 1657 1658 *saved_pt = pvo->pvo_pte; 1659 1660 pvo->pvo_pte.pte_lo &= ~PTE_RPGN; 1661 } 1662 1663 pvo->pvo_pte.pte_lo |= pa; 1664 1665 if (!pmap_pte_spill(pvo->pvo_vaddr)) 1666 panic("pmap_pa_map: could not spill pvo %p", pvo); 1667 1668 if (depth_p != NULL) 1669 (*depth_p)++; 1670} 1671 1672static void 1673pmap_pa_unmap(struct pvo_entry *pvo, struct pte *saved_pt, int *depth_p) 1674{ 1675 struct pte *pt; 1676 1677 pt = pmap_pvo_to_pte(pvo, -1); 1678 1679 if (pt != NULL) { 1680 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1681 PVO_PTEGIDX_CLR(pvo); 1682 pmap_pte_overflow++; 1683 } 1684 1685 pvo->pvo_pte.pte_lo &= ~PTE_RPGN; 1686 1687 /* 1688 * If there is a saved PTE and it's valid, restore it and return. 1689 */ 1690 if (saved_pt != NULL && (saved_pt->pte_lo & PTE_RPGN) != 0) { 1691 if (depth_p != NULL && --(*depth_p) == 0) 1692 panic("pmap_pa_unmap: restoring but depth == 0"); 1693 1694 pvo->pvo_pte = *saved_pt; 1695 1696 if (!pmap_pte_spill(pvo->pvo_vaddr)) 1697 panic("pmap_pa_unmap: could not spill pvo %p", pvo); 1698 } 1699} 1700 1701static void 1702pmap_syncicache(vm_offset_t pa, vm_size_t len) 1703{ 1704 __syncicache((void *)pa, len); 1705} 1706 1707static void 1708tlbia(void) 1709{ 1710 caddr_t i; 1711 1712 SYNC(); 1713 for (i = 0; i < (caddr_t)0x00040000; i += 0x00001000) { 1714 TLBIE(i); 1715 EIEIO(); 1716 } 1717 TLBSYNC(); 1718 SYNC(); 1719} 1720 1721static int 1722pmap_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head, 1723 vm_offset_t va, vm_offset_t pa, u_int pte_lo, int flags) 1724{ 1725 struct pvo_entry *pvo; 1726 u_int sr; 1727 int first; 1728 u_int ptegidx; 1729 int i; 1730 int bootstrap; 1731 1732 pmap_pvo_enter_calls++; 1733 first = 0; 1734 1735 bootstrap = 0; 1736 1737 /* 1738 * Compute the PTE Group index. 1739 */ 1740 va &= ~ADDR_POFF; 1741 sr = va_to_sr(pm->pm_sr, va); 1742 ptegidx = va_to_pteg(sr, va); 1743 1744 /* 1745 * Remove any existing mapping for this page. Reuse the pvo entry if 1746 * there is a mapping. 1747 */ 1748 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1749 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1750 if ((pvo->pvo_pte.pte_lo & PTE_RPGN) == pa && 1751 (pvo->pvo_pte.pte_lo & PTE_PP) == 1752 (pte_lo & PTE_PP)) { 1753 return (0); 1754 } 1755 pmap_pvo_remove(pvo, -1); 1756 break; 1757 } 1758 } 1759 1760 /* 1761 * If we aren't overwriting a mapping, try to allocate. 1762 */ 1763 if (pmap_initialized) { 1764 pvo = uma_zalloc(zone, M_NOWAIT); 1765 } else { 1766 if (pmap_bpvo_pool_index >= BPVO_POOL_SIZE) { 1767 panic("pmap_enter: bpvo pool exhausted, %d, %d, %d", 1768 pmap_bpvo_pool_index, BPVO_POOL_SIZE, 1769 BPVO_POOL_SIZE * sizeof(struct pvo_entry)); 1770 } 1771 pvo = &pmap_bpvo_pool[pmap_bpvo_pool_index]; 1772 pmap_bpvo_pool_index++; 1773 bootstrap = 1; 1774 } 1775 1776 if (pvo == NULL) { 1777 return (ENOMEM); 1778 } 1779 1780 pmap_pvo_entries++; 1781 pvo->pvo_vaddr = va; 1782 pvo->pvo_pmap = pm; 1783 LIST_INSERT_HEAD(&pmap_pvo_table[ptegidx], pvo, pvo_olink); 1784 pvo->pvo_vaddr &= ~ADDR_POFF; 1785 if (flags & VM_PROT_EXECUTE) 1786 pvo->pvo_vaddr |= PVO_EXECUTABLE; 1787 if (flags & PVO_WIRED) 1788 pvo->pvo_vaddr |= PVO_WIRED; 1789 if (pvo_head != &pmap_pvo_kunmanaged) 1790 pvo->pvo_vaddr |= PVO_MANAGED; 1791 if (bootstrap) 1792 pvo->pvo_vaddr |= PVO_BOOTSTRAP; 1793 pmap_pte_create(&pvo->pvo_pte, sr, va, pa | pte_lo); 1794 1795 /* 1796 * Remember if the list was empty and therefore will be the first 1797 * item. 1798 */ 1799 if (LIST_FIRST(pvo_head) == NULL) 1800 first = 1; 1801 1802 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 1803 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1804 pvo->pvo_pmap->pm_stats.wired_count++; 1805 pvo->pvo_pmap->pm_stats.resident_count++; 1806 1807 /* 1808 * We hope this succeeds but it isn't required. 1809 */ 1810 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 1811 if (i >= 0) { 1812 PVO_PTEGIDX_SET(pvo, i); 1813 } else { 1814 panic("pmap_pvo_enter: overflow"); 1815 pmap_pte_overflow++; 1816 } 1817 1818 return (first ? ENOENT : 0); 1819} 1820 1821static void 1822pmap_pvo_remove(struct pvo_entry *pvo, int pteidx) 1823{ 1824 struct pte *pt; 1825 1826 /* 1827 * If there is an active pte entry, we need to deactivate it (and 1828 * save the ref & cfg bits). 1829 */ 1830 pt = pmap_pvo_to_pte(pvo, pteidx); 1831 if (pt != NULL) { 1832 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1833 PVO_PTEGIDX_CLR(pvo); 1834 } else { 1835 pmap_pte_overflow--; 1836 } 1837 1838 /* 1839 * Update our statistics. 1840 */ 1841 pvo->pvo_pmap->pm_stats.resident_count--; 1842 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1843 pvo->pvo_pmap->pm_stats.wired_count--; 1844 1845 /* 1846 * Save the REF/CHG bits into their cache if the page is managed. 1847 */ 1848 if (pvo->pvo_vaddr & PVO_MANAGED) { 1849 struct vm_page *pg; 1850 1851 pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pte_lo & PTE_RPGN); 1852 if (pg != NULL) { 1853 pmap_attr_save(pg, pvo->pvo_pte.pte_lo & 1854 (PTE_REF | PTE_CHG)); 1855 } 1856 } 1857 1858 /* 1859 * Remove this PVO from the PV list. 1860 */ 1861 LIST_REMOVE(pvo, pvo_vlink); 1862 1863 /* 1864 * Remove this from the overflow list and return it to the pool 1865 * if we aren't going to reuse it. 1866 */ 1867 LIST_REMOVE(pvo, pvo_olink); 1868 if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP)) 1869 uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? pmap_mpvo_zone : 1870 pmap_upvo_zone, pvo); 1871 pmap_pvo_entries--; 1872 pmap_pvo_remove_calls++; 1873} 1874 1875static __inline int 1876pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 1877{ 1878 int pteidx; 1879 1880 /* 1881 * We can find the actual pte entry without searching by grabbing 1882 * the PTEG index from 3 unused bits in pte_lo[11:9] and by 1883 * noticing the HID bit. 1884 */ 1885 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 1886 if (pvo->pvo_pte.pte_hi & PTE_HID) 1887 pteidx ^= pmap_pteg_mask * 8; 1888 1889 return (pteidx); 1890} 1891 1892static struct pvo_entry * 1893pmap_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p) 1894{ 1895 struct pvo_entry *pvo; 1896 int ptegidx; 1897 u_int sr; 1898 1899 va &= ~ADDR_POFF; 1900 sr = va_to_sr(pm->pm_sr, va); 1901 ptegidx = va_to_pteg(sr, va); 1902 1903 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1904 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1905 if (pteidx_p) 1906 *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx); 1907 return (pvo); 1908 } 1909 } 1910 1911 return (NULL); 1912} 1913 1914static struct pte * 1915pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 1916{ 1917 struct pte *pt; 1918 1919 /* 1920 * If we haven't been supplied the ptegidx, calculate it. 1921 */ 1922 if (pteidx == -1) { 1923 int ptegidx; 1924 u_int sr; 1925 1926 sr = va_to_sr(pvo->pvo_pmap->pm_sr, pvo->pvo_vaddr); 1927 ptegidx = va_to_pteg(sr, pvo->pvo_vaddr); 1928 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1929 } 1930 1931 pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7]; 1932 1933 if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 1934 panic("pmap_pvo_to_pte: pvo %p has valid pte in pvo but no " 1935 "valid pte index", pvo); 1936 } 1937 1938 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 1939 panic("pmap_pvo_to_pte: pvo %p has valid pte index in pvo " 1940 "pvo but no valid pte", pvo); 1941 } 1942 1943 if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 1944 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) { 1945 panic("pmap_pvo_to_pte: pvo %p has valid pte in " 1946 "pmap_pteg_table %p but invalid in pvo", pvo, pt); 1947 } 1948 1949 if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) 1950 != 0) { 1951 panic("pmap_pvo_to_pte: pvo %p pte does not match " 1952 "pte %p in pmap_pteg_table", pvo, pt); 1953 } 1954 1955 return (pt); 1956 } 1957 1958 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1959 panic("pmap_pvo_to_pte: pvo %p has invalid pte %p in " 1960 "pmap_pteg_table but valid in pvo", pvo, pt); 1961 } 1962 1963 return (NULL); 1964} 1965 1966static void * 1967pmap_pvo_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 1968{ 1969 vm_page_t m; 1970 1971 if (bytes != PAGE_SIZE) 1972 panic("pmap_pvo_allocf: benno was shortsighted. hit him."); 1973 1974 *flags = UMA_SLAB_PRIV; 1975 m = vm_page_alloc(pmap_pvo_obj, pmap_pvo_count, VM_ALLOC_SYSTEM); 1976 if (m == NULL) 1977 return (NULL); 1978 pmap_pvo_count++; 1979 return ((void *)VM_PAGE_TO_PHYS(m)); 1980} 1981 1982/* 1983 * XXX: THIS STUFF SHOULD BE IN pte.c? 1984 */ 1985int 1986pmap_pte_spill(vm_offset_t addr) 1987{ 1988 struct pvo_entry *source_pvo, *victim_pvo; 1989 struct pvo_entry *pvo; 1990 int ptegidx, i, j; 1991 u_int sr; 1992 struct pteg *pteg; 1993 struct pte *pt; 1994 1995 pmap_pte_spills++; 1996 1997 sr = mfsrin(addr); 1998 ptegidx = va_to_pteg(sr, addr); 1999 2000 /* 2001 * Have to substitute some entry. Use the primary hash for this. 2002 * Use low bits of timebase as random generator. 2003 */ 2004 pteg = &pmap_pteg_table[ptegidx]; 2005 __asm __volatile("mftb %0" : "=r"(i)); 2006 i &= 7; 2007 pt = &pteg->pt[i]; 2008 2009 source_pvo = NULL; 2010 victim_pvo = NULL; 2011 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2012 /* 2013 * We need to find a pvo entry for this address. 2014 */ 2015 PMAP_PVO_CHECK(pvo); 2016 if (source_pvo == NULL && 2017 pmap_pte_match(&pvo->pvo_pte, sr, addr, 2018 pvo->pvo_pte.pte_hi & PTE_HID)) { 2019 /* 2020 * Now found an entry to be spilled into the pteg. 2021 * The PTE is now valid, so we know it's active. 2022 */ 2023 j = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 2024 2025 if (j >= 0) { 2026 PVO_PTEGIDX_SET(pvo, j); 2027 pmap_pte_overflow--; 2028 PMAP_PVO_CHECK(pvo); 2029 return (1); 2030 } 2031 2032 source_pvo = pvo; 2033 2034 if (victim_pvo != NULL) 2035 break; 2036 } 2037 2038 /* 2039 * We also need the pvo entry of the victim we are replacing 2040 * so save the R & C bits of the PTE. 2041 */ 2042 if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 2043 pmap_pte_compare(pt, &pvo->pvo_pte)) { 2044 victim_pvo = pvo; 2045 if (source_pvo != NULL) 2046 break; 2047 } 2048 } 2049 2050 if (source_pvo == NULL) 2051 return (0); 2052 2053 if (victim_pvo == NULL) { 2054 if ((pt->pte_hi & PTE_HID) == 0) 2055 panic("pmap_pte_spill: victim p-pte (%p) has no pvo" 2056 "entry", pt); 2057 2058 /* 2059 * If this is a secondary PTE, we need to search it's primary 2060 * pvo bucket for the matching PVO. 2061 */ 2062 LIST_FOREACH(pvo, &pmap_pvo_table[ptegidx ^ pmap_pteg_mask], 2063 pvo_olink) { 2064 PMAP_PVO_CHECK(pvo); 2065 /* 2066 * We also need the pvo entry of the victim we are 2067 * replacing so save the R & C bits of the PTE. 2068 */ 2069 if (pmap_pte_compare(pt, &pvo->pvo_pte)) { 2070 victim_pvo = pvo; 2071 break; 2072 } 2073 } 2074 2075 if (victim_pvo == NULL) 2076 panic("pmap_pte_spill: victim s-pte (%p) has no pvo" 2077 "entry", pt); 2078 } 2079 2080 /* 2081 * We are invalidating the TLB entry for the EA we are replacing even 2082 * though it's valid. If we don't, we lose any ref/chg bit changes 2083 * contained in the TLB entry. 2084 */ 2085 source_pvo->pvo_pte.pte_hi &= ~PTE_HID; 2086 2087 pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr); 2088 pmap_pte_set(pt, &source_pvo->pvo_pte); 2089 2090 PVO_PTEGIDX_CLR(victim_pvo); 2091 PVO_PTEGIDX_SET(source_pvo, i); 2092 pmap_pte_replacements++; 2093 2094 PMAP_PVO_CHECK(victim_pvo); 2095 PMAP_PVO_CHECK(source_pvo); 2096 2097 return (1); 2098} 2099 2100static int 2101pmap_pte_insert(u_int ptegidx, struct pte *pvo_pt) 2102{ 2103 struct pte *pt; 2104 int i; 2105 2106 /* 2107 * First try primary hash. 2108 */ 2109 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 2110 if ((pt->pte_hi & PTE_VALID) == 0) { 2111 pvo_pt->pte_hi &= ~PTE_HID; 2112 pmap_pte_set(pt, pvo_pt); 2113 return (i); 2114 } 2115 } 2116 2117 /* 2118 * Now try secondary hash. 2119 */ 2120 ptegidx ^= pmap_pteg_mask; 2121 ptegidx++; 2122 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 2123 if ((pt->pte_hi & PTE_VALID) == 0) { 2124 pvo_pt->pte_hi |= PTE_HID; 2125 pmap_pte_set(pt, pvo_pt); 2126 return (i); 2127 } 2128 } 2129 2130 panic("pmap_pte_insert: overflow"); 2131 return (-1); 2132} 2133 2134static boolean_t 2135pmap_query_bit(vm_page_t m, int ptebit) 2136{ 2137 struct pvo_entry *pvo; 2138 struct pte *pt; 2139 2140 if (pmap_attr_fetch(m) & ptebit) 2141 return (TRUE); 2142 2143 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2144 PMAP_PVO_CHECK(pvo); /* sanity check */ 2145 2146 /* 2147 * See if we saved the bit off. If so, cache it and return 2148 * success. 2149 */ 2150 if (pvo->pvo_pte.pte_lo & ptebit) { 2151 pmap_attr_save(m, ptebit); 2152 PMAP_PVO_CHECK(pvo); /* sanity check */ 2153 return (TRUE); 2154 } 2155 } 2156 2157 /* 2158 * No luck, now go through the hard part of looking at the PTEs 2159 * themselves. Sync so that any pending REF/CHG bits are flushed to 2160 * the PTEs. 2161 */ 2162 SYNC(); 2163 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2164 PMAP_PVO_CHECK(pvo); /* sanity check */ 2165 2166 /* 2167 * See if this pvo has a valid PTE. if so, fetch the 2168 * REF/CHG bits from the valid PTE. If the appropriate 2169 * ptebit is set, cache it and return success. 2170 */ 2171 pt = pmap_pvo_to_pte(pvo, -1); 2172 if (pt != NULL) { 2173 pmap_pte_synch(pt, &pvo->pvo_pte); 2174 if (pvo->pvo_pte.pte_lo & ptebit) { 2175 pmap_attr_save(m, ptebit); 2176 PMAP_PVO_CHECK(pvo); /* sanity check */ 2177 return (TRUE); 2178 } 2179 } 2180 } 2181 2182 return (TRUE); 2183} 2184 2185static u_int 2186pmap_clear_bit(vm_page_t m, int ptebit, int *origbit) 2187{ 2188 u_int count; 2189 struct pvo_entry *pvo; 2190 struct pte *pt; 2191 int rv; 2192 2193 /* 2194 * Clear the cached value. 2195 */ 2196 rv = pmap_attr_fetch(m); 2197 pmap_attr_clear(m, ptebit); 2198 2199 /* 2200 * Sync so that any pending REF/CHG bits are flushed to the PTEs (so 2201 * we can reset the right ones). note that since the pvo entries and 2202 * list heads are accessed via BAT0 and are never placed in the page 2203 * table, we don't have to worry about further accesses setting the 2204 * REF/CHG bits. 2205 */ 2206 SYNC(); 2207 2208 /* 2209 * For each pvo entry, clear the pvo's ptebit. If this pvo has a 2210 * valid pte clear the ptebit from the valid pte. 2211 */ 2212 count = 0; 2213 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { 2214 PMAP_PVO_CHECK(pvo); /* sanity check */ 2215 pt = pmap_pvo_to_pte(pvo, -1); 2216 if (pt != NULL) { 2217 pmap_pte_synch(pt, &pvo->pvo_pte); 2218 if (pvo->pvo_pte.pte_lo & ptebit) { 2219 count++; 2220 pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit); 2221 } 2222 } 2223 rv |= pvo->pvo_pte.pte_lo; 2224 pvo->pvo_pte.pte_lo &= ~ptebit; 2225 PMAP_PVO_CHECK(pvo); /* sanity check */ 2226 } 2227 2228 if (origbit != NULL) { 2229 *origbit = rv; 2230 } 2231 2232 return (count); 2233} 2234 2235/* 2236 * Return true if the physical range is encompassed by the battable[idx] 2237 */ 2238static int 2239pmap_bat_mapped(int idx, vm_offset_t pa, vm_size_t size) 2240{ 2241 u_int prot; 2242 u_int32_t start; 2243 u_int32_t end; 2244 u_int32_t bat_ble; 2245 2246 /* 2247 * Return immediately if not a valid mapping 2248 */ 2249 if (!battable[idx].batu & BAT_Vs) 2250 return (EINVAL); 2251 2252 /* 2253 * The BAT entry must be cache-inhibited, guarded, and r/w 2254 * so it can function as an i/o page 2255 */ 2256 prot = battable[idx].batl & (BAT_I|BAT_G|BAT_PP_RW); 2257 if (prot != (BAT_I|BAT_G|BAT_PP_RW)) 2258 return (EPERM); 2259 2260 /* 2261 * The address should be within the BAT range. Assume that the 2262 * start address in the BAT has the correct alignment (thus 2263 * not requiring masking) 2264 */ 2265 start = battable[idx].batl & BAT_PBS; 2266 bat_ble = (battable[idx].batu & ~(BAT_EBS)) | 0x03; 2267 end = start | (bat_ble << 15) | 0x7fff; 2268 2269 if ((pa < start) || ((pa + size) > end)) 2270 return (ERANGE); 2271 2272 return (0); 2273} 2274 2275 2276/* 2277 * Map a set of physical memory pages into the kernel virtual 2278 * address space. Return a pointer to where it is mapped. This 2279 * routine is intended to be used for mapping device memory, 2280 * NOT real memory. 2281 */ 2282void * 2283pmap_mapdev(vm_offset_t pa, vm_size_t size) 2284{ 2285 vm_offset_t va, tmpva, ppa, offset; 2286 int i; 2287 2288 ppa = trunc_page(pa); 2289 offset = pa & PAGE_MASK; 2290 size = roundup(offset + size, PAGE_SIZE); 2291 2292 GIANT_REQUIRED; 2293 2294 /* 2295 * If the physical address lies within a valid BAT table entry, 2296 * return the 1:1 mapping. This currently doesn't work 2297 * for regions that overlap 256M BAT segments. 2298 */ 2299 for (i = 0; i < 16; i++) { 2300 if (pmap_bat_mapped(i, pa, size) == 0) 2301 return ((void *) pa); 2302 } 2303 2304 va = kmem_alloc_pageable(kernel_map, size); 2305 if (!va) 2306 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 2307 2308 for (tmpva = va; size > 0;) { 2309 pmap_kenter(tmpva, ppa); 2310 TLBIE(tmpva); /* XXX or should it be invalidate-all ? */ 2311 size -= PAGE_SIZE; 2312 tmpva += PAGE_SIZE; 2313 ppa += PAGE_SIZE; 2314 } 2315 2316 return ((void *)(va + offset)); 2317} 2318 2319void 2320pmap_unmapdev(vm_offset_t va, vm_size_t size) 2321{ 2322 vm_offset_t base, offset; 2323 2324 /* 2325 * If this is outside kernel virtual space, then it's a 2326 * battable entry and doesn't require unmapping 2327 */ 2328 if ((va >= VM_MIN_KERNEL_ADDRESS) && (va <= VM_MAX_KERNEL_ADDRESS)) { 2329 base = trunc_page(va); 2330 offset = va & PAGE_MASK; 2331 size = roundup(offset + size, PAGE_SIZE); 2332 kmem_free(kernel_map, base, size); 2333 } 2334}
|