396static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; 397static struct rwlock pvh_global_lock; 398 399int l1_mem_types[] = { 400 ARM_L1S_STRONG_ORD, 401 ARM_L1S_DEVICE_NOSHARE, 402 ARM_L1S_DEVICE_SHARE, 403 ARM_L1S_NRML_NOCACHE, 404 ARM_L1S_NRML_IWT_OWT, 405 ARM_L1S_NRML_IWB_OWB, 406 ARM_L1S_NRML_IWBA_OWBA 407}; 408 409int l2l_mem_types[] = { 410 ARM_L2L_STRONG_ORD, 411 ARM_L2L_DEVICE_NOSHARE, 412 ARM_L2L_DEVICE_SHARE, 413 ARM_L2L_NRML_NOCACHE, 414 ARM_L2L_NRML_IWT_OWT, 415 ARM_L2L_NRML_IWB_OWB, 416 ARM_L2L_NRML_IWBA_OWBA 417}; 418 419int l2s_mem_types[] = { 420 ARM_L2S_STRONG_ORD, 421 ARM_L2S_DEVICE_NOSHARE, 422 ARM_L2S_DEVICE_SHARE, 423 ARM_L2S_NRML_NOCACHE, 424 ARM_L2S_NRML_IWT_OWT, 425 ARM_L2S_NRML_IWB_OWB, 426 ARM_L2S_NRML_IWBA_OWBA 427}; 428 429/* 430 * This list exists for the benefit of pmap_map_chunk(). It keeps track 431 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 432 * find them as necessary. 433 * 434 * Note that the data on this list MUST remain valid after initarm() returns, 435 * as pmap_bootstrap() uses it to contruct L2 table metadata. 436 */ 437SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 438 439static void 440pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 441{ 442 int i; 443 444 l1->l1_kva = l1pt; 445 l1->l1_domain_use_count = 0; 446 l1->l1_domain_first = 0; 447 448 for (i = 0; i < PMAP_DOMAINS; i++) 449 l1->l1_domain_free[i] = i + 1; 450 451 /* 452 * Copy the kernel's L1 entries to each new L1. 453 */ 454 if (l1pt != pmap_kernel()->pm_l1->l1_kva) 455 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 456 457 if ((l1->l1_physaddr = pmap_extract(pmap_kernel(), (vm_offset_t)l1pt)) == 0) 458 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 459 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 460 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 461} 462 463static vm_offset_t 464kernel_pt_lookup(vm_paddr_t pa) 465{ 466 struct pv_addr *pv; 467 468 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 469 if (pv->pv_pa == pa) 470 return (pv->pv_va); 471 } 472 return (0); 473} 474 475void 476pmap_pte_init_mmu_v6(void) 477{ 478 479 if (PTE_PAGETABLE >= 3) 480 pmap_needs_pte_sync = 1; 481 pte_l1_s_cache_mode = l1_mem_types[PTE_CACHE]; 482 pte_l2_l_cache_mode = l2l_mem_types[PTE_CACHE]; 483 pte_l2_s_cache_mode = l2s_mem_types[PTE_CACHE]; 484 485 pte_l1_s_cache_mode_pt = l1_mem_types[PTE_PAGETABLE]; 486 pte_l2_l_cache_mode_pt = l2l_mem_types[PTE_PAGETABLE]; 487 pte_l2_s_cache_mode_pt = l2s_mem_types[PTE_PAGETABLE]; 488 489} 490 491/* 492 * Allocate an L1 translation table for the specified pmap. 493 * This is called at pmap creation time. 494 */ 495static void 496pmap_alloc_l1(pmap_t pm) 497{ 498 struct l1_ttable *l1; 499 u_int8_t domain; 500 501 /* 502 * Remove the L1 at the head of the LRU list 503 */ 504 mtx_lock(&l1_lru_lock); 505 l1 = TAILQ_FIRST(&l1_lru_list); 506 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 507 508 /* 509 * Pick the first available domain number, and update 510 * the link to the next number. 511 */ 512 domain = l1->l1_domain_first; 513 l1->l1_domain_first = l1->l1_domain_free[domain]; 514 515 /* 516 * If there are still free domain numbers in this L1, 517 * put it back on the TAIL of the LRU list. 518 */ 519 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 520 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 521 522 mtx_unlock(&l1_lru_lock); 523 524 /* 525 * Fix up the relevant bits in the pmap structure 526 */ 527 pm->pm_l1 = l1; 528 pm->pm_domain = domain + 1; 529} 530 531/* 532 * Free an L1 translation table. 533 * This is called at pmap destruction time. 534 */ 535static void 536pmap_free_l1(pmap_t pm) 537{ 538 struct l1_ttable *l1 = pm->pm_l1; 539 540 mtx_lock(&l1_lru_lock); 541 542 /* 543 * If this L1 is currently on the LRU list, remove it. 544 */ 545 if (l1->l1_domain_use_count < PMAP_DOMAINS) 546 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 547 548 /* 549 * Free up the domain number which was allocated to the pmap 550 */ 551 l1->l1_domain_free[pm->pm_domain - 1] = l1->l1_domain_first; 552 l1->l1_domain_first = pm->pm_domain - 1; 553 l1->l1_domain_use_count--; 554 555 /* 556 * The L1 now must have at least 1 free domain, so add 557 * it back to the LRU list. If the use count is zero, 558 * put it at the head of the list, otherwise it goes 559 * to the tail. 560 */ 561 if (l1->l1_domain_use_count == 0) { 562 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 563 } else 564 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 565 566 mtx_unlock(&l1_lru_lock); 567} 568 569/* 570 * Returns a pointer to the L2 bucket associated with the specified pmap 571 * and VA, or NULL if no L2 bucket exists for the address. 572 */ 573static PMAP_INLINE struct l2_bucket * 574pmap_get_l2_bucket(pmap_t pm, vm_offset_t va) 575{ 576 struct l2_dtable *l2; 577 struct l2_bucket *l2b; 578 u_short l1idx; 579 580 l1idx = L1_IDX(va); 581 582 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 583 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 584 return (NULL); 585 586 return (l2b); 587} 588 589/* 590 * Returns a pointer to the L2 bucket associated with the specified pmap 591 * and VA. 592 * 593 * If no L2 bucket exists, perform the necessary allocations to put an L2 594 * bucket/page table in place. 595 * 596 * Note that if a new L2 bucket/page was allocated, the caller *must* 597 * increment the bucket occupancy counter appropriately *before* 598 * releasing the pmap's lock to ensure no other thread or cpu deallocates 599 * the bucket/page in the meantime. 600 */ 601static struct l2_bucket * 602pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) 603{ 604 struct l2_dtable *l2; 605 struct l2_bucket *l2b; 606 u_short l1idx; 607 608 l1idx = L1_IDX(va); 609 610 PMAP_ASSERT_LOCKED(pm); 611 rw_assert(&pvh_global_lock, RA_WLOCKED); 612 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 613 /* 614 * No mapping at this address, as there is 615 * no entry in the L1 table. 616 * Need to allocate a new l2_dtable. 617 */ 618 PMAP_UNLOCK(pm); 619 rw_wunlock(&pvh_global_lock); 620 if ((l2 = uma_zalloc(l2table_zone, M_NOWAIT)) == NULL) { 621 rw_wlock(&pvh_global_lock); 622 PMAP_LOCK(pm); 623 return (NULL); 624 } 625 rw_wlock(&pvh_global_lock); 626 PMAP_LOCK(pm); 627 if (pm->pm_l2[L2_IDX(l1idx)] != NULL) { 628 /* 629 * Someone already allocated the l2_dtable while 630 * we were doing the same. 631 */ 632 uma_zfree(l2table_zone, l2); 633 l2 = pm->pm_l2[L2_IDX(l1idx)]; 634 } else { 635 bzero(l2, sizeof(*l2)); 636 /* 637 * Link it into the parent pmap 638 */ 639 pm->pm_l2[L2_IDX(l1idx)] = l2; 640 } 641 } 642 643 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 644 645 /* 646 * Fetch pointer to the L2 page table associated with the address. 647 */ 648 if (l2b->l2b_kva == NULL) { 649 pt_entry_t *ptep; 650 651 /* 652 * No L2 page table has been allocated. Chances are, this 653 * is because we just allocated the l2_dtable, above. 654 */ 655 PMAP_UNLOCK(pm); 656 rw_wunlock(&pvh_global_lock); 657 ptep = uma_zalloc(l2zone, M_NOWAIT); 658 rw_wlock(&pvh_global_lock); 659 PMAP_LOCK(pm); 660 if (l2b->l2b_kva != 0) { 661 /* We lost the race. */ 662 uma_zfree(l2zone, ptep); 663 return (l2b); 664 } 665 l2b->l2b_phys = vtophys(ptep); 666 if (ptep == NULL) { 667 /* 668 * Oops, no more L2 page tables available at this 669 * time. We may need to deallocate the l2_dtable 670 * if we allocated a new one above. 671 */ 672 if (l2->l2_occupancy == 0) { 673 pm->pm_l2[L2_IDX(l1idx)] = NULL; 674 uma_zfree(l2table_zone, l2); 675 } 676 return (NULL); 677 } 678 679 l2->l2_occupancy++; 680 l2b->l2b_kva = ptep; 681 l2b->l2b_l1idx = l1idx; 682 } 683 684 return (l2b); 685} 686 687static PMAP_INLINE void 688pmap_free_l2_ptp(pt_entry_t *l2) 689{ 690 uma_zfree(l2zone, l2); 691} 692/* 693 * One or more mappings in the specified L2 descriptor table have just been 694 * invalidated. 695 * 696 * Garbage collect the metadata and descriptor table itself if necessary. 697 * 698 * The pmap lock must be acquired when this is called (not necessary 699 * for the kernel pmap). 700 */ 701static void 702pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 703{ 704 struct l2_dtable *l2; 705 pd_entry_t *pl1pd, l1pd; 706 pt_entry_t *ptep; 707 u_short l1idx; 708 709 710 /* 711 * Update the bucket's reference count according to how many 712 * PTEs the caller has just invalidated. 713 */ 714 l2b->l2b_occupancy -= count; 715 716 /* 717 * Note: 718 * 719 * Level 2 page tables allocated to the kernel pmap are never freed 720 * as that would require checking all Level 1 page tables and 721 * removing any references to the Level 2 page table. See also the 722 * comment elsewhere about never freeing bootstrap L2 descriptors. 723 * 724 * We make do with just invalidating the mapping in the L2 table. 725 * 726 * This isn't really a big deal in practice and, in fact, leads 727 * to a performance win over time as we don't need to continually 728 * alloc/free. 729 */ 730 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 731 return; 732 733 /* 734 * There are no more valid mappings in this level 2 page table. 735 * Go ahead and NULL-out the pointer in the bucket, then 736 * free the page table. 737 */ 738 l1idx = l2b->l2b_l1idx; 739 ptep = l2b->l2b_kva; 740 l2b->l2b_kva = NULL; 741 742 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 743 744 /* 745 * If the L1 slot matches the pmap's domain 746 * number, then invalidate it. 747 */ 748 l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); 749 if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { 750 *pl1pd = 0; 751 PTE_SYNC(pl1pd); 752 } 753 754 /* 755 * Release the L2 descriptor table back to the pool cache. 756 */ 757 pmap_free_l2_ptp(ptep); 758 759 /* 760 * Update the reference count in the associated l2_dtable 761 */ 762 l2 = pm->pm_l2[L2_IDX(l1idx)]; 763 if (--l2->l2_occupancy > 0) 764 return; 765 766 /* 767 * There are no more valid mappings in any of the Level 1 768 * slots managed by this l2_dtable. Go ahead and NULL-out 769 * the pointer in the parent pmap and free the l2_dtable. 770 */ 771 pm->pm_l2[L2_IDX(l1idx)] = NULL; 772 uma_zfree(l2table_zone, l2); 773} 774 775/* 776 * Pool cache constructors for L2 descriptor tables, metadata and pmap 777 * structures. 778 */ 779static int 780pmap_l2ptp_ctor(void *mem, int size, void *arg, int flags) 781{ 782 struct l2_bucket *l2b; 783 pt_entry_t *ptep, pte; 784 vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; 785 786 /* 787 * The mappings for these page tables were initially made using 788 * pmap_kenter() by the pool subsystem. Therefore, the cache- 789 * mode will not be right for page table mappings. To avoid 790 * polluting the pmap_kenter() code with a special case for 791 * page tables, we simply fix up the cache-mode here if it's not 792 * correct. 793 */ 794 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 795 ptep = &l2b->l2b_kva[l2pte_index(va)]; 796 pte = *ptep; 797 798 cpu_idcache_wbinv_range(va, PAGE_SIZE); 799 pmap_l2cache_wbinv_range(va, pte & L2_S_FRAME, PAGE_SIZE); 800 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 801 /* 802 * Page tables must have the cache-mode set to 803 * Write-Thru. 804 */ 805 *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 806 PTE_SYNC(ptep); 807 cpu_tlb_flushD_SE(va); 808 cpu_cpwait(); 809 } 810 811 memset(mem, 0, L2_TABLE_SIZE_REAL); 812 return (0); 813} 814 815/* 816 * Modify pte bits for all ptes corresponding to the given physical address. 817 * We use `maskbits' rather than `clearbits' because we're always passing 818 * constants and the latter would require an extra inversion at run-time. 819 */ 820static int 821pmap_clearbit(struct vm_page *pg, u_int maskbits) 822{ 823 struct l2_bucket *l2b; 824 struct pv_entry *pv; 825 pt_entry_t *ptep, npte, opte; 826 pmap_t pm; 827 vm_offset_t va; 828 u_int oflags; 829 int count = 0; 830 831 rw_wlock(&pvh_global_lock); 832 833 if (maskbits & PVF_WRITE) 834 maskbits |= PVF_MOD; 835 /* 836 * Clear saved attributes (modify, reference) 837 */ 838 pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 839 840 if (TAILQ_EMPTY(&pg->md.pv_list)) { 841 rw_wunlock(&pvh_global_lock); 842 return (0); 843 } 844 845 /* 846 * Loop over all current mappings setting/clearing as appropos 847 */ 848 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 849 va = pv->pv_va; 850 pm = pv->pv_pmap; 851 oflags = pv->pv_flags; 852 pv->pv_flags &= ~maskbits; 853 854 PMAP_LOCK(pm); 855 856 l2b = pmap_get_l2_bucket(pm, va); 857 858 ptep = &l2b->l2b_kva[l2pte_index(va)]; 859 npte = opte = *ptep; 860 861 if ((maskbits & (PVF_WRITE|PVF_MOD)) && L2_S_WRITABLE(opte)) { 862 vm_page_dirty(pg); 863 864 /* make the pte read only */ 865 npte |= L2_APX; 866 } 867 868 if (maskbits & PVF_REF) { 869 /* 870 * Make the PTE invalid so that we will take a 871 * page fault the next time the mapping is 872 * referenced. 873 */ 874 npte &= ~L2_TYPE_MASK; 875 npte |= L2_TYPE_INV; 876 } 877 878 CTR4(KTR_PMAP,"clearbit: pmap:%p bits:%x pte:%x->%x", 879 pm, maskbits, opte, npte); 880 if (npte != opte) { 881 count++; 882 *ptep = npte; 883 PTE_SYNC(ptep); 884 /* Flush the TLB entry if a current pmap. */ 885 if (PV_BEEN_EXECD(oflags)) 886 cpu_tlb_flushID_SE(pv->pv_va); 887 else if (PV_BEEN_REFD(oflags)) 888 cpu_tlb_flushD_SE(pv->pv_va); 889 } 890 891 PMAP_UNLOCK(pm); 892 893 } 894 895 if (maskbits & PVF_WRITE) 896 vm_page_aflag_clear(pg, PGA_WRITEABLE); 897 rw_wunlock(&pvh_global_lock); 898 return (count); 899} 900 901/* 902 * main pv_entry manipulation functions: 903 * pmap_enter_pv: enter a mapping onto a vm_page list 904 * pmap_remove_pv: remove a mappiing from a vm_page list 905 * 906 * NOTE: pmap_enter_pv expects to lock the pvh itself 907 * pmap_remove_pv expects the caller to lock the pvh before calling 908 */ 909 910/* 911 * pmap_enter_pv: enter a mapping onto a vm_page's PV list 912 * 913 * => caller should hold the proper lock on pvh_global_lock 914 * => caller should have pmap locked 915 * => we will (someday) gain the lock on the vm_page's PV list 916 * => caller should adjust ptp's wire_count before calling 917 * => caller should not adjust pmap's wire_count 918 */ 919static void 920pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 921 vm_offset_t va, u_int flags) 922{ 923 924 rw_assert(&pvh_global_lock, RA_WLOCKED); 925 926 PMAP_ASSERT_LOCKED(pm); 927 pve->pv_pmap = pm; 928 pve->pv_va = va; 929 pve->pv_flags = flags; 930 931 TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); 932 TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); 933 pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); 934 if (pve->pv_flags & PVF_WIRED) 935 ++pm->pm_stats.wired_count; 936 vm_page_aflag_set(pg, PGA_REFERENCED); 937} 938 939/* 940 * 941 * pmap_find_pv: Find a pv entry 942 * 943 * => caller should hold lock on vm_page 944 */ 945static PMAP_INLINE struct pv_entry * 946pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 947{ 948 struct pv_entry *pv; 949 950 rw_assert(&pvh_global_lock, RA_WLOCKED); 951 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) 952 if (pm == pv->pv_pmap && va == pv->pv_va) 953 break; 954 return (pv); 955} 956 957/* 958 * vector_page_setprot: 959 * 960 * Manipulate the protection of the vector page. 961 */ 962void 963vector_page_setprot(int prot) 964{ 965 struct l2_bucket *l2b; 966 pt_entry_t *ptep; 967 968 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 969 970 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 971 972 pmap_set_prot(ptep, prot|VM_PROT_EXECUTE, 0); 973 974 cpu_tlb_flushD_SE(vector_page); 975 cpu_cpwait(); 976} 977 978static void 979pmap_set_prot(pt_entry_t *ptep, vm_prot_t prot, uint8_t user) 980{ 981 982 *ptep &= ~L2_S_PROT_MASK; 983 984 if (!(prot & VM_PROT_EXECUTE)) 985 *ptep |= L2_XN; 986 987 *ptep |= L2_S_PROT_R; 988 989 if (user) 990 *ptep |= L2_S_PROT_U; 991 992 if (prot & VM_PROT_WRITE) 993 *ptep &= ~(L2_APX); 994} 995 996/* 997 * pmap_remove_pv: try to remove a mapping from a pv_list 998 * 999 * => caller should hold proper lock on pmap_main_lock 1000 * => pmap should be locked 1001 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1002 * => caller should adjust ptp's wire_count and free PTP if needed 1003 * => caller should NOT adjust pmap's wire_count 1004 * => we return the removed pve 1005 */ 1006 1007static void 1008pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) 1009{ 1010 1011 rw_assert(&pvh_global_lock, RA_WLOCKED); 1012 PMAP_ASSERT_LOCKED(pm); 1013 1014 TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); 1015 TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); 1016 1017 if (pve->pv_flags & PVF_WIRED) 1018 --pm->pm_stats.wired_count; 1019 1020 if (pg->md.pvh_attrs & PVF_MOD) 1021 vm_page_dirty(pg); 1022 1023 if (TAILQ_FIRST(&pg->md.pv_list) == NULL) 1024 pg->md.pvh_attrs &= ~PVF_REF; 1025 else 1026 vm_page_aflag_set(pg, PGA_REFERENCED); 1027 1028 if (pve->pv_flags & PVF_WRITE) { 1029 TAILQ_FOREACH(pve, &pg->md.pv_list, pv_list) 1030 if (pve->pv_flags & PVF_WRITE) 1031 break; 1032 if (!pve) { 1033 pg->md.pvh_attrs &= ~PVF_MOD; 1034 vm_page_aflag_clear(pg, PGA_WRITEABLE); 1035 } 1036 } 1037} 1038 1039static struct pv_entry * 1040pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1041{ 1042 struct pv_entry *pve; 1043 1044 rw_assert(&pvh_global_lock, RA_WLOCKED); 1045 pve = TAILQ_FIRST(&pg->md.pv_list); 1046 1047 while (pve) { 1048 if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ 1049 pmap_nuke_pv(pg, pm, pve); 1050 break; 1051 } 1052 pve = TAILQ_NEXT(pve, pv_list); 1053 } 1054 1055 return(pve); /* return removed pve */ 1056} 1057 1058/* 1059 * 1060 * pmap_modify_pv: Update pv flags 1061 * 1062 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1063 * => caller should NOT adjust pmap's wire_count 1064 * => we return the old flags 1065 * 1066 * Modify a physical-virtual mapping in the pv table 1067 */ 1068static u_int 1069pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va, 1070 u_int clr_mask, u_int set_mask) 1071{ 1072 struct pv_entry *npv; 1073 u_int flags, oflags; 1074 1075 PMAP_ASSERT_LOCKED(pm); 1076 rw_assert(&pvh_global_lock, RA_WLOCKED); 1077 if ((npv = pmap_find_pv(pg, pm, va)) == NULL) 1078 return (0); 1079 1080 /* 1081 * There is at least one VA mapping this page. 1082 */ 1083 1084 if (clr_mask & (PVF_REF | PVF_MOD)) 1085 pg->md.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1086 1087 oflags = npv->pv_flags; 1088 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1089 1090 if ((flags ^ oflags) & PVF_WIRED) { 1091 if (flags & PVF_WIRED) 1092 ++pm->pm_stats.wired_count; 1093 else 1094 --pm->pm_stats.wired_count; 1095 } 1096 if ((oflags & PVF_WRITE) && !(flags & PVF_WRITE)) { 1097 TAILQ_FOREACH(npv, &pg->md.pv_list, pv_list) { 1098 if (npv->pv_flags & PVF_WRITE) 1099 break; 1100 } 1101 if (!npv) { 1102 pg->md.pvh_attrs &= ~PVF_MOD; 1103 vm_page_aflag_clear(pg, PGA_WRITEABLE); 1104 } 1105 } 1106 1107 return (oflags); 1108} 1109 1110/* Function to set the debug level of the pmap code */ 1111#ifdef PMAP_DEBUG 1112void 1113pmap_debug(int level) 1114{ 1115 pmap_debug_level = level; 1116 dprintf("pmap_debug: level=%d\n", pmap_debug_level); 1117} 1118#endif /* PMAP_DEBUG */ 1119 1120void 1121pmap_pinit0(struct pmap *pmap) 1122{ 1123 PDEBUG(1, printf("pmap_pinit0: pmap = %08x\n", (u_int32_t) pmap)); 1124 1125 dprintf("pmap_pinit0: pmap = %08x, pm_pdir = %08x\n", 1126 (u_int32_t) pmap, (u_int32_t) pmap->pm_pdir); 1127 bcopy(kernel_pmap, pmap, sizeof(*pmap)); 1128 bzero(&pmap->pm_mtx, sizeof(pmap->pm_mtx)); 1129 PMAP_LOCK_INIT(pmap); 1130} 1131 1132/* 1133 * Initialize a vm_page's machine-dependent fields. 1134 */ 1135void 1136pmap_page_init(vm_page_t m) 1137{ 1138 1139 TAILQ_INIT(&m->md.pv_list); 1140 m->md.pv_memattr = VM_MEMATTR_DEFAULT; 1141} 1142 1143/* 1144 * Initialize the pmap module. 1145 * Called by vm_init, to initialize any structures that the pmap 1146 * system needs to map virtual memory. 1147 */ 1148void 1149pmap_init(void) 1150{ 1151 int shpgperproc = PMAP_SHPGPERPROC; 1152 1153 PDEBUG(1, printf("pmap_init: phys_start = %08x\n", PHYSADDR)); 1154 1155 l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, 1156 NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1157 l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), NULL, 1158 NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1159 1160 /* 1161 * Initialize the PV entry allocator. 1162 */ 1163 pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, 1164 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1165 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 1166 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
| 395static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; 396static struct rwlock pvh_global_lock; 397 398int l1_mem_types[] = { 399 ARM_L1S_STRONG_ORD, 400 ARM_L1S_DEVICE_NOSHARE, 401 ARM_L1S_DEVICE_SHARE, 402 ARM_L1S_NRML_NOCACHE, 403 ARM_L1S_NRML_IWT_OWT, 404 ARM_L1S_NRML_IWB_OWB, 405 ARM_L1S_NRML_IWBA_OWBA 406}; 407 408int l2l_mem_types[] = { 409 ARM_L2L_STRONG_ORD, 410 ARM_L2L_DEVICE_NOSHARE, 411 ARM_L2L_DEVICE_SHARE, 412 ARM_L2L_NRML_NOCACHE, 413 ARM_L2L_NRML_IWT_OWT, 414 ARM_L2L_NRML_IWB_OWB, 415 ARM_L2L_NRML_IWBA_OWBA 416}; 417 418int l2s_mem_types[] = { 419 ARM_L2S_STRONG_ORD, 420 ARM_L2S_DEVICE_NOSHARE, 421 ARM_L2S_DEVICE_SHARE, 422 ARM_L2S_NRML_NOCACHE, 423 ARM_L2S_NRML_IWT_OWT, 424 ARM_L2S_NRML_IWB_OWB, 425 ARM_L2S_NRML_IWBA_OWBA 426}; 427 428/* 429 * This list exists for the benefit of pmap_map_chunk(). It keeps track 430 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 431 * find them as necessary. 432 * 433 * Note that the data on this list MUST remain valid after initarm() returns, 434 * as pmap_bootstrap() uses it to contruct L2 table metadata. 435 */ 436SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 437 438static void 439pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 440{ 441 int i; 442 443 l1->l1_kva = l1pt; 444 l1->l1_domain_use_count = 0; 445 l1->l1_domain_first = 0; 446 447 for (i = 0; i < PMAP_DOMAINS; i++) 448 l1->l1_domain_free[i] = i + 1; 449 450 /* 451 * Copy the kernel's L1 entries to each new L1. 452 */ 453 if (l1pt != pmap_kernel()->pm_l1->l1_kva) 454 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 455 456 if ((l1->l1_physaddr = pmap_extract(pmap_kernel(), (vm_offset_t)l1pt)) == 0) 457 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 458 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 459 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 460} 461 462static vm_offset_t 463kernel_pt_lookup(vm_paddr_t pa) 464{ 465 struct pv_addr *pv; 466 467 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 468 if (pv->pv_pa == pa) 469 return (pv->pv_va); 470 } 471 return (0); 472} 473 474void 475pmap_pte_init_mmu_v6(void) 476{ 477 478 if (PTE_PAGETABLE >= 3) 479 pmap_needs_pte_sync = 1; 480 pte_l1_s_cache_mode = l1_mem_types[PTE_CACHE]; 481 pte_l2_l_cache_mode = l2l_mem_types[PTE_CACHE]; 482 pte_l2_s_cache_mode = l2s_mem_types[PTE_CACHE]; 483 484 pte_l1_s_cache_mode_pt = l1_mem_types[PTE_PAGETABLE]; 485 pte_l2_l_cache_mode_pt = l2l_mem_types[PTE_PAGETABLE]; 486 pte_l2_s_cache_mode_pt = l2s_mem_types[PTE_PAGETABLE]; 487 488} 489 490/* 491 * Allocate an L1 translation table for the specified pmap. 492 * This is called at pmap creation time. 493 */ 494static void 495pmap_alloc_l1(pmap_t pm) 496{ 497 struct l1_ttable *l1; 498 u_int8_t domain; 499 500 /* 501 * Remove the L1 at the head of the LRU list 502 */ 503 mtx_lock(&l1_lru_lock); 504 l1 = TAILQ_FIRST(&l1_lru_list); 505 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 506 507 /* 508 * Pick the first available domain number, and update 509 * the link to the next number. 510 */ 511 domain = l1->l1_domain_first; 512 l1->l1_domain_first = l1->l1_domain_free[domain]; 513 514 /* 515 * If there are still free domain numbers in this L1, 516 * put it back on the TAIL of the LRU list. 517 */ 518 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 519 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 520 521 mtx_unlock(&l1_lru_lock); 522 523 /* 524 * Fix up the relevant bits in the pmap structure 525 */ 526 pm->pm_l1 = l1; 527 pm->pm_domain = domain + 1; 528} 529 530/* 531 * Free an L1 translation table. 532 * This is called at pmap destruction time. 533 */ 534static void 535pmap_free_l1(pmap_t pm) 536{ 537 struct l1_ttable *l1 = pm->pm_l1; 538 539 mtx_lock(&l1_lru_lock); 540 541 /* 542 * If this L1 is currently on the LRU list, remove it. 543 */ 544 if (l1->l1_domain_use_count < PMAP_DOMAINS) 545 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 546 547 /* 548 * Free up the domain number which was allocated to the pmap 549 */ 550 l1->l1_domain_free[pm->pm_domain - 1] = l1->l1_domain_first; 551 l1->l1_domain_first = pm->pm_domain - 1; 552 l1->l1_domain_use_count--; 553 554 /* 555 * The L1 now must have at least 1 free domain, so add 556 * it back to the LRU list. If the use count is zero, 557 * put it at the head of the list, otherwise it goes 558 * to the tail. 559 */ 560 if (l1->l1_domain_use_count == 0) { 561 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 562 } else 563 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 564 565 mtx_unlock(&l1_lru_lock); 566} 567 568/* 569 * Returns a pointer to the L2 bucket associated with the specified pmap 570 * and VA, or NULL if no L2 bucket exists for the address. 571 */ 572static PMAP_INLINE struct l2_bucket * 573pmap_get_l2_bucket(pmap_t pm, vm_offset_t va) 574{ 575 struct l2_dtable *l2; 576 struct l2_bucket *l2b; 577 u_short l1idx; 578 579 l1idx = L1_IDX(va); 580 581 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 582 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 583 return (NULL); 584 585 return (l2b); 586} 587 588/* 589 * Returns a pointer to the L2 bucket associated with the specified pmap 590 * and VA. 591 * 592 * If no L2 bucket exists, perform the necessary allocations to put an L2 593 * bucket/page table in place. 594 * 595 * Note that if a new L2 bucket/page was allocated, the caller *must* 596 * increment the bucket occupancy counter appropriately *before* 597 * releasing the pmap's lock to ensure no other thread or cpu deallocates 598 * the bucket/page in the meantime. 599 */ 600static struct l2_bucket * 601pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) 602{ 603 struct l2_dtable *l2; 604 struct l2_bucket *l2b; 605 u_short l1idx; 606 607 l1idx = L1_IDX(va); 608 609 PMAP_ASSERT_LOCKED(pm); 610 rw_assert(&pvh_global_lock, RA_WLOCKED); 611 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 612 /* 613 * No mapping at this address, as there is 614 * no entry in the L1 table. 615 * Need to allocate a new l2_dtable. 616 */ 617 PMAP_UNLOCK(pm); 618 rw_wunlock(&pvh_global_lock); 619 if ((l2 = uma_zalloc(l2table_zone, M_NOWAIT)) == NULL) { 620 rw_wlock(&pvh_global_lock); 621 PMAP_LOCK(pm); 622 return (NULL); 623 } 624 rw_wlock(&pvh_global_lock); 625 PMAP_LOCK(pm); 626 if (pm->pm_l2[L2_IDX(l1idx)] != NULL) { 627 /* 628 * Someone already allocated the l2_dtable while 629 * we were doing the same. 630 */ 631 uma_zfree(l2table_zone, l2); 632 l2 = pm->pm_l2[L2_IDX(l1idx)]; 633 } else { 634 bzero(l2, sizeof(*l2)); 635 /* 636 * Link it into the parent pmap 637 */ 638 pm->pm_l2[L2_IDX(l1idx)] = l2; 639 } 640 } 641 642 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 643 644 /* 645 * Fetch pointer to the L2 page table associated with the address. 646 */ 647 if (l2b->l2b_kva == NULL) { 648 pt_entry_t *ptep; 649 650 /* 651 * No L2 page table has been allocated. Chances are, this 652 * is because we just allocated the l2_dtable, above. 653 */ 654 PMAP_UNLOCK(pm); 655 rw_wunlock(&pvh_global_lock); 656 ptep = uma_zalloc(l2zone, M_NOWAIT); 657 rw_wlock(&pvh_global_lock); 658 PMAP_LOCK(pm); 659 if (l2b->l2b_kva != 0) { 660 /* We lost the race. */ 661 uma_zfree(l2zone, ptep); 662 return (l2b); 663 } 664 l2b->l2b_phys = vtophys(ptep); 665 if (ptep == NULL) { 666 /* 667 * Oops, no more L2 page tables available at this 668 * time. We may need to deallocate the l2_dtable 669 * if we allocated a new one above. 670 */ 671 if (l2->l2_occupancy == 0) { 672 pm->pm_l2[L2_IDX(l1idx)] = NULL; 673 uma_zfree(l2table_zone, l2); 674 } 675 return (NULL); 676 } 677 678 l2->l2_occupancy++; 679 l2b->l2b_kva = ptep; 680 l2b->l2b_l1idx = l1idx; 681 } 682 683 return (l2b); 684} 685 686static PMAP_INLINE void 687pmap_free_l2_ptp(pt_entry_t *l2) 688{ 689 uma_zfree(l2zone, l2); 690} 691/* 692 * One or more mappings in the specified L2 descriptor table have just been 693 * invalidated. 694 * 695 * Garbage collect the metadata and descriptor table itself if necessary. 696 * 697 * The pmap lock must be acquired when this is called (not necessary 698 * for the kernel pmap). 699 */ 700static void 701pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 702{ 703 struct l2_dtable *l2; 704 pd_entry_t *pl1pd, l1pd; 705 pt_entry_t *ptep; 706 u_short l1idx; 707 708 709 /* 710 * Update the bucket's reference count according to how many 711 * PTEs the caller has just invalidated. 712 */ 713 l2b->l2b_occupancy -= count; 714 715 /* 716 * Note: 717 * 718 * Level 2 page tables allocated to the kernel pmap are never freed 719 * as that would require checking all Level 1 page tables and 720 * removing any references to the Level 2 page table. See also the 721 * comment elsewhere about never freeing bootstrap L2 descriptors. 722 * 723 * We make do with just invalidating the mapping in the L2 table. 724 * 725 * This isn't really a big deal in practice and, in fact, leads 726 * to a performance win over time as we don't need to continually 727 * alloc/free. 728 */ 729 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 730 return; 731 732 /* 733 * There are no more valid mappings in this level 2 page table. 734 * Go ahead and NULL-out the pointer in the bucket, then 735 * free the page table. 736 */ 737 l1idx = l2b->l2b_l1idx; 738 ptep = l2b->l2b_kva; 739 l2b->l2b_kva = NULL; 740 741 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 742 743 /* 744 * If the L1 slot matches the pmap's domain 745 * number, then invalidate it. 746 */ 747 l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); 748 if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { 749 *pl1pd = 0; 750 PTE_SYNC(pl1pd); 751 } 752 753 /* 754 * Release the L2 descriptor table back to the pool cache. 755 */ 756 pmap_free_l2_ptp(ptep); 757 758 /* 759 * Update the reference count in the associated l2_dtable 760 */ 761 l2 = pm->pm_l2[L2_IDX(l1idx)]; 762 if (--l2->l2_occupancy > 0) 763 return; 764 765 /* 766 * There are no more valid mappings in any of the Level 1 767 * slots managed by this l2_dtable. Go ahead and NULL-out 768 * the pointer in the parent pmap and free the l2_dtable. 769 */ 770 pm->pm_l2[L2_IDX(l1idx)] = NULL; 771 uma_zfree(l2table_zone, l2); 772} 773 774/* 775 * Pool cache constructors for L2 descriptor tables, metadata and pmap 776 * structures. 777 */ 778static int 779pmap_l2ptp_ctor(void *mem, int size, void *arg, int flags) 780{ 781 struct l2_bucket *l2b; 782 pt_entry_t *ptep, pte; 783 vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; 784 785 /* 786 * The mappings for these page tables were initially made using 787 * pmap_kenter() by the pool subsystem. Therefore, the cache- 788 * mode will not be right for page table mappings. To avoid 789 * polluting the pmap_kenter() code with a special case for 790 * page tables, we simply fix up the cache-mode here if it's not 791 * correct. 792 */ 793 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 794 ptep = &l2b->l2b_kva[l2pte_index(va)]; 795 pte = *ptep; 796 797 cpu_idcache_wbinv_range(va, PAGE_SIZE); 798 pmap_l2cache_wbinv_range(va, pte & L2_S_FRAME, PAGE_SIZE); 799 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 800 /* 801 * Page tables must have the cache-mode set to 802 * Write-Thru. 803 */ 804 *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 805 PTE_SYNC(ptep); 806 cpu_tlb_flushD_SE(va); 807 cpu_cpwait(); 808 } 809 810 memset(mem, 0, L2_TABLE_SIZE_REAL); 811 return (0); 812} 813 814/* 815 * Modify pte bits for all ptes corresponding to the given physical address. 816 * We use `maskbits' rather than `clearbits' because we're always passing 817 * constants and the latter would require an extra inversion at run-time. 818 */ 819static int 820pmap_clearbit(struct vm_page *pg, u_int maskbits) 821{ 822 struct l2_bucket *l2b; 823 struct pv_entry *pv; 824 pt_entry_t *ptep, npte, opte; 825 pmap_t pm; 826 vm_offset_t va; 827 u_int oflags; 828 int count = 0; 829 830 rw_wlock(&pvh_global_lock); 831 832 if (maskbits & PVF_WRITE) 833 maskbits |= PVF_MOD; 834 /* 835 * Clear saved attributes (modify, reference) 836 */ 837 pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 838 839 if (TAILQ_EMPTY(&pg->md.pv_list)) { 840 rw_wunlock(&pvh_global_lock); 841 return (0); 842 } 843 844 /* 845 * Loop over all current mappings setting/clearing as appropos 846 */ 847 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 848 va = pv->pv_va; 849 pm = pv->pv_pmap; 850 oflags = pv->pv_flags; 851 pv->pv_flags &= ~maskbits; 852 853 PMAP_LOCK(pm); 854 855 l2b = pmap_get_l2_bucket(pm, va); 856 857 ptep = &l2b->l2b_kva[l2pte_index(va)]; 858 npte = opte = *ptep; 859 860 if ((maskbits & (PVF_WRITE|PVF_MOD)) && L2_S_WRITABLE(opte)) { 861 vm_page_dirty(pg); 862 863 /* make the pte read only */ 864 npte |= L2_APX; 865 } 866 867 if (maskbits & PVF_REF) { 868 /* 869 * Make the PTE invalid so that we will take a 870 * page fault the next time the mapping is 871 * referenced. 872 */ 873 npte &= ~L2_TYPE_MASK; 874 npte |= L2_TYPE_INV; 875 } 876 877 CTR4(KTR_PMAP,"clearbit: pmap:%p bits:%x pte:%x->%x", 878 pm, maskbits, opte, npte); 879 if (npte != opte) { 880 count++; 881 *ptep = npte; 882 PTE_SYNC(ptep); 883 /* Flush the TLB entry if a current pmap. */ 884 if (PV_BEEN_EXECD(oflags)) 885 cpu_tlb_flushID_SE(pv->pv_va); 886 else if (PV_BEEN_REFD(oflags)) 887 cpu_tlb_flushD_SE(pv->pv_va); 888 } 889 890 PMAP_UNLOCK(pm); 891 892 } 893 894 if (maskbits & PVF_WRITE) 895 vm_page_aflag_clear(pg, PGA_WRITEABLE); 896 rw_wunlock(&pvh_global_lock); 897 return (count); 898} 899 900/* 901 * main pv_entry manipulation functions: 902 * pmap_enter_pv: enter a mapping onto a vm_page list 903 * pmap_remove_pv: remove a mappiing from a vm_page list 904 * 905 * NOTE: pmap_enter_pv expects to lock the pvh itself 906 * pmap_remove_pv expects the caller to lock the pvh before calling 907 */ 908 909/* 910 * pmap_enter_pv: enter a mapping onto a vm_page's PV list 911 * 912 * => caller should hold the proper lock on pvh_global_lock 913 * => caller should have pmap locked 914 * => we will (someday) gain the lock on the vm_page's PV list 915 * => caller should adjust ptp's wire_count before calling 916 * => caller should not adjust pmap's wire_count 917 */ 918static void 919pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 920 vm_offset_t va, u_int flags) 921{ 922 923 rw_assert(&pvh_global_lock, RA_WLOCKED); 924 925 PMAP_ASSERT_LOCKED(pm); 926 pve->pv_pmap = pm; 927 pve->pv_va = va; 928 pve->pv_flags = flags; 929 930 TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); 931 TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); 932 pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); 933 if (pve->pv_flags & PVF_WIRED) 934 ++pm->pm_stats.wired_count; 935 vm_page_aflag_set(pg, PGA_REFERENCED); 936} 937 938/* 939 * 940 * pmap_find_pv: Find a pv entry 941 * 942 * => caller should hold lock on vm_page 943 */ 944static PMAP_INLINE struct pv_entry * 945pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 946{ 947 struct pv_entry *pv; 948 949 rw_assert(&pvh_global_lock, RA_WLOCKED); 950 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) 951 if (pm == pv->pv_pmap && va == pv->pv_va) 952 break; 953 return (pv); 954} 955 956/* 957 * vector_page_setprot: 958 * 959 * Manipulate the protection of the vector page. 960 */ 961void 962vector_page_setprot(int prot) 963{ 964 struct l2_bucket *l2b; 965 pt_entry_t *ptep; 966 967 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 968 969 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 970 971 pmap_set_prot(ptep, prot|VM_PROT_EXECUTE, 0); 972 973 cpu_tlb_flushD_SE(vector_page); 974 cpu_cpwait(); 975} 976 977static void 978pmap_set_prot(pt_entry_t *ptep, vm_prot_t prot, uint8_t user) 979{ 980 981 *ptep &= ~L2_S_PROT_MASK; 982 983 if (!(prot & VM_PROT_EXECUTE)) 984 *ptep |= L2_XN; 985 986 *ptep |= L2_S_PROT_R; 987 988 if (user) 989 *ptep |= L2_S_PROT_U; 990 991 if (prot & VM_PROT_WRITE) 992 *ptep &= ~(L2_APX); 993} 994 995/* 996 * pmap_remove_pv: try to remove a mapping from a pv_list 997 * 998 * => caller should hold proper lock on pmap_main_lock 999 * => pmap should be locked 1000 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1001 * => caller should adjust ptp's wire_count and free PTP if needed 1002 * => caller should NOT adjust pmap's wire_count 1003 * => we return the removed pve 1004 */ 1005 1006static void 1007pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) 1008{ 1009 1010 rw_assert(&pvh_global_lock, RA_WLOCKED); 1011 PMAP_ASSERT_LOCKED(pm); 1012 1013 TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); 1014 TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); 1015 1016 if (pve->pv_flags & PVF_WIRED) 1017 --pm->pm_stats.wired_count; 1018 1019 if (pg->md.pvh_attrs & PVF_MOD) 1020 vm_page_dirty(pg); 1021 1022 if (TAILQ_FIRST(&pg->md.pv_list) == NULL) 1023 pg->md.pvh_attrs &= ~PVF_REF; 1024 else 1025 vm_page_aflag_set(pg, PGA_REFERENCED); 1026 1027 if (pve->pv_flags & PVF_WRITE) { 1028 TAILQ_FOREACH(pve, &pg->md.pv_list, pv_list) 1029 if (pve->pv_flags & PVF_WRITE) 1030 break; 1031 if (!pve) { 1032 pg->md.pvh_attrs &= ~PVF_MOD; 1033 vm_page_aflag_clear(pg, PGA_WRITEABLE); 1034 } 1035 } 1036} 1037 1038static struct pv_entry * 1039pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1040{ 1041 struct pv_entry *pve; 1042 1043 rw_assert(&pvh_global_lock, RA_WLOCKED); 1044 pve = TAILQ_FIRST(&pg->md.pv_list); 1045 1046 while (pve) { 1047 if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ 1048 pmap_nuke_pv(pg, pm, pve); 1049 break; 1050 } 1051 pve = TAILQ_NEXT(pve, pv_list); 1052 } 1053 1054 return(pve); /* return removed pve */ 1055} 1056 1057/* 1058 * 1059 * pmap_modify_pv: Update pv flags 1060 * 1061 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1062 * => caller should NOT adjust pmap's wire_count 1063 * => we return the old flags 1064 * 1065 * Modify a physical-virtual mapping in the pv table 1066 */ 1067static u_int 1068pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va, 1069 u_int clr_mask, u_int set_mask) 1070{ 1071 struct pv_entry *npv; 1072 u_int flags, oflags; 1073 1074 PMAP_ASSERT_LOCKED(pm); 1075 rw_assert(&pvh_global_lock, RA_WLOCKED); 1076 if ((npv = pmap_find_pv(pg, pm, va)) == NULL) 1077 return (0); 1078 1079 /* 1080 * There is at least one VA mapping this page. 1081 */ 1082 1083 if (clr_mask & (PVF_REF | PVF_MOD)) 1084 pg->md.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1085 1086 oflags = npv->pv_flags; 1087 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1088 1089 if ((flags ^ oflags) & PVF_WIRED) { 1090 if (flags & PVF_WIRED) 1091 ++pm->pm_stats.wired_count; 1092 else 1093 --pm->pm_stats.wired_count; 1094 } 1095 if ((oflags & PVF_WRITE) && !(flags & PVF_WRITE)) { 1096 TAILQ_FOREACH(npv, &pg->md.pv_list, pv_list) { 1097 if (npv->pv_flags & PVF_WRITE) 1098 break; 1099 } 1100 if (!npv) { 1101 pg->md.pvh_attrs &= ~PVF_MOD; 1102 vm_page_aflag_clear(pg, PGA_WRITEABLE); 1103 } 1104 } 1105 1106 return (oflags); 1107} 1108 1109/* Function to set the debug level of the pmap code */ 1110#ifdef PMAP_DEBUG 1111void 1112pmap_debug(int level) 1113{ 1114 pmap_debug_level = level; 1115 dprintf("pmap_debug: level=%d\n", pmap_debug_level); 1116} 1117#endif /* PMAP_DEBUG */ 1118 1119void 1120pmap_pinit0(struct pmap *pmap) 1121{ 1122 PDEBUG(1, printf("pmap_pinit0: pmap = %08x\n", (u_int32_t) pmap)); 1123 1124 dprintf("pmap_pinit0: pmap = %08x, pm_pdir = %08x\n", 1125 (u_int32_t) pmap, (u_int32_t) pmap->pm_pdir); 1126 bcopy(kernel_pmap, pmap, sizeof(*pmap)); 1127 bzero(&pmap->pm_mtx, sizeof(pmap->pm_mtx)); 1128 PMAP_LOCK_INIT(pmap); 1129} 1130 1131/* 1132 * Initialize a vm_page's machine-dependent fields. 1133 */ 1134void 1135pmap_page_init(vm_page_t m) 1136{ 1137 1138 TAILQ_INIT(&m->md.pv_list); 1139 m->md.pv_memattr = VM_MEMATTR_DEFAULT; 1140} 1141 1142/* 1143 * Initialize the pmap module. 1144 * Called by vm_init, to initialize any structures that the pmap 1145 * system needs to map virtual memory. 1146 */ 1147void 1148pmap_init(void) 1149{ 1150 int shpgperproc = PMAP_SHPGPERPROC; 1151 1152 PDEBUG(1, printf("pmap_init: phys_start = %08x\n", PHYSADDR)); 1153 1154 l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, 1155 NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1156 l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), NULL, 1157 NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1158 1159 /* 1160 * Initialize the PV entry allocator. 1161 */ 1162 pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, 1163 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1164 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 1165 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
|
1168 pv_entry_high_water = 9 * (pv_entry_max / 10); 1169 1170 /* 1171 * Now it is safe to enable pv_table recording. 1172 */ 1173 PDEBUG(1, printf("pmap_init: done!\n")); 1174} 1175 1176int 1177pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) 1178{ 1179 struct l2_dtable *l2; 1180 struct l2_bucket *l2b; 1181 pd_entry_t *pl1pd, l1pd; 1182 pt_entry_t *ptep, pte; 1183 vm_paddr_t pa; 1184 u_int l1idx; 1185 int rv = 0; 1186 1187 l1idx = L1_IDX(va); 1188 rw_wlock(&pvh_global_lock); 1189 PMAP_LOCK(pm); 1190 1191 /* 1192 * If there is no l2_dtable for this address, then the process 1193 * has no business accessing it. 1194 * 1195 * Note: This will catch userland processes trying to access 1196 * kernel addresses. 1197 */ 1198 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1199 if (l2 == NULL) 1200 goto out; 1201 1202 /* 1203 * Likewise if there is no L2 descriptor table 1204 */ 1205 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1206 if (l2b->l2b_kva == NULL) 1207 goto out; 1208 1209 /* 1210 * Check the PTE itself. 1211 */ 1212 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1213 pte = *ptep; 1214 if (pte == 0) 1215 goto out; 1216 1217 /* 1218 * Catch a userland access to the vector page mapped at 0x0 1219 */ 1220 if (user && ((pte & L2_S_PROT_MASK) == L2_S_PROT_U)) 1221 goto out; 1222 if (va == vector_page) 1223 goto out; 1224 1225 pa = l2pte_pa(pte); 1226 CTR5(KTR_PMAP, "pmap_fault_fix: pmap:%p va:%x pte:0x%x ftype:%x user:%x", 1227 pm, va, pte, ftype, user); 1228 if ((ftype & VM_PROT_WRITE) && !(L2_S_WRITABLE(pte))) { 1229 /* 1230 * This looks like a good candidate for "page modified" 1231 * emulation... 1232 */ 1233 struct pv_entry *pv; 1234 struct vm_page *pg; 1235 1236 /* Extract the physical address of the page */ 1237 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 1238 goto out; 1239 } 1240 /* Get the current flags for this page. */ 1241 1242 pv = pmap_find_pv(pg, pm, va); 1243 if (pv == NULL) { 1244 goto out; 1245 } 1246 1247 /* 1248 * Do the flags say this page is writable? If not then it 1249 * is a genuine write fault. If yes then the write fault is 1250 * our fault as we did not reflect the write access in the 1251 * PTE. Now we know a write has occurred we can correct this 1252 * and also set the modified bit 1253 */ 1254 if ((pv->pv_flags & PVF_WRITE) == 0) { 1255 goto out; 1256 } 1257 pg->md.pvh_attrs |= PVF_REF | PVF_MOD; 1258 vm_page_dirty(pg); 1259 pv->pv_flags |= PVF_REF | PVF_MOD; 1260 1261 /* Re-enable write permissions for the page */ 1262 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 1263 pmap_set_prot(ptep, VM_PROT_WRITE, *ptep & L2_S_PROT_U); 1264 CTR1(KTR_PMAP, "pmap_fault_fix: new pte:0x%x", pte); 1265 PTE_SYNC(ptep); 1266 rv = 1; 1267 } else if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { 1268 /* 1269 * This looks like a good candidate for "page referenced" 1270 * emulation. 1271 */ 1272 struct pv_entry *pv; 1273 struct vm_page *pg; 1274 1275 /* Extract the physical address of the page */ 1276 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 1277 goto out; 1278 /* Get the current flags for this page. */ 1279 1280 pv = pmap_find_pv(pg, pm, va); 1281 if (pv == NULL) 1282 goto out; 1283 1284 pg->md.pvh_attrs |= PVF_REF; 1285 pv->pv_flags |= PVF_REF; 1286 1287 1288 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 1289 PTE_SYNC(ptep); 1290 rv = 1; 1291 } 1292 1293 /* 1294 * We know there is a valid mapping here, so simply 1295 * fix up the L1 if necessary. 1296 */ 1297 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1298 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; 1299 if (*pl1pd != l1pd) { 1300 *pl1pd = l1pd; 1301 PTE_SYNC(pl1pd); 1302 rv = 1; 1303 } 1304 1305#ifdef DEBUG 1306 /* 1307 * If 'rv == 0' at this point, it generally indicates that there is a 1308 * stale TLB entry for the faulting address. This happens when two or 1309 * more processes are sharing an L1. Since we don't flush the TLB on 1310 * a context switch between such processes, we can take domain faults 1311 * for mappings which exist at the same VA in both processes. EVEN IF 1312 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 1313 * example. 1314 * 1315 * This is extremely likely to happen if pmap_enter() updated the L1 1316 * entry for a recently entered mapping. In this case, the TLB is 1317 * flushed for the new mapping, but there may still be TLB entries for 1318 * other mappings belonging to other processes in the 1MB range 1319 * covered by the L1 entry. 1320 * 1321 * Since 'rv == 0', we know that the L1 already contains the correct 1322 * value, so the fault must be due to a stale TLB entry. 1323 * 1324 * Since we always need to flush the TLB anyway in the case where we 1325 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 1326 * stale TLB entries dynamically. 1327 * 1328 * However, the above condition can ONLY happen if the current L1 is 1329 * being shared. If it happens when the L1 is unshared, it indicates 1330 * that other parts of the pmap are not doing their job WRT managing 1331 * the TLB. 1332 */ 1333 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { 1334 printf("fixup: pm %p, va 0x%08x, ftype %d - nothing to do!\n", 1335 pm, va, ftype); 1336 printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", 1337 l2, l2b, ptep, pl1pd); 1338 printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", 1339 pte, l1pd, last_fault_code); 1340#ifdef DDB 1341 Debugger(); 1342#endif 1343 } 1344#endif 1345 1346 cpu_tlb_flushID_SE(va); 1347 cpu_cpwait(); 1348 1349 rv = 1; 1350 1351out: 1352 rw_wunlock(&pvh_global_lock); 1353 PMAP_UNLOCK(pm); 1354 return (rv); 1355} 1356 1357void 1358pmap_postinit(void) 1359{ 1360 struct l2_bucket *l2b; 1361 struct l1_ttable *l1; 1362 pd_entry_t *pl1pt; 1363 pt_entry_t *ptep, pte; 1364 vm_offset_t va, eva; 1365 u_int loop, needed; 1366 1367 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 1368 needed -= 1; 1369 l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); 1370 1371 for (loop = 0; loop < needed; loop++, l1++) { 1372 /* Allocate a L1 page table */ 1373 va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_VMPMAP, 0, 0x0, 1374 0xffffffff, L1_TABLE_SIZE, 0); 1375 1376 if (va == 0) 1377 panic("Cannot allocate L1 KVM"); 1378 1379 eva = va + L1_TABLE_SIZE; 1380 pl1pt = (pd_entry_t *)va; 1381 1382 while (va < eva) { 1383 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1384 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1385 pte = *ptep; 1386 pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 1387 *ptep = pte; 1388 PTE_SYNC(ptep); 1389 cpu_tlb_flushD_SE(va); 1390 1391 va += PAGE_SIZE; 1392 } 1393 pmap_init_l1(l1, pl1pt); 1394 } 1395#ifdef DEBUG 1396 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 1397 needed); 1398#endif 1399} 1400 1401/* 1402 * This is used to stuff certain critical values into the PCB where they 1403 * can be accessed quickly from cpu_switch() et al. 1404 */ 1405void 1406pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) 1407{ 1408 struct l2_bucket *l2b; 1409 1410 pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; 1411 pcb->pcb_dacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 1412 (DOMAIN_CLIENT << (pm->pm_domain * 2)); 1413 1414 if (vector_page < KERNBASE) { 1415 pcb->pcb_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 1416 l2b = pmap_get_l2_bucket(pm, vector_page); 1417 pcb->pcb_l1vec = l2b->l2b_phys | L1_C_PROTO | 1418 L1_C_DOM(pm->pm_domain) | L1_C_DOM(PMAP_DOMAIN_KERNEL); 1419 } else 1420 pcb->pcb_pl1vec = NULL; 1421} 1422 1423void 1424pmap_activate(struct thread *td) 1425{ 1426 pmap_t pm; 1427 struct pcb *pcb; 1428 1429 pm = vmspace_pmap(td->td_proc->p_vmspace); 1430 pcb = td->td_pcb; 1431 1432 critical_enter(); 1433 pmap_set_pcb_pagedir(pm, pcb); 1434 1435 if (td == curthread) { 1436 u_int cur_dacr, cur_ttb; 1437 1438 __asm __volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); 1439 __asm __volatile("mrc p15, 0, %0, c3, c0, 0" : "=r"(cur_dacr)); 1440 1441 cur_ttb &= ~(L1_TABLE_SIZE - 1); 1442 1443 if (cur_ttb == (u_int)pcb->pcb_pagedir && 1444 cur_dacr == pcb->pcb_dacr) { 1445 /* 1446 * No need to switch address spaces. 1447 */ 1448 critical_exit(); 1449 return; 1450 } 1451 1452 1453 /* 1454 * We MUST, I repeat, MUST fix up the L1 entry corresponding 1455 * to 'vector_page' in the incoming L1 table before switching 1456 * to it otherwise subsequent interrupts/exceptions (including 1457 * domain faults!) will jump into hyperspace. 1458 */ 1459 if (pcb->pcb_pl1vec) { 1460 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 1461 } 1462 1463 cpu_domains(pcb->pcb_dacr); 1464 cpu_setttb(pcb->pcb_pagedir); 1465 } 1466 critical_exit(); 1467} 1468 1469static int 1470pmap_set_pt_cache_mode(pd_entry_t *kl1, vm_offset_t va) 1471{ 1472 pd_entry_t *pdep, pde; 1473 pt_entry_t *ptep, pte; 1474 vm_offset_t pa; 1475 int rv = 0; 1476 1477 /* 1478 * Make sure the descriptor itself has the correct cache mode 1479 */ 1480 pdep = &kl1[L1_IDX(va)]; 1481 pde = *pdep; 1482 1483 if (l1pte_section_p(pde)) { 1484 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 1485 *pdep = (pde & ~L1_S_CACHE_MASK) | 1486 pte_l1_s_cache_mode_pt; 1487 PTE_SYNC(pdep); 1488 rv = 1; 1489 } 1490 } else { 1491 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 1492 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 1493 if (ptep == NULL) 1494 panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); 1495 1496 ptep = &ptep[l2pte_index(va)]; 1497 pte = *ptep; 1498 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1499 *ptep = (pte & ~L2_S_CACHE_MASK) | 1500 pte_l2_s_cache_mode_pt; 1501 PTE_SYNC(ptep); 1502 rv = 1; 1503 } 1504 } 1505 1506 return (rv); 1507} 1508 1509static void 1510pmap_alloc_specials(vm_offset_t *availp, int pages, vm_offset_t *vap, 1511 pt_entry_t **ptep) 1512{ 1513 vm_offset_t va = *availp; 1514 struct l2_bucket *l2b; 1515 1516 if (ptep) { 1517 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1518 if (l2b == NULL) 1519 panic("pmap_alloc_specials: no l2b for 0x%x", va); 1520 1521 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 1522 } 1523 1524 *vap = va; 1525 *availp = va + (PAGE_SIZE * pages); 1526} 1527 1528/* 1529 * Bootstrap the system enough to run with virtual memory. 1530 * 1531 * On the arm this is called after mapping has already been enabled 1532 * and just syncs the pmap module with what has already been done. 1533 * [We can't call it easily with mapping off since the kernel is not 1534 * mapped with PA == VA, hence we would have to relocate every address 1535 * from the linked base (virtual) address "KERNBASE" to the actual 1536 * (physical) address starting relative to 0] 1537 */ 1538#define PMAP_STATIC_L2_SIZE 16 1539 1540void 1541pmap_bootstrap(vm_offset_t firstaddr, struct pv_addr *l1pt) 1542{ 1543 static struct l1_ttable static_l1; 1544 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 1545 struct l1_ttable *l1 = &static_l1; 1546 struct l2_dtable *l2; 1547 struct l2_bucket *l2b; 1548 pd_entry_t pde; 1549 pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va; 1550 pt_entry_t *ptep; 1551 vm_paddr_t pa; 1552 vm_offset_t va; 1553 vm_size_t size; 1554 int l1idx, l2idx, l2next = 0; 1555 1556 PDEBUG(1, printf("firstaddr = %08x, lastaddr = %08x\n", 1557 firstaddr, vm_max_kernel_address)); 1558 1559 virtual_avail = firstaddr; 1560 kernel_pmap->pm_l1 = l1; 1561 kernel_l1pa = l1pt->pv_pa; 1562 1563 /* 1564 * Scan the L1 translation table created by initarm() and create 1565 * the required metadata for all valid mappings found in it. 1566 */ 1567 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 1568 pde = kernel_l1pt[l1idx]; 1569 1570 /* 1571 * We're only interested in Coarse mappings. 1572 * pmap_extract() can deal with section mappings without 1573 * recourse to checking L2 metadata. 1574 */ 1575 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 1576 continue; 1577 1578 /* 1579 * Lookup the KVA of this L2 descriptor table 1580 */ 1581 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 1582 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 1583 1584 if (ptep == NULL) { 1585 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 1586 (u_int)l1idx << L1_S_SHIFT, (long unsigned int)pa); 1587 } 1588 1589 /* 1590 * Fetch the associated L2 metadata structure. 1591 * Allocate a new one if necessary. 1592 */ 1593 if ((l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]) == NULL) { 1594 if (l2next == PMAP_STATIC_L2_SIZE) 1595 panic("pmap_bootstrap: out of static L2s"); 1596 kernel_pmap->pm_l2[L2_IDX(l1idx)] = l2 = 1597 &static_l2[l2next++]; 1598 } 1599 1600 /* 1601 * One more L1 slot tracked... 1602 */ 1603 l2->l2_occupancy++; 1604 1605 /* 1606 * Fill in the details of the L2 descriptor in the 1607 * appropriate bucket. 1608 */ 1609 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1610 l2b->l2b_kva = ptep; 1611 l2b->l2b_phys = pa; 1612 l2b->l2b_l1idx = l1idx; 1613 1614 /* 1615 * Establish an initial occupancy count for this descriptor 1616 */ 1617 for (l2idx = 0; 1618 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1619 l2idx++) { 1620 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 1621 l2b->l2b_occupancy++; 1622 } 1623 } 1624 1625 /* 1626 * Make sure the descriptor itself has the correct cache mode. 1627 * If not, fix it, but whine about the problem. Port-meisters 1628 * should consider this a clue to fix up their initarm() 1629 * function. :) 1630 */ 1631 if (pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)ptep)) { 1632 printf("pmap_bootstrap: WARNING! wrong cache mode for " 1633 "L2 pte @ %p\n", ptep); 1634 } 1635 } 1636 1637 1638 /* 1639 * Ensure the primary (kernel) L1 has the correct cache mode for 1640 * a page table. Bitch if it is not correctly set. 1641 */ 1642 for (va = (vm_offset_t)kernel_l1pt; 1643 va < ((vm_offset_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { 1644 if (pmap_set_pt_cache_mode(kernel_l1pt, va)) 1645 printf("pmap_bootstrap: WARNING! wrong cache mode for " 1646 "primary L1 @ 0x%x\n", va); 1647 } 1648 1649 cpu_dcache_wbinv_all(); 1650 cpu_l2cache_wbinv_all(); 1651 cpu_tlb_flushID(); 1652 cpu_cpwait(); 1653 1654 PMAP_LOCK_INIT(kernel_pmap); 1655 CPU_FILL(&kernel_pmap->pm_active); 1656 kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; 1657 TAILQ_INIT(&kernel_pmap->pm_pvlist); 1658 1659 /* 1660 * Initialize the global pv list lock. 1661 */ 1662 rw_init(&pvh_global_lock, "pmap pv global"); 1663 1664 /* 1665 * Reserve some special page table entries/VA space for temporary 1666 * mapping of pages. 1667 */ 1668 1669 pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); 1670 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte); 1671 pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); 1672 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte); 1673 size = ((vm_max_kernel_address - pmap_curmaxkvaddr) + L1_S_OFFSET) / 1674 L1_S_SIZE; 1675 pmap_alloc_specials(&virtual_avail, 1676 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 1677 &pmap_kernel_l2ptp_kva, NULL); 1678 1679 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 1680 pmap_alloc_specials(&virtual_avail, 1681 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 1682 &pmap_kernel_l2dtable_kva, NULL); 1683 1684 pmap_alloc_specials(&virtual_avail, 1685 1, (vm_offset_t*)&_tmppt, NULL); 1686 pmap_alloc_specials(&virtual_avail, 1687 MAXDUMPPGS, (vm_offset_t *)&crashdumpmap, NULL); 1688 SLIST_INIT(&l1_list); 1689 TAILQ_INIT(&l1_lru_list); 1690 mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); 1691 pmap_init_l1(l1, kernel_l1pt); 1692 cpu_dcache_wbinv_all(); 1693 cpu_l2cache_wbinv_all(); 1694 1695 virtual_avail = round_page(virtual_avail); 1696 virtual_end = vm_max_kernel_address; 1697 kernel_vm_end = pmap_curmaxkvaddr; 1698 arm_nocache_startaddr = vm_max_kernel_address; 1699 mtx_init(&cmtx, "TMP mappings mtx", NULL, MTX_DEF); 1700 1701 pmap_set_pcb_pagedir(kernel_pmap, thread0.td_pcb); 1702} 1703 1704/*************************************************** 1705 * Pmap allocation/deallocation routines. 1706 ***************************************************/ 1707 1708/* 1709 * Release any resources held by the given physical map. 1710 * Called when a pmap initialized by pmap_pinit is being released. 1711 * Should only be called if the map contains no valid mappings. 1712 */ 1713void 1714pmap_release(pmap_t pmap) 1715{ 1716 struct pcb *pcb; 1717 1718 cpu_idcache_wbinv_all(); 1719 cpu_l2cache_wbinv_all(); 1720 cpu_tlb_flushID(); 1721 cpu_cpwait(); 1722 if (vector_page < KERNBASE) { 1723 struct pcb *curpcb = PCPU_GET(curpcb); 1724 pcb = thread0.td_pcb; 1725 if (pmap_is_current(pmap)) { 1726 /* 1727 * Frob the L1 entry corresponding to the vector 1728 * page so that it contains the kernel pmap's domain 1729 * number. This will ensure pmap_remove() does not 1730 * pull the current vector page out from under us. 1731 */ 1732 critical_enter(); 1733 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 1734 cpu_domains(pcb->pcb_dacr); 1735 cpu_setttb(pcb->pcb_pagedir); 1736 critical_exit(); 1737 } 1738 pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); 1739 /* 1740 * Make sure cpu_switch(), et al, DTRT. This is safe to do 1741 * since this process has no remaining mappings of its own. 1742 */ 1743 curpcb->pcb_pl1vec = pcb->pcb_pl1vec; 1744 curpcb->pcb_l1vec = pcb->pcb_l1vec; 1745 curpcb->pcb_dacr = pcb->pcb_dacr; 1746 curpcb->pcb_pagedir = pcb->pcb_pagedir; 1747 1748 } 1749 pmap_free_l1(pmap); 1750 PMAP_LOCK_DESTROY(pmap); 1751 1752 dprintf("pmap_release()\n"); 1753} 1754 1755 1756 1757/* 1758 * Helper function for pmap_grow_l2_bucket() 1759 */ 1760static __inline int 1761pmap_grow_map(vm_offset_t va, pt_entry_t cache_mode, vm_paddr_t *pap) 1762{ 1763 struct l2_bucket *l2b; 1764 pt_entry_t *ptep; 1765 vm_paddr_t pa; 1766 struct vm_page *pg; 1767 1768 pg = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 1769 if (pg == NULL) 1770 return (1); 1771 pa = VM_PAGE_TO_PHYS(pg); 1772 1773 if (pap) 1774 *pap = pa; 1775 1776 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1777 1778 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1779 *ptep = L2_S_PROTO | pa | cache_mode; 1780 pmap_set_prot(ptep, VM_PROT_READ | VM_PROT_WRITE, 0); 1781 PTE_SYNC(ptep); 1782 1783 return (0); 1784} 1785 1786/* 1787 * This is the same as pmap_alloc_l2_bucket(), except that it is only 1788 * used by pmap_growkernel(). 1789 */ 1790static __inline struct l2_bucket * 1791pmap_grow_l2_bucket(pmap_t pm, vm_offset_t va) 1792{ 1793 struct l2_dtable *l2; 1794 struct l2_bucket *l2b; 1795 struct l1_ttable *l1; 1796 pd_entry_t *pl1pd; 1797 u_short l1idx; 1798 vm_offset_t nva; 1799 1800 l1idx = L1_IDX(va); 1801 1802 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 1803 /* 1804 * No mapping at this address, as there is 1805 * no entry in the L1 table. 1806 * Need to allocate a new l2_dtable. 1807 */ 1808 nva = pmap_kernel_l2dtable_kva; 1809 if ((nva & PAGE_MASK) == 0) { 1810 /* 1811 * Need to allocate a backing page 1812 */ 1813 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 1814 return (NULL); 1815 } 1816 1817 l2 = (struct l2_dtable *)nva; 1818 nva += sizeof(struct l2_dtable); 1819 1820 if ((nva & PAGE_MASK) < (pmap_kernel_l2dtable_kva & 1821 PAGE_MASK)) { 1822 /* 1823 * The new l2_dtable straddles a page boundary. 1824 * Map in another page to cover it. 1825 */ 1826 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 1827 return (NULL); 1828 } 1829 1830 pmap_kernel_l2dtable_kva = nva; 1831 1832 /* 1833 * Link it into the parent pmap 1834 */ 1835 pm->pm_l2[L2_IDX(l1idx)] = l2; 1836 memset(l2, 0, sizeof(*l2)); 1837 } 1838 1839 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1840 1841 /* 1842 * Fetch pointer to the L2 page table associated with the address. 1843 */ 1844 if (l2b->l2b_kva == NULL) { 1845 pt_entry_t *ptep; 1846 1847 /* 1848 * No L2 page table has been allocated. Chances are, this 1849 * is because we just allocated the l2_dtable, above. 1850 */ 1851 nva = pmap_kernel_l2ptp_kva; 1852 ptep = (pt_entry_t *)nva; 1853 if ((nva & PAGE_MASK) == 0) { 1854 /* 1855 * Need to allocate a backing page 1856 */ 1857 if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, 1858 &pmap_kernel_l2ptp_phys)) 1859 return (NULL); 1860 } 1861 memset(ptep, 0, L2_TABLE_SIZE_REAL); 1862 l2->l2_occupancy++; 1863 l2b->l2b_kva = ptep; 1864 l2b->l2b_l1idx = l1idx; 1865 l2b->l2b_phys = pmap_kernel_l2ptp_phys; 1866 1867 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 1868 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 1869 } 1870 1871 /* Distribute new L1 entry to all other L1s */ 1872 SLIST_FOREACH(l1, &l1_list, l1_link) { 1873 pl1pd = &l1->l1_kva[L1_IDX(va)]; 1874 *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | 1875 L1_C_PROTO; 1876 PTE_SYNC(pl1pd); 1877 } 1878 1879 return (l2b); 1880} 1881 1882 1883/* 1884 * grow the number of kernel page table entries, if needed 1885 */ 1886void 1887pmap_growkernel(vm_offset_t addr) 1888{ 1889 pmap_t kpm = pmap_kernel(); 1890 1891 if (addr <= pmap_curmaxkvaddr) 1892 return; /* we are OK */ 1893 1894 /* 1895 * whoops! we need to add kernel PTPs 1896 */ 1897 1898 /* Map 1MB at a time */ 1899 for (; pmap_curmaxkvaddr < addr; pmap_curmaxkvaddr += L1_S_SIZE) 1900 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 1901 1902 /* 1903 * flush out the cache, expensive but growkernel will happen so 1904 * rarely 1905 */ 1906 cpu_dcache_wbinv_all(); 1907 cpu_l2cache_wbinv_all(); 1908 cpu_tlb_flushD(); 1909 cpu_cpwait(); 1910 kernel_vm_end = pmap_curmaxkvaddr; 1911} 1912 1913 1914/* 1915 * Remove all pages from specified address space 1916 * this aids process exit speeds. Also, this code 1917 * is special cased for current process only, but 1918 * can have the more generic (and slightly slower) 1919 * mode enabled. This is much faster than pmap_remove 1920 * in the case of running down an entire address space. 1921 */ 1922void 1923pmap_remove_pages(pmap_t pmap) 1924{ 1925 struct pv_entry *pv, *npv; 1926 struct l2_bucket *l2b = NULL; 1927 vm_page_t m; 1928 pt_entry_t *pt; 1929 1930 rw_wlock(&pvh_global_lock); 1931 PMAP_LOCK(pmap); 1932 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 1933 if (pv->pv_flags & PVF_WIRED) { 1934 /* Cannot remove wired pages now. */ 1935 npv = TAILQ_NEXT(pv, pv_plist); 1936 continue; 1937 } 1938 pmap->pm_stats.resident_count--; 1939 l2b = pmap_get_l2_bucket(pmap, pv->pv_va); 1940 KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); 1941 pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1942 m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK); 1943 KASSERT((vm_offset_t)m >= KERNBASE, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); 1944 *pt = 0; 1945 PTE_SYNC(pt); 1946 npv = TAILQ_NEXT(pv, pv_plist); 1947 pmap_nuke_pv(m, pmap, pv); 1948 if (TAILQ_EMPTY(&m->md.pv_list)) 1949 vm_page_aflag_clear(m, PGA_WRITEABLE); 1950 pmap_free_pv_entry(pv); 1951 pmap_free_l2_bucket(pmap, l2b, 1); 1952 } 1953 rw_wunlock(&pvh_global_lock); 1954 cpu_tlb_flushID(); 1955 cpu_cpwait(); 1956 PMAP_UNLOCK(pmap); 1957} 1958 1959 1960/*************************************************** 1961 * Low level mapping routines..... 1962 ***************************************************/ 1963 1964#ifdef ARM_HAVE_SUPERSECTIONS 1965/* Map a super section into the KVA. */ 1966 1967void 1968pmap_kenter_supersection(vm_offset_t va, uint64_t pa, int flags) 1969{ 1970 pd_entry_t pd = L1_S_PROTO | L1_S_SUPERSEC | (pa & L1_SUP_FRAME) | 1971 (((pa >> 32) & 0xf) << 20) | L1_S_PROT(PTE_KERNEL, 1972 VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); 1973 struct l1_ttable *l1; 1974 vm_offset_t va0, va_end; 1975 1976 KASSERT(((va | pa) & L1_SUP_OFFSET) == 0, 1977 ("Not a valid super section mapping")); 1978 if (flags & SECTION_CACHE) 1979 pd |= pte_l1_s_cache_mode; 1980 else if (flags & SECTION_PT) 1981 pd |= pte_l1_s_cache_mode_pt; 1982 1983 va0 = va & L1_SUP_FRAME; 1984 va_end = va + L1_SUP_SIZE; 1985 SLIST_FOREACH(l1, &l1_list, l1_link) { 1986 va = va0; 1987 for (; va < va_end; va += L1_S_SIZE) { 1988 l1->l1_kva[L1_IDX(va)] = pd; 1989 PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); 1990 } 1991 } 1992} 1993#endif 1994 1995/* Map a section into the KVA. */ 1996 1997void 1998pmap_kenter_section(vm_offset_t va, vm_offset_t pa, int flags) 1999{ 2000 pd_entry_t pd = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, 2001 VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); 2002 struct l1_ttable *l1; 2003 2004 KASSERT(((va | pa) & L1_S_OFFSET) == 0, 2005 ("Not a valid section mapping")); 2006 if (flags & SECTION_CACHE) 2007 pd |= pte_l1_s_cache_mode; 2008 else if (flags & SECTION_PT) 2009 pd |= pte_l1_s_cache_mode_pt; 2010 2011 SLIST_FOREACH(l1, &l1_list, l1_link) { 2012 l1->l1_kva[L1_IDX(va)] = pd; 2013 PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); 2014 } 2015} 2016 2017/* 2018 * Make a temporary mapping for a physical address. This is only intended 2019 * to be used for panic dumps. 2020 */ 2021void * 2022pmap_kenter_temp(vm_paddr_t pa, int i) 2023{ 2024 vm_offset_t va; 2025 2026 va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); 2027 pmap_kenter(va, pa); 2028 return ((void *)crashdumpmap); 2029} 2030 2031/* 2032 * add a wired page to the kva 2033 * note that in order for the mapping to take effect -- you 2034 * should do a invltlb after doing the pmap_kenter... 2035 */ 2036static PMAP_INLINE void 2037pmap_kenter_internal(vm_offset_t va, vm_offset_t pa, int flags) 2038{ 2039 struct l2_bucket *l2b; 2040 pt_entry_t *pte; 2041 pt_entry_t opte; 2042 2043 PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", 2044 (uint32_t) va, (uint32_t) pa)); 2045 2046 2047 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2048 if (l2b == NULL) 2049 l2b = pmap_grow_l2_bucket(pmap_kernel(), va); 2050 KASSERT(l2b != NULL, ("No L2 Bucket")); 2051 2052 pte = &l2b->l2b_kva[l2pte_index(va)]; 2053 opte = *pte; 2054 if (l2pte_valid(opte)) { 2055 cpu_tlb_flushD_SE(va); 2056 cpu_cpwait(); 2057 } else { 2058 if (opte == 0) 2059 l2b->l2b_occupancy++; 2060 } 2061 2062 if (flags & KENTER_CACHE) { 2063 *pte = L2_S_PROTO | pa | pte_l2_s_cache_mode; 2064 pmap_set_prot(pte, VM_PROT_READ | VM_PROT_WRITE, 2065 flags & KENTER_USER); 2066 } else { 2067 *pte = L2_S_PROTO | pa; 2068 pmap_set_prot(pte, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 2069 0); 2070 } 2071 2072 PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", 2073 (uint32_t) pte, opte, *pte)); 2074 PTE_SYNC(pte); 2075 cpu_cpwait(); 2076} 2077 2078void 2079pmap_kenter(vm_offset_t va, vm_paddr_t pa) 2080{ 2081 pmap_kenter_internal(va, pa, KENTER_CACHE); 2082} 2083 2084void 2085pmap_kenter_nocache(vm_offset_t va, vm_paddr_t pa) 2086{ 2087 2088 pmap_kenter_internal(va, pa, 0); 2089} 2090 2091void 2092pmap_kenter_user(vm_offset_t va, vm_paddr_t pa) 2093{ 2094 2095 pmap_kenter_internal(va, pa, KENTER_CACHE|KENTER_USER); 2096 /* 2097 * Call pmap_fault_fixup now, to make sure we'll have no exception 2098 * at the first use of the new address, or bad things will happen, 2099 * as we use one of these addresses in the exception handlers. 2100 */ 2101 pmap_fault_fixup(pmap_kernel(), va, VM_PROT_READ|VM_PROT_WRITE, 1); 2102} 2103 2104vm_paddr_t 2105pmap_kextract(vm_offset_t va) 2106{ 2107 2108 return (pmap_extract_locked(kernel_pmap, va)); 2109} 2110 2111/* 2112 * remove a page from the kernel pagetables 2113 */ 2114void 2115pmap_kremove(vm_offset_t va) 2116{ 2117 struct l2_bucket *l2b; 2118 pt_entry_t *pte, opte; 2119 2120 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2121 if (!l2b) 2122 return; 2123 KASSERT(l2b != NULL, ("No L2 Bucket")); 2124 pte = &l2b->l2b_kva[l2pte_index(va)]; 2125 opte = *pte; 2126 if (l2pte_valid(opte)) { 2127 va = va & ~PAGE_MASK; 2128 cpu_tlb_flushD_SE(va); 2129 cpu_cpwait(); 2130 *pte = 0; 2131 PTE_SYNC(pte); 2132 } 2133} 2134 2135 2136/* 2137 * Used to map a range of physical addresses into kernel 2138 * virtual address space. 2139 * 2140 * The value passed in '*virt' is a suggested virtual address for 2141 * the mapping. Architectures which can support a direct-mapped 2142 * physical to virtual region can return the appropriate address 2143 * within that region, leaving '*virt' unchanged. Other 2144 * architectures should map the pages starting at '*virt' and 2145 * update '*virt' with the first usable address after the mapped 2146 * region. 2147 */ 2148vm_offset_t 2149pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 2150{ 2151 vm_offset_t sva = *virt; 2152 vm_offset_t va = sva; 2153 2154 PDEBUG(1, printf("pmap_map: virt = %08x, start = %08x, end = %08x, " 2155 "prot = %d\n", (uint32_t) *virt, (uint32_t) start, (uint32_t) end, 2156 prot)); 2157 2158 while (start < end) { 2159 pmap_kenter(va, start); 2160 va += PAGE_SIZE; 2161 start += PAGE_SIZE; 2162 } 2163 *virt = va; 2164 return (sva); 2165} 2166 2167/* 2168 * Add a list of wired pages to the kva 2169 * this routine is only used for temporary 2170 * kernel mappings that do not need to have 2171 * page modification or references recorded. 2172 * Note that old mappings are simply written 2173 * over. The page *must* be wired. 2174 */ 2175void 2176pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 2177{ 2178 int i; 2179 2180 for (i = 0; i < count; i++) { 2181 pmap_kenter_internal(va, VM_PAGE_TO_PHYS(m[i]), 2182 KENTER_CACHE); 2183 va += PAGE_SIZE; 2184 } 2185} 2186 2187 2188/* 2189 * this routine jerks page mappings from the 2190 * kernel -- it is meant only for temporary mappings. 2191 */ 2192void 2193pmap_qremove(vm_offset_t va, int count) 2194{ 2195 int i; 2196 2197 for (i = 0; i < count; i++) { 2198 if (vtophys(va)) 2199 pmap_kremove(va); 2200 2201 va += PAGE_SIZE; 2202 } 2203} 2204 2205 2206/* 2207 * pmap_object_init_pt preloads the ptes for a given object 2208 * into the specified pmap. This eliminates the blast of soft 2209 * faults on process startup and immediately after an mmap. 2210 */ 2211void 2212pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 2213 vm_pindex_t pindex, vm_size_t size) 2214{ 2215 2216 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2217 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2218 ("pmap_object_init_pt: non-device object")); 2219} 2220 2221 2222/* 2223 * pmap_is_prefaultable: 2224 * 2225 * Return whether or not the specified virtual address is elgible 2226 * for prefault. 2227 */ 2228boolean_t 2229pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2230{ 2231 pd_entry_t *pde; 2232 pt_entry_t *pte; 2233 2234 if (!pmap_get_pde_pte(pmap, addr, &pde, &pte)) 2235 return (FALSE); 2236 KASSERT(pte != NULL, ("Valid mapping but no pte ?")); 2237 if (*pte == 0) 2238 return (TRUE); 2239 return (FALSE); 2240} 2241 2242/* 2243 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 2244 * Returns TRUE if the mapping exists, else FALSE. 2245 * 2246 * NOTE: This function is only used by a couple of arm-specific modules. 2247 * It is not safe to take any pmap locks here, since we could be right 2248 * in the middle of debugging the pmap anyway... 2249 * 2250 * It is possible for this routine to return FALSE even though a valid 2251 * mapping does exist. This is because we don't lock, so the metadata 2252 * state may be inconsistent. 2253 * 2254 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 2255 * a "section" mapping. 2256 */ 2257boolean_t 2258pmap_get_pde_pte(pmap_t pm, vm_offset_t va, pd_entry_t **pdp, pt_entry_t **ptp) 2259{ 2260 struct l2_dtable *l2; 2261 pd_entry_t *pl1pd, l1pd; 2262 pt_entry_t *ptep; 2263 u_short l1idx; 2264 2265 if (pm->pm_l1 == NULL) 2266 return (FALSE); 2267 2268 l1idx = L1_IDX(va); 2269 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 2270 l1pd = *pl1pd; 2271 2272 if (l1pte_section_p(l1pd)) { 2273 *ptp = NULL; 2274 return (TRUE); 2275 } 2276 2277 if (pm->pm_l2 == NULL) 2278 return (FALSE); 2279 2280 l2 = pm->pm_l2[L2_IDX(l1idx)]; 2281 2282 if (l2 == NULL || 2283 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 2284 return (FALSE); 2285 } 2286 2287 *ptp = &ptep[l2pte_index(va)]; 2288 return (TRUE); 2289} 2290 2291/* 2292 * Routine: pmap_remove_all 2293 * Function: 2294 * Removes this physical page from 2295 * all physical maps in which it resides. 2296 * Reflects back modify bits to the pager. 2297 * 2298 * Notes: 2299 * Original versions of this routine were very 2300 * inefficient because they iteratively called 2301 * pmap_remove (slow...) 2302 */ 2303void 2304pmap_remove_all(vm_page_t m) 2305{ 2306 pv_entry_t pv; 2307 pt_entry_t *ptep; 2308 struct l2_bucket *l2b; 2309 boolean_t flush = FALSE; 2310 pmap_t curpm; 2311 int flags = 0; 2312 2313 KASSERT((m->flags & PG_FICTITIOUS) == 0, 2314 ("pmap_remove_all: page %p is fictitious", m)); 2315 2316 if (TAILQ_EMPTY(&m->md.pv_list)) 2317 return; 2318 rw_wlock(&pvh_global_lock); 2319 curpm = vmspace_pmap(curproc->p_vmspace); 2320 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 2321 if (flush == FALSE && (pv->pv_pmap == curpm || 2322 pv->pv_pmap == pmap_kernel())) 2323 flush = TRUE; 2324 2325 PMAP_LOCK(pv->pv_pmap); 2326 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 2327 KASSERT(l2b != NULL, ("No l2 bucket")); 2328 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2329 if (L2_S_WRITABLE(*ptep)) 2330 vm_page_dirty(m); 2331 *ptep = 0; 2332 if (pmap_is_current(pv->pv_pmap)) 2333 PTE_SYNC(ptep); 2334 pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); 2335 pv->pv_pmap->pm_stats.resident_count--; 2336 flags |= pv->pv_flags; 2337 pmap_nuke_pv(m, pv->pv_pmap, pv); 2338 PMAP_UNLOCK(pv->pv_pmap); 2339 pmap_free_pv_entry(pv); 2340 } 2341 m->md.pvh_attrs &= ~(PVF_MOD | PVF_REF); 2342 2343 if (flush) { 2344 if (PV_BEEN_EXECD(flags)) 2345 cpu_tlb_flushID(); 2346 else 2347 cpu_tlb_flushD(); 2348 } 2349 vm_page_aflag_clear(m, PGA_WRITEABLE); 2350 rw_wunlock(&pvh_global_lock); 2351} 2352 2353int 2354pmap_change_attr(vm_offset_t sva, vm_size_t len, int mode) 2355{ 2356 vm_offset_t base, offset, tmpva; 2357 vm_size_t size; 2358 struct l2_bucket *l2b; 2359 pt_entry_t *ptep, pte; 2360 vm_offset_t next_bucket; 2361 2362 PMAP_LOCK(kernel_pmap); 2363 2364 base = trunc_page(sva); 2365 offset = sva & PAGE_MASK; 2366 size = roundup(offset + len, PAGE_SIZE); 2367 2368#ifdef checkit 2369 /* 2370 * Only supported on kernel virtual addresses, including the direct 2371 * map but excluding the recursive map. 2372 */ 2373 if (base < DMAP_MIN_ADDRESS) { 2374 PMAP_UNLOCK(kernel_pmap); 2375 return (EINVAL); 2376 } 2377#endif 2378 for (tmpva = base; tmpva < base + size; ) { 2379 next_bucket = L2_NEXT_BUCKET(tmpva); 2380 if (next_bucket > base + size) 2381 next_bucket = base + size; 2382 2383 l2b = pmap_get_l2_bucket(kernel_pmap, tmpva); 2384 if (l2b == NULL) { 2385 tmpva = next_bucket; 2386 continue; 2387 } 2388 2389 ptep = &l2b->l2b_kva[l2pte_index(tmpva)]; 2390 2391 if (*ptep == 0) { 2392 PMAP_UNLOCK(kernel_pmap); 2393 return(EINVAL); 2394 } 2395 2396 pte = *ptep &~ L2_S_CACHE_MASK; 2397 cpu_idcache_wbinv_range(tmpva, PAGE_SIZE); 2398 pmap_l2cache_wbinv_range(tmpva, pte & L2_S_FRAME, PAGE_SIZE); 2399 *ptep = pte; 2400 cpu_tlb_flushID_SE(tmpva); 2401 2402 dprintf("%s: for va:%x ptep:%x pte:%x\n", 2403 __func__, tmpva, (uint32_t)ptep, pte); 2404 tmpva += PAGE_SIZE; 2405 } 2406 2407 PMAP_UNLOCK(kernel_pmap); 2408 2409 return (0); 2410} 2411 2412/* 2413 * Set the physical protection on the 2414 * specified range of this map as requested. 2415 */ 2416void 2417pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 2418{ 2419 struct l2_bucket *l2b; 2420 pt_entry_t *ptep, pte; 2421 vm_offset_t next_bucket; 2422 u_int flags; 2423 int flush; 2424 2425 if ((prot & VM_PROT_READ) == 0) { 2426 pmap_remove(pm, sva, eva); 2427 return; 2428 } 2429 2430 if (prot & VM_PROT_WRITE) { 2431 /* 2432 * If this is a read->write transition, just ignore it and let 2433 * vm_fault() take care of it later. 2434 */ 2435 return; 2436 } 2437 2438 rw_wlock(&pvh_global_lock); 2439 PMAP_LOCK(pm); 2440 2441 /* 2442 * OK, at this point, we know we're doing write-protect operation. 2443 * If the pmap is active, write-back the range. 2444 */ 2445 2446 flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; 2447 flags = 0; 2448 2449 while (sva < eva) { 2450 next_bucket = L2_NEXT_BUCKET(sva); 2451 if (next_bucket > eva) 2452 next_bucket = eva; 2453 2454 l2b = pmap_get_l2_bucket(pm, sva); 2455 if (l2b == NULL) { 2456 sva = next_bucket; 2457 continue; 2458 } 2459 2460 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 2461 2462 while (sva < next_bucket) { 2463 if ((pte = *ptep) != 0 && L2_S_WRITABLE(pte)) { 2464 struct vm_page *pg; 2465 u_int f; 2466 2467 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 2468 pmap_set_prot(ptep, prot, !(pm == pmap_kernel())); 2469 PTE_SYNC(ptep); 2470 2471 f = pmap_modify_pv(pg, pm, sva, 2472 PVF_WRITE, 0); 2473 if (f & PVF_WRITE) 2474 vm_page_dirty(pg); 2475 2476 if (flush >= 0) { 2477 flush++; 2478 flags |= f; 2479 } else 2480 if (PV_BEEN_EXECD(f)) 2481 cpu_tlb_flushID_SE(sva); 2482 else 2483 if (PV_BEEN_REFD(f)) 2484 cpu_tlb_flushD_SE(sva); 2485 } 2486 2487 sva += PAGE_SIZE; 2488 ptep++; 2489 } 2490 } 2491 2492 2493 if (flush) { 2494 if (PV_BEEN_EXECD(flags)) 2495 cpu_tlb_flushID(); 2496 else 2497 if (PV_BEEN_REFD(flags)) 2498 cpu_tlb_flushD(); 2499 } 2500 rw_wunlock(&pvh_global_lock); 2501 2502 PMAP_UNLOCK(pm); 2503} 2504 2505 2506/* 2507 * Insert the given physical page (p) at 2508 * the specified virtual address (v) in the 2509 * target physical map with the protection requested. 2510 * 2511 * If specified, the page will be wired down, meaning 2512 * that the related pte can not be reclaimed. 2513 * 2514 * NB: This is the only routine which MAY NOT lazy-evaluate 2515 * or lose information. That is, this routine must actually 2516 * insert this page into the given map NOW. 2517 */ 2518 2519void 2520pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, 2521 vm_prot_t prot, boolean_t wired) 2522{ 2523 2524 rw_wlock(&pvh_global_lock); 2525 PMAP_LOCK(pmap); 2526 pmap_enter_locked(pmap, va, m, prot, wired, M_WAITOK); 2527 PMAP_UNLOCK(pmap); 2528 rw_wunlock(&pvh_global_lock); 2529} 2530 2531/* 2532 * The pvh global and pmap locks must be held. 2533 */ 2534static void 2535pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 2536 boolean_t wired, int flags) 2537{ 2538 struct l2_bucket *l2b = NULL; 2539 struct vm_page *opg; 2540 struct pv_entry *pve = NULL; 2541 pt_entry_t *ptep, npte, opte; 2542 u_int nflags; 2543 u_int oflags; 2544 vm_paddr_t pa; 2545 u_char user; 2546 2547 PMAP_ASSERT_LOCKED(pmap); 2548 rw_assert(&pvh_global_lock, RA_WLOCKED); 2549 if (va == vector_page) { 2550 pa = systempage.pv_pa; 2551 m = NULL; 2552 } else { 2553 KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 || 2554 (flags & M_NOWAIT) != 0, 2555 ("pmap_enter_locked: page %p is not busy", m)); 2556 pa = VM_PAGE_TO_PHYS(m); 2557 } 2558 2559 user = 0; 2560 /* 2561 * Make sure userland mappings get the right permissions 2562 */ 2563 if (pmap != pmap_kernel() && va != vector_page) 2564 user = 1; 2565 2566 nflags = 0; 2567 2568 if (prot & VM_PROT_WRITE) 2569 nflags |= PVF_WRITE; 2570 if (prot & VM_PROT_EXECUTE) 2571 nflags |= PVF_EXEC; 2572 if (wired) 2573 nflags |= PVF_WIRED; 2574 2575 PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " 2576 "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired)); 2577 2578 if (pmap == pmap_kernel()) { 2579 l2b = pmap_get_l2_bucket(pmap, va); 2580 if (l2b == NULL) 2581 l2b = pmap_grow_l2_bucket(pmap, va); 2582 } else { 2583do_l2b_alloc: 2584 l2b = pmap_alloc_l2_bucket(pmap, va); 2585 if (l2b == NULL) { 2586 if (flags & M_WAITOK) { 2587 PMAP_UNLOCK(pmap); 2588 rw_wunlock(&pvh_global_lock); 2589 VM_WAIT; 2590 rw_wlock(&pvh_global_lock); 2591 PMAP_LOCK(pmap); 2592 goto do_l2b_alloc; 2593 } 2594 return; 2595 } 2596 } 2597 2598 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2599 2600 opte = *ptep; 2601 npte = pa; 2602 oflags = 0; 2603 if (opte) { 2604 /* 2605 * There is already a mapping at this address. 2606 * If the physical address is different, lookup the 2607 * vm_page. 2608 */ 2609 if (l2pte_pa(opte) != pa) 2610 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 2611 else 2612 opg = m; 2613 } else 2614 opg = NULL; 2615 2616 if ((prot & (VM_PROT_ALL)) || 2617 (!m || m->md.pvh_attrs & PVF_REF)) { 2618 /* 2619 * - The access type indicates that we don't need 2620 * to do referenced emulation. 2621 * OR 2622 * - The physical page has already been referenced 2623 * so no need to re-do referenced emulation here. 2624 */ 2625 npte |= L2_S_PROTO; 2626#ifdef SMP 2627 npte |= L2_SHARED; 2628#endif 2629 2630 nflags |= PVF_REF; 2631 2632 if (m && ((prot & VM_PROT_WRITE) != 0 || 2633 (m->md.pvh_attrs & PVF_MOD))) { 2634 /* 2635 * This is a writable mapping, and the 2636 * page's mod state indicates it has 2637 * already been modified. Make it 2638 * writable from the outset. 2639 */ 2640 nflags |= PVF_MOD; 2641 if (!(m->md.pvh_attrs & PVF_MOD)) 2642 vm_page_dirty(m); 2643 } 2644 if (m && opte) 2645 vm_page_aflag_set(m, PGA_REFERENCED); 2646 } else { 2647 /* 2648 * Need to do page referenced emulation. 2649 */ 2650 npte |= L2_TYPE_INV; 2651 } 2652 2653 npte |= L2_S_PROT_R; 2654 2655 if (prot & VM_PROT_WRITE) { 2656 npte &= ~(L2_APX); 2657 2658 if (m != NULL && 2659 (m->oflags & VPO_UNMANAGED) == 0) 2660 vm_page_aflag_set(m, PGA_WRITEABLE); 2661 } 2662 2663 if (user) 2664 npte |= L2_S_PROT_U; 2665 2666 2667 if (!(prot & VM_PROT_EXECUTE) && m) 2668 npte |= L2_XN; 2669 2670 if (m->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) 2671 npte |= pte_l2_s_cache_mode; 2672 2673 if (m && m == opg) { 2674 /* 2675 * We're changing the attrs of an existing mapping. 2676 */ 2677 oflags = pmap_modify_pv(m, pmap, va, 2678 PVF_WRITE | PVF_EXEC | PVF_WIRED | 2679 PVF_MOD | PVF_REF, nflags); 2680 } else { 2681 /* 2682 * New mapping, or changing the backing page 2683 * of an existing mapping. 2684 */ 2685 if (opg) { 2686 /* 2687 * Replacing an existing mapping with a new one. 2688 * It is part of our managed memory so we 2689 * must remove it from the PV list 2690 */ 2691 if ((pve = pmap_remove_pv(opg, pmap, va))) { 2692 oflags = pve->pv_flags; 2693 2694 if (m && ((m->oflags & VPO_UNMANAGED))) { 2695 pmap_free_pv_entry(pve); 2696 pve = NULL; 2697 } 2698 } 2699 } 2700 2701 if ((m && !(m->oflags & VPO_UNMANAGED))) { 2702 if ((!pve) && (pve = pmap_get_pv_entry()) == NULL) 2703 panic("pmap_enter: no pv entries"); 2704 2705 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 2706 ("pmap_enter: managed mapping within the clean submap")); 2707 KASSERT(pve != NULL, ("No pv")); 2708 pmap_enter_pv(m, pve, pmap, va, nflags); 2709 } 2710 } 2711 2712 /* 2713 * Keep the stats up to date 2714 */ 2715 if (opte == 0) { 2716 l2b->l2b_occupancy++; 2717 pmap->pm_stats.resident_count++; 2718 } 2719 2720 CTR5(KTR_PMAP,"enter: pmap:%p va:%x prot:%x pte:%x->%x", 2721 pmap, va, prot, opte, npte); 2722 /* 2723 * If this is just a wiring change, the two PTEs will be 2724 * identical, so there's no need to update the page table. 2725 */ 2726 if (npte != opte) { 2727 boolean_t is_cached = pmap_is_current(pmap); 2728 2729 *ptep = npte; 2730 PTE_SYNC(ptep); 2731 if (is_cached) { 2732 /* 2733 * We only need to frob the cache/tlb if this pmap 2734 * is current 2735 */ 2736 if (L1_IDX(va) != L1_IDX(vector_page) && 2737 l2pte_valid(npte)) { 2738 /* 2739 * This mapping is likely to be accessed as 2740 * soon as we return to userland. Fix up the 2741 * L1 entry to avoid taking another 2742 * page/domain fault. 2743 */ 2744 pd_entry_t *pl1pd, l1pd; 2745 2746 pl1pd = &pmap->pm_l1->l1_kva[L1_IDX(va)]; 2747 l1pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | 2748 L1_C_PROTO; 2749 if (*pl1pd != l1pd) { 2750 *pl1pd = l1pd; 2751 PTE_SYNC(pl1pd); 2752 } 2753 } 2754 } 2755 2756 if (PV_BEEN_EXECD(oflags)) 2757 cpu_tlb_flushID_SE(va); 2758 else if (PV_BEEN_REFD(oflags)) 2759 cpu_tlb_flushD_SE(va); 2760 } 2761 2762 if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) 2763 cpu_icache_sync_range(va, PAGE_SIZE); 2764} 2765 2766/* 2767 * Maps a sequence of resident pages belonging to the same object. 2768 * The sequence begins with the given page m_start. This page is 2769 * mapped at the given virtual address start. Each subsequent page is 2770 * mapped at a virtual address that is offset from start by the same 2771 * amount as the page is offset from m_start within the object. The 2772 * last page in the sequence is the page with the largest offset from 2773 * m_start that can be mapped at a virtual address less than the given 2774 * virtual address end. Not every virtual page between start and end 2775 * is mapped; only those for which a resident page exists with the 2776 * corresponding offset from m_start are mapped. 2777 */ 2778void 2779pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2780 vm_page_t m_start, vm_prot_t prot) 2781{ 2782 vm_page_t m; 2783 vm_pindex_t diff, psize; 2784 2785 psize = atop(end - start); 2786 m = m_start; 2787 rw_wlock(&pvh_global_lock); 2788 PMAP_LOCK(pmap); 2789 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2790 pmap_enter_locked(pmap, start + ptoa(diff), m, prot & 2791 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE, M_NOWAIT); 2792 m = TAILQ_NEXT(m, listq); 2793 } 2794 PMAP_UNLOCK(pmap); 2795 rw_wunlock(&pvh_global_lock); 2796} 2797 2798/* 2799 * this code makes some *MAJOR* assumptions: 2800 * 1. Current pmap & pmap exists. 2801 * 2. Not wired. 2802 * 3. Read access. 2803 * 4. No page table pages. 2804 * but is *MUCH* faster than pmap_enter... 2805 */ 2806 2807void 2808pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2809{ 2810 2811 rw_wlock(&pvh_global_lock); 2812 PMAP_LOCK(pmap); 2813 pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 2814 FALSE, M_NOWAIT); 2815 PMAP_UNLOCK(pmap); 2816 rw_wunlock(&pvh_global_lock); 2817} 2818 2819/* 2820 * Routine: pmap_change_wiring 2821 * Function: Change the wiring attribute for a map/virtual-address 2822 * pair. 2823 * In/out conditions: 2824 * The mapping must already exist in the pmap. 2825 */ 2826void 2827pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 2828{ 2829 struct l2_bucket *l2b; 2830 pt_entry_t *ptep, pte; 2831 vm_page_t pg; 2832 2833 rw_wlock(&pvh_global_lock); 2834 PMAP_LOCK(pmap); 2835 l2b = pmap_get_l2_bucket(pmap, va); 2836 KASSERT(l2b, ("No l2b bucket in pmap_change_wiring")); 2837 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2838 pte = *ptep; 2839 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 2840 if (pg) 2841 pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired); 2842 rw_wunlock(&pvh_global_lock); 2843 PMAP_UNLOCK(pmap); 2844} 2845 2846 2847/* 2848 * Copy the range specified by src_addr/len 2849 * from the source map to the range dst_addr/len 2850 * in the destination map. 2851 * 2852 * This routine is only advisory and need not do anything. 2853 */ 2854void 2855pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2856 vm_size_t len, vm_offset_t src_addr) 2857{ 2858} 2859 2860 2861/* 2862 * Routine: pmap_extract 2863 * Function: 2864 * Extract the physical page address associated 2865 * with the given map/virtual_address pair. 2866 */ 2867vm_paddr_t 2868pmap_extract(pmap_t pmap, vm_offset_t va) 2869{ 2870 vm_paddr_t pa; 2871 2872 PMAP_LOCK(pmap); 2873 pa = pmap_extract_locked(pmap, va); 2874 PMAP_UNLOCK(pmap); 2875 return (pa); 2876} 2877 2878static vm_paddr_t 2879pmap_extract_locked(pmap_t pmap, vm_offset_t va) 2880{ 2881 struct l2_dtable *l2; 2882 pd_entry_t l1pd; 2883 pt_entry_t *ptep, pte; 2884 vm_paddr_t pa; 2885 u_int l1idx; 2886 2887 if (pmap != kernel_pmap) 2888 PMAP_ASSERT_LOCKED(pmap); 2889 l1idx = L1_IDX(va); 2890 l1pd = pmap->pm_l1->l1_kva[l1idx]; 2891 if (l1pte_section_p(l1pd)) { 2892 /* 2893 * These should only happen for the kernel pmap. 2894 */ 2895 KASSERT(pmap == kernel_pmap, ("unexpected section")); 2896 /* XXX: what to do about the bits > 32 ? */ 2897 if (l1pd & L1_S_SUPERSEC) 2898 pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); 2899 else 2900 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 2901 } else { 2902 /* 2903 * Note that we can't rely on the validity of the L1 2904 * descriptor as an indication that a mapping exists. 2905 * We have to look it up in the L2 dtable. 2906 */ 2907 l2 = pmap->pm_l2[L2_IDX(l1idx)]; 2908 if (l2 == NULL || 2909 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) 2910 return (0); 2911 pte = ptep[l2pte_index(va)]; 2912 if (pte == 0) 2913 return (0); 2914 switch (pte & L2_TYPE_MASK) { 2915 case L2_TYPE_L: 2916 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 2917 break; 2918 default: 2919 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2920 break; 2921 } 2922 } 2923 return (pa); 2924} 2925 2926/* 2927 * Atomically extract and hold the physical page with the given 2928 * pmap and virtual address pair if that mapping permits the given 2929 * protection. 2930 * 2931 */ 2932vm_page_t 2933pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 2934{ 2935 struct l2_dtable *l2; 2936 pd_entry_t l1pd; 2937 pt_entry_t *ptep, pte; 2938 vm_paddr_t pa, paddr; 2939 vm_page_t m = NULL; 2940 u_int l1idx; 2941 l1idx = L1_IDX(va); 2942 paddr = 0; 2943 2944 PMAP_LOCK(pmap); 2945retry: 2946 l1pd = pmap->pm_l1->l1_kva[l1idx]; 2947 if (l1pte_section_p(l1pd)) { 2948 /* 2949 * These should only happen for pmap_kernel() 2950 */ 2951 KASSERT(pmap == pmap_kernel(), ("huh")); 2952 /* XXX: what to do about the bits > 32 ? */ 2953 if (l1pd & L1_S_SUPERSEC) 2954 pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); 2955 else 2956 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 2957 if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) 2958 goto retry; 2959 if (L1_S_WRITABLE(l1pd) || (prot & VM_PROT_WRITE) == 0) { 2960 m = PHYS_TO_VM_PAGE(pa); 2961 vm_page_hold(m); 2962 } 2963 } else { 2964 /* 2965 * Note that we can't rely on the validity of the L1 2966 * descriptor as an indication that a mapping exists. 2967 * We have to look it up in the L2 dtable. 2968 */ 2969 l2 = pmap->pm_l2[L2_IDX(l1idx)]; 2970 2971 if (l2 == NULL || 2972 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 2973 PMAP_UNLOCK(pmap); 2974 return (NULL); 2975 } 2976 2977 ptep = &ptep[l2pte_index(va)]; 2978 pte = *ptep; 2979 2980 if (pte == 0) { 2981 PMAP_UNLOCK(pmap); 2982 return (NULL); 2983 } else if ((prot & VM_PROT_WRITE) && (pte & L2_APX)) { 2984 PMAP_UNLOCK(pmap); 2985 return (NULL); 2986 } else { 2987 switch (pte & L2_TYPE_MASK) { 2988 case L2_TYPE_L: 2989 panic("extract and hold section mapping"); 2990 break; 2991 default: 2992 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2993 break; 2994 } 2995 if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) 2996 goto retry; 2997 m = PHYS_TO_VM_PAGE(pa); 2998 vm_page_hold(m); 2999 } 3000 3001 } 3002 3003 PMAP_UNLOCK(pmap); 3004 PA_UNLOCK_COND(paddr); 3005 return (m); 3006} 3007 3008/* 3009 * Initialize a preallocated and zeroed pmap structure, 3010 * such as one in a vmspace structure. 3011 */ 3012 3013int 3014pmap_pinit(pmap_t pmap) 3015{ 3016 PDEBUG(1, printf("pmap_pinit: pmap = %08x\n", (uint32_t) pmap)); 3017 3018 PMAP_LOCK_INIT(pmap); 3019 pmap_alloc_l1(pmap); 3020 bzero(pmap->pm_l2, sizeof(pmap->pm_l2)); 3021 3022 CPU_ZERO(&pmap->pm_active); 3023 3024 TAILQ_INIT(&pmap->pm_pvlist); 3025 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 3026 pmap->pm_stats.resident_count = 1; 3027 if (vector_page < KERNBASE) { 3028 pmap_enter(pmap, vector_page, 3029 VM_PROT_READ, PHYS_TO_VM_PAGE(systempage.pv_pa), 3030 VM_PROT_READ, 1); 3031 } 3032 return (1); 3033} 3034 3035 3036/*************************************************** 3037 * page management routines. 3038 ***************************************************/ 3039 3040 3041static void 3042pmap_free_pv_entry(pv_entry_t pv) 3043{ 3044 pv_entry_count--; 3045 uma_zfree(pvzone, pv); 3046} 3047 3048 3049/* 3050 * get a new pv_entry, allocating a block from the system 3051 * when needed. 3052 * the memory allocation is performed bypassing the malloc code 3053 * because of the possibility of allocations at interrupt time. 3054 */ 3055static pv_entry_t 3056pmap_get_pv_entry(void) 3057{ 3058 pv_entry_t ret_value; 3059 3060 pv_entry_count++; 3061 if (pv_entry_count > pv_entry_high_water) 3062 pagedaemon_wakeup(); 3063 ret_value = uma_zalloc(pvzone, M_NOWAIT); 3064 return ret_value; 3065} 3066 3067/* 3068 * Remove the given range of addresses from the specified map. 3069 * 3070 * It is assumed that the start and end are properly 3071 * rounded to the page size. 3072 */ 3073#define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3074void 3075pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 3076{ 3077 struct l2_bucket *l2b; 3078 vm_offset_t next_bucket; 3079 pt_entry_t *ptep; 3080 u_int total; 3081 u_int mappings, is_exec, is_refd; 3082 int flushall = 0; 3083 3084 3085 /* 3086 * we lock in the pmap => pv_head direction 3087 */ 3088 3089 rw_wlock(&pvh_global_lock); 3090 PMAP_LOCK(pm); 3091 total = 0; 3092 while (sva < eva) { 3093 /* 3094 * Do one L2 bucket's worth at a time. 3095 */ 3096 next_bucket = L2_NEXT_BUCKET(sva); 3097 if (next_bucket > eva) 3098 next_bucket = eva; 3099 3100 l2b = pmap_get_l2_bucket(pm, sva); 3101 if (l2b == NULL) { 3102 sva = next_bucket; 3103 continue; 3104 } 3105 3106 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3107 mappings = 0; 3108 3109 while (sva < next_bucket) { 3110 struct vm_page *pg; 3111 pt_entry_t pte; 3112 vm_paddr_t pa; 3113 3114 pte = *ptep; 3115 3116 if (pte == 0) { 3117 /* 3118 * Nothing here, move along 3119 */ 3120 sva += PAGE_SIZE; 3121 ptep++; 3122 continue; 3123 } 3124 3125 pm->pm_stats.resident_count--; 3126 pa = l2pte_pa(pte); 3127 is_exec = 0; 3128 is_refd = 1; 3129 3130 /* 3131 * Update flags. In a number of circumstances, 3132 * we could cluster a lot of these and do a 3133 * number of sequential pages in one go. 3134 */ 3135 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 3136 struct pv_entry *pve; 3137 3138 pve = pmap_remove_pv(pg, pm, sva); 3139 if (pve) { 3140 is_exec = PV_BEEN_EXECD(pve->pv_flags); 3141 is_refd = PV_BEEN_REFD(pve->pv_flags); 3142 pmap_free_pv_entry(pve); 3143 } 3144 } 3145 3146 if (pmap_is_current(pm)) { 3147 total++; 3148 if (total < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3149 if (is_exec) 3150 cpu_tlb_flushID_SE(sva); 3151 else if (is_refd) 3152 cpu_tlb_flushD_SE(sva); 3153 } else if (total == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3154 flushall = 1; 3155 } 3156 } 3157 *ptep = 0; 3158 PTE_SYNC(ptep); 3159 3160 sva += PAGE_SIZE; 3161 ptep++; 3162 mappings++; 3163 } 3164 3165 pmap_free_l2_bucket(pm, l2b, mappings); 3166 } 3167 3168 rw_wunlock(&pvh_global_lock); 3169 if (flushall) 3170 cpu_tlb_flushID(); 3171 PMAP_UNLOCK(pm); 3172} 3173 3174/* 3175 * pmap_zero_page() 3176 * 3177 * Zero a given physical page by mapping it at a page hook point. 3178 * In doing the zero page op, the page we zero is mapped cachable, as with 3179 * StrongARM accesses to non-cached pages are non-burst making writing 3180 * _any_ bulk data very slow. 3181 */ 3182static void 3183pmap_zero_page_gen(vm_page_t pg, int off, int size) 3184{ 3185 3186 vm_paddr_t phys = VM_PAGE_TO_PHYS(pg); 3187 if (!TAILQ_EMPTY(&pg->md.pv_list)) 3188 panic("pmap_zero_page: page has mappings"); 3189 3190 mtx_lock(&cmtx); 3191 /* 3192 * Hook in the page, zero it, invalidate the TLB as needed. 3193 * 3194 * Note the temporary zero-page mapping must be a non-cached page in 3195 * order to work without corruption when write-allocate is enabled. 3196 */ 3197 *cdst_pte = L2_S_PROTO | phys | pte_l2_s_cache_mode; 3198 pmap_set_prot(cdst_pte, VM_PROT_WRITE, 0); 3199 PTE_SYNC(cdst_pte); 3200 cpu_tlb_flushD_SE(cdstp); 3201 cpu_cpwait(); 3202 if (off || size != PAGE_SIZE) 3203 bzero((void *)(cdstp + off), size); 3204 else 3205 bzero_page(cdstp); 3206 3207 /* 3208 * Although aliasing is not possible if we use 3209 * cdstp temporary mappings with memory that 3210 * will be mapped later as non-cached or with write-through 3211 * caches we might end up overwriting it when calling wbinv_all 3212 * So make sure caches are clean after copy operation 3213 */ 3214 cpu_idcache_wbinv_range(cdstp, size); 3215 pmap_l2cache_wbinv_range(cdstp, phys, size); 3216 3217 mtx_unlock(&cmtx); 3218} 3219 3220/* 3221 * pmap_zero_page zeros the specified hardware page by mapping 3222 * the page into KVM and using bzero to clear its contents. 3223 */ 3224void 3225pmap_zero_page(vm_page_t m) 3226{ 3227 pmap_zero_page_gen(m, 0, PAGE_SIZE); 3228} 3229 3230 3231/* 3232 * pmap_zero_page_area zeros the specified hardware page by mapping 3233 * the page into KVM and using bzero to clear its contents. 3234 * 3235 * off and size may not cover an area beyond a single hardware page. 3236 */ 3237void 3238pmap_zero_page_area(vm_page_t m, int off, int size) 3239{ 3240 3241 pmap_zero_page_gen(m, off, size); 3242} 3243 3244 3245/* 3246 * pmap_zero_page_idle zeros the specified hardware page by mapping 3247 * the page into KVM and using bzero to clear its contents. This 3248 * is intended to be called from the vm_pagezero process only and 3249 * outside of Giant. 3250 */ 3251void 3252pmap_zero_page_idle(vm_page_t m) 3253{ 3254 3255 pmap_zero_page(m); 3256} 3257 3258/* 3259 * pmap_copy_page copies the specified (machine independent) 3260 * page by mapping the page into virtual memory and using 3261 * bcopy to copy the page, one machine dependent page at a 3262 * time. 3263 */ 3264 3265/* 3266 * pmap_copy_page() 3267 * 3268 * Copy one physical page into another, by mapping the pages into 3269 * hook points. The same comment regarding cachability as in 3270 * pmap_zero_page also applies here. 3271 */ 3272void 3273pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) 3274{ 3275 /* 3276 * Hold the source page's lock for the duration of the copy 3277 * so that no other mappings can be created while we have a 3278 * potentially aliased mapping. 3279 * Map the pages into the page hook points, copy them, and purge 3280 * the cache for the appropriate page. Invalidate the TLB 3281 * as required. 3282 */ 3283 mtx_lock(&cmtx); 3284 3285 /* For ARMv6 using System bit is deprecated and mapping with AP 3286 * bits set to 0x0 makes page not accessible. csrc_pte is mapped 3287 * read/write until proper mapping defines are created for ARMv6. 3288 */ 3289 *csrc_pte = L2_S_PROTO | src | pte_l2_s_cache_mode; 3290 pmap_set_prot(csrc_pte, VM_PROT_READ, 0); 3291 PTE_SYNC(csrc_pte); 3292 3293 *cdst_pte = L2_S_PROTO | dst | pte_l2_s_cache_mode; 3294 pmap_set_prot(cdst_pte, VM_PROT_READ | VM_PROT_WRITE, 0); 3295 PTE_SYNC(cdst_pte); 3296 3297 cpu_tlb_flushD_SE(csrcp); 3298 cpu_tlb_flushD_SE(cdstp); 3299 cpu_cpwait(); 3300 3301 /* 3302 * Although aliasing is not possible if we use 3303 * cdstp temporary mappings with memory that 3304 * will be mapped later as non-cached or with write-through 3305 * caches we might end up overwriting it when calling wbinv_all 3306 * So make sure caches are clean after copy operation 3307 */ 3308 bcopy_page(csrcp, cdstp); 3309 3310 cpu_idcache_wbinv_range(cdstp, PAGE_SIZE); 3311 pmap_l2cache_wbinv_range(cdstp, dst, PAGE_SIZE); 3312 3313 mtx_unlock(&cmtx); 3314} 3315 3316void 3317pmap_copy_page(vm_page_t src, vm_page_t dst) 3318{ 3319 3320 if (_arm_memcpy && PAGE_SIZE >= _min_memcpy_size && 3321 _arm_memcpy((void *)VM_PAGE_TO_PHYS(dst), 3322 (void *)VM_PAGE_TO_PHYS(src), PAGE_SIZE, IS_PHYSICAL) == 0) 3323 return; 3324 3325 pmap_copy_page_generic(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); 3326} 3327 3328/* 3329 * this routine returns true if a physical page resides 3330 * in the given pmap. 3331 */ 3332boolean_t 3333pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 3334{ 3335 pv_entry_t pv; 3336 int loops = 0; 3337 boolean_t rv; 3338 3339 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3340 ("pmap_page_exists_quick: page %p is not managed", m)); 3341 rv = FALSE; 3342 rw_wlock(&pvh_global_lock); 3343 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 3344 if (pv->pv_pmap == pmap) { 3345 rv = TRUE; 3346 break; 3347 } 3348 loops++; 3349 if (loops >= 16) 3350 break; 3351 } 3352 3353 rw_wunlock(&pvh_global_lock); 3354 return (rv); 3355} 3356 3357/* 3358 * pmap_page_wired_mappings: 3359 * 3360 * Return the number of managed mappings to the given physical page 3361 * that are wired. 3362 */ 3363int 3364pmap_page_wired_mappings(vm_page_t m) 3365{ 3366 pv_entry_t pv; 3367 int count; 3368 3369 count = 0; 3370 if ((m->flags & PG_FICTITIOUS) != 0) 3371 return (count); 3372 rw_wlock(&pvh_global_lock); 3373 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) 3374 if ((pv->pv_flags & PVF_WIRED) != 0) 3375 count++; 3376 rw_wunlock(&pvh_global_lock); 3377 return (count); 3378} 3379 3380/* 3381 * pmap_is_referenced: 3382 * 3383 * Return whether or not the specified physical page was referenced 3384 * in any physical maps. 3385 */ 3386boolean_t 3387pmap_is_referenced(vm_page_t m) 3388{ 3389 3390 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3391 ("pmap_is_referenced: page %p is not managed", m)); 3392 return ((m->md.pvh_attrs & PVF_REF) != 0); 3393} 3394 3395/* 3396 * pmap_ts_referenced: 3397 * 3398 * Return the count of reference bits for a page, clearing all of them. 3399 */ 3400int 3401pmap_ts_referenced(vm_page_t m) 3402{ 3403 3404 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3405 ("pmap_ts_referenced: page %p is not managed", m)); 3406 return (pmap_clearbit(m, PVF_REF)); 3407} 3408 3409 3410boolean_t 3411pmap_is_modified(vm_page_t m) 3412{ 3413 3414 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3415 ("pmap_is_modified: page %p is not managed", m)); 3416 if (m->md.pvh_attrs & PVF_MOD) 3417 return (TRUE); 3418 3419 return(FALSE); 3420} 3421 3422 3423/* 3424 * Clear the modify bits on the specified physical page. 3425 */ 3426void 3427pmap_clear_modify(vm_page_t m) 3428{ 3429 3430 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3431 ("pmap_clear_modify: page %p is not managed", m)); 3432 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 3433 KASSERT((m->oflags & VPO_BUSY) == 0, 3434 ("pmap_clear_modify: page %p is busy", m)); 3435 3436 /* 3437 * If the page is not PGA_WRITEABLE, then no mappings can be modified. 3438 * If the object containing the page is locked and the page is not 3439 * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set. 3440 */ 3441 if ((m->aflags & PGA_WRITEABLE) == 0) 3442 return; 3443 3444 if (m->md.pvh_attrs & PVF_MOD) 3445 pmap_clearbit(m, PVF_MOD); 3446} 3447 3448 3449/* 3450 * pmap_clear_reference: 3451 * 3452 * Clear the reference bit on the specified physical page. 3453 */ 3454void 3455pmap_clear_reference(vm_page_t m) 3456{ 3457 3458 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3459 ("pmap_clear_reference: page %p is not managed", m)); 3460 if (m->md.pvh_attrs & PVF_REF) 3461 pmap_clearbit(m, PVF_REF); 3462} 3463 3464 3465/* 3466 * Clear the write and modified bits in each of the given page's mappings. 3467 */ 3468void 3469pmap_remove_write(vm_page_t m) 3470{ 3471 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3472 ("pmap_remove_write: page %p is not managed", m)); 3473 3474 /* 3475 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by 3476 * another thread while the object is locked. Thus, if PGA_WRITEABLE 3477 * is clear, no page table entries need updating. 3478 */ 3479 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 3480 if ((m->oflags & VPO_BUSY) != 0 || 3481 (m->aflags & PGA_WRITEABLE) != 0) 3482 pmap_clearbit(m, PVF_WRITE); 3483} 3484 3485 3486/* 3487 * perform the pmap work for mincore 3488 */ 3489int 3490pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 3491{ 3492 struct l2_bucket *l2b; 3493 pt_entry_t *ptep, pte; 3494 vm_paddr_t pa; 3495 vm_page_t m; 3496 int val; 3497 boolean_t managed; 3498 3499 PMAP_LOCK(pmap); 3500retry: 3501 l2b = pmap_get_l2_bucket(pmap, addr); 3502 if (l2b == NULL) { 3503 val = 0; 3504 goto out; 3505 } 3506 ptep = &l2b->l2b_kva[l2pte_index(addr)]; 3507 pte = *ptep; 3508 if (!l2pte_valid(pte)) { 3509 val = 0; 3510 goto out; 3511 } 3512 val = MINCORE_INCORE; 3513 if (L2_S_WRITABLE(pte)) 3514 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 3515 managed = FALSE; 3516 pa = l2pte_pa(pte); 3517 m = PHYS_TO_VM_PAGE(pa); 3518 if (m != NULL && (m->oflags & VPO_UNMANAGED) == 0) 3519 managed = TRUE; 3520 if (managed) { 3521 /* 3522 * The ARM pmap tries to maintain a per-mapping 3523 * reference bit. The trouble is that it's kept in 3524 * the PV entry, not the PTE, so it's costly to access 3525 * here. You would need to acquire the pvh global 3526 * lock, call pmap_find_pv(), and introduce a custom 3527 * version of vm_page_pa_tryrelock() that releases and 3528 * reacquires the pvh global lock. In the end, I 3529 * doubt it's worthwhile. This may falsely report 3530 * the given address as referenced. 3531 */ 3532 if ((m->md.pvh_attrs & PVF_REF) != 0) 3533 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 3534 } 3535 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 3536 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { 3537 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ 3538 if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) 3539 goto retry; 3540 } else 3541out: 3542 PA_UNLOCK_COND(*locked_pa); 3543 PMAP_UNLOCK(pmap); 3544 return (val); 3545} 3546 3547void 3548pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 3549{ 3550} 3551 3552/* 3553 * Increase the starting virtual address of the given mapping if a 3554 * different alignment might result in more superpage mappings. 3555 */ 3556void 3557pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 3558 vm_offset_t *addr, vm_size_t size) 3559{ 3560} 3561 3562 3563/* 3564 * Map a set of physical memory pages into the kernel virtual 3565 * address space. Return a pointer to where it is mapped. This 3566 * routine is intended to be used for mapping device memory, 3567 * NOT real memory. 3568 */ 3569void * 3570pmap_mapdev(vm_offset_t pa, vm_size_t size) 3571{ 3572 vm_offset_t va, tmpva, offset; 3573 3574 offset = pa & PAGE_MASK; 3575 size = roundup(size, PAGE_SIZE); 3576 3577 GIANT_REQUIRED; 3578 3579 va = kmem_alloc_nofault(kernel_map, size); 3580 if (!va) 3581 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 3582 for (tmpva = va; size > 0;) { 3583 pmap_kenter_internal(tmpva, pa, 0); 3584 size -= PAGE_SIZE; 3585 tmpva += PAGE_SIZE; 3586 pa += PAGE_SIZE; 3587 } 3588 3589 return ((void *)(va + offset)); 3590} 3591 3592/* 3593 * pmap_map_section: 3594 * 3595 * Create a single section mapping. 3596 */ 3597void 3598pmap_map_section(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 3599 int prot, int cache) 3600{ 3601 pd_entry_t *pde = (pd_entry_t *) l1pt; 3602 pd_entry_t fl; 3603 3604 KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("ouin2")); 3605 3606 fl = l1_mem_types[cache]; 3607 3608 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3609 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 3610 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 3611} 3612 3613/* 3614 * pmap_link_l2pt: 3615 * 3616 * Link the L2 page table specified by l2pv.pv_pa into the L1 3617 * page table at the slot for "va". 3618 */ 3619void 3620pmap_link_l2pt(vm_offset_t l1pt, vm_offset_t va, struct pv_addr *l2pv) 3621{ 3622 pd_entry_t *pde = (pd_entry_t *) l1pt, proto; 3623 u_int slot = va >> L1_S_SHIFT; 3624 3625 proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; 3626 3627#ifdef VERBOSE_INIT_ARM 3628 printf("pmap_link_l2pt: pa=0x%x va=0x%x\n", l2pv->pv_pa, l2pv->pv_va); 3629#endif 3630 3631 pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); 3632 PTE_SYNC(&pde[slot]); 3633 3634 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3635 3636} 3637 3638/* 3639 * pmap_map_entry 3640 * 3641 * Create a single page mapping. 3642 */ 3643void 3644pmap_map_entry(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, 3645 int cache) 3646{ 3647 pd_entry_t *pde = (pd_entry_t *) l1pt; 3648 pt_entry_t fl; 3649 pt_entry_t *pte; 3650 3651 KASSERT(((va | pa) & PAGE_MASK) == 0, ("ouin")); 3652 3653 fl = l2s_mem_types[cache]; 3654 3655 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3656 panic("pmap_map_entry: no L2 table for VA 0x%08x", va); 3657 3658 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 3659 3660 if (pte == NULL) 3661 panic("pmap_map_entry: can't find L2 table for VA 0x%08x", va); 3662 3663 pte[l2pte_index(va)] = L2_S_PROTO | pa | fl; 3664 pmap_set_prot(&pte[l2pte_index(va)], prot, 0); 3665 PTE_SYNC(&pte[l2pte_index(va)]); 3666} 3667 3668/* 3669 * pmap_map_chunk: 3670 * 3671 * Map a chunk of memory using the most efficient mappings 3672 * possible (section. large page, small page) into the 3673 * provided L1 and L2 tables at the specified virtual address. 3674 */ 3675vm_size_t 3676pmap_map_chunk(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 3677 vm_size_t size, int prot, int type) 3678{ 3679 pd_entry_t *pde = (pd_entry_t *) l1pt; 3680 pt_entry_t *pte, f1, f2s, f2l; 3681 vm_size_t resid; 3682 int i; 3683 3684 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 3685 3686 if (l1pt == 0) 3687 panic("pmap_map_chunk: no L1 table provided"); 3688 3689#ifdef VERBOSE_INIT_ARM 3690 printf("pmap_map_chunk: pa=0x%x va=0x%x size=0x%x resid=0x%x " 3691 "prot=0x%x type=%d\n", pa, va, size, resid, prot, type); 3692#endif 3693 3694 f1 = l1_mem_types[type]; 3695 f2l = l2l_mem_types[type]; 3696 f2s = l2s_mem_types[type]; 3697 3698 size = resid; 3699 3700 while (resid > 0) { 3701 /* See if we can use a section mapping. */ 3702 if (L1_S_MAPPABLE_P(va, pa, resid)) { 3703#ifdef VERBOSE_INIT_ARM 3704 printf("S"); 3705#endif 3706 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3707 L1_S_PROT(PTE_KERNEL, prot) | f1 | 3708 L1_S_DOM(PMAP_DOMAIN_KERNEL); 3709 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 3710 va += L1_S_SIZE; 3711 pa += L1_S_SIZE; 3712 resid -= L1_S_SIZE; 3713 continue; 3714 } 3715 3716 /* 3717 * Ok, we're going to use an L2 table. Make sure 3718 * one is actually in the corresponding L1 slot 3719 * for the current VA. 3720 */ 3721 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3722 panic("pmap_map_chunk: no L2 table for VA 0x%08x", va); 3723 3724 pte = (pt_entry_t *) kernel_pt_lookup( 3725 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 3726 if (pte == NULL) 3727 panic("pmap_map_chunk: can't find L2 table for VA" 3728 "0x%08x", va); 3729 /* See if we can use a L2 large page mapping. */ 3730 if (L2_L_MAPPABLE_P(va, pa, resid)) { 3731#ifdef VERBOSE_INIT_ARM 3732 printf("L"); 3733#endif 3734 for (i = 0; i < 16; i++) { 3735 pte[l2pte_index(va) + i] = 3736 L2_L_PROTO | pa | 3737 L2_L_PROT(PTE_KERNEL, prot) | f2l; 3738 PTE_SYNC(&pte[l2pte_index(va) + i]); 3739 } 3740 va += L2_L_SIZE; 3741 pa += L2_L_SIZE; 3742 resid -= L2_L_SIZE; 3743 continue; 3744 } 3745 3746 /* Use a small page mapping. */ 3747#ifdef VERBOSE_INIT_ARM 3748 printf("P"); 3749#endif 3750 pte[l2pte_index(va)] = L2_S_PROTO | pa | f2s; 3751 pmap_set_prot(&pte[l2pte_index(va)], prot, 0); 3752 PTE_SYNC(&pte[l2pte_index(va)]); 3753 va += PAGE_SIZE; 3754 pa += PAGE_SIZE; 3755 resid -= PAGE_SIZE; 3756 } 3757#ifdef VERBOSE_INIT_ARM 3758 printf("\n"); 3759#endif 3760 return (size); 3761 3762} 3763 3764/********************** Static device map routines ***************************/ 3765 3766static const struct pmap_devmap *pmap_devmap_table; 3767 3768/* 3769 * Register the devmap table. This is provided in case early console 3770 * initialization needs to register mappings created by bootstrap code 3771 * before pmap_devmap_bootstrap() is called. 3772 */ 3773void 3774pmap_devmap_register(const struct pmap_devmap *table) 3775{ 3776 3777 pmap_devmap_table = table; 3778} 3779 3780/* 3781 * Map all of the static regions in the devmap table, and remember 3782 * the devmap table so other parts of the kernel can look up entries 3783 * later. 3784 */ 3785void 3786pmap_devmap_bootstrap(vm_offset_t l1pt, const struct pmap_devmap *table) 3787{ 3788 int i; 3789 3790 pmap_devmap_table = table; 3791 3792 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 3793#ifdef VERBOSE_INIT_ARM 3794 printf("devmap: %08x -> %08x @ %08x\n", 3795 pmap_devmap_table[i].pd_pa, 3796 pmap_devmap_table[i].pd_pa + 3797 pmap_devmap_table[i].pd_size - 1, 3798 pmap_devmap_table[i].pd_va); 3799#endif 3800 pmap_map_chunk(l1pt, pmap_devmap_table[i].pd_va, 3801 pmap_devmap_table[i].pd_pa, 3802 pmap_devmap_table[i].pd_size, 3803 pmap_devmap_table[i].pd_prot, 3804 pmap_devmap_table[i].pd_cache); 3805 } 3806} 3807 3808const struct pmap_devmap * 3809pmap_devmap_find_pa(vm_paddr_t pa, vm_size_t size) 3810{ 3811 int i; 3812 3813 if (pmap_devmap_table == NULL) 3814 return (NULL); 3815 3816 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 3817 if (pa >= pmap_devmap_table[i].pd_pa && 3818 pa + size <= pmap_devmap_table[i].pd_pa + 3819 pmap_devmap_table[i].pd_size) 3820 return (&pmap_devmap_table[i]); 3821 } 3822 3823 return (NULL); 3824} 3825 3826const struct pmap_devmap * 3827pmap_devmap_find_va(vm_offset_t va, vm_size_t size) 3828{ 3829 int i; 3830 3831 if (pmap_devmap_table == NULL) 3832 return (NULL); 3833 3834 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 3835 if (va >= pmap_devmap_table[i].pd_va && 3836 va + size <= pmap_devmap_table[i].pd_va + 3837 pmap_devmap_table[i].pd_size) 3838 return (&pmap_devmap_table[i]); 3839 } 3840 3841 return (NULL); 3842} 3843 3844int 3845pmap_dmap_iscurrent(pmap_t pmap) 3846{ 3847 return(pmap_is_current(pmap)); 3848} 3849 3850void 3851pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 3852{ 3853 /* 3854 * Remember the memattr in a field that gets used to set the appropriate 3855 * bits in the PTEs as mappings are established. 3856 */ 3857 m->md.pv_memattr = ma; 3858 3859 /* 3860 * It appears that this function can only be called before any mappings 3861 * for the page are established on ARM. If this ever changes, this code 3862 * will need to walk the pv_list and make each of the existing mappings 3863 * uncacheable, being careful to sync caches and PTEs (and maybe 3864 * invalidate TLB?) for any current mapping it modifies. 3865 */ 3866 if (m->md.pv_kva != 0 || TAILQ_FIRST(&m->md.pv_list) != NULL) 3867 panic("Can't change memattr on page with existing mappings"); 3868}
| 1167 pv_entry_high_water = 9 * (pv_entry_max / 10); 1168 1169 /* 1170 * Now it is safe to enable pv_table recording. 1171 */ 1172 PDEBUG(1, printf("pmap_init: done!\n")); 1173} 1174 1175int 1176pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) 1177{ 1178 struct l2_dtable *l2; 1179 struct l2_bucket *l2b; 1180 pd_entry_t *pl1pd, l1pd; 1181 pt_entry_t *ptep, pte; 1182 vm_paddr_t pa; 1183 u_int l1idx; 1184 int rv = 0; 1185 1186 l1idx = L1_IDX(va); 1187 rw_wlock(&pvh_global_lock); 1188 PMAP_LOCK(pm); 1189 1190 /* 1191 * If there is no l2_dtable for this address, then the process 1192 * has no business accessing it. 1193 * 1194 * Note: This will catch userland processes trying to access 1195 * kernel addresses. 1196 */ 1197 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1198 if (l2 == NULL) 1199 goto out; 1200 1201 /* 1202 * Likewise if there is no L2 descriptor table 1203 */ 1204 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1205 if (l2b->l2b_kva == NULL) 1206 goto out; 1207 1208 /* 1209 * Check the PTE itself. 1210 */ 1211 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1212 pte = *ptep; 1213 if (pte == 0) 1214 goto out; 1215 1216 /* 1217 * Catch a userland access to the vector page mapped at 0x0 1218 */ 1219 if (user && ((pte & L2_S_PROT_MASK) == L2_S_PROT_U)) 1220 goto out; 1221 if (va == vector_page) 1222 goto out; 1223 1224 pa = l2pte_pa(pte); 1225 CTR5(KTR_PMAP, "pmap_fault_fix: pmap:%p va:%x pte:0x%x ftype:%x user:%x", 1226 pm, va, pte, ftype, user); 1227 if ((ftype & VM_PROT_WRITE) && !(L2_S_WRITABLE(pte))) { 1228 /* 1229 * This looks like a good candidate for "page modified" 1230 * emulation... 1231 */ 1232 struct pv_entry *pv; 1233 struct vm_page *pg; 1234 1235 /* Extract the physical address of the page */ 1236 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 1237 goto out; 1238 } 1239 /* Get the current flags for this page. */ 1240 1241 pv = pmap_find_pv(pg, pm, va); 1242 if (pv == NULL) { 1243 goto out; 1244 } 1245 1246 /* 1247 * Do the flags say this page is writable? If not then it 1248 * is a genuine write fault. If yes then the write fault is 1249 * our fault as we did not reflect the write access in the 1250 * PTE. Now we know a write has occurred we can correct this 1251 * and also set the modified bit 1252 */ 1253 if ((pv->pv_flags & PVF_WRITE) == 0) { 1254 goto out; 1255 } 1256 pg->md.pvh_attrs |= PVF_REF | PVF_MOD; 1257 vm_page_dirty(pg); 1258 pv->pv_flags |= PVF_REF | PVF_MOD; 1259 1260 /* Re-enable write permissions for the page */ 1261 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 1262 pmap_set_prot(ptep, VM_PROT_WRITE, *ptep & L2_S_PROT_U); 1263 CTR1(KTR_PMAP, "pmap_fault_fix: new pte:0x%x", pte); 1264 PTE_SYNC(ptep); 1265 rv = 1; 1266 } else if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { 1267 /* 1268 * This looks like a good candidate for "page referenced" 1269 * emulation. 1270 */ 1271 struct pv_entry *pv; 1272 struct vm_page *pg; 1273 1274 /* Extract the physical address of the page */ 1275 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 1276 goto out; 1277 /* Get the current flags for this page. */ 1278 1279 pv = pmap_find_pv(pg, pm, va); 1280 if (pv == NULL) 1281 goto out; 1282 1283 pg->md.pvh_attrs |= PVF_REF; 1284 pv->pv_flags |= PVF_REF; 1285 1286 1287 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 1288 PTE_SYNC(ptep); 1289 rv = 1; 1290 } 1291 1292 /* 1293 * We know there is a valid mapping here, so simply 1294 * fix up the L1 if necessary. 1295 */ 1296 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1297 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; 1298 if (*pl1pd != l1pd) { 1299 *pl1pd = l1pd; 1300 PTE_SYNC(pl1pd); 1301 rv = 1; 1302 } 1303 1304#ifdef DEBUG 1305 /* 1306 * If 'rv == 0' at this point, it generally indicates that there is a 1307 * stale TLB entry for the faulting address. This happens when two or 1308 * more processes are sharing an L1. Since we don't flush the TLB on 1309 * a context switch between such processes, we can take domain faults 1310 * for mappings which exist at the same VA in both processes. EVEN IF 1311 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 1312 * example. 1313 * 1314 * This is extremely likely to happen if pmap_enter() updated the L1 1315 * entry for a recently entered mapping. In this case, the TLB is 1316 * flushed for the new mapping, but there may still be TLB entries for 1317 * other mappings belonging to other processes in the 1MB range 1318 * covered by the L1 entry. 1319 * 1320 * Since 'rv == 0', we know that the L1 already contains the correct 1321 * value, so the fault must be due to a stale TLB entry. 1322 * 1323 * Since we always need to flush the TLB anyway in the case where we 1324 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 1325 * stale TLB entries dynamically. 1326 * 1327 * However, the above condition can ONLY happen if the current L1 is 1328 * being shared. If it happens when the L1 is unshared, it indicates 1329 * that other parts of the pmap are not doing their job WRT managing 1330 * the TLB. 1331 */ 1332 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { 1333 printf("fixup: pm %p, va 0x%08x, ftype %d - nothing to do!\n", 1334 pm, va, ftype); 1335 printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", 1336 l2, l2b, ptep, pl1pd); 1337 printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", 1338 pte, l1pd, last_fault_code); 1339#ifdef DDB 1340 Debugger(); 1341#endif 1342 } 1343#endif 1344 1345 cpu_tlb_flushID_SE(va); 1346 cpu_cpwait(); 1347 1348 rv = 1; 1349 1350out: 1351 rw_wunlock(&pvh_global_lock); 1352 PMAP_UNLOCK(pm); 1353 return (rv); 1354} 1355 1356void 1357pmap_postinit(void) 1358{ 1359 struct l2_bucket *l2b; 1360 struct l1_ttable *l1; 1361 pd_entry_t *pl1pt; 1362 pt_entry_t *ptep, pte; 1363 vm_offset_t va, eva; 1364 u_int loop, needed; 1365 1366 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 1367 needed -= 1; 1368 l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); 1369 1370 for (loop = 0; loop < needed; loop++, l1++) { 1371 /* Allocate a L1 page table */ 1372 va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_VMPMAP, 0, 0x0, 1373 0xffffffff, L1_TABLE_SIZE, 0); 1374 1375 if (va == 0) 1376 panic("Cannot allocate L1 KVM"); 1377 1378 eva = va + L1_TABLE_SIZE; 1379 pl1pt = (pd_entry_t *)va; 1380 1381 while (va < eva) { 1382 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1383 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1384 pte = *ptep; 1385 pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 1386 *ptep = pte; 1387 PTE_SYNC(ptep); 1388 cpu_tlb_flushD_SE(va); 1389 1390 va += PAGE_SIZE; 1391 } 1392 pmap_init_l1(l1, pl1pt); 1393 } 1394#ifdef DEBUG 1395 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 1396 needed); 1397#endif 1398} 1399 1400/* 1401 * This is used to stuff certain critical values into the PCB where they 1402 * can be accessed quickly from cpu_switch() et al. 1403 */ 1404void 1405pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) 1406{ 1407 struct l2_bucket *l2b; 1408 1409 pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; 1410 pcb->pcb_dacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 1411 (DOMAIN_CLIENT << (pm->pm_domain * 2)); 1412 1413 if (vector_page < KERNBASE) { 1414 pcb->pcb_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 1415 l2b = pmap_get_l2_bucket(pm, vector_page); 1416 pcb->pcb_l1vec = l2b->l2b_phys | L1_C_PROTO | 1417 L1_C_DOM(pm->pm_domain) | L1_C_DOM(PMAP_DOMAIN_KERNEL); 1418 } else 1419 pcb->pcb_pl1vec = NULL; 1420} 1421 1422void 1423pmap_activate(struct thread *td) 1424{ 1425 pmap_t pm; 1426 struct pcb *pcb; 1427 1428 pm = vmspace_pmap(td->td_proc->p_vmspace); 1429 pcb = td->td_pcb; 1430 1431 critical_enter(); 1432 pmap_set_pcb_pagedir(pm, pcb); 1433 1434 if (td == curthread) { 1435 u_int cur_dacr, cur_ttb; 1436 1437 __asm __volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); 1438 __asm __volatile("mrc p15, 0, %0, c3, c0, 0" : "=r"(cur_dacr)); 1439 1440 cur_ttb &= ~(L1_TABLE_SIZE - 1); 1441 1442 if (cur_ttb == (u_int)pcb->pcb_pagedir && 1443 cur_dacr == pcb->pcb_dacr) { 1444 /* 1445 * No need to switch address spaces. 1446 */ 1447 critical_exit(); 1448 return; 1449 } 1450 1451 1452 /* 1453 * We MUST, I repeat, MUST fix up the L1 entry corresponding 1454 * to 'vector_page' in the incoming L1 table before switching 1455 * to it otherwise subsequent interrupts/exceptions (including 1456 * domain faults!) will jump into hyperspace. 1457 */ 1458 if (pcb->pcb_pl1vec) { 1459 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 1460 } 1461 1462 cpu_domains(pcb->pcb_dacr); 1463 cpu_setttb(pcb->pcb_pagedir); 1464 } 1465 critical_exit(); 1466} 1467 1468static int 1469pmap_set_pt_cache_mode(pd_entry_t *kl1, vm_offset_t va) 1470{ 1471 pd_entry_t *pdep, pde; 1472 pt_entry_t *ptep, pte; 1473 vm_offset_t pa; 1474 int rv = 0; 1475 1476 /* 1477 * Make sure the descriptor itself has the correct cache mode 1478 */ 1479 pdep = &kl1[L1_IDX(va)]; 1480 pde = *pdep; 1481 1482 if (l1pte_section_p(pde)) { 1483 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 1484 *pdep = (pde & ~L1_S_CACHE_MASK) | 1485 pte_l1_s_cache_mode_pt; 1486 PTE_SYNC(pdep); 1487 rv = 1; 1488 } 1489 } else { 1490 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 1491 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 1492 if (ptep == NULL) 1493 panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); 1494 1495 ptep = &ptep[l2pte_index(va)]; 1496 pte = *ptep; 1497 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1498 *ptep = (pte & ~L2_S_CACHE_MASK) | 1499 pte_l2_s_cache_mode_pt; 1500 PTE_SYNC(ptep); 1501 rv = 1; 1502 } 1503 } 1504 1505 return (rv); 1506} 1507 1508static void 1509pmap_alloc_specials(vm_offset_t *availp, int pages, vm_offset_t *vap, 1510 pt_entry_t **ptep) 1511{ 1512 vm_offset_t va = *availp; 1513 struct l2_bucket *l2b; 1514 1515 if (ptep) { 1516 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1517 if (l2b == NULL) 1518 panic("pmap_alloc_specials: no l2b for 0x%x", va); 1519 1520 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 1521 } 1522 1523 *vap = va; 1524 *availp = va + (PAGE_SIZE * pages); 1525} 1526 1527/* 1528 * Bootstrap the system enough to run with virtual memory. 1529 * 1530 * On the arm this is called after mapping has already been enabled 1531 * and just syncs the pmap module with what has already been done. 1532 * [We can't call it easily with mapping off since the kernel is not 1533 * mapped with PA == VA, hence we would have to relocate every address 1534 * from the linked base (virtual) address "KERNBASE" to the actual 1535 * (physical) address starting relative to 0] 1536 */ 1537#define PMAP_STATIC_L2_SIZE 16 1538 1539void 1540pmap_bootstrap(vm_offset_t firstaddr, struct pv_addr *l1pt) 1541{ 1542 static struct l1_ttable static_l1; 1543 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 1544 struct l1_ttable *l1 = &static_l1; 1545 struct l2_dtable *l2; 1546 struct l2_bucket *l2b; 1547 pd_entry_t pde; 1548 pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va; 1549 pt_entry_t *ptep; 1550 vm_paddr_t pa; 1551 vm_offset_t va; 1552 vm_size_t size; 1553 int l1idx, l2idx, l2next = 0; 1554 1555 PDEBUG(1, printf("firstaddr = %08x, lastaddr = %08x\n", 1556 firstaddr, vm_max_kernel_address)); 1557 1558 virtual_avail = firstaddr; 1559 kernel_pmap->pm_l1 = l1; 1560 kernel_l1pa = l1pt->pv_pa; 1561 1562 /* 1563 * Scan the L1 translation table created by initarm() and create 1564 * the required metadata for all valid mappings found in it. 1565 */ 1566 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 1567 pde = kernel_l1pt[l1idx]; 1568 1569 /* 1570 * We're only interested in Coarse mappings. 1571 * pmap_extract() can deal with section mappings without 1572 * recourse to checking L2 metadata. 1573 */ 1574 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 1575 continue; 1576 1577 /* 1578 * Lookup the KVA of this L2 descriptor table 1579 */ 1580 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 1581 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 1582 1583 if (ptep == NULL) { 1584 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 1585 (u_int)l1idx << L1_S_SHIFT, (long unsigned int)pa); 1586 } 1587 1588 /* 1589 * Fetch the associated L2 metadata structure. 1590 * Allocate a new one if necessary. 1591 */ 1592 if ((l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]) == NULL) { 1593 if (l2next == PMAP_STATIC_L2_SIZE) 1594 panic("pmap_bootstrap: out of static L2s"); 1595 kernel_pmap->pm_l2[L2_IDX(l1idx)] = l2 = 1596 &static_l2[l2next++]; 1597 } 1598 1599 /* 1600 * One more L1 slot tracked... 1601 */ 1602 l2->l2_occupancy++; 1603 1604 /* 1605 * Fill in the details of the L2 descriptor in the 1606 * appropriate bucket. 1607 */ 1608 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1609 l2b->l2b_kva = ptep; 1610 l2b->l2b_phys = pa; 1611 l2b->l2b_l1idx = l1idx; 1612 1613 /* 1614 * Establish an initial occupancy count for this descriptor 1615 */ 1616 for (l2idx = 0; 1617 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1618 l2idx++) { 1619 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 1620 l2b->l2b_occupancy++; 1621 } 1622 } 1623 1624 /* 1625 * Make sure the descriptor itself has the correct cache mode. 1626 * If not, fix it, but whine about the problem. Port-meisters 1627 * should consider this a clue to fix up their initarm() 1628 * function. :) 1629 */ 1630 if (pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)ptep)) { 1631 printf("pmap_bootstrap: WARNING! wrong cache mode for " 1632 "L2 pte @ %p\n", ptep); 1633 } 1634 } 1635 1636 1637 /* 1638 * Ensure the primary (kernel) L1 has the correct cache mode for 1639 * a page table. Bitch if it is not correctly set. 1640 */ 1641 for (va = (vm_offset_t)kernel_l1pt; 1642 va < ((vm_offset_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { 1643 if (pmap_set_pt_cache_mode(kernel_l1pt, va)) 1644 printf("pmap_bootstrap: WARNING! wrong cache mode for " 1645 "primary L1 @ 0x%x\n", va); 1646 } 1647 1648 cpu_dcache_wbinv_all(); 1649 cpu_l2cache_wbinv_all(); 1650 cpu_tlb_flushID(); 1651 cpu_cpwait(); 1652 1653 PMAP_LOCK_INIT(kernel_pmap); 1654 CPU_FILL(&kernel_pmap->pm_active); 1655 kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; 1656 TAILQ_INIT(&kernel_pmap->pm_pvlist); 1657 1658 /* 1659 * Initialize the global pv list lock. 1660 */ 1661 rw_init(&pvh_global_lock, "pmap pv global"); 1662 1663 /* 1664 * Reserve some special page table entries/VA space for temporary 1665 * mapping of pages. 1666 */ 1667 1668 pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); 1669 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte); 1670 pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); 1671 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte); 1672 size = ((vm_max_kernel_address - pmap_curmaxkvaddr) + L1_S_OFFSET) / 1673 L1_S_SIZE; 1674 pmap_alloc_specials(&virtual_avail, 1675 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 1676 &pmap_kernel_l2ptp_kva, NULL); 1677 1678 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 1679 pmap_alloc_specials(&virtual_avail, 1680 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 1681 &pmap_kernel_l2dtable_kva, NULL); 1682 1683 pmap_alloc_specials(&virtual_avail, 1684 1, (vm_offset_t*)&_tmppt, NULL); 1685 pmap_alloc_specials(&virtual_avail, 1686 MAXDUMPPGS, (vm_offset_t *)&crashdumpmap, NULL); 1687 SLIST_INIT(&l1_list); 1688 TAILQ_INIT(&l1_lru_list); 1689 mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); 1690 pmap_init_l1(l1, kernel_l1pt); 1691 cpu_dcache_wbinv_all(); 1692 cpu_l2cache_wbinv_all(); 1693 1694 virtual_avail = round_page(virtual_avail); 1695 virtual_end = vm_max_kernel_address; 1696 kernel_vm_end = pmap_curmaxkvaddr; 1697 arm_nocache_startaddr = vm_max_kernel_address; 1698 mtx_init(&cmtx, "TMP mappings mtx", NULL, MTX_DEF); 1699 1700 pmap_set_pcb_pagedir(kernel_pmap, thread0.td_pcb); 1701} 1702 1703/*************************************************** 1704 * Pmap allocation/deallocation routines. 1705 ***************************************************/ 1706 1707/* 1708 * Release any resources held by the given physical map. 1709 * Called when a pmap initialized by pmap_pinit is being released. 1710 * Should only be called if the map contains no valid mappings. 1711 */ 1712void 1713pmap_release(pmap_t pmap) 1714{ 1715 struct pcb *pcb; 1716 1717 cpu_idcache_wbinv_all(); 1718 cpu_l2cache_wbinv_all(); 1719 cpu_tlb_flushID(); 1720 cpu_cpwait(); 1721 if (vector_page < KERNBASE) { 1722 struct pcb *curpcb = PCPU_GET(curpcb); 1723 pcb = thread0.td_pcb; 1724 if (pmap_is_current(pmap)) { 1725 /* 1726 * Frob the L1 entry corresponding to the vector 1727 * page so that it contains the kernel pmap's domain 1728 * number. This will ensure pmap_remove() does not 1729 * pull the current vector page out from under us. 1730 */ 1731 critical_enter(); 1732 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 1733 cpu_domains(pcb->pcb_dacr); 1734 cpu_setttb(pcb->pcb_pagedir); 1735 critical_exit(); 1736 } 1737 pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); 1738 /* 1739 * Make sure cpu_switch(), et al, DTRT. This is safe to do 1740 * since this process has no remaining mappings of its own. 1741 */ 1742 curpcb->pcb_pl1vec = pcb->pcb_pl1vec; 1743 curpcb->pcb_l1vec = pcb->pcb_l1vec; 1744 curpcb->pcb_dacr = pcb->pcb_dacr; 1745 curpcb->pcb_pagedir = pcb->pcb_pagedir; 1746 1747 } 1748 pmap_free_l1(pmap); 1749 PMAP_LOCK_DESTROY(pmap); 1750 1751 dprintf("pmap_release()\n"); 1752} 1753 1754 1755 1756/* 1757 * Helper function for pmap_grow_l2_bucket() 1758 */ 1759static __inline int 1760pmap_grow_map(vm_offset_t va, pt_entry_t cache_mode, vm_paddr_t *pap) 1761{ 1762 struct l2_bucket *l2b; 1763 pt_entry_t *ptep; 1764 vm_paddr_t pa; 1765 struct vm_page *pg; 1766 1767 pg = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 1768 if (pg == NULL) 1769 return (1); 1770 pa = VM_PAGE_TO_PHYS(pg); 1771 1772 if (pap) 1773 *pap = pa; 1774 1775 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1776 1777 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1778 *ptep = L2_S_PROTO | pa | cache_mode; 1779 pmap_set_prot(ptep, VM_PROT_READ | VM_PROT_WRITE, 0); 1780 PTE_SYNC(ptep); 1781 1782 return (0); 1783} 1784 1785/* 1786 * This is the same as pmap_alloc_l2_bucket(), except that it is only 1787 * used by pmap_growkernel(). 1788 */ 1789static __inline struct l2_bucket * 1790pmap_grow_l2_bucket(pmap_t pm, vm_offset_t va) 1791{ 1792 struct l2_dtable *l2; 1793 struct l2_bucket *l2b; 1794 struct l1_ttable *l1; 1795 pd_entry_t *pl1pd; 1796 u_short l1idx; 1797 vm_offset_t nva; 1798 1799 l1idx = L1_IDX(va); 1800 1801 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 1802 /* 1803 * No mapping at this address, as there is 1804 * no entry in the L1 table. 1805 * Need to allocate a new l2_dtable. 1806 */ 1807 nva = pmap_kernel_l2dtable_kva; 1808 if ((nva & PAGE_MASK) == 0) { 1809 /* 1810 * Need to allocate a backing page 1811 */ 1812 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 1813 return (NULL); 1814 } 1815 1816 l2 = (struct l2_dtable *)nva; 1817 nva += sizeof(struct l2_dtable); 1818 1819 if ((nva & PAGE_MASK) < (pmap_kernel_l2dtable_kva & 1820 PAGE_MASK)) { 1821 /* 1822 * The new l2_dtable straddles a page boundary. 1823 * Map in another page to cover it. 1824 */ 1825 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 1826 return (NULL); 1827 } 1828 1829 pmap_kernel_l2dtable_kva = nva; 1830 1831 /* 1832 * Link it into the parent pmap 1833 */ 1834 pm->pm_l2[L2_IDX(l1idx)] = l2; 1835 memset(l2, 0, sizeof(*l2)); 1836 } 1837 1838 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1839 1840 /* 1841 * Fetch pointer to the L2 page table associated with the address. 1842 */ 1843 if (l2b->l2b_kva == NULL) { 1844 pt_entry_t *ptep; 1845 1846 /* 1847 * No L2 page table has been allocated. Chances are, this 1848 * is because we just allocated the l2_dtable, above. 1849 */ 1850 nva = pmap_kernel_l2ptp_kva; 1851 ptep = (pt_entry_t *)nva; 1852 if ((nva & PAGE_MASK) == 0) { 1853 /* 1854 * Need to allocate a backing page 1855 */ 1856 if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, 1857 &pmap_kernel_l2ptp_phys)) 1858 return (NULL); 1859 } 1860 memset(ptep, 0, L2_TABLE_SIZE_REAL); 1861 l2->l2_occupancy++; 1862 l2b->l2b_kva = ptep; 1863 l2b->l2b_l1idx = l1idx; 1864 l2b->l2b_phys = pmap_kernel_l2ptp_phys; 1865 1866 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 1867 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 1868 } 1869 1870 /* Distribute new L1 entry to all other L1s */ 1871 SLIST_FOREACH(l1, &l1_list, l1_link) { 1872 pl1pd = &l1->l1_kva[L1_IDX(va)]; 1873 *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | 1874 L1_C_PROTO; 1875 PTE_SYNC(pl1pd); 1876 } 1877 1878 return (l2b); 1879} 1880 1881 1882/* 1883 * grow the number of kernel page table entries, if needed 1884 */ 1885void 1886pmap_growkernel(vm_offset_t addr) 1887{ 1888 pmap_t kpm = pmap_kernel(); 1889 1890 if (addr <= pmap_curmaxkvaddr) 1891 return; /* we are OK */ 1892 1893 /* 1894 * whoops! we need to add kernel PTPs 1895 */ 1896 1897 /* Map 1MB at a time */ 1898 for (; pmap_curmaxkvaddr < addr; pmap_curmaxkvaddr += L1_S_SIZE) 1899 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 1900 1901 /* 1902 * flush out the cache, expensive but growkernel will happen so 1903 * rarely 1904 */ 1905 cpu_dcache_wbinv_all(); 1906 cpu_l2cache_wbinv_all(); 1907 cpu_tlb_flushD(); 1908 cpu_cpwait(); 1909 kernel_vm_end = pmap_curmaxkvaddr; 1910} 1911 1912 1913/* 1914 * Remove all pages from specified address space 1915 * this aids process exit speeds. Also, this code 1916 * is special cased for current process only, but 1917 * can have the more generic (and slightly slower) 1918 * mode enabled. This is much faster than pmap_remove 1919 * in the case of running down an entire address space. 1920 */ 1921void 1922pmap_remove_pages(pmap_t pmap) 1923{ 1924 struct pv_entry *pv, *npv; 1925 struct l2_bucket *l2b = NULL; 1926 vm_page_t m; 1927 pt_entry_t *pt; 1928 1929 rw_wlock(&pvh_global_lock); 1930 PMAP_LOCK(pmap); 1931 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 1932 if (pv->pv_flags & PVF_WIRED) { 1933 /* Cannot remove wired pages now. */ 1934 npv = TAILQ_NEXT(pv, pv_plist); 1935 continue; 1936 } 1937 pmap->pm_stats.resident_count--; 1938 l2b = pmap_get_l2_bucket(pmap, pv->pv_va); 1939 KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); 1940 pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1941 m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK); 1942 KASSERT((vm_offset_t)m >= KERNBASE, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); 1943 *pt = 0; 1944 PTE_SYNC(pt); 1945 npv = TAILQ_NEXT(pv, pv_plist); 1946 pmap_nuke_pv(m, pmap, pv); 1947 if (TAILQ_EMPTY(&m->md.pv_list)) 1948 vm_page_aflag_clear(m, PGA_WRITEABLE); 1949 pmap_free_pv_entry(pv); 1950 pmap_free_l2_bucket(pmap, l2b, 1); 1951 } 1952 rw_wunlock(&pvh_global_lock); 1953 cpu_tlb_flushID(); 1954 cpu_cpwait(); 1955 PMAP_UNLOCK(pmap); 1956} 1957 1958 1959/*************************************************** 1960 * Low level mapping routines..... 1961 ***************************************************/ 1962 1963#ifdef ARM_HAVE_SUPERSECTIONS 1964/* Map a super section into the KVA. */ 1965 1966void 1967pmap_kenter_supersection(vm_offset_t va, uint64_t pa, int flags) 1968{ 1969 pd_entry_t pd = L1_S_PROTO | L1_S_SUPERSEC | (pa & L1_SUP_FRAME) | 1970 (((pa >> 32) & 0xf) << 20) | L1_S_PROT(PTE_KERNEL, 1971 VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); 1972 struct l1_ttable *l1; 1973 vm_offset_t va0, va_end; 1974 1975 KASSERT(((va | pa) & L1_SUP_OFFSET) == 0, 1976 ("Not a valid super section mapping")); 1977 if (flags & SECTION_CACHE) 1978 pd |= pte_l1_s_cache_mode; 1979 else if (flags & SECTION_PT) 1980 pd |= pte_l1_s_cache_mode_pt; 1981 1982 va0 = va & L1_SUP_FRAME; 1983 va_end = va + L1_SUP_SIZE; 1984 SLIST_FOREACH(l1, &l1_list, l1_link) { 1985 va = va0; 1986 for (; va < va_end; va += L1_S_SIZE) { 1987 l1->l1_kva[L1_IDX(va)] = pd; 1988 PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); 1989 } 1990 } 1991} 1992#endif 1993 1994/* Map a section into the KVA. */ 1995 1996void 1997pmap_kenter_section(vm_offset_t va, vm_offset_t pa, int flags) 1998{ 1999 pd_entry_t pd = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, 2000 VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); 2001 struct l1_ttable *l1; 2002 2003 KASSERT(((va | pa) & L1_S_OFFSET) == 0, 2004 ("Not a valid section mapping")); 2005 if (flags & SECTION_CACHE) 2006 pd |= pte_l1_s_cache_mode; 2007 else if (flags & SECTION_PT) 2008 pd |= pte_l1_s_cache_mode_pt; 2009 2010 SLIST_FOREACH(l1, &l1_list, l1_link) { 2011 l1->l1_kva[L1_IDX(va)] = pd; 2012 PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); 2013 } 2014} 2015 2016/* 2017 * Make a temporary mapping for a physical address. This is only intended 2018 * to be used for panic dumps. 2019 */ 2020void * 2021pmap_kenter_temp(vm_paddr_t pa, int i) 2022{ 2023 vm_offset_t va; 2024 2025 va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); 2026 pmap_kenter(va, pa); 2027 return ((void *)crashdumpmap); 2028} 2029 2030/* 2031 * add a wired page to the kva 2032 * note that in order for the mapping to take effect -- you 2033 * should do a invltlb after doing the pmap_kenter... 2034 */ 2035static PMAP_INLINE void 2036pmap_kenter_internal(vm_offset_t va, vm_offset_t pa, int flags) 2037{ 2038 struct l2_bucket *l2b; 2039 pt_entry_t *pte; 2040 pt_entry_t opte; 2041 2042 PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", 2043 (uint32_t) va, (uint32_t) pa)); 2044 2045 2046 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2047 if (l2b == NULL) 2048 l2b = pmap_grow_l2_bucket(pmap_kernel(), va); 2049 KASSERT(l2b != NULL, ("No L2 Bucket")); 2050 2051 pte = &l2b->l2b_kva[l2pte_index(va)]; 2052 opte = *pte; 2053 if (l2pte_valid(opte)) { 2054 cpu_tlb_flushD_SE(va); 2055 cpu_cpwait(); 2056 } else { 2057 if (opte == 0) 2058 l2b->l2b_occupancy++; 2059 } 2060 2061 if (flags & KENTER_CACHE) { 2062 *pte = L2_S_PROTO | pa | pte_l2_s_cache_mode; 2063 pmap_set_prot(pte, VM_PROT_READ | VM_PROT_WRITE, 2064 flags & KENTER_USER); 2065 } else { 2066 *pte = L2_S_PROTO | pa; 2067 pmap_set_prot(pte, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 2068 0); 2069 } 2070 2071 PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", 2072 (uint32_t) pte, opte, *pte)); 2073 PTE_SYNC(pte); 2074 cpu_cpwait(); 2075} 2076 2077void 2078pmap_kenter(vm_offset_t va, vm_paddr_t pa) 2079{ 2080 pmap_kenter_internal(va, pa, KENTER_CACHE); 2081} 2082 2083void 2084pmap_kenter_nocache(vm_offset_t va, vm_paddr_t pa) 2085{ 2086 2087 pmap_kenter_internal(va, pa, 0); 2088} 2089 2090void 2091pmap_kenter_user(vm_offset_t va, vm_paddr_t pa) 2092{ 2093 2094 pmap_kenter_internal(va, pa, KENTER_CACHE|KENTER_USER); 2095 /* 2096 * Call pmap_fault_fixup now, to make sure we'll have no exception 2097 * at the first use of the new address, or bad things will happen, 2098 * as we use one of these addresses in the exception handlers. 2099 */ 2100 pmap_fault_fixup(pmap_kernel(), va, VM_PROT_READ|VM_PROT_WRITE, 1); 2101} 2102 2103vm_paddr_t 2104pmap_kextract(vm_offset_t va) 2105{ 2106 2107 return (pmap_extract_locked(kernel_pmap, va)); 2108} 2109 2110/* 2111 * remove a page from the kernel pagetables 2112 */ 2113void 2114pmap_kremove(vm_offset_t va) 2115{ 2116 struct l2_bucket *l2b; 2117 pt_entry_t *pte, opte; 2118 2119 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2120 if (!l2b) 2121 return; 2122 KASSERT(l2b != NULL, ("No L2 Bucket")); 2123 pte = &l2b->l2b_kva[l2pte_index(va)]; 2124 opte = *pte; 2125 if (l2pte_valid(opte)) { 2126 va = va & ~PAGE_MASK; 2127 cpu_tlb_flushD_SE(va); 2128 cpu_cpwait(); 2129 *pte = 0; 2130 PTE_SYNC(pte); 2131 } 2132} 2133 2134 2135/* 2136 * Used to map a range of physical addresses into kernel 2137 * virtual address space. 2138 * 2139 * The value passed in '*virt' is a suggested virtual address for 2140 * the mapping. Architectures which can support a direct-mapped 2141 * physical to virtual region can return the appropriate address 2142 * within that region, leaving '*virt' unchanged. Other 2143 * architectures should map the pages starting at '*virt' and 2144 * update '*virt' with the first usable address after the mapped 2145 * region. 2146 */ 2147vm_offset_t 2148pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 2149{ 2150 vm_offset_t sva = *virt; 2151 vm_offset_t va = sva; 2152 2153 PDEBUG(1, printf("pmap_map: virt = %08x, start = %08x, end = %08x, " 2154 "prot = %d\n", (uint32_t) *virt, (uint32_t) start, (uint32_t) end, 2155 prot)); 2156 2157 while (start < end) { 2158 pmap_kenter(va, start); 2159 va += PAGE_SIZE; 2160 start += PAGE_SIZE; 2161 } 2162 *virt = va; 2163 return (sva); 2164} 2165 2166/* 2167 * Add a list of wired pages to the kva 2168 * this routine is only used for temporary 2169 * kernel mappings that do not need to have 2170 * page modification or references recorded. 2171 * Note that old mappings are simply written 2172 * over. The page *must* be wired. 2173 */ 2174void 2175pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 2176{ 2177 int i; 2178 2179 for (i = 0; i < count; i++) { 2180 pmap_kenter_internal(va, VM_PAGE_TO_PHYS(m[i]), 2181 KENTER_CACHE); 2182 va += PAGE_SIZE; 2183 } 2184} 2185 2186 2187/* 2188 * this routine jerks page mappings from the 2189 * kernel -- it is meant only for temporary mappings. 2190 */ 2191void 2192pmap_qremove(vm_offset_t va, int count) 2193{ 2194 int i; 2195 2196 for (i = 0; i < count; i++) { 2197 if (vtophys(va)) 2198 pmap_kremove(va); 2199 2200 va += PAGE_SIZE; 2201 } 2202} 2203 2204 2205/* 2206 * pmap_object_init_pt preloads the ptes for a given object 2207 * into the specified pmap. This eliminates the blast of soft 2208 * faults on process startup and immediately after an mmap. 2209 */ 2210void 2211pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 2212 vm_pindex_t pindex, vm_size_t size) 2213{ 2214 2215 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2216 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 2217 ("pmap_object_init_pt: non-device object")); 2218} 2219 2220 2221/* 2222 * pmap_is_prefaultable: 2223 * 2224 * Return whether or not the specified virtual address is elgible 2225 * for prefault. 2226 */ 2227boolean_t 2228pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2229{ 2230 pd_entry_t *pde; 2231 pt_entry_t *pte; 2232 2233 if (!pmap_get_pde_pte(pmap, addr, &pde, &pte)) 2234 return (FALSE); 2235 KASSERT(pte != NULL, ("Valid mapping but no pte ?")); 2236 if (*pte == 0) 2237 return (TRUE); 2238 return (FALSE); 2239} 2240 2241/* 2242 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 2243 * Returns TRUE if the mapping exists, else FALSE. 2244 * 2245 * NOTE: This function is only used by a couple of arm-specific modules. 2246 * It is not safe to take any pmap locks here, since we could be right 2247 * in the middle of debugging the pmap anyway... 2248 * 2249 * It is possible for this routine to return FALSE even though a valid 2250 * mapping does exist. This is because we don't lock, so the metadata 2251 * state may be inconsistent. 2252 * 2253 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 2254 * a "section" mapping. 2255 */ 2256boolean_t 2257pmap_get_pde_pte(pmap_t pm, vm_offset_t va, pd_entry_t **pdp, pt_entry_t **ptp) 2258{ 2259 struct l2_dtable *l2; 2260 pd_entry_t *pl1pd, l1pd; 2261 pt_entry_t *ptep; 2262 u_short l1idx; 2263 2264 if (pm->pm_l1 == NULL) 2265 return (FALSE); 2266 2267 l1idx = L1_IDX(va); 2268 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 2269 l1pd = *pl1pd; 2270 2271 if (l1pte_section_p(l1pd)) { 2272 *ptp = NULL; 2273 return (TRUE); 2274 } 2275 2276 if (pm->pm_l2 == NULL) 2277 return (FALSE); 2278 2279 l2 = pm->pm_l2[L2_IDX(l1idx)]; 2280 2281 if (l2 == NULL || 2282 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 2283 return (FALSE); 2284 } 2285 2286 *ptp = &ptep[l2pte_index(va)]; 2287 return (TRUE); 2288} 2289 2290/* 2291 * Routine: pmap_remove_all 2292 * Function: 2293 * Removes this physical page from 2294 * all physical maps in which it resides. 2295 * Reflects back modify bits to the pager. 2296 * 2297 * Notes: 2298 * Original versions of this routine were very 2299 * inefficient because they iteratively called 2300 * pmap_remove (slow...) 2301 */ 2302void 2303pmap_remove_all(vm_page_t m) 2304{ 2305 pv_entry_t pv; 2306 pt_entry_t *ptep; 2307 struct l2_bucket *l2b; 2308 boolean_t flush = FALSE; 2309 pmap_t curpm; 2310 int flags = 0; 2311 2312 KASSERT((m->flags & PG_FICTITIOUS) == 0, 2313 ("pmap_remove_all: page %p is fictitious", m)); 2314 2315 if (TAILQ_EMPTY(&m->md.pv_list)) 2316 return; 2317 rw_wlock(&pvh_global_lock); 2318 curpm = vmspace_pmap(curproc->p_vmspace); 2319 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 2320 if (flush == FALSE && (pv->pv_pmap == curpm || 2321 pv->pv_pmap == pmap_kernel())) 2322 flush = TRUE; 2323 2324 PMAP_LOCK(pv->pv_pmap); 2325 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 2326 KASSERT(l2b != NULL, ("No l2 bucket")); 2327 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2328 if (L2_S_WRITABLE(*ptep)) 2329 vm_page_dirty(m); 2330 *ptep = 0; 2331 if (pmap_is_current(pv->pv_pmap)) 2332 PTE_SYNC(ptep); 2333 pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); 2334 pv->pv_pmap->pm_stats.resident_count--; 2335 flags |= pv->pv_flags; 2336 pmap_nuke_pv(m, pv->pv_pmap, pv); 2337 PMAP_UNLOCK(pv->pv_pmap); 2338 pmap_free_pv_entry(pv); 2339 } 2340 m->md.pvh_attrs &= ~(PVF_MOD | PVF_REF); 2341 2342 if (flush) { 2343 if (PV_BEEN_EXECD(flags)) 2344 cpu_tlb_flushID(); 2345 else 2346 cpu_tlb_flushD(); 2347 } 2348 vm_page_aflag_clear(m, PGA_WRITEABLE); 2349 rw_wunlock(&pvh_global_lock); 2350} 2351 2352int 2353pmap_change_attr(vm_offset_t sva, vm_size_t len, int mode) 2354{ 2355 vm_offset_t base, offset, tmpva; 2356 vm_size_t size; 2357 struct l2_bucket *l2b; 2358 pt_entry_t *ptep, pte; 2359 vm_offset_t next_bucket; 2360 2361 PMAP_LOCK(kernel_pmap); 2362 2363 base = trunc_page(sva); 2364 offset = sva & PAGE_MASK; 2365 size = roundup(offset + len, PAGE_SIZE); 2366 2367#ifdef checkit 2368 /* 2369 * Only supported on kernel virtual addresses, including the direct 2370 * map but excluding the recursive map. 2371 */ 2372 if (base < DMAP_MIN_ADDRESS) { 2373 PMAP_UNLOCK(kernel_pmap); 2374 return (EINVAL); 2375 } 2376#endif 2377 for (tmpva = base; tmpva < base + size; ) { 2378 next_bucket = L2_NEXT_BUCKET(tmpva); 2379 if (next_bucket > base + size) 2380 next_bucket = base + size; 2381 2382 l2b = pmap_get_l2_bucket(kernel_pmap, tmpva); 2383 if (l2b == NULL) { 2384 tmpva = next_bucket; 2385 continue; 2386 } 2387 2388 ptep = &l2b->l2b_kva[l2pte_index(tmpva)]; 2389 2390 if (*ptep == 0) { 2391 PMAP_UNLOCK(kernel_pmap); 2392 return(EINVAL); 2393 } 2394 2395 pte = *ptep &~ L2_S_CACHE_MASK; 2396 cpu_idcache_wbinv_range(tmpva, PAGE_SIZE); 2397 pmap_l2cache_wbinv_range(tmpva, pte & L2_S_FRAME, PAGE_SIZE); 2398 *ptep = pte; 2399 cpu_tlb_flushID_SE(tmpva); 2400 2401 dprintf("%s: for va:%x ptep:%x pte:%x\n", 2402 __func__, tmpva, (uint32_t)ptep, pte); 2403 tmpva += PAGE_SIZE; 2404 } 2405 2406 PMAP_UNLOCK(kernel_pmap); 2407 2408 return (0); 2409} 2410 2411/* 2412 * Set the physical protection on the 2413 * specified range of this map as requested. 2414 */ 2415void 2416pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 2417{ 2418 struct l2_bucket *l2b; 2419 pt_entry_t *ptep, pte; 2420 vm_offset_t next_bucket; 2421 u_int flags; 2422 int flush; 2423 2424 if ((prot & VM_PROT_READ) == 0) { 2425 pmap_remove(pm, sva, eva); 2426 return; 2427 } 2428 2429 if (prot & VM_PROT_WRITE) { 2430 /* 2431 * If this is a read->write transition, just ignore it and let 2432 * vm_fault() take care of it later. 2433 */ 2434 return; 2435 } 2436 2437 rw_wlock(&pvh_global_lock); 2438 PMAP_LOCK(pm); 2439 2440 /* 2441 * OK, at this point, we know we're doing write-protect operation. 2442 * If the pmap is active, write-back the range. 2443 */ 2444 2445 flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; 2446 flags = 0; 2447 2448 while (sva < eva) { 2449 next_bucket = L2_NEXT_BUCKET(sva); 2450 if (next_bucket > eva) 2451 next_bucket = eva; 2452 2453 l2b = pmap_get_l2_bucket(pm, sva); 2454 if (l2b == NULL) { 2455 sva = next_bucket; 2456 continue; 2457 } 2458 2459 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 2460 2461 while (sva < next_bucket) { 2462 if ((pte = *ptep) != 0 && L2_S_WRITABLE(pte)) { 2463 struct vm_page *pg; 2464 u_int f; 2465 2466 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 2467 pmap_set_prot(ptep, prot, !(pm == pmap_kernel())); 2468 PTE_SYNC(ptep); 2469 2470 f = pmap_modify_pv(pg, pm, sva, 2471 PVF_WRITE, 0); 2472 if (f & PVF_WRITE) 2473 vm_page_dirty(pg); 2474 2475 if (flush >= 0) { 2476 flush++; 2477 flags |= f; 2478 } else 2479 if (PV_BEEN_EXECD(f)) 2480 cpu_tlb_flushID_SE(sva); 2481 else 2482 if (PV_BEEN_REFD(f)) 2483 cpu_tlb_flushD_SE(sva); 2484 } 2485 2486 sva += PAGE_SIZE; 2487 ptep++; 2488 } 2489 } 2490 2491 2492 if (flush) { 2493 if (PV_BEEN_EXECD(flags)) 2494 cpu_tlb_flushID(); 2495 else 2496 if (PV_BEEN_REFD(flags)) 2497 cpu_tlb_flushD(); 2498 } 2499 rw_wunlock(&pvh_global_lock); 2500 2501 PMAP_UNLOCK(pm); 2502} 2503 2504 2505/* 2506 * Insert the given physical page (p) at 2507 * the specified virtual address (v) in the 2508 * target physical map with the protection requested. 2509 * 2510 * If specified, the page will be wired down, meaning 2511 * that the related pte can not be reclaimed. 2512 * 2513 * NB: This is the only routine which MAY NOT lazy-evaluate 2514 * or lose information. That is, this routine must actually 2515 * insert this page into the given map NOW. 2516 */ 2517 2518void 2519pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, 2520 vm_prot_t prot, boolean_t wired) 2521{ 2522 2523 rw_wlock(&pvh_global_lock); 2524 PMAP_LOCK(pmap); 2525 pmap_enter_locked(pmap, va, m, prot, wired, M_WAITOK); 2526 PMAP_UNLOCK(pmap); 2527 rw_wunlock(&pvh_global_lock); 2528} 2529 2530/* 2531 * The pvh global and pmap locks must be held. 2532 */ 2533static void 2534pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 2535 boolean_t wired, int flags) 2536{ 2537 struct l2_bucket *l2b = NULL; 2538 struct vm_page *opg; 2539 struct pv_entry *pve = NULL; 2540 pt_entry_t *ptep, npte, opte; 2541 u_int nflags; 2542 u_int oflags; 2543 vm_paddr_t pa; 2544 u_char user; 2545 2546 PMAP_ASSERT_LOCKED(pmap); 2547 rw_assert(&pvh_global_lock, RA_WLOCKED); 2548 if (va == vector_page) { 2549 pa = systempage.pv_pa; 2550 m = NULL; 2551 } else { 2552 KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 || 2553 (flags & M_NOWAIT) != 0, 2554 ("pmap_enter_locked: page %p is not busy", m)); 2555 pa = VM_PAGE_TO_PHYS(m); 2556 } 2557 2558 user = 0; 2559 /* 2560 * Make sure userland mappings get the right permissions 2561 */ 2562 if (pmap != pmap_kernel() && va != vector_page) 2563 user = 1; 2564 2565 nflags = 0; 2566 2567 if (prot & VM_PROT_WRITE) 2568 nflags |= PVF_WRITE; 2569 if (prot & VM_PROT_EXECUTE) 2570 nflags |= PVF_EXEC; 2571 if (wired) 2572 nflags |= PVF_WIRED; 2573 2574 PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " 2575 "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired)); 2576 2577 if (pmap == pmap_kernel()) { 2578 l2b = pmap_get_l2_bucket(pmap, va); 2579 if (l2b == NULL) 2580 l2b = pmap_grow_l2_bucket(pmap, va); 2581 } else { 2582do_l2b_alloc: 2583 l2b = pmap_alloc_l2_bucket(pmap, va); 2584 if (l2b == NULL) { 2585 if (flags & M_WAITOK) { 2586 PMAP_UNLOCK(pmap); 2587 rw_wunlock(&pvh_global_lock); 2588 VM_WAIT; 2589 rw_wlock(&pvh_global_lock); 2590 PMAP_LOCK(pmap); 2591 goto do_l2b_alloc; 2592 } 2593 return; 2594 } 2595 } 2596 2597 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2598 2599 opte = *ptep; 2600 npte = pa; 2601 oflags = 0; 2602 if (opte) { 2603 /* 2604 * There is already a mapping at this address. 2605 * If the physical address is different, lookup the 2606 * vm_page. 2607 */ 2608 if (l2pte_pa(opte) != pa) 2609 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 2610 else 2611 opg = m; 2612 } else 2613 opg = NULL; 2614 2615 if ((prot & (VM_PROT_ALL)) || 2616 (!m || m->md.pvh_attrs & PVF_REF)) { 2617 /* 2618 * - The access type indicates that we don't need 2619 * to do referenced emulation. 2620 * OR 2621 * - The physical page has already been referenced 2622 * so no need to re-do referenced emulation here. 2623 */ 2624 npte |= L2_S_PROTO; 2625#ifdef SMP 2626 npte |= L2_SHARED; 2627#endif 2628 2629 nflags |= PVF_REF; 2630 2631 if (m && ((prot & VM_PROT_WRITE) != 0 || 2632 (m->md.pvh_attrs & PVF_MOD))) { 2633 /* 2634 * This is a writable mapping, and the 2635 * page's mod state indicates it has 2636 * already been modified. Make it 2637 * writable from the outset. 2638 */ 2639 nflags |= PVF_MOD; 2640 if (!(m->md.pvh_attrs & PVF_MOD)) 2641 vm_page_dirty(m); 2642 } 2643 if (m && opte) 2644 vm_page_aflag_set(m, PGA_REFERENCED); 2645 } else { 2646 /* 2647 * Need to do page referenced emulation. 2648 */ 2649 npte |= L2_TYPE_INV; 2650 } 2651 2652 npte |= L2_S_PROT_R; 2653 2654 if (prot & VM_PROT_WRITE) { 2655 npte &= ~(L2_APX); 2656 2657 if (m != NULL && 2658 (m->oflags & VPO_UNMANAGED) == 0) 2659 vm_page_aflag_set(m, PGA_WRITEABLE); 2660 } 2661 2662 if (user) 2663 npte |= L2_S_PROT_U; 2664 2665 2666 if (!(prot & VM_PROT_EXECUTE) && m) 2667 npte |= L2_XN; 2668 2669 if (m->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) 2670 npte |= pte_l2_s_cache_mode; 2671 2672 if (m && m == opg) { 2673 /* 2674 * We're changing the attrs of an existing mapping. 2675 */ 2676 oflags = pmap_modify_pv(m, pmap, va, 2677 PVF_WRITE | PVF_EXEC | PVF_WIRED | 2678 PVF_MOD | PVF_REF, nflags); 2679 } else { 2680 /* 2681 * New mapping, or changing the backing page 2682 * of an existing mapping. 2683 */ 2684 if (opg) { 2685 /* 2686 * Replacing an existing mapping with a new one. 2687 * It is part of our managed memory so we 2688 * must remove it from the PV list 2689 */ 2690 if ((pve = pmap_remove_pv(opg, pmap, va))) { 2691 oflags = pve->pv_flags; 2692 2693 if (m && ((m->oflags & VPO_UNMANAGED))) { 2694 pmap_free_pv_entry(pve); 2695 pve = NULL; 2696 } 2697 } 2698 } 2699 2700 if ((m && !(m->oflags & VPO_UNMANAGED))) { 2701 if ((!pve) && (pve = pmap_get_pv_entry()) == NULL) 2702 panic("pmap_enter: no pv entries"); 2703 2704 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 2705 ("pmap_enter: managed mapping within the clean submap")); 2706 KASSERT(pve != NULL, ("No pv")); 2707 pmap_enter_pv(m, pve, pmap, va, nflags); 2708 } 2709 } 2710 2711 /* 2712 * Keep the stats up to date 2713 */ 2714 if (opte == 0) { 2715 l2b->l2b_occupancy++; 2716 pmap->pm_stats.resident_count++; 2717 } 2718 2719 CTR5(KTR_PMAP,"enter: pmap:%p va:%x prot:%x pte:%x->%x", 2720 pmap, va, prot, opte, npte); 2721 /* 2722 * If this is just a wiring change, the two PTEs will be 2723 * identical, so there's no need to update the page table. 2724 */ 2725 if (npte != opte) { 2726 boolean_t is_cached = pmap_is_current(pmap); 2727 2728 *ptep = npte; 2729 PTE_SYNC(ptep); 2730 if (is_cached) { 2731 /* 2732 * We only need to frob the cache/tlb if this pmap 2733 * is current 2734 */ 2735 if (L1_IDX(va) != L1_IDX(vector_page) && 2736 l2pte_valid(npte)) { 2737 /* 2738 * This mapping is likely to be accessed as 2739 * soon as we return to userland. Fix up the 2740 * L1 entry to avoid taking another 2741 * page/domain fault. 2742 */ 2743 pd_entry_t *pl1pd, l1pd; 2744 2745 pl1pd = &pmap->pm_l1->l1_kva[L1_IDX(va)]; 2746 l1pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | 2747 L1_C_PROTO; 2748 if (*pl1pd != l1pd) { 2749 *pl1pd = l1pd; 2750 PTE_SYNC(pl1pd); 2751 } 2752 } 2753 } 2754 2755 if (PV_BEEN_EXECD(oflags)) 2756 cpu_tlb_flushID_SE(va); 2757 else if (PV_BEEN_REFD(oflags)) 2758 cpu_tlb_flushD_SE(va); 2759 } 2760 2761 if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) 2762 cpu_icache_sync_range(va, PAGE_SIZE); 2763} 2764 2765/* 2766 * Maps a sequence of resident pages belonging to the same object. 2767 * The sequence begins with the given page m_start. This page is 2768 * mapped at the given virtual address start. Each subsequent page is 2769 * mapped at a virtual address that is offset from start by the same 2770 * amount as the page is offset from m_start within the object. The 2771 * last page in the sequence is the page with the largest offset from 2772 * m_start that can be mapped at a virtual address less than the given 2773 * virtual address end. Not every virtual page between start and end 2774 * is mapped; only those for which a resident page exists with the 2775 * corresponding offset from m_start are mapped. 2776 */ 2777void 2778pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2779 vm_page_t m_start, vm_prot_t prot) 2780{ 2781 vm_page_t m; 2782 vm_pindex_t diff, psize; 2783 2784 psize = atop(end - start); 2785 m = m_start; 2786 rw_wlock(&pvh_global_lock); 2787 PMAP_LOCK(pmap); 2788 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2789 pmap_enter_locked(pmap, start + ptoa(diff), m, prot & 2790 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE, M_NOWAIT); 2791 m = TAILQ_NEXT(m, listq); 2792 } 2793 PMAP_UNLOCK(pmap); 2794 rw_wunlock(&pvh_global_lock); 2795} 2796 2797/* 2798 * this code makes some *MAJOR* assumptions: 2799 * 1. Current pmap & pmap exists. 2800 * 2. Not wired. 2801 * 3. Read access. 2802 * 4. No page table pages. 2803 * but is *MUCH* faster than pmap_enter... 2804 */ 2805 2806void 2807pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2808{ 2809 2810 rw_wlock(&pvh_global_lock); 2811 PMAP_LOCK(pmap); 2812 pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 2813 FALSE, M_NOWAIT); 2814 PMAP_UNLOCK(pmap); 2815 rw_wunlock(&pvh_global_lock); 2816} 2817 2818/* 2819 * Routine: pmap_change_wiring 2820 * Function: Change the wiring attribute for a map/virtual-address 2821 * pair. 2822 * In/out conditions: 2823 * The mapping must already exist in the pmap. 2824 */ 2825void 2826pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 2827{ 2828 struct l2_bucket *l2b; 2829 pt_entry_t *ptep, pte; 2830 vm_page_t pg; 2831 2832 rw_wlock(&pvh_global_lock); 2833 PMAP_LOCK(pmap); 2834 l2b = pmap_get_l2_bucket(pmap, va); 2835 KASSERT(l2b, ("No l2b bucket in pmap_change_wiring")); 2836 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2837 pte = *ptep; 2838 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 2839 if (pg) 2840 pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired); 2841 rw_wunlock(&pvh_global_lock); 2842 PMAP_UNLOCK(pmap); 2843} 2844 2845 2846/* 2847 * Copy the range specified by src_addr/len 2848 * from the source map to the range dst_addr/len 2849 * in the destination map. 2850 * 2851 * This routine is only advisory and need not do anything. 2852 */ 2853void 2854pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2855 vm_size_t len, vm_offset_t src_addr) 2856{ 2857} 2858 2859 2860/* 2861 * Routine: pmap_extract 2862 * Function: 2863 * Extract the physical page address associated 2864 * with the given map/virtual_address pair. 2865 */ 2866vm_paddr_t 2867pmap_extract(pmap_t pmap, vm_offset_t va) 2868{ 2869 vm_paddr_t pa; 2870 2871 PMAP_LOCK(pmap); 2872 pa = pmap_extract_locked(pmap, va); 2873 PMAP_UNLOCK(pmap); 2874 return (pa); 2875} 2876 2877static vm_paddr_t 2878pmap_extract_locked(pmap_t pmap, vm_offset_t va) 2879{ 2880 struct l2_dtable *l2; 2881 pd_entry_t l1pd; 2882 pt_entry_t *ptep, pte; 2883 vm_paddr_t pa; 2884 u_int l1idx; 2885 2886 if (pmap != kernel_pmap) 2887 PMAP_ASSERT_LOCKED(pmap); 2888 l1idx = L1_IDX(va); 2889 l1pd = pmap->pm_l1->l1_kva[l1idx]; 2890 if (l1pte_section_p(l1pd)) { 2891 /* 2892 * These should only happen for the kernel pmap. 2893 */ 2894 KASSERT(pmap == kernel_pmap, ("unexpected section")); 2895 /* XXX: what to do about the bits > 32 ? */ 2896 if (l1pd & L1_S_SUPERSEC) 2897 pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); 2898 else 2899 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 2900 } else { 2901 /* 2902 * Note that we can't rely on the validity of the L1 2903 * descriptor as an indication that a mapping exists. 2904 * We have to look it up in the L2 dtable. 2905 */ 2906 l2 = pmap->pm_l2[L2_IDX(l1idx)]; 2907 if (l2 == NULL || 2908 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) 2909 return (0); 2910 pte = ptep[l2pte_index(va)]; 2911 if (pte == 0) 2912 return (0); 2913 switch (pte & L2_TYPE_MASK) { 2914 case L2_TYPE_L: 2915 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 2916 break; 2917 default: 2918 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2919 break; 2920 } 2921 } 2922 return (pa); 2923} 2924 2925/* 2926 * Atomically extract and hold the physical page with the given 2927 * pmap and virtual address pair if that mapping permits the given 2928 * protection. 2929 * 2930 */ 2931vm_page_t 2932pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 2933{ 2934 struct l2_dtable *l2; 2935 pd_entry_t l1pd; 2936 pt_entry_t *ptep, pte; 2937 vm_paddr_t pa, paddr; 2938 vm_page_t m = NULL; 2939 u_int l1idx; 2940 l1idx = L1_IDX(va); 2941 paddr = 0; 2942 2943 PMAP_LOCK(pmap); 2944retry: 2945 l1pd = pmap->pm_l1->l1_kva[l1idx]; 2946 if (l1pte_section_p(l1pd)) { 2947 /* 2948 * These should only happen for pmap_kernel() 2949 */ 2950 KASSERT(pmap == pmap_kernel(), ("huh")); 2951 /* XXX: what to do about the bits > 32 ? */ 2952 if (l1pd & L1_S_SUPERSEC) 2953 pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); 2954 else 2955 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 2956 if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) 2957 goto retry; 2958 if (L1_S_WRITABLE(l1pd) || (prot & VM_PROT_WRITE) == 0) { 2959 m = PHYS_TO_VM_PAGE(pa); 2960 vm_page_hold(m); 2961 } 2962 } else { 2963 /* 2964 * Note that we can't rely on the validity of the L1 2965 * descriptor as an indication that a mapping exists. 2966 * We have to look it up in the L2 dtable. 2967 */ 2968 l2 = pmap->pm_l2[L2_IDX(l1idx)]; 2969 2970 if (l2 == NULL || 2971 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 2972 PMAP_UNLOCK(pmap); 2973 return (NULL); 2974 } 2975 2976 ptep = &ptep[l2pte_index(va)]; 2977 pte = *ptep; 2978 2979 if (pte == 0) { 2980 PMAP_UNLOCK(pmap); 2981 return (NULL); 2982 } else if ((prot & VM_PROT_WRITE) && (pte & L2_APX)) { 2983 PMAP_UNLOCK(pmap); 2984 return (NULL); 2985 } else { 2986 switch (pte & L2_TYPE_MASK) { 2987 case L2_TYPE_L: 2988 panic("extract and hold section mapping"); 2989 break; 2990 default: 2991 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2992 break; 2993 } 2994 if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) 2995 goto retry; 2996 m = PHYS_TO_VM_PAGE(pa); 2997 vm_page_hold(m); 2998 } 2999 3000 } 3001 3002 PMAP_UNLOCK(pmap); 3003 PA_UNLOCK_COND(paddr); 3004 return (m); 3005} 3006 3007/* 3008 * Initialize a preallocated and zeroed pmap structure, 3009 * such as one in a vmspace structure. 3010 */ 3011 3012int 3013pmap_pinit(pmap_t pmap) 3014{ 3015 PDEBUG(1, printf("pmap_pinit: pmap = %08x\n", (uint32_t) pmap)); 3016 3017 PMAP_LOCK_INIT(pmap); 3018 pmap_alloc_l1(pmap); 3019 bzero(pmap->pm_l2, sizeof(pmap->pm_l2)); 3020 3021 CPU_ZERO(&pmap->pm_active); 3022 3023 TAILQ_INIT(&pmap->pm_pvlist); 3024 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 3025 pmap->pm_stats.resident_count = 1; 3026 if (vector_page < KERNBASE) { 3027 pmap_enter(pmap, vector_page, 3028 VM_PROT_READ, PHYS_TO_VM_PAGE(systempage.pv_pa), 3029 VM_PROT_READ, 1); 3030 } 3031 return (1); 3032} 3033 3034 3035/*************************************************** 3036 * page management routines. 3037 ***************************************************/ 3038 3039 3040static void 3041pmap_free_pv_entry(pv_entry_t pv) 3042{ 3043 pv_entry_count--; 3044 uma_zfree(pvzone, pv); 3045} 3046 3047 3048/* 3049 * get a new pv_entry, allocating a block from the system 3050 * when needed. 3051 * the memory allocation is performed bypassing the malloc code 3052 * because of the possibility of allocations at interrupt time. 3053 */ 3054static pv_entry_t 3055pmap_get_pv_entry(void) 3056{ 3057 pv_entry_t ret_value; 3058 3059 pv_entry_count++; 3060 if (pv_entry_count > pv_entry_high_water) 3061 pagedaemon_wakeup(); 3062 ret_value = uma_zalloc(pvzone, M_NOWAIT); 3063 return ret_value; 3064} 3065 3066/* 3067 * Remove the given range of addresses from the specified map. 3068 * 3069 * It is assumed that the start and end are properly 3070 * rounded to the page size. 3071 */ 3072#define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3073void 3074pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 3075{ 3076 struct l2_bucket *l2b; 3077 vm_offset_t next_bucket; 3078 pt_entry_t *ptep; 3079 u_int total; 3080 u_int mappings, is_exec, is_refd; 3081 int flushall = 0; 3082 3083 3084 /* 3085 * we lock in the pmap => pv_head direction 3086 */ 3087 3088 rw_wlock(&pvh_global_lock); 3089 PMAP_LOCK(pm); 3090 total = 0; 3091 while (sva < eva) { 3092 /* 3093 * Do one L2 bucket's worth at a time. 3094 */ 3095 next_bucket = L2_NEXT_BUCKET(sva); 3096 if (next_bucket > eva) 3097 next_bucket = eva; 3098 3099 l2b = pmap_get_l2_bucket(pm, sva); 3100 if (l2b == NULL) { 3101 sva = next_bucket; 3102 continue; 3103 } 3104 3105 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3106 mappings = 0; 3107 3108 while (sva < next_bucket) { 3109 struct vm_page *pg; 3110 pt_entry_t pte; 3111 vm_paddr_t pa; 3112 3113 pte = *ptep; 3114 3115 if (pte == 0) { 3116 /* 3117 * Nothing here, move along 3118 */ 3119 sva += PAGE_SIZE; 3120 ptep++; 3121 continue; 3122 } 3123 3124 pm->pm_stats.resident_count--; 3125 pa = l2pte_pa(pte); 3126 is_exec = 0; 3127 is_refd = 1; 3128 3129 /* 3130 * Update flags. In a number of circumstances, 3131 * we could cluster a lot of these and do a 3132 * number of sequential pages in one go. 3133 */ 3134 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 3135 struct pv_entry *pve; 3136 3137 pve = pmap_remove_pv(pg, pm, sva); 3138 if (pve) { 3139 is_exec = PV_BEEN_EXECD(pve->pv_flags); 3140 is_refd = PV_BEEN_REFD(pve->pv_flags); 3141 pmap_free_pv_entry(pve); 3142 } 3143 } 3144 3145 if (pmap_is_current(pm)) { 3146 total++; 3147 if (total < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3148 if (is_exec) 3149 cpu_tlb_flushID_SE(sva); 3150 else if (is_refd) 3151 cpu_tlb_flushD_SE(sva); 3152 } else if (total == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3153 flushall = 1; 3154 } 3155 } 3156 *ptep = 0; 3157 PTE_SYNC(ptep); 3158 3159 sva += PAGE_SIZE; 3160 ptep++; 3161 mappings++; 3162 } 3163 3164 pmap_free_l2_bucket(pm, l2b, mappings); 3165 } 3166 3167 rw_wunlock(&pvh_global_lock); 3168 if (flushall) 3169 cpu_tlb_flushID(); 3170 PMAP_UNLOCK(pm); 3171} 3172 3173/* 3174 * pmap_zero_page() 3175 * 3176 * Zero a given physical page by mapping it at a page hook point. 3177 * In doing the zero page op, the page we zero is mapped cachable, as with 3178 * StrongARM accesses to non-cached pages are non-burst making writing 3179 * _any_ bulk data very slow. 3180 */ 3181static void 3182pmap_zero_page_gen(vm_page_t pg, int off, int size) 3183{ 3184 3185 vm_paddr_t phys = VM_PAGE_TO_PHYS(pg); 3186 if (!TAILQ_EMPTY(&pg->md.pv_list)) 3187 panic("pmap_zero_page: page has mappings"); 3188 3189 mtx_lock(&cmtx); 3190 /* 3191 * Hook in the page, zero it, invalidate the TLB as needed. 3192 * 3193 * Note the temporary zero-page mapping must be a non-cached page in 3194 * order to work without corruption when write-allocate is enabled. 3195 */ 3196 *cdst_pte = L2_S_PROTO | phys | pte_l2_s_cache_mode; 3197 pmap_set_prot(cdst_pte, VM_PROT_WRITE, 0); 3198 PTE_SYNC(cdst_pte); 3199 cpu_tlb_flushD_SE(cdstp); 3200 cpu_cpwait(); 3201 if (off || size != PAGE_SIZE) 3202 bzero((void *)(cdstp + off), size); 3203 else 3204 bzero_page(cdstp); 3205 3206 /* 3207 * Although aliasing is not possible if we use 3208 * cdstp temporary mappings with memory that 3209 * will be mapped later as non-cached or with write-through 3210 * caches we might end up overwriting it when calling wbinv_all 3211 * So make sure caches are clean after copy operation 3212 */ 3213 cpu_idcache_wbinv_range(cdstp, size); 3214 pmap_l2cache_wbinv_range(cdstp, phys, size); 3215 3216 mtx_unlock(&cmtx); 3217} 3218 3219/* 3220 * pmap_zero_page zeros the specified hardware page by mapping 3221 * the page into KVM and using bzero to clear its contents. 3222 */ 3223void 3224pmap_zero_page(vm_page_t m) 3225{ 3226 pmap_zero_page_gen(m, 0, PAGE_SIZE); 3227} 3228 3229 3230/* 3231 * pmap_zero_page_area zeros the specified hardware page by mapping 3232 * the page into KVM and using bzero to clear its contents. 3233 * 3234 * off and size may not cover an area beyond a single hardware page. 3235 */ 3236void 3237pmap_zero_page_area(vm_page_t m, int off, int size) 3238{ 3239 3240 pmap_zero_page_gen(m, off, size); 3241} 3242 3243 3244/* 3245 * pmap_zero_page_idle zeros the specified hardware page by mapping 3246 * the page into KVM and using bzero to clear its contents. This 3247 * is intended to be called from the vm_pagezero process only and 3248 * outside of Giant. 3249 */ 3250void 3251pmap_zero_page_idle(vm_page_t m) 3252{ 3253 3254 pmap_zero_page(m); 3255} 3256 3257/* 3258 * pmap_copy_page copies the specified (machine independent) 3259 * page by mapping the page into virtual memory and using 3260 * bcopy to copy the page, one machine dependent page at a 3261 * time. 3262 */ 3263 3264/* 3265 * pmap_copy_page() 3266 * 3267 * Copy one physical page into another, by mapping the pages into 3268 * hook points. The same comment regarding cachability as in 3269 * pmap_zero_page also applies here. 3270 */ 3271void 3272pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) 3273{ 3274 /* 3275 * Hold the source page's lock for the duration of the copy 3276 * so that no other mappings can be created while we have a 3277 * potentially aliased mapping. 3278 * Map the pages into the page hook points, copy them, and purge 3279 * the cache for the appropriate page. Invalidate the TLB 3280 * as required. 3281 */ 3282 mtx_lock(&cmtx); 3283 3284 /* For ARMv6 using System bit is deprecated and mapping with AP 3285 * bits set to 0x0 makes page not accessible. csrc_pte is mapped 3286 * read/write until proper mapping defines are created for ARMv6. 3287 */ 3288 *csrc_pte = L2_S_PROTO | src | pte_l2_s_cache_mode; 3289 pmap_set_prot(csrc_pte, VM_PROT_READ, 0); 3290 PTE_SYNC(csrc_pte); 3291 3292 *cdst_pte = L2_S_PROTO | dst | pte_l2_s_cache_mode; 3293 pmap_set_prot(cdst_pte, VM_PROT_READ | VM_PROT_WRITE, 0); 3294 PTE_SYNC(cdst_pte); 3295 3296 cpu_tlb_flushD_SE(csrcp); 3297 cpu_tlb_flushD_SE(cdstp); 3298 cpu_cpwait(); 3299 3300 /* 3301 * Although aliasing is not possible if we use 3302 * cdstp temporary mappings with memory that 3303 * will be mapped later as non-cached or with write-through 3304 * caches we might end up overwriting it when calling wbinv_all 3305 * So make sure caches are clean after copy operation 3306 */ 3307 bcopy_page(csrcp, cdstp); 3308 3309 cpu_idcache_wbinv_range(cdstp, PAGE_SIZE); 3310 pmap_l2cache_wbinv_range(cdstp, dst, PAGE_SIZE); 3311 3312 mtx_unlock(&cmtx); 3313} 3314 3315void 3316pmap_copy_page(vm_page_t src, vm_page_t dst) 3317{ 3318 3319 if (_arm_memcpy && PAGE_SIZE >= _min_memcpy_size && 3320 _arm_memcpy((void *)VM_PAGE_TO_PHYS(dst), 3321 (void *)VM_PAGE_TO_PHYS(src), PAGE_SIZE, IS_PHYSICAL) == 0) 3322 return; 3323 3324 pmap_copy_page_generic(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); 3325} 3326 3327/* 3328 * this routine returns true if a physical page resides 3329 * in the given pmap. 3330 */ 3331boolean_t 3332pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 3333{ 3334 pv_entry_t pv; 3335 int loops = 0; 3336 boolean_t rv; 3337 3338 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3339 ("pmap_page_exists_quick: page %p is not managed", m)); 3340 rv = FALSE; 3341 rw_wlock(&pvh_global_lock); 3342 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 3343 if (pv->pv_pmap == pmap) { 3344 rv = TRUE; 3345 break; 3346 } 3347 loops++; 3348 if (loops >= 16) 3349 break; 3350 } 3351 3352 rw_wunlock(&pvh_global_lock); 3353 return (rv); 3354} 3355 3356/* 3357 * pmap_page_wired_mappings: 3358 * 3359 * Return the number of managed mappings to the given physical page 3360 * that are wired. 3361 */ 3362int 3363pmap_page_wired_mappings(vm_page_t m) 3364{ 3365 pv_entry_t pv; 3366 int count; 3367 3368 count = 0; 3369 if ((m->flags & PG_FICTITIOUS) != 0) 3370 return (count); 3371 rw_wlock(&pvh_global_lock); 3372 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) 3373 if ((pv->pv_flags & PVF_WIRED) != 0) 3374 count++; 3375 rw_wunlock(&pvh_global_lock); 3376 return (count); 3377} 3378 3379/* 3380 * pmap_is_referenced: 3381 * 3382 * Return whether or not the specified physical page was referenced 3383 * in any physical maps. 3384 */ 3385boolean_t 3386pmap_is_referenced(vm_page_t m) 3387{ 3388 3389 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3390 ("pmap_is_referenced: page %p is not managed", m)); 3391 return ((m->md.pvh_attrs & PVF_REF) != 0); 3392} 3393 3394/* 3395 * pmap_ts_referenced: 3396 * 3397 * Return the count of reference bits for a page, clearing all of them. 3398 */ 3399int 3400pmap_ts_referenced(vm_page_t m) 3401{ 3402 3403 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3404 ("pmap_ts_referenced: page %p is not managed", m)); 3405 return (pmap_clearbit(m, PVF_REF)); 3406} 3407 3408 3409boolean_t 3410pmap_is_modified(vm_page_t m) 3411{ 3412 3413 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3414 ("pmap_is_modified: page %p is not managed", m)); 3415 if (m->md.pvh_attrs & PVF_MOD) 3416 return (TRUE); 3417 3418 return(FALSE); 3419} 3420 3421 3422/* 3423 * Clear the modify bits on the specified physical page. 3424 */ 3425void 3426pmap_clear_modify(vm_page_t m) 3427{ 3428 3429 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3430 ("pmap_clear_modify: page %p is not managed", m)); 3431 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 3432 KASSERT((m->oflags & VPO_BUSY) == 0, 3433 ("pmap_clear_modify: page %p is busy", m)); 3434 3435 /* 3436 * If the page is not PGA_WRITEABLE, then no mappings can be modified. 3437 * If the object containing the page is locked and the page is not 3438 * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set. 3439 */ 3440 if ((m->aflags & PGA_WRITEABLE) == 0) 3441 return; 3442 3443 if (m->md.pvh_attrs & PVF_MOD) 3444 pmap_clearbit(m, PVF_MOD); 3445} 3446 3447 3448/* 3449 * pmap_clear_reference: 3450 * 3451 * Clear the reference bit on the specified physical page. 3452 */ 3453void 3454pmap_clear_reference(vm_page_t m) 3455{ 3456 3457 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3458 ("pmap_clear_reference: page %p is not managed", m)); 3459 if (m->md.pvh_attrs & PVF_REF) 3460 pmap_clearbit(m, PVF_REF); 3461} 3462 3463 3464/* 3465 * Clear the write and modified bits in each of the given page's mappings. 3466 */ 3467void 3468pmap_remove_write(vm_page_t m) 3469{ 3470 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3471 ("pmap_remove_write: page %p is not managed", m)); 3472 3473 /* 3474 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by 3475 * another thread while the object is locked. Thus, if PGA_WRITEABLE 3476 * is clear, no page table entries need updating. 3477 */ 3478 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 3479 if ((m->oflags & VPO_BUSY) != 0 || 3480 (m->aflags & PGA_WRITEABLE) != 0) 3481 pmap_clearbit(m, PVF_WRITE); 3482} 3483 3484 3485/* 3486 * perform the pmap work for mincore 3487 */ 3488int 3489pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 3490{ 3491 struct l2_bucket *l2b; 3492 pt_entry_t *ptep, pte; 3493 vm_paddr_t pa; 3494 vm_page_t m; 3495 int val; 3496 boolean_t managed; 3497 3498 PMAP_LOCK(pmap); 3499retry: 3500 l2b = pmap_get_l2_bucket(pmap, addr); 3501 if (l2b == NULL) { 3502 val = 0; 3503 goto out; 3504 } 3505 ptep = &l2b->l2b_kva[l2pte_index(addr)]; 3506 pte = *ptep; 3507 if (!l2pte_valid(pte)) { 3508 val = 0; 3509 goto out; 3510 } 3511 val = MINCORE_INCORE; 3512 if (L2_S_WRITABLE(pte)) 3513 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 3514 managed = FALSE; 3515 pa = l2pte_pa(pte); 3516 m = PHYS_TO_VM_PAGE(pa); 3517 if (m != NULL && (m->oflags & VPO_UNMANAGED) == 0) 3518 managed = TRUE; 3519 if (managed) { 3520 /* 3521 * The ARM pmap tries to maintain a per-mapping 3522 * reference bit. The trouble is that it's kept in 3523 * the PV entry, not the PTE, so it's costly to access 3524 * here. You would need to acquire the pvh global 3525 * lock, call pmap_find_pv(), and introduce a custom 3526 * version of vm_page_pa_tryrelock() that releases and 3527 * reacquires the pvh global lock. In the end, I 3528 * doubt it's worthwhile. This may falsely report 3529 * the given address as referenced. 3530 */ 3531 if ((m->md.pvh_attrs & PVF_REF) != 0) 3532 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 3533 } 3534 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 3535 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { 3536 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ 3537 if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) 3538 goto retry; 3539 } else 3540out: 3541 PA_UNLOCK_COND(*locked_pa); 3542 PMAP_UNLOCK(pmap); 3543 return (val); 3544} 3545 3546void 3547pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 3548{ 3549} 3550 3551/* 3552 * Increase the starting virtual address of the given mapping if a 3553 * different alignment might result in more superpage mappings. 3554 */ 3555void 3556pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 3557 vm_offset_t *addr, vm_size_t size) 3558{ 3559} 3560 3561 3562/* 3563 * Map a set of physical memory pages into the kernel virtual 3564 * address space. Return a pointer to where it is mapped. This 3565 * routine is intended to be used for mapping device memory, 3566 * NOT real memory. 3567 */ 3568void * 3569pmap_mapdev(vm_offset_t pa, vm_size_t size) 3570{ 3571 vm_offset_t va, tmpva, offset; 3572 3573 offset = pa & PAGE_MASK; 3574 size = roundup(size, PAGE_SIZE); 3575 3576 GIANT_REQUIRED; 3577 3578 va = kmem_alloc_nofault(kernel_map, size); 3579 if (!va) 3580 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 3581 for (tmpva = va; size > 0;) { 3582 pmap_kenter_internal(tmpva, pa, 0); 3583 size -= PAGE_SIZE; 3584 tmpva += PAGE_SIZE; 3585 pa += PAGE_SIZE; 3586 } 3587 3588 return ((void *)(va + offset)); 3589} 3590 3591/* 3592 * pmap_map_section: 3593 * 3594 * Create a single section mapping. 3595 */ 3596void 3597pmap_map_section(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 3598 int prot, int cache) 3599{ 3600 pd_entry_t *pde = (pd_entry_t *) l1pt; 3601 pd_entry_t fl; 3602 3603 KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("ouin2")); 3604 3605 fl = l1_mem_types[cache]; 3606 3607 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3608 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 3609 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 3610} 3611 3612/* 3613 * pmap_link_l2pt: 3614 * 3615 * Link the L2 page table specified by l2pv.pv_pa into the L1 3616 * page table at the slot for "va". 3617 */ 3618void 3619pmap_link_l2pt(vm_offset_t l1pt, vm_offset_t va, struct pv_addr *l2pv) 3620{ 3621 pd_entry_t *pde = (pd_entry_t *) l1pt, proto; 3622 u_int slot = va >> L1_S_SHIFT; 3623 3624 proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; 3625 3626#ifdef VERBOSE_INIT_ARM 3627 printf("pmap_link_l2pt: pa=0x%x va=0x%x\n", l2pv->pv_pa, l2pv->pv_va); 3628#endif 3629 3630 pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); 3631 PTE_SYNC(&pde[slot]); 3632 3633 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3634 3635} 3636 3637/* 3638 * pmap_map_entry 3639 * 3640 * Create a single page mapping. 3641 */ 3642void 3643pmap_map_entry(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, 3644 int cache) 3645{ 3646 pd_entry_t *pde = (pd_entry_t *) l1pt; 3647 pt_entry_t fl; 3648 pt_entry_t *pte; 3649 3650 KASSERT(((va | pa) & PAGE_MASK) == 0, ("ouin")); 3651 3652 fl = l2s_mem_types[cache]; 3653 3654 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3655 panic("pmap_map_entry: no L2 table for VA 0x%08x", va); 3656 3657 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 3658 3659 if (pte == NULL) 3660 panic("pmap_map_entry: can't find L2 table for VA 0x%08x", va); 3661 3662 pte[l2pte_index(va)] = L2_S_PROTO | pa | fl; 3663 pmap_set_prot(&pte[l2pte_index(va)], prot, 0); 3664 PTE_SYNC(&pte[l2pte_index(va)]); 3665} 3666 3667/* 3668 * pmap_map_chunk: 3669 * 3670 * Map a chunk of memory using the most efficient mappings 3671 * possible (section. large page, small page) into the 3672 * provided L1 and L2 tables at the specified virtual address. 3673 */ 3674vm_size_t 3675pmap_map_chunk(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 3676 vm_size_t size, int prot, int type) 3677{ 3678 pd_entry_t *pde = (pd_entry_t *) l1pt; 3679 pt_entry_t *pte, f1, f2s, f2l; 3680 vm_size_t resid; 3681 int i; 3682 3683 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 3684 3685 if (l1pt == 0) 3686 panic("pmap_map_chunk: no L1 table provided"); 3687 3688#ifdef VERBOSE_INIT_ARM 3689 printf("pmap_map_chunk: pa=0x%x va=0x%x size=0x%x resid=0x%x " 3690 "prot=0x%x type=%d\n", pa, va, size, resid, prot, type); 3691#endif 3692 3693 f1 = l1_mem_types[type]; 3694 f2l = l2l_mem_types[type]; 3695 f2s = l2s_mem_types[type]; 3696 3697 size = resid; 3698 3699 while (resid > 0) { 3700 /* See if we can use a section mapping. */ 3701 if (L1_S_MAPPABLE_P(va, pa, resid)) { 3702#ifdef VERBOSE_INIT_ARM 3703 printf("S"); 3704#endif 3705 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3706 L1_S_PROT(PTE_KERNEL, prot) | f1 | 3707 L1_S_DOM(PMAP_DOMAIN_KERNEL); 3708 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 3709 va += L1_S_SIZE; 3710 pa += L1_S_SIZE; 3711 resid -= L1_S_SIZE; 3712 continue; 3713 } 3714 3715 /* 3716 * Ok, we're going to use an L2 table. Make sure 3717 * one is actually in the corresponding L1 slot 3718 * for the current VA. 3719 */ 3720 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3721 panic("pmap_map_chunk: no L2 table for VA 0x%08x", va); 3722 3723 pte = (pt_entry_t *) kernel_pt_lookup( 3724 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 3725 if (pte == NULL) 3726 panic("pmap_map_chunk: can't find L2 table for VA" 3727 "0x%08x", va); 3728 /* See if we can use a L2 large page mapping. */ 3729 if (L2_L_MAPPABLE_P(va, pa, resid)) { 3730#ifdef VERBOSE_INIT_ARM 3731 printf("L"); 3732#endif 3733 for (i = 0; i < 16; i++) { 3734 pte[l2pte_index(va) + i] = 3735 L2_L_PROTO | pa | 3736 L2_L_PROT(PTE_KERNEL, prot) | f2l; 3737 PTE_SYNC(&pte[l2pte_index(va) + i]); 3738 } 3739 va += L2_L_SIZE; 3740 pa += L2_L_SIZE; 3741 resid -= L2_L_SIZE; 3742 continue; 3743 } 3744 3745 /* Use a small page mapping. */ 3746#ifdef VERBOSE_INIT_ARM 3747 printf("P"); 3748#endif 3749 pte[l2pte_index(va)] = L2_S_PROTO | pa | f2s; 3750 pmap_set_prot(&pte[l2pte_index(va)], prot, 0); 3751 PTE_SYNC(&pte[l2pte_index(va)]); 3752 va += PAGE_SIZE; 3753 pa += PAGE_SIZE; 3754 resid -= PAGE_SIZE; 3755 } 3756#ifdef VERBOSE_INIT_ARM 3757 printf("\n"); 3758#endif 3759 return (size); 3760 3761} 3762 3763/********************** Static device map routines ***************************/ 3764 3765static const struct pmap_devmap *pmap_devmap_table; 3766 3767/* 3768 * Register the devmap table. This is provided in case early console 3769 * initialization needs to register mappings created by bootstrap code 3770 * before pmap_devmap_bootstrap() is called. 3771 */ 3772void 3773pmap_devmap_register(const struct pmap_devmap *table) 3774{ 3775 3776 pmap_devmap_table = table; 3777} 3778 3779/* 3780 * Map all of the static regions in the devmap table, and remember 3781 * the devmap table so other parts of the kernel can look up entries 3782 * later. 3783 */ 3784void 3785pmap_devmap_bootstrap(vm_offset_t l1pt, const struct pmap_devmap *table) 3786{ 3787 int i; 3788 3789 pmap_devmap_table = table; 3790 3791 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 3792#ifdef VERBOSE_INIT_ARM 3793 printf("devmap: %08x -> %08x @ %08x\n", 3794 pmap_devmap_table[i].pd_pa, 3795 pmap_devmap_table[i].pd_pa + 3796 pmap_devmap_table[i].pd_size - 1, 3797 pmap_devmap_table[i].pd_va); 3798#endif 3799 pmap_map_chunk(l1pt, pmap_devmap_table[i].pd_va, 3800 pmap_devmap_table[i].pd_pa, 3801 pmap_devmap_table[i].pd_size, 3802 pmap_devmap_table[i].pd_prot, 3803 pmap_devmap_table[i].pd_cache); 3804 } 3805} 3806 3807const struct pmap_devmap * 3808pmap_devmap_find_pa(vm_paddr_t pa, vm_size_t size) 3809{ 3810 int i; 3811 3812 if (pmap_devmap_table == NULL) 3813 return (NULL); 3814 3815 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 3816 if (pa >= pmap_devmap_table[i].pd_pa && 3817 pa + size <= pmap_devmap_table[i].pd_pa + 3818 pmap_devmap_table[i].pd_size) 3819 return (&pmap_devmap_table[i]); 3820 } 3821 3822 return (NULL); 3823} 3824 3825const struct pmap_devmap * 3826pmap_devmap_find_va(vm_offset_t va, vm_size_t size) 3827{ 3828 int i; 3829 3830 if (pmap_devmap_table == NULL) 3831 return (NULL); 3832 3833 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 3834 if (va >= pmap_devmap_table[i].pd_va && 3835 va + size <= pmap_devmap_table[i].pd_va + 3836 pmap_devmap_table[i].pd_size) 3837 return (&pmap_devmap_table[i]); 3838 } 3839 3840 return (NULL); 3841} 3842 3843int 3844pmap_dmap_iscurrent(pmap_t pmap) 3845{ 3846 return(pmap_is_current(pmap)); 3847} 3848 3849void 3850pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 3851{ 3852 /* 3853 * Remember the memattr in a field that gets used to set the appropriate 3854 * bits in the PTEs as mappings are established. 3855 */ 3856 m->md.pv_memattr = ma; 3857 3858 /* 3859 * It appears that this function can only be called before any mappings 3860 * for the page are established on ARM. If this ever changes, this code 3861 * will need to walk the pv_list and make each of the existing mappings 3862 * uncacheable, being careful to sync caches and PTEs (and maybe 3863 * invalidate TLB?) for any current mapping it modifies. 3864 */ 3865 if (m->md.pv_kva != 0 || TAILQ_FIRST(&m->md.pv_list) != NULL) 3866 panic("Can't change memattr on page with existing mappings"); 3867}
|