/linux-master/arch/powerpc/mm/nohash/ |
H A D | book3e_pgtable.c | 78 pud_t *pudp; local 86 pudp = pud_alloc(&init_mm, p4dp, ea); 87 if (!pudp) 89 pmdp = pmd_alloc(&init_mm, pudp, ea); 99 pudp = early_alloc_pgtable(PUD_TABLE_SIZE); 100 p4d_populate(&init_mm, p4dp, pudp); 102 pudp = pud_offset(p4dp, ea); 103 if (pud_none(*pudp)) { 105 pud_populate(&init_mm, pudp, pmdp); 107 pmdp = pmd_offset(pudp, e [all...] |
/linux-master/arch/arc/include/asm/ |
H A D | pgalloc.h | 75 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) argument 77 set_p4d(p4dp, __p4d((unsigned long)pudp)); 86 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) argument 88 set_pud(pudp, __pud((unsigned long)pmdp));
|
/linux-master/arch/arm64/include/asm/ |
H A D | pgalloc.h | 24 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) argument 26 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); 29 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) argument 34 __pud_populate(pudp, __pa(pmdp), pudval); 37 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) argument 45 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) argument 48 set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); 51 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) argument 56 __p4d_populate(p4dp, __pa(pudp), p4dval); 66 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_ argument [all...] |
H A D | tlb.h | 101 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, argument 104 struct ptdesc *ptdesc = virt_to_ptdesc(pudp);
|
/linux-master/arch/powerpc/mm/kasan/ |
H A D | init_book3e_64.c | 36 pud_t *pudp; local 43 pudp = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); 44 memcpy(pudp, kasan_early_shadow_pud, PUD_TABLE_SIZE); 45 p4d_populate(&init_mm, p4dp, pudp); 47 pudp = pud_offset(p4dp, ea); 48 if (kasan_pmd_table(*pudp)) { 51 pud_populate(&init_mm, pudp, pmdp); 53 pmdp = pmd_offset(pudp, ea);
|
/linux-master/arch/x86/include/asm/ |
H A D | pgtable-3level.h | 53 static inline void native_set_pud(pud_t *pudp, pud_t pud) argument 56 pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd); 58 pxx_xchg64(pud, pudp, native_pud_val(pud)); 81 static inline void native_pud_clear(pud_t *pudp) argument 85 static inline void pud_clear(pud_t *pudp) argument 87 set_pud(pudp, __pud(0)); 113 static inline pud_t native_pudp_get_and_clear(pud_t *pudp) argument 115 return pxx_xchg64(pud, pudp, 0ULL);
|
/linux-master/arch/arm64/mm/ |
H A D | fixmap.c | 54 static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, argument 58 pud_t pud = READ_ONCE(*pudp); 62 __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); 64 pmdp = pmd_offset_kimg(pudp, addr); 76 pud_t *pudp; local 91 pudp = pud_offset_kimg(p4dp, addr); 92 early_fixmap_init_pmd(pudp, addr, end);
|
H A D | kasan_init.c | 76 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, argument 79 if (pud_none(READ_ONCE(*pudp))) { 83 __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); 86 return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); 132 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, argument 136 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); 148 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); local 152 kasan_pmd_populate(pudp, addr, next, node, early); 153 } while (pudp 260 pud_t *pudp = (pud_t *)__phys_to_kimg(__pgd_to_phys(pgd)); local 284 pud_t *pudp = (pud_t *)__phys_to_kimg(__pgd_to_phys(pgd)); local [all...] |
H A D | mmu.c | 238 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, argument 245 pmdp = pmd_set_fixmap_offset(pudp, addr); 275 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, argument 281 pud_t pud = READ_ONCE(*pudp); 295 __pud_populate(pudp, pmd_phys, pudval); 296 pud = READ_ONCE(*pudp); 310 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); 323 pud_t *pudp; local 338 pudp = pud_set_fixmap_offset(p4dp, addr); 340 pud_t old_pud = READ_ONCE(*pudp); 860 unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, unsigned long end, bool free_mapped, struct vmem_altmap *altmap) argument 898 pud_t *pudp, pud; local 1008 free_empty_pmd_table(pud_t *pudp, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling) argument 1052 pud_t *pudp, pud; local 1182 pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot) argument 1216 pud_clear_huge(pud_t *pudp) argument 1251 pud_free_pmd_page(pud_t *pudp, unsigned long addr) argument [all...] |
H A D | hugetlbpage.c | 101 pud_t *pudp; local 106 pudp = pud_offset(p4dp, addr); 107 pmdp = pmd_offset(pudp, addr); 274 pud_t *pudp; local 283 pudp = pud_alloc(mm, p4dp, addr); 284 if (!pudp) 288 ptep = (pte_t *)pudp; 290 pmdp = pmd_alloc(mm, pudp, addr); 297 if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp))) 298 ptep = huge_pmd_share(mm, vma, addr, pudp); 315 pud_t *pudp, pud; local [all...] |
H A D | pageattr.c | 217 pud_t *pudp, pud; local 230 pudp = pud_offset(p4dp, addr); 231 pud = READ_ONCE(*pudp); 237 pmdp = pmd_offset(pudp, addr);
|
/linux-master/arch/s390/mm/ |
H A D | pageattr.c | 175 static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, argument 183 pmdp = pmd_offset(pudp, addr); 211 static int split_pud_page(pud_t *pudp, unsigned long addr) argument 221 pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; 222 ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); 223 nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); 234 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); 240 static void modify_pud_page(pud_t *pudp, unsigned long addr, argument 243 pud_t new = *pudp; 261 pgt_set((unsigned long *)pudp, pud_va 269 pud_t *pudp; local [all...] |
H A D | hugetlbpage.c | 182 pud_t *pudp = (pud_t *) ptep; local 185 pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY)); 196 pud_t *pudp; local 202 pudp = pud_alloc(mm, p4dp, addr); 203 if (pudp) { 205 return (pte_t *) pudp; 207 pmdp = pmd_alloc(mm, pudp, addr); 218 pud_t *pudp; local 225 pudp = pud_offset(p4dp, addr); 226 if (pud_present(*pudp)) { [all...] |
/linux-master/include/linux/ |
H A D | page_table_check.h | 23 void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud); 86 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, argument 92 __page_table_check_pud_set(mm, pudp, pud); 137 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, argument
|
/linux-master/arch/powerpc/mm/book3s64/ |
H A D | pgtable.c | 69 pud_t *pudp, pud_t entry, int dirty) 73 WARN_ON(!pud_devmap(*pudp)); 74 assert_spin_locked(pud_lockptr(vma->vm_mm, pudp)); 76 changed = !pud_same(*(pudp), entry); 82 __ptep_set_access_flags(vma, pudp_ptep(pudp), 96 unsigned long address, pud_t *pudp) 98 return __pudp_test_and_clear_young(vma->vm_mm, address, pudp); 123 pud_t *pudp, pud_t pud) 131 WARN_ON(pte_hw_valid(pud_pte(*pudp))); 132 assert_spin_locked(pud_lockptr(mm, pudp)); 68 pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t entry, int dirty) argument 95 pudp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pud_t *pudp) argument 122 set_pud_at(struct mm_struct *mm, unsigned long addr, pud_t *pudp, pud_t pud) argument 196 pudp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long addr, pud_t *pudp, int full) argument [all...] |
H A D | radix_pgtable.c | 78 pud_t *pudp; local 85 pudp = early_alloc_pgtable(PAGE_SIZE, nid, 87 p4d_populate(&init_mm, p4dp, pudp); 89 pudp = pud_offset(p4dp, ea); 91 ptep = (pte_t *)pudp; 94 if (pud_none(*pudp)) { 97 pud_populate(&init_mm, pudp, pmdp); 99 pmdp = pmd_offset(pudp, ea); 130 pud_t *pudp; local 153 pudp 191 pud_t *pudp; local 1014 vmemmap_pmd_alloc(pud_t *pudp, int node, unsigned long address) argument 1361 radix__pud_hugepage_update(struct mm_struct *mm, unsigned long addr, pud_t *pudp, unsigned long clr, unsigned long set) argument 1454 radix__pudp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, pud_t *pudp) argument [all...] |
/linux-master/arch/loongarch/mm/ |
H A D | kasan_init.c | 119 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, bool early) argument 121 if (__pud_none(early, READ_ONCE(*pudp))) { 126 pud_populate(&init_mm, pudp, (pmd_t *)__va(pmd_phys)); 129 return pmd_offset(pudp, addr); 160 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, argument 164 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); 176 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); local 180 kasan_pmd_populate(pudp, addr, next, node, early); 181 } while (pudp++, addr = next, addr != end);
|
/linux-master/arch/riscv/mm/ |
H A D | kasan_init.c | 84 pud_t *pudp, *p; local 92 pudp = pud_offset(p4d, vaddr); 97 if (pud_none(pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) && 101 set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL)); 107 kasan_populate_pmd(pudp, vaddr, next); 108 } while (pudp++, vaddr = next, vaddr != end); 168 pud_t *pudp, *base_pud; local 172 pudp = (pud_t *)p4dp; 175 pudp = base_pud + pud_index(vaddr); 182 pud_clear(pudp); 238 pud_t *pudp, *base_pud; local [all...] |
H A D | pageattr.c | 96 static int __split_linear_mapping_pmd(pud_t *pudp, argument 102 pmdp = pmd_offset(pudp, vaddr); 138 pud_t *pudp; local 142 pudp = pud_offset(p4dp, vaddr); 151 if (pud_leaf(pudp_get(pudp))) { 153 unsigned long pfn = _pud_pfn(pudp_get(pudp)); 154 pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK); 169 set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE)); 172 ret = __split_linear_mapping_pmd(pudp, vaddr, next); 175 } while (pudp [all...] |
/linux-master/arch/sh/include/asm/ |
H A D | pgalloc.h | 17 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
|
/linux-master/arch/powerpc/include/asm/nohash/64/ |
H A D | pgtable.h | 109 static inline void pud_set(pud_t *pudp, unsigned long val) argument 111 *pudp = __pud(val); 114 static inline void pud_clear(pud_t *pudp) argument 116 *pudp = __pud(0);
|
/linux-master/arch/sparc/include/asm/ |
H A D | pgalloc_32.h | 29 static inline void pud_set(pud_t * pudp, pmd_t * pmdp) argument 33 set_pte((pte_t *)pudp, __pte((SRMMU_ET_PTD | (pa >> 4))));
|
/linux-master/arch/arm/include/asm/ |
H A D | pgtable-3level.h | 123 #define pud_clear(pudp) \ 125 *pudp = __pud(0); \ 126 clean_pmd_entry(pudp); \ 129 #define set_pud(pudp, pud) \ 131 *pudp = pud; \ 132 flush_pmd_entry(pudp); \
|
/linux-master/mm/ |
H A D | debug_vm_pgtable.c | 66 pud_t *pudp; member in struct:pgtable_debug_args 371 set_pud_at(args->mm, vaddr, args->pudp, pud); 373 pudp_set_wrprotect(args->mm, vaddr, args->pudp); 374 pud = READ_ONCE(*args->pudp); 378 pudp_huge_get_and_clear(args->mm, vaddr, args->pudp); 379 pud = READ_ONCE(*args->pudp); 386 set_pud_at(args->mm, vaddr, args->pudp, pud); 390 pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1); 391 pud = READ_ONCE(*args->pudp); 395 pudp_huge_get_and_clear_full(args->vma, vaddr, args->pudp, [all...] |
/linux-master/include/asm-generic/ |
H A D | tlb.h | 674 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0) 677 #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ 680 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \ 720 #define pud_free_tlb(tlb, pudp, address) \ 724 __pud_free_tlb(tlb, pudp, address); \ 729 #define p4d_free_tlb(tlb, pudp, address) \ 733 __p4d_free_tlb(tlb, pudp, address); \
|