/linux-master/arch/powerpc/mm/nohash/ |
H A D | book3e_pgtable.c | 77 p4d_t *p4dp; local 85 p4dp = p4d_offset(pgdp, ea); 86 pudp = pud_alloc(&init_mm, p4dp, ea); 97 p4dp = p4d_offset(pgdp, ea); 98 if (p4d_none(*p4dp)) { 100 p4d_populate(&init_mm, p4dp, pudp); 102 pudp = pud_offset(p4dp, ea);
|
/linux-master/arch/arm64/include/asm/ |
H A D | pgalloc.h | 45 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) argument 48 set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); 51 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) argument 56 __p4d_populate(p4dp, __pa(pudp), p4dval); 66 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) argument 74 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot) argument 77 set_pgd(pgdp, __pgd(__phys_to_pgd_val(p4dp) | prot)); 80 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, p4d_t *p4dp) argument 85 __pgd_populate(pgdp, __pa(p4dp), pgdval); 107 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_ argument [all...] |
H A D | pgtable.h | 799 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) argument 801 if (in_swapper_pgdir(p4dp)) { 802 set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); 806 WRITE_ONCE(*p4dp, p4d); 811 static inline void p4d_clear(p4d_t *p4dp) argument 814 set_p4d(p4dp, __p4d(0)); 824 static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) argument 826 return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr); 834 static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) argument 838 return p4d_page_paddr(READ_ONCE(*p4dp)) 842 pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long addr) argument 850 pud_offset(p4d_t *p4dp, unsigned long addr) argument 863 pud_set_fixmap_offset(p4d_t *p4dp, unsigned long addr) argument 877 pud_offset_kimg(p4d_t *p4dp, u64 addr) argument [all...] |
/linux-master/arch/riscv/mm/ |
H A D | kasan_init.c | 115 p4d_t *p4dp, *p; local 123 p4dp = p4d_offset(pgd, vaddr); 128 if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) && 132 set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL)); 138 kasan_populate_pud(p4dp, vaddr, next); 139 } while (p4dp++, vaddr = next, vaddr != end); 165 static void __init kasan_early_clear_pud(p4d_t *p4dp, argument 172 pudp = (pud_t *)p4dp; 174 base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); 193 p4d_t *p4dp, *base_p4 local 234 kasan_early_populate_pud(p4d_t *p4dp, unsigned long vaddr, unsigned long end) argument 267 p4d_t *p4dp, *base_p4d; local [all...] |
H A D | pageattr.c | 135 static int __split_linear_mapping_pud(p4d_t *p4dp, argument 142 pudp = pud_offset(p4dp, vaddr); 183 p4d_t *p4dp; local 187 p4dp = p4d_offset(pgdp, vaddr); 200 if (p4d_leaf(p4dp_get(p4dp))) { 202 unsigned long pfn = _p4d_pfn(p4dp_get(p4dp)); 203 pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK); 227 set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE)); 230 ret = __split_linear_mapping_pud(p4dp, vaddr, next); 233 } while (p4dp [all...] |
/linux-master/arch/arm64/mm/ |
H A D | fixmap.c | 72 static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, argument 75 p4d_t p4d = READ_ONCE(*p4dp); 89 __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); 91 pudp = pud_offset_kimg(p4dp, addr); 107 p4d_t *p4dp = p4d_offset_kimg(pgdp, addr); local 109 early_fixmap_init_pud(p4dp, addr, end);
|
H A D | pageattr.c | 216 p4d_t *p4dp; local 226 p4dp = p4d_offset(pgdp, addr); 227 if (p4d_none(READ_ONCE(*p4dp))) 230 pudp = pud_offset(p4dp, addr);
|
H A D | kasan_init.c | 89 static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, argument 92 if (p4d_none(READ_ONCE(*p4dp))) { 96 __p4d_populate(p4dp, pud_phys, P4D_TYPE_TABLE); 99 return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr); 144 static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr, argument 148 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); 160 p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early); local 164 kasan_pud_populate(p4dp, addr, next, node, early); 165 } while (p4dp [all...] |
H A D | mmu.c | 316 static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, argument 322 p4d_t p4d = READ_ONCE(*p4dp); 333 __p4d_populate(p4dp, pud_phys, p4dval); 334 p4d = READ_ONCE(*p4dp); 338 pudp = pud_set_fixmap_offset(p4dp, addr); 378 p4d_t *p4dp; local 393 p4dp = p4d_set_fixmap_offset(pgdp, addr); 395 p4d_t old_p4d = READ_ONCE(*p4dp); 399 alloc_init_pud(p4dp, addr, next, phys, prot, 403 p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); 893 unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, unsigned long end, bool free_mapped, struct vmem_altmap *altmap) argument 931 p4d_t *p4dp, p4d; local 1048 free_empty_pud_table(p4d_t *p4dp, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling) argument 1092 p4d_t *p4dp, p4d; local 1211 p4d_clear_huge(p4d_t *p4dp) argument [all...] |
H A D | hugetlbpage.c | 100 p4d_t *p4dp; local 105 p4dp = p4d_offset(pgdp, addr); 106 pudp = pud_offset(p4dp, addr); 273 p4d_t *p4dp; local 279 p4dp = p4d_alloc(mm, pgdp, addr); 280 if (!p4dp) 283 pudp = pud_alloc(mm, p4dp, addr); 314 p4d_t *p4dp; local 322 p4dp = p4d_offset(pgdp, addr); 323 if (!p4d_present(READ_ONCE(*p4dp))) [all...] |
/linux-master/arch/powerpc/include/asm/nohash/64/ |
H A D | pgtable-4k.h | 67 static inline void p4d_clear(p4d_t *p4dp) argument 69 *p4dp = __p4d(0);
|
H A D | pgtable.h | 143 static inline void p4d_set(p4d_t *p4dp, unsigned long val) argument 145 *p4dp = __p4d(val);
|
/linux-master/arch/arc/include/asm/ |
H A D | pgalloc.h | 75 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) argument 77 set_p4d(p4dp, __p4d((unsigned long)pudp));
|
H A D | pgtable-levels.h | 113 #define set_p4d(p4dp, p4d) (*(p4dp) = p4d)
|
/linux-master/arch/powerpc/mm/kasan/ |
H A D | init_book3e_64.c | 35 p4d_t *p4dp; local 41 p4dp = p4d_offset(pgdp, ea); 42 if (kasan_pud_table(*p4dp)) { 45 p4d_populate(&init_mm, p4dp, pudp); 47 pudp = pud_offset(p4dp, ea);
|
/linux-master/arch/loongarch/mm/ |
H A D | kasan_init.c | 132 static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, bool early) argument 134 if (__p4d_none(early, READ_ONCE(*p4dp))) { 139 p4d_populate(&init_mm, p4dp, (pud_t *)__va(pud_phys)); 142 return pud_offset(p4dp, addr); 172 static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr, argument 176 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); 188 p4d_t *p4dp = p4d_offset(pgdp, addr); local 192 kasan_pud_populate(p4dp, addr, next, node, early); 193 } while (p4dp++, addr = next, addr != end);
|
/linux-master/arch/s390/mm/ |
H A D | hugetlbpage.c | 195 p4d_t *p4dp; local 200 p4dp = p4d_alloc(mm, pgdp, addr); 201 if (p4dp) { 202 pudp = pud_alloc(mm, p4dp, addr); 217 p4d_t *p4dp; local 223 p4dp = p4d_offset(pgdp, addr); 224 if (p4d_present(*p4dp)) { 225 pudp = pud_offset(p4dp, addr);
|
H A D | pageattr.c | 302 p4d_t *p4dp; local 305 p4dp = p4d_offset(pgd, addr); 307 if (p4d_none(*p4dp)) 310 rc = walk_pud_level(p4dp, addr, next, flags); 311 p4dp++;
|
/linux-master/arch/x86/include/asm/ |
H A D | pgtable_64.h | 142 static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) argument 148 WRITE_ONCE(*p4dp, p4d); 153 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); 154 WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd)));
|
/linux-master/arch/powerpc/mm/ |
H A D | pgtable.c | 385 p4d_t p4d, *p4dp; local 406 p4dp = p4d_offset(pgdp, ea); 407 p4d = READ_ONCE(*p4dp); 414 ret_pte = (pte_t *)p4dp;
|
/linux-master/arch/powerpc/mm/book3s64/ |
H A D | radix_pgtable.c | 77 p4d_t *p4dp; local 83 p4dp = p4d_offset(pgdp, ea); 84 if (p4d_none(*p4dp)) { 87 p4d_populate(&init_mm, p4dp, pudp); 89 pudp = pud_offset(p4dp, ea); 129 p4d_t *p4dp; local 152 p4dp = p4d_offset(pgdp, ea); 153 pudp = pud_alloc(&init_mm, p4dp, ea); 190 p4d_t *p4dp; local 203 p4dp 997 vmemmap_pud_alloc(p4d_t *p4dp, int node, unsigned long address) argument [all...] |
/linux-master/arch/arm/mm/ |
H A D | kasan_init.c | 128 p4d_t *p4dp; local 155 p4dp = p4d_offset(pgdp, addr); 156 pudp = pud_offset(p4dp, addr);
|
/linux-master/arch/x86/boot/compressed/ |
H A D | ident_map_64.c | 260 p4d_t *p4dp; local 283 p4dp = p4d_offset(pgdp, address); 284 pudp = pud_offset(p4dp, address);
|
/linux-master/arch/sparc/mm/ |
H A D | srmmu.c | 498 p4d_t *p4dp; local 506 p4dp = p4d_offset(pgdp, virt_addr); 507 pudp = pud_offset(p4dp, virt_addr); 537 p4d_t *p4dp; local 544 p4dp = p4d_offset(pgdp, virt_addr); 545 pudp = pud_offset(p4dp, virt_addr); 684 p4d_t *p4dp; local 691 p4dp = p4d_offset(pgdp, start); 692 pudp = pud_offset(p4dp, start); 719 p4d_t *p4dp; local 778 p4d_t *p4dp; local [all...] |
H A D | fault_64.c | 82 p4d_t *p4dp; local 91 p4dp = p4d_offset(pgdp, tpc); 92 if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) 94 pudp = pud_offset(p4dp, tpc);
|