/linux-master/arch/x86/mm/ |
H A D | fault.c | 180 p4d_t *p4d, *p4d_k; local 195 p4d = p4d_offset(pgd, address); 200 pud = pud_offset(p4d, address); 299 p4d_t *p4d; local 312 p4d = p4d_offset(pgd, address); 313 pud = pud_offset(p4d, address); 355 p4d_t *p4d; local 368 p4d = p4d_offset(pgd, address); 369 if (bad_address(p4d)) 372 pr_cont("P4D %lx ", p4d_val(*p4d)); 981 p4d_t *p4d; local [all...] |
H A D | ident_map.c | 68 p4d_t *p4d = p4d_page + p4d_index(addr); local 75 if (p4d_present(*p4d)) { 76 pud = pud_offset(p4d, 0); 91 set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); 114 p4d_t *p4d; local 121 p4d = p4d_offset(pgd, 0); 122 result = ident_p4d_init(info, p4d, addr, next); 128 p4d = (p4d_t *)info->alloc_pgt_page(info->context); 129 if (!p4d) 131 result = ident_p4d_init(info, p4d, add [all...] |
/linux-master/arch/x86/entry/vsyscall/ |
H A D | vsyscall_64.c | 337 p4d_t *p4d; local 343 p4d = p4d_offset(pgd, VSYSCALL_ADDR); 345 set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER)); 347 pud = pud_offset(p4d, VSYSCALL_ADDR);
|
/linux-master/arch/s390/mm/ |
H A D | gmap.c | 558 p4d_t *p4d; local 596 p4d = p4d_offset(pgd, vmaddr); 597 VM_BUG_ON(p4d_none(*p4d)); 598 pud = pud_offset(p4d, vmaddr);
|
/linux-master/arch/x86/include/asm/ |
H A D | pgtable_types.h | 342 typedef struct { p4dval_t p4d; } p4d_t; member in struct:__anon5 349 static inline p4dval_t native_p4d_val(p4d_t p4d) argument 351 return p4d.p4d; 361 static inline p4dval_t native_p4d_val(p4d_t p4d) argument 363 return native_pgd_val(p4d.pgd); 384 return (pud_t) { .p4d.pgd = native_make_pgd(val) }; 389 return native_pgd_val(pud.p4d.pgd); 408 return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; 413 return native_pgd_val(pmd.pud.p4d 417 p4d_pfn_mask(p4d_t p4d) argument 423 p4d_flags_mask(p4d_t p4d) argument 428 p4d_flags(p4d_t p4d) argument [all...] |
/linux-master/arch/riscv/mm/ |
H A D | init.c | 968 * early_pmd/pud/p4d and the address returned is the physical one. 1442 p4d_t *p4d; local 1446 lvl = "p4d"; 1447 p4d = p4d_alloc(&init_mm, pgd, addr); 1448 if (!p4d) 1455 pud = pud_alloc(&init_mm, p4d, addr);
|
H A D | pgtable.c | 32 pud_t *pud_offset(p4d_t *p4d, unsigned long address) argument 35 return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); 37 return (pud_t *)p4d; 50 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) argument 55 void p4d_clear_huge(p4d_t *p4d) argument
|
/linux-master/mm/ |
H A D | hugetlb.c | 7443 p4d_t *p4d = p4d_offset(pgd, addr); local 7444 pud_t *pud = pud_offset(p4d, addr); 7488 p4d_t *p4d; local 7493 p4d = p4d_alloc(mm, pgd, addr); 7494 if (!p4d) 7496 pud = pud_alloc(mm, p4d, addr); 7531 p4d_t *p4d; local 7538 p4d = p4d_offset(pgd, addr); 7539 if (!p4d_present(*p4d)) 7542 pud = pud_offset(p4d, add [all...] |
H A D | memory-failure.c | 393 p4d_t *p4d; local 403 p4d = p4d_offset(pgd, address); 404 if (!p4d_present(*p4d)) 406 pud = pud_offset(p4d, address);
|
H A D | gup.c | 775 p4d_t *p4d; local 777 p4d = p4d_offset(pgdp, address); 778 if (p4d_none(*p4d)) 780 BUILD_BUG_ON(p4d_huge(*p4d)); 781 if (unlikely(p4d_bad(*p4d))) 784 return follow_pud_mask(vma, address, p4d, flags, ctx); 864 p4d_t *p4d; local 880 p4d = p4d_offset(pgd, address); 881 if (p4d_none(*p4d)) 883 pud = pud_offset(p4d, addres 3044 gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end, unsigned int flags, struct page **pages, int *nr) argument 3080 p4d_t p4d = READ_ONCE(*p4dp); local [all...] |
H A D | vmalloc.c | 201 static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, argument 208 pud = pud_alloc_track(&init_mm, p4d, addr, mask); 227 static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, argument 246 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 249 return p4d_set_huge(p4d, phys_addr, prot); 256 p4d_t *p4d; local 259 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 260 if (!p4d) 265 if (vmap_try_huge_p4d(p4d, add 379 vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pgtbl_mod_mask *mask) argument 405 p4d_t *p4d; local 527 vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr, pgtbl_mod_mask *mask) argument 549 p4d_t *p4d; local 736 p4d_t *p4d; local [all...] |
H A D | memory.c | 230 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, argument 239 pud = pud_offset(p4d, addr); 258 pud = pud_offset(p4d, start); 259 p4d_clear(p4d); 268 p4d_t *p4d; local 273 p4d = p4d_offset(pgd, addr); 276 if (p4d_none_or_clear_bad(p4d)) 278 free_pud_range(tlb, p4d, addr, next, floor, ceiling); 279 } while (p4d++, addr = next, addr != end); 292 p4d 493 p4d_t *p4d = p4d_offset(pgd, addr); local 1730 zap_pud_range(struct mmu_gather *tlb, struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr, unsigned long end, struct zap_details *details) argument 1764 p4d_t *p4d; local 1949 p4d_t *p4d; local 2538 remap_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, unsigned long pfn, pgprot_t prot) argument 2564 p4d_t *p4d; local 2797 apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, pte_fn_t fn, void *data, bool create, pgtbl_mod_mask *mask) argument 2838 p4d_t *p4d; local 5364 p4d_t *p4d; local 5826 __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) argument 5894 p4d_t *p4d; local [all...] |
H A D | userfaultfd.c | 435 p4d_t *p4d; local 439 p4d = p4d_alloc(mm, pgd, address); 440 if (!p4d) 442 pud = pud_alloc(mm, p4d, address);
|
/linux-master/include/linux/ |
H A D | mm.h | 2720 static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, argument 2729 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address); 2818 static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, argument 2821 return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? 2822 NULL : pud_offset(p4d, address); 3871 pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
|
/linux-master/arch/x86/kvm/mmu/ |
H A D | mmu.c | 3089 p4d_t p4d; local 3121 p4d = READ_ONCE(*p4d_offset(&pgd, hva)); 3122 if (p4d_none(p4d) || !p4d_present(p4d)) 3125 pud = READ_ONCE(*pud_offset(&p4d, hva));
|
/linux-master/arch/x86/kernel/ |
H A D | head64.c | 150 p4dval_t *p4d; local 182 p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt); 183 p4d[MAX_PTRS_PER_P4D - 1] += load_delta; 185 pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; 208 p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd; 211 pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; 212 pgd[i + 1] = (pgdval_t)p4d + pgtable_flags; 215 p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; 216 p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; 286 p4dval_t p4d, *p4d_ local [all...] |
/linux-master/arch/riscv/include/asm/ |
H A D | pgalloc.h | 42 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) argument 47 set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); 51 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, argument 57 set_p4d_safe(p4d, 62 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) argument 65 unsigned long pfn = virt_to_pfn(p4d); 72 p4d_t *p4d) 75 unsigned long pfn = virt_to_pfn(p4d); 126 static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d) argument 128 BUG_ON((unsigned long)p4d 71 pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) argument 133 p4d_free(struct mm_struct *mm, p4d_t *p4d) argument 139 __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, unsigned long addr) argument [all...] |
/linux-master/arch/loongarch/include/asm/ |
H A D | pgtable.h | 133 * Empty pgd/p4d entries point to the invalid_pud_table. 135 static inline int p4d_none(p4d_t p4d) argument 137 return p4d_val(p4d) == (unsigned long)invalid_pud_table; 140 static inline int p4d_bad(p4d_t p4d) argument 142 return p4d_val(p4d) & ~PAGE_MASK; 145 static inline int p4d_present(p4d_t p4d) argument 147 return p4d_val(p4d) != (unsigned long)invalid_pud_table; 155 static inline pud_t *p4d_pgtable(p4d_t p4d) argument 157 return (pud_t *)p4d_val(p4d); 160 static inline void set_p4d(p4d_t *p4d, p4d_ argument [all...] |
/linux-master/arch/arm/mm/ |
H A D | fault.c | 52 p4d_t *p4d; local 57 p4d = p4d_offset(pgd, addr); 58 if (p4d_none(*p4d)) 61 if (p4d_bad(*p4d)) { 66 pud = pud_offset(p4d, addr); 439 p4d_t *p4d, *p4d_k; local 454 p4d = p4d_offset(pgd, addr); 459 if (!p4d_present(*p4d)) 460 set_p4d(p4d, *p4d_k); 462 pud = pud_offset(p4d, add [all...] |
/linux-master/arch/s390/include/asm/ |
H A D | page.h | 88 typedef struct { unsigned long p4d; } p4d_t; member in struct:__anon11 110 static inline unsigned long p4d_val(p4d_t p4d) argument 112 return p4d.p4d;
|
/linux-master/arch/sparc/mm/ |
H A D | srmmu.c | 271 p4d_t *p4d; local 306 p4d = p4d_offset(pgd, vaddr); 307 pud = pud_offset(p4d, vaddr); 901 p4d_t *p4d; local 964 p4d = p4d_offset(pgd, PKMAP_BASE); 965 pud = pud_offset(p4d, PKMAP_BASE);
|
/linux-master/arch/powerpc/mm/book3s64/ |
H A D | radix_pgtable.c | 658 static void free_pud_table(pud_t *pud_start, p4d_t *p4d) argument 670 p4d_clear(p4d); 836 p4d_t *p4d; local 844 p4d = p4d_offset(pgd, addr); 845 if (!p4d_present(*p4d)) 848 if (p4d_leaf(*p4d)) { 859 pud_base = p4d_pgtable(*p4d); 861 free_pud_table(pud_base, p4d); 1054 p4d_t *p4d; local 1063 p4d 1139 p4d_t *p4d; local 1171 p4d_t *p4d; local 1234 p4d_t *p4d; local [all...] |
/linux-master/arch/powerpc/include/asm/book3s/64/ |
H A D | pgtable.h | 935 #define p4d_write(p4d) pte_write(p4d_pte(p4d)) 942 static inline int p4d_none(p4d_t p4d) argument 944 return !p4d_raw(p4d); 947 static inline int p4d_present(p4d_t p4d) argument 949 return !!(p4d_raw(p4d) & cpu_to_be64(_PAGE_PRESENT)); 952 static inline pte_t p4d_pte(p4d_t p4d) argument 954 return __pte_raw(p4d_raw(p4d)); 962 static inline int p4d_bad(p4d_t p4d) argument 965 return radix__p4d_bad(p4d); 970 p4d_access_permitted(p4d_t p4d, bool write) argument 980 p4d_pgtable(p4d_t p4d) argument [all...] |
H A D | radix.h | 258 static inline int radix__p4d_bad(p4d_t p4d) argument 260 return !!(p4d_val(p4d) & RADIX_P4D_BAD_BITS);
|
H A D | hash.h | 151 static inline int hash__p4d_bad(p4d_t p4d) argument 153 return (p4d_val(p4d) == 0);
|