/linux-master/include/asm-generic/ |
H A D | pgtable-nopud.h | 12 * Having the pud type consist of a p4d gets the size right, and allows 13 * us to conceptually access the p4d entry that this pud is folded into 16 typedef struct { p4d_t p4d; } pud_t; member in struct:__anon167 26 * into the p4d entry) 28 static inline int p4d_none(p4d_t p4d) { return 0; } argument 29 static inline int p4d_bad(p4d_t p4d) { return 0; } argument 30 static inline int p4d_present(p4d_t p4d) { return 1; } argument 31 static inline void p4d_clear(p4d_t *p4d) { } argument 32 #define pud_ERROR(pud) (p4d_ERROR((pud).p4d)) 34 #define p4d_populate(mm, p4d, pu 42 pud_offset(p4d_t *p4d, unsigned long address) argument [all...] |
H A D | pgtable-nop4d.h | 18 * setup: the p4d is never bad, and a p4d always exists (as it's folded 25 #define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd)) 27 #define pgd_populate(mm, pgd, p4d) do { } while (0) 28 #define pgd_populate_safe(mm, pgd, p4d) do { } while (0) 47 * allocating and freeing a p4d is trivial: the 1-entry p4d is
|
/linux-master/arch/powerpc/include/asm/nohash/64/ |
H A D | pgtable-4k.h | 56 #define p4d_none(p4d) (!p4d_val(p4d)) 57 #define p4d_bad(p4d) (p4d_val(p4d) == 0) 58 #define p4d_present(p4d) (p4d_val(p4d) != 0) 62 static inline pud_t *p4d_pgtable(p4d_t p4d) argument 64 return (pud_t *) (p4d_val(p4d) & ~P4D_MASKED_BITS); 72 static inline pte_t p4d_pte(p4d_t p4d) argument 74 return __pte(p4d_val(p4d)); [all...] |
/linux-master/arch/sh/mm/ |
H A D | hugetlbpage.c | 28 p4d_t *p4d; local 35 p4d = p4d_alloc(mm, pgd, addr); 36 if (p4d) { 37 pud = pud_alloc(mm, p4d, addr); 53 p4d_t *p4d; local 60 p4d = p4d_offset(pgd, addr); 61 if (p4d) { 62 pud = pud_offset(p4d, addr);
|
H A D | tlbex_32.c | 26 p4d_t *p4d; local 46 p4d = p4d_offset(pgd, address); 47 if (p4d_none_or_clear_bad(p4d)) 49 pud = pud_offset(p4d, address);
|
/linux-master/arch/mips/mm/ |
H A D | hugetlbpage.c | 28 p4d_t *p4d; local 33 p4d = p4d_alloc(mm, pgd, addr); 34 pud = pud_alloc(mm, p4d, addr); 45 p4d_t *p4d; local 51 p4d = p4d_offset(pgd, addr); 52 if (p4d_present(*p4d)) { 53 pud = pud_offset(p4d, addr);
|
/linux-master/arch/riscv/include/asm/ |
H A D | pgalloc.h | 42 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) argument 47 set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); 51 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, argument 57 set_p4d_safe(p4d, 62 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) argument 65 unsigned long pfn = virt_to_pfn(p4d); 72 p4d_t *p4d) 75 unsigned long pfn = virt_to_pfn(p4d); 126 static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d) argument 128 BUG_ON((unsigned long)p4d 71 pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) argument 133 p4d_free(struct mm_struct *mm, p4d_t *p4d) argument 139 __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, unsigned long addr) argument [all...] |
H A D | pgtable-64.h | 27 /* p4d is folded into pgd in case of 4-level page table */ 48 unsigned long p4d; member in struct:__anon25 51 #define p4d_val(x) ((x).p4d) 276 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) 278 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) argument 281 WRITE_ONCE(*p4dp, p4d); 283 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); 286 static inline int p4d_none(p4d_t p4d) argument 289 return (p4d_val(p4d) == 0); 294 static inline int p4d_present(p4d_t p4d) argument 302 p4d_bad(p4d_t p4d) argument 310 p4d_clear(p4d_t *p4d) argument 321 _p4d_pfn(p4d_t p4d) argument 326 p4d_pgtable(p4d_t p4d) argument 335 p4d_page(p4d_t p4d) argument [all...] |
/linux-master/arch/loongarch/mm/ |
H A D | hugetlbpage.c | 20 p4d_t *p4d; local 25 p4d = p4d_alloc(mm, pgd, addr); 26 pud = pud_alloc(mm, p4d, addr); 37 p4d_t *p4d; local 43 p4d = p4d_offset(pgd, addr); 44 if (p4d_present(*p4d)) { 45 pud = pud_offset(p4d, addr);
|
/linux-master/mm/kasan/ |
H A D | init.c | 45 static inline bool kasan_pud_table(p4d_t p4d) argument 47 return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); 50 static inline bool kasan_pud_table(p4d_t p4d) argument 146 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, argument 149 pud_t *pud = pud_offset(p4d, addr); 191 p4d_t *p4d = p4d_offset(pgd, addr); local 200 p4d_populate(&init_mm, p4d, 202 pud = pud_offset(p4d, addr); 211 if (p4d_none(*p4d)) { 215 p = pud_alloc(&init_mm, p4d, add 248 p4d_t *p4d; local 322 kasan_free_pud(pud_t *pud_start, p4d_t *p4d) argument 339 p4d_t *p4d; local 427 kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr, unsigned long end) argument 466 p4d_t *p4d; local [all...] |
/linux-master/arch/x86/mm/ |
H A D | ident_map.c | 68 p4d_t *p4d = p4d_page + p4d_index(addr); local 75 if (p4d_present(*p4d)) { 76 pud = pud_offset(p4d, 0); 91 set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); 114 p4d_t *p4d; local 121 p4d = p4d_offset(pgd, 0); 122 result = ident_p4d_init(info, p4d, addr, next); 128 p4d = (p4d_t *)info->alloc_pgt_page(info->context); 129 if (!p4d) 131 result = ident_p4d_init(info, p4d, add [all...] |
H A D | kasan_init_64.c | 103 static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, argument 109 if (p4d_none(*p4d)) { 112 p4d_populate(&init_mm, p4d, p); 115 pud = pud_offset(p4d, addr); 127 p4d_t *p4d; local 135 p4d = p4d_offset(pgd, addr); 138 kasan_populate_p4d(p4d, addr, next, nid); 139 } while (p4d++, addr = next, addr != end); 178 * With folded p4d, pgd_clear() is nop, use p4d_clear() 194 unsigned long p4d; local 209 p4d_t *p4d, p4d_entry; local 249 p4d_t *p4d; local [all...] |
H A D | init_64.c | 73 DEFINE_POPULATE(p4d_populate, p4d, pud, init) 74 DEFINE_POPULATE(pgd_populate, pgd, p4d, init) 88 DEFINE_ENTRY(p4d, p4d, init) 180 * With folded p4d, pgd_none() is always false, we need to 181 * handle synchronization on p4d level. 192 p4d_t *p4d; local 196 p4d = p4d_offset(pgd, addr); 201 if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) 202 BUG_ON(p4d_pgtable(*p4d) 252 p4d_t *p4d = (p4d_t *)spp_getpage(); local 261 fill_pud(p4d_t *p4d, unsigned long vaddr) argument 312 p4d_t *p4d = p4d_page + p4d_index(vaddr); local 346 p4d_t *p4d; local 370 p4d_t *p4d; local 689 p4d_t *p4d = p4d_page + p4d_index(vaddr); local 741 p4d_t *p4d; local 1041 free_pud_table(pud_t *pud_start, p4d_t *p4d) argument 1191 p4d_t *p4d; local 1225 p4d_t *p4d; local 1293 p4d_t *p4d; local 1564 p4d_t *p4d; local [all...] |
H A D | pgtable_32.c | 30 p4d_t *p4d; local 40 p4d = p4d_offset(pgd, vaddr); 41 if (p4d_none(*p4d)) { 45 pud = pud_offset(p4d, vaddr);
|
/linux-master/arch/arm/mm/ |
H A D | pgd.c | 145 p4d_t *p4d; local 157 p4d = p4d_offset(pgd, 0); 158 if (p4d_none_or_clear_bad(p4d)) 161 pud = pud_offset(p4d, 0); 178 p4d_clear(p4d); 182 p4d_free(mm, p4d); 193 p4d = p4d_offset(pgd, 0); 194 if (p4d_none_or_clear_bad(p4d)) 196 pud = pud_offset(p4d, 0); 203 p4d_clear(p4d); [all...] |
/linux-master/mm/ |
H A D | pgalloc-track.h | 19 static inline pud_t *pud_alloc_track(struct mm_struct *mm, p4d_t *p4d, argument 23 if (unlikely(p4d_none(*p4d))) { 24 if (__pud_alloc(mm, p4d, address)) 29 return pud_offset(p4d, address);
|
/linux-master/arch/x86/include/asm/ |
H A D | pgalloc.h | 113 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) argument 116 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 119 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) argument 122 set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud))); 134 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) argument 138 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 139 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 142 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) argument 146 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); 147 set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); 159 p4d_free(struct mm_struct *mm, p4d_t *p4d) argument 170 __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, unsigned long address) argument [all...] |
/linux-master/arch/mips/include/asm/ |
H A D | pgtable-64.h | 184 static inline int p4d_none(p4d_t p4d) argument 186 return p4d_val(p4d) == (unsigned long)invalid_pud_table; 189 static inline int p4d_bad(p4d_t p4d) argument 191 if (unlikely(p4d_val(p4d) & ~PAGE_MASK)) 197 static inline int p4d_present(p4d_t p4d) argument 199 return p4d_val(p4d) != (unsigned long)invalid_pud_table; 207 static inline pud_t *p4d_pgtable(p4d_t p4d) argument 209 return (pud_t *)p4d_val(p4d); 212 #define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d)) 217 set_p4d(p4d_t *p4d, p4d_t p4dval) argument [all...] |
/linux-master/arch/x86/power/ |
H A D | hibernate_64.c | 32 p4d_t *p4d = NULL; local 55 p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); 56 if (!p4d) 72 if (p4d) { 74 pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot)); 76 set_p4d(p4d + p4d_index(restore_jump_address), new_p4d); 79 /* No p4d for 4-level paging: point the pgd to the pud page table */
|
/linux-master/arch/parisc/mm/ |
H A D | fixmap.c | 17 p4d_t *p4d = p4d_offset(pgd, vaddr); local 18 pud_t *pud = pud_offset(p4d, vaddr);
|
H A D | hugetlbpage.c | 51 p4d_t *p4d; local 64 p4d = p4d_offset(pgd, addr); 65 pud = pud_alloc(mm, p4d, addr); 78 p4d_t *p4d; local 87 p4d = p4d_offset(pgd, addr); 88 if (!p4d_none(*p4d)) { 89 pud = pud_offset(p4d, addr);
|
/linux-master/arch/s390/mm/ |
H A D | vmem.c | 315 static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, argument 326 pud = pud_offset(p4d, addr); 369 static void try_free_pud_table(p4d_t *p4d, unsigned long start) argument 374 pud = pud_offset(p4d, start); 379 vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER, NULL); 380 p4d_clear(p4d); 388 p4d_t *p4d; local 391 p4d = p4d_offset(pgd, addr); 392 for (; addr < end; addr = next, p4d++) { 395 if (p4d_none(*p4d)) 416 p4d_t *p4d; local 434 p4d_t *p4d; local 571 p4d_t *p4d; local [all...] |
/linux-master/arch/powerpc/mm/ |
H A D | pgtable_64.c | 101 struct page *p4d_page(p4d_t p4d) argument 103 if (p4d_leaf(p4d)) { 105 VM_WARN_ON(!p4d_huge(p4d)); 106 return pte_page(p4d_pte(p4d)); 108 return virt_to_page(p4d_pgtable(p4d));
|
/linux-master/arch/riscv/mm/ |
H A D | pgtable.c | 32 pud_t *pud_offset(p4d_t *p4d, unsigned long address) argument 35 return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); 37 return (pud_t *)p4d; 50 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) argument 55 void p4d_clear_huge(p4d_t *p4d) argument
|
/linux-master/arch/arc/include/asm/ |
H A D | pgtable-levels.h | 111 #define p4d_pgtable(p4d) ((pud_t *)(p4d_val(p4d) & PAGE_MASK)) 112 #define p4d_page(p4d) virt_to_page(p4d_pgtable(p4d)) 113 #define set_p4d(p4dp, p4d) (*(p4dp) = p4d)
|