arm64/mm/mmu.c

275 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
281 	pud_t pud = READ_ONCE(*pudp);
299 		__pud_populate(pudp, pmd_phys, pudval);
302 		pmdp = pmd_set_fixmap_offset(pudp, addr);
331 	pud_t *pudp;
341 		pudp = pud_set_fixmap(pud_phys);
342 		init_clear_pgtable(pudp);
343 		pudp += pud_index(addr);
347 		pudp = pud_set_fixmap_offset(p4dp, addr);
351 		pud_t old_pud = READ_ONCE(*pudp);
361 			pud_set_huge(pudp, phys, prot);
368 						      READ_ONCE(pud_val(*pudp))));
370 			alloc_init_cont_pmd(pudp, addr, next, phys, prot,
374 			       pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
377 	} while (pudp++, addr = next, addr != end);
873 static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr,
882 		pmdp = pmd_offset(pudp, addr);
911 	pud_t *pudp, pud;
915 		pudp = pud_offset(p4dp, addr);
916 		pud = READ_ONCE(*pudp);
922 			pud_clear(pudp);
935 		unmap_hotplug_pmd_range(pudp, addr, next, free_mapped, altmap);
1021 static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
1030 		pmdp = pmd_offset(pudp, addr);
1050 	pmdp = pmd_offset(pudp, 0UL);
1056 	pud_clear(pudp);
1065 	pud_t *pudp, pud;
1070 		pudp = pud_offset(p4dp, addr);
1071 		pud = READ_ONCE(*pudp);
1076 		free_empty_pmd_table(pudp, addr, next, floor, ceiling);
1090 	pudp = pud_offset(p4dp, 0UL);
1092 		if (!pud_none(READ_ONCE(pudp[i])))
1098 	free_hotplug_pgtable_page(virt_to_page(pudp));
1195 int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
1200 	if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)),
1205 	set_pud(pudp, new_pud);
1229 int pud_clear_huge(pud_t *pudp)
1231 	if (!pud_sect(READ_ONCE(*pudp)))
1233 	pud_clear(pudp);
1264 int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
1271 	pud = READ_ONCE(*pudp);
1278 	table = pmd_offset(pudp, addr);
1286 	pud_clear(pudp);