arm64/mm/mmu.c

175 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
180 	ptep = pte_set_fixmap_offset(pmdp, addr);
199 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
206 	pmd_t pmd = READ_ONCE(*pmdp);
217 		__pmd_populate(pmdp, pte_phys, pmdval);
218 		pmd = READ_ONCE(*pmdp);
232 		init_pte(pmdp, addr, next, phys, __prot);
243 	pmd_t *pmdp;
245 	pmdp = pmd_set_fixmap_offset(pudp, addr);
247 		pmd_t old_pmd = READ_ONCE(*pmdp);
254 			pmd_set_huge(pmdp, phys, prot);
261 						      READ_ONCE(pmd_val(*pmdp))));
263 			alloc_init_cont_pte(pmdp, addr, next, phys, prot,
267 			       pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
270 	} while (pmdp++, addr = next, addr != end);
839 static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr,
846 		ptep = pte_offset_kernel(pmdp, addr);
865 	pmd_t *pmdp, pmd;
869 		pmdp = pmd_offset(pudp, addr);
870 		pmd = READ_ONCE(*pmdp);
876 			pmd_clear(pmdp);
889 		unmap_hotplug_pte_range(pmdp, addr, next, free_mapped, altmap);
971 static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
979 		ptep = pte_offset_kernel(pmdp, addr);
997 	ptep = pte_offset_kernel(pmdp, 0UL);
1003 	pmd_clear(pmdp);
1012 	pmd_t *pmdp, pmd;
1017 		pmdp = pmd_offset(pudp, addr);
1018 		pmd = READ_ONCE(*pmdp);
1023 		free_empty_pte_table(pmdp, addr, next, floor, ceiling);
1037 	pmdp = pmd_offset(pudp, 0UL);
1039 		if (!pmd_none(READ_ONCE(pmdp[i])))
1045 	free_hotplug_pgtable_page(virt_to_page(pmdp));
1147 void __meminit vmemmap_set_pmd(pmd_t *pmdp, void *p, int node,
1150 	pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));
1153 int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node,
1156 	vmemmap_verify((pte_t *)pmdp, node, addr, next);
1196 int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
1201 	if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)),
1206 	set_pmd(pmdp, new_pmd);
1224 int pmd_clear_huge(pmd_t *pmdp)
1226 	if (!pmd_sect(READ_ONCE(*pmdp)))
1228 	pmd_clear(pmdp);
1232 int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
1237 	pmd = READ_ONCE(*pmdp);
1244 	table = pte_offset_kernel(pmdp, addr);
1245 	pmd_clear(pmdp);
1254 	pmd_t *pmdp;
1266 	pmdp = table;
1270 		pmd_free_pte_page(pmdp, next);
1271 	} while (pmdp++, next += PMD_SIZE, next != end);