arm64/mm/hugetlbpage.c

82 static int find_num_contig(struct mm_struct *mm, unsigned long addr,
85 	pgd_t *pgdp = pgd_offset(mm, addr);
91 	p4dp = p4d_offset(pgdp, addr);
92 	pudp = pud_offset(p4dp, addr);
93 	pmdp = pmd_offset(pudp, addr);
161 			     unsigned long addr,
169 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
170 		pte_t pte = __ptep_get_and_clear(mm, addr, ptep);
187 				    unsigned long addr,
192 	pte_t orig_pte = get_clear_contig(mm, addr, ptep, pgsize, ncontig);
195 	flush_tlb_range(&vma, addr, addr + (pgsize * ncontig));
209 			     unsigned long addr,
215 	unsigned long i, saddr = addr;
217 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
218 		__ptep_get_and_clear(mm, addr, ptep);
220 	flush_tlb_range(&vma, saddr, addr);
223 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
235 		for (i = 0; i < ncontig; i++, ptep++, addr += pgsize)
236 			__set_ptes(mm, addr, ptep, pte, 1);
241 		__set_ptes(mm, addr, ptep, pte, 1);
249 	clear_flush(mm, addr, ptep, pgsize, ncontig);
251 	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
252 		__set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
256 		      unsigned long addr, unsigned long sz)
264 	pgdp = pgd_offset(mm, addr);
265 	p4dp = p4d_alloc(mm, pgdp, addr);
269 	pudp = pud_alloc(mm, p4dp, addr);
276 		pmdp = pmd_alloc(mm, pudp, addr);
280 		WARN_ON(addr & (sz - 1));
281 		ptep = pte_alloc_huge(mm, pmdp, addr);
283 		if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
284 			ptep = huge_pmd_share(mm, vma, addr, pudp);
286 			ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
288 		pmdp = pmd_alloc(mm, pudp, addr);
289 		WARN_ON(addr & (sz - 1));
297 		       unsigned long addr, unsigned long sz)
304 	pgdp = pgd_offset(mm, addr);
308 	p4dp = p4d_offset(pgdp, addr);
312 	pudp = pud_offset(p4dp, addr);
322 		addr &= CONT_PMD_MASK;
324 	pmdp = pmd_offset(pudp, addr);
333 		return pte_offset_huge(pmdp, (addr & CONT_PTE_MASK));
376 void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
384 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
385 		__pte_clear(mm, addr, ptep);
389 			      unsigned long addr, pte_t *ptep)
396 		return __ptep_get_and_clear(mm, addr, ptep);
398 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
400 	return get_clear_contig(mm, addr, ptep, pgsize, ncontig);
433 			       unsigned long addr, pte_t *ptep,
444 		return __ptep_set_access_flags(vma, addr, ptep, pte, dirty);
446 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
452 	orig_pte = get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
462 	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
463 		__set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
469 			     unsigned long addr, pte_t *ptep)
478 		__ptep_set_wrprotect(mm, addr, ptep);
482 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
485 	pte = get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
491 	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
492 		__set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
496 			    unsigned long addr, pte_t *ptep)
503 		return ptep_clear_flush(vma, addr, ptep);
505 	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
506 	return get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
527 pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
536 			return huge_ptep_clear_flush(vma, addr, ptep);
538 	return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
541 void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
546 	set_huge_pte_at(vma->vm_mm, addr, ptep, pte, psize);