linux-master/mm/memory.c

780 		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
783 	unsigned long vm_flags = dst_vma->vm_flags;
875 		pte_marker marker = copy_pte_marker(entry, dst_vma);
882 	if (!userfaultfd_wp(dst_vma))
901 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
919 	folio_add_new_anon_rmap(new_folio, dst_vma, addr);
920 	folio_add_lru_vma(new_folio, dst_vma);
924 	pte = mk_pte(&new_folio->page, dst_vma->vm_page_prot);
925 	pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
926 	if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
929 	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
933 static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
950 	if (!userfaultfd_wp(dst_vma))
953 	set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
964 copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1008 		__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte,
1024 			err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
1036 	__copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1);
1064 copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1068 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1129 						  dst_vma, src_vma,
1151 		ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte,
1209 copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1213 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1229 					    addr, dst_vma, src_vma);
1238 		if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1246 copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1250 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1275 		if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1283 copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1287 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1299 		if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1312 vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1315 	 * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
1320 	if (userfaultfd_wp(dst_vma))
1339 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1345 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1351 	if (!vma_needs_copy(dst_vma, src_vma))
1355 		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
1397 		if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1399 			untrack_pfn_clear(dst_vma);