linux-master/mm/mremap.c

3  *	mm/mremap.c
11 #include <linux/mm.h>
35 static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
41 	pgd = pgd_offset(mm, addr);
56 static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
61 	pud = get_old_pud(mm, addr);
72 static pud_t *alloc_new_pud(struct mm_struct *mm, struct vm_area_struct *vma,
78 	pgd = pgd_offset(mm, addr);
79 	p4d = p4d_alloc(mm, pgd, addr);
83 	return pud_alloc(mm, p4d, addr);
86 static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
92 	pud = alloc_new_pud(mm, vma, addr);
96 	pmd = pmd_alloc(mm, pud, addr);
141 	struct mm_struct *mm = vma->vm_mm;
173 	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
178 	new_pte = pte_offset_map_nolock(mm, new_pmd, new_addr, &new_ptl);
194 		pte = ptep_get_and_clear(mm, old_addr, old_pte);
210 		set_pte_at(mm, new_addr, new_pte, pte);
240 	struct mm_struct *mm = vma->vm_mm;
276 	new_ptl = pmd_lockptr(mm, new_pmd);
286 	pmd_populate(mm, new_pmd, pmd_pgtable(pmd));
308 	struct mm_struct *mm = vma->vm_mm;
325 	new_ptl = pud_lockptr(mm, new_pud);
335 	pud_populate(mm, new_pud, pud_pgtable(pud));
357 	struct mm_struct *mm = vma->vm_mm;
372 	new_ptl = pud_lockptr(mm, new_pud);
384 	set_pud_at(mm, new_addr, new_pud, pud);
657 	struct mm_struct *mm = vma->vm_mm;
673 	if (mm->map_count >= sysctl_max_map_count - 3)
701 		if (security_vm_enough_memory_mm(mm, to_account >> PAGE_SHIFT))
761 	hiwater_vm = mm->hiwater_vm;
762 	vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT);
784 	vma_iter_init(&vmi, mm, old_addr);
785 	if (do_vmi_munmap(&vmi, mm, old_addr, old_len, uf_unmap, false) < 0) {
793 		mm->locked_vm += new_len >> PAGE_SHIFT;
797 	mm->hiwater_vm = hiwater_vm;
816 	struct mm_struct *mm = current->mm;
820 	vma = vma_lookup(mm, addr);
857 	if (!mlock_future_ok(mm, vma->vm_flags, new_len - old_len))
860 	if (!may_expand_vm(mm, vma->vm_flags,
873 	struct mm_struct *mm = current->mm;
902 	if ((mm->map_count + 2) >= sysctl_max_map_count - 3)
915 	if (unlikely(!can_modify_mm(mm, addr, addr + old_len)))
924 		ret = do_munmap(mm, new_addr, new_len, uf_unmap_early);
930 		ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap);
944 		!may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) {
997 	struct mm_struct *mm = current->mm;
1047 	if (mmap_write_lock_killable(current->mm))
1049 	vma = vma_lookup(mm, addr);
1090 	if (unlikely(!can_modify_mm(mm, addr, addr + old_len))) {
1102 		VMA_ITERATOR(vmi, mm, addr + new_len);
1109 		ret = do_vmi_munmap(&vmi, mm, addr + new_len, old_len - new_len,
1135 			VMA_ITERATOR(vmi, mm, vma->vm_end);
1139 				if (security_vm_enough_memory_mm(mm, pages)) {
1162 			vm_stat_account(mm, vma->vm_flags, pages);
1164 				mm->locked_vm += pages;
1198 	mmap_write_unlock(current->mm);
1202 	userfaultfd_unmap_complete(mm, &uf_unmap_early);
1204 	userfaultfd_unmap_complete(mm, &uf_unmap);