linux-master/mm/userfaultfd.c

185 	struct folio *folio;
215 	folio = page_folio(page);
219 			folio_add_lru(folio);
220 		folio_add_file_rmap_pte(folio, page, dst_vma);
222 		folio_add_new_anon_rmap(folio, dst_vma, dst_addr);
223 		folio_add_lru_vma(folio, dst_vma);
230 	inc_mm_counter(dst_mm, mm_counter(folio));
248 				 struct folio **foliop)
252 	struct folio *folio;
256 		folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, dst_vma,
258 		if (!folio)
261 		kaddr = kmap_local_folio(folio, 0);
286 			*foliop = folio;
291 		flush_dcache_folio(folio);
293 		folio = *foliop;
302 	__folio_mark_uptodate(folio);
305 	if (mem_cgroup_charge(folio, dst_vma->vm_mm, GFP_KERNEL))
309 				       &folio->page, true, flags);
315 	folio_put(folio);
323 	struct folio *folio;
326 	folio = vma_alloc_zeroed_movable_folio(dst_vma, dst_addr);
327 	if (!folio)
330 	if (mem_cgroup_charge(folio, dst_vma->vm_mm, GFP_KERNEL))
335 	 * zeroing out the folio become visible before mapping the page
338 	__folio_mark_uptodate(folio);
341 				       &folio->page, true, 0);
347 	folio_put(folio);
393 	struct folio *folio;
397 	ret = shmem_get_folio(inode, pgoff, &folio, SGP_NOALLOC);
398 	/* Our caller expects us to return -EFAULT if we failed to find folio */
403 	if (!folio) {
408 	page = folio_file_page(folio, pgoff);
419 	folio_unlock(folio);
424 	folio_unlock(folio);
425 	folio_put(folio);
507 	struct folio *folio;
528 	folio = NULL;
601 					       src_addr, flags, &folio);
611 			BUG_ON(!folio);
613 			err = copy_folio_from_user(folio,
623 			BUG_ON(folio);
642 	if (folio)
643 		folio_put(folio);
664 						struct folio **foliop)
715 	struct folio *folio;
730 	folio = NULL;
816 				       src_addr, flags, &folio);
824 			BUG_ON(!folio);
826 			kaddr = kmap_local_folio(folio, 0);
835 			flush_dcache_folio(folio);
838 			BUG_ON(folio);
856 	if (folio)
857 		folio_put(folio);
1037 			    struct folio *src_folio)
1145 	struct folio *src_folio = NULL;
1202 	/* If PTE changed after we locked the folio them start over */
1218 		 * Pin and lock both source folio and anon_vma. Since we are in
1223 			struct folio *folio;
1236 			folio = vm_normal_folio(src_vma, src_addr, orig_src_pte);
1237 			if (!folio || !PageAnonExclusive(&folio->page)) {
1243 			folio_get(folio);
1244 			src_folio = folio;
1272 			/* have to reacquire the folio after it got split */
1282 			 * without the folio lock. Serialize against it with
1283 			 * the anon_vma lock, the folio lock is not enough.
1590  * first obtaining the folio lock (the only current instance is
1591  * folio_referenced), they will have to verify if the folio->mapping
1700 				struct folio *folio = pfn_folio(pmd_pfn(*src_pmd));
1702 				if (!folio || (!is_huge_zero_page(&folio->page) &&
1703 					       !PageAnonExclusive(&folio->page))) {
1711 				/* The folio will be split by move_pages_pte() */