linux-master/mm/gup.c

1192 		unsigned int gup_flags, struct vm_area_struct **vma,
1204 	if (gup_flags & FOLL_WRITE)
1232 		if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(entry)))
1236 	ret = try_grab_page(*page, gup_flags);
1347 					  unsigned long gup_flags)
1353 	if ((gup_flags & (FOLL_PIN | FOLL_LONGTERM)) !=
1364 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
1367 	int write = (gup_flags & FOLL_WRITE);
1368 	int foreign = (gup_flags & FOLL_REMOTE);
1374 	if ((gup_flags & FOLL_ANON) && !vma_anon)
1377 	if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma))
1385 		    !writable_file_mapping_allowed(vma, gup_flags))
1389 			if (!(gup_flags & FOLL_FORCE))
1407 		if (!(gup_flags & FOLL_FORCE))
1469  * @gup_flags:	flags modifying pin behaviour
1502  * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
1510  * A caller using such a combination of @gup_flags must therefore hold the
1516  * you need some special @gup_flags.
1520 		unsigned int gup_flags, struct page **pages,
1532 	VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
1536 		unsigned int foll_flags = gup_flags;
1545 			if (gup_flags & FOLL_MADV_POPULATE) {
1551 				if (check_vma_flags(vma, gup_flags)) {
1560 						gup_flags, &vma,
1572 			ret = check_vma_flags(vma, gup_flags);
1988 	int gup_flags;
2008 	gup_flags = FOLL_TOUCH;
2018 		gup_flags |= FOLL_WRITE;
2020 		gup_flags |= FOLL_FORCE;
2023 		gup_flags |= FOLL_UNLOCKABLE;
2029 	ret = __get_user_pages(mm, start, nr_pages, gup_flags,
2060 	int gup_flags;
2076 	gup_flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_UNLOCKABLE |
2079 		gup_flags |= FOLL_WRITE;
2082 				      gup_flags);
2550 				  unsigned int gup_flags)
2555 	if (!(gup_flags & FOLL_LONGTERM))
2557 					       locked, gup_flags);
2563 							  gup_flags);
2583 	unsigned int gup_flags = *gup_flags_p;
2592 	if (WARN_ON_ONCE(gup_flags & INTERNAL_GUP_FLAGS))
2595 	gup_flags |= to_set;
2601 		gup_flags |= FOLL_UNLOCKABLE;
2605 	if (WARN_ON_ONCE((gup_flags & (FOLL_PIN | FOLL_GET)) ==
2610 	if (WARN_ON_ONCE(!(gup_flags & FOLL_PIN) && (gup_flags & FOLL_LONGTERM)))
2614 	if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages))
2618 	if (WARN_ON_ONCE((gup_flags & FOLL_LONGTERM) &&
2619 			 (gup_flags & FOLL_PCI_P2PDMA)))
2622 	*gup_flags_p = gup_flags;
2632  * @gup_flags:	flags modifying lookup behaviour
2666  * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
2685 		unsigned int gup_flags, struct page **pages,
2690 	if (!is_valid_gup_args(pages, locked, &gup_flags,
2696 				       gup_flags);
2703 			   unsigned int gup_flags, struct page **pages,
2714  * @gup_flags:  flags modifying lookup behaviour
2725 		    unsigned int gup_flags, struct page **pages)
2729 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH))
2733 				       &locked, gup_flags);
2749  * get_user_pages_fast should be used instead if specific gup_flags
2753 			     struct page **pages, unsigned int gup_flags)
2757 	if (!is_valid_gup_args(pages, NULL, &gup_flags,
2762 				       &locked, gup_flags);
3391 		unsigned int gup_flags, struct page **pages)
3401 	if (gup_flags & FOLL_PIN) {
3419 	gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned);
3426 	if (gup_flags & FOLL_PIN) {
3438 		unsigned int gup_flags, struct page **pages)
3445 	if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
3451 	if (gup_flags & FOLL_PIN)
3454 	if (!(gup_flags & FOLL_FAST_ONLY))
3466 	nr_pinned = gup_fast(start, end, gup_flags, pages);
3467 	if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
3475 				    gup_flags | FOLL_TOUCH | FOLL_UNLOCKABLE);
3492  * @gup_flags:  flags modifying pin behaviour
3507 			     unsigned int gup_flags, struct page **pages)
3516 	if (!is_valid_gup_args(pages, NULL, &gup_flags,
3520 	return gup_fast_fallback(start, nr_pages, gup_flags, pages);
3528  * @gup_flags:  flags modifying pin behaviour
3541 			unsigned int gup_flags, struct page **pages)
3549 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET))
3551 	return gup_fast_fallback(start, nr_pages, gup_flags, pages);
3560  * @gup_flags:  flags modifying pin behaviour
3575 			unsigned int gup_flags, struct page **pages)
3577 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
3579 	return gup_fast_fallback(start, nr_pages, gup_flags, pages);
3589  * @gup_flags:	flags modifying lookup behaviour
3608 			   unsigned int gup_flags, struct page **pages,
3613 	if (!is_valid_gup_args(pages, locked, &gup_flags,
3618 				     gup_flags);
3627  * @gup_flags:	flags modifying lookup behaviour
3641 		    unsigned int gup_flags, struct page **pages)
3645 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))
3648 				     pages, &locked, gup_flags);
3661 			     struct page **pages, unsigned int gup_flags)
3665 	if (!is_valid_gup_args(pages, NULL, &gup_flags,
3670 				     &locked, gup_flags);