iommu/iommufd/pages.c

708 static void pfn_reader_user_init(struct pfn_reader_user *user,
711 	user->upages = NULL;
712 	user->upages_start = 0;
713 	user->upages_end = 0;
714 	user->locked = -1;
716 	user->gup_flags = FOLL_LONGTERM;
718 		user->gup_flags |= FOLL_WRITE;
721 static void pfn_reader_user_destroy(struct pfn_reader_user *user,
724 	if (user->locked != -1) {
725 		if (user->locked)
729 		user->locked = -1;
732 	kfree(user->upages);
733 	user->upages = NULL;
736 static int pfn_reader_user_pin(struct pfn_reader_user *user,
750 	if (!user->upages) {
752 		user->upages_len =
753 			(last_index - start_index + 1) * sizeof(*user->upages);
754 		user->upages = temp_kmalloc(&user->upages_len, NULL, 0);
755 		if (!user->upages)
759 	if (user->locked == -1) {
769 		user->locked = 0;
773 		       user->upages_len / sizeof(*user->upages));
781 		rc = pin_user_pages_fast(uptr, npages, user->gup_flags,
782 					 user->upages);
784 		if (!user->locked) {
786 			user->locked = 1;
789 					   user->gup_flags, user->upages,
790 					   &user->locked);
798 	user->upages_start = start_index;
799 	user->upages_end = start_index + rc;
831 			       bool inc, struct pfn_reader_user *user)
836 	if (user && user->locked) {
838 		user->locked = 0;
840 	} else if ((!user || !user->upages) &&
858 			    bool inc, struct pfn_reader_user *user)
872 		rc = update_mm_locked_vm(pages, npages, inc, user);
898  * and processed. If the user lacked the limit then the error unwind will unpin
903 static int pfn_reader_user_update_pinned(struct pfn_reader_user *user,
923 	return do_update_pinned(pages, npages, inc, user);
944 	struct pfn_reader_user user;
949 	return pfn_reader_user_update_pinned(&pfns->user, pfns->pages);
1010 	if (start_index >= pfns->user.upages_end) {
1011 		rc = pfn_reader_user_pin(&pfns->user, pfns->pages, start_index,
1018 			 pfns->user.upages +
1019 				 (start_index - pfns->user.upages_start),
1020 			 pfns->user.upages_end - start_index);
1073 	pfn_reader_user_init(&pfns->user, pages);
1095 	if (pfns->user.upages_end > pfns->batch_end_index) {
1096 		size_t npages = pfns->user.upages_end - pfns->batch_end_index;
1099 		unpin_user_pages(pfns->user.upages + (pfns->batch_end_index -
1100 						      pfns->user.upages_start),
1103 		pfns->user.upages_end = pfns->batch_end_index;
1116 	pfn_reader_user_destroy(&pfns->user, pfns->pages);
1633 				   struct pfn_reader_user *user,
1642 		user->upages = out_pages + (cur_index - start_index);
1643 		rc = pfn_reader_user_pin(user, pages, cur_index, last_index);
1646 		cur_index = user->upages_end;
1676 	struct pfn_reader_user user;
1681 	pfn_reader_user_init(&user, pages);
1682 	user.upages_len = (last_index - start_index + 1) * sizeof(*out_pages);
1710 		rc = iopt_pages_fill_from_mm(pages, &user, span.start_hole,
1723 	rc = pfn_reader_user_update_pinned(&user, pages);
1726 	user.upages = NULL;
1727 	pfn_reader_user_destroy(&user, pages);
1733 	user.upages = NULL;
1734 	pfn_reader_user_destroy(&user, pages);