iommu/iommufd/pages.c

262  * PFNs. This is used as a temporary holding memory for shuttling pfns from one
264  * work on the largest possible grouping of pfns. eg fewer lock/unlock cycles,
268 	unsigned long *pfns;
279 	batch->pfns[0] = 0;
297 	batch->pfns[0] = batch->pfns[batch->end - 1] +
310 	batch->pfns[0] += skip_pfns;
318 	const size_t elmsz = sizeof(*batch->pfns) + sizeof(*batch->npfns);
321 	batch->pfns = temp_kmalloc(&size, backup, backup_len);
322 	if (!batch->pfns)
327 	batch->npfns = (u32 *)(batch->pfns + batch->array_size);
345 	if (batch->pfns != backup)
346 		kfree(batch->pfns);
355 	    pfn == batch->pfns[batch->end - 1] + batch->npfns[batch->end - 1] &&
364 	batch->pfns[batch->end] = pfn;
371  * Fill the batch with pfns from the domain. When the batch is full, or it
425 /* Continues reading a domain until we reach a discontinuity in the pfns. */
499 				PFN_PHYS(batch->pfns[cur]) + page_offset,
503 				       PFN_PHYS(batch->pfns[cur]) + page_offset,
642 			pfn_to_page(batch->pfns[cur] + first_page_off),
677 		copy_data_page(pfn_to_page(batch->pfns[cur] + npage), data,
933  * This iterator reads the pfns optimizing to load according to the
947 static int pfn_reader_update_pinned(struct pfn_reader *pfns)
949 	return pfn_reader_user_update_pinned(&pfns->user, pfns->pages);
956 static void pfn_reader_unpin(struct pfn_reader *pfns)
958 	unsigned long last = pfns->batch_end_index - 1;
959 	unsigned long start = pfns->batch_start_index;
961 	struct iopt_pages *pages = pfns->pages;
970 		batch_unpin(&pfns->batch, pages, span.start_hole - start,
976 static int pfn_reader_fill_span(struct pfn_reader *pfns)
978 	struct interval_tree_double_span_iter *span = &pfns->span;
979 	unsigned long start_index = pfns->batch_end_index;
988 		batch_from_xarray(&pfns->batch, &pfns->pages->pinned_pfns,
999 		area = iopt_pages_find_domain_area(pfns->pages, start_index);
1005 			&pfns->batch, area->storage_domain, area, start_index,
1010 	if (start_index >= pfns->user.upages_end) {
1011 		rc = pfn_reader_user_pin(&pfns->user, pfns->pages, start_index,
1017 	batch_from_pages(&pfns->batch,
1018 			 pfns->user.upages +
1019 				 (start_index - pfns->user.upages_start),
1020 			 pfns->user.upages_end - start_index);
1024 static bool pfn_reader_done(struct pfn_reader *pfns)
1026 	return pfns->batch_start_index == pfns->last_index + 1;
1029 static int pfn_reader_next(struct pfn_reader *pfns)
1033 	batch_clear(&pfns->batch);
1034 	pfns->batch_start_index = pfns->batch_end_index;
1036 	while (pfns->batch_end_index != pfns->last_index + 1) {
1037 		unsigned int npfns = pfns->batch.total_pfns;
1040 		    WARN_ON(interval_tree_double_span_iter_done(&pfns->span)))
1043 		rc = pfn_reader_fill_span(pfns);
1047 		if (WARN_ON(!pfns->batch.total_pfns))
1050 		pfns->batch_end_index =
1051 			pfns->batch_start_index + pfns->batch.total_pfns;
1052 		if (pfns->batch_end_index == pfns->span.last_used + 1)
1053 			interval_tree_double_span_iter_next(&pfns->span);
1056 		if (npfns == pfns->batch.total_pfns)
1062 static int pfn_reader_init(struct pfn_reader *pfns, struct iopt_pages *pages,
1069 	pfns->pages = pages;
1070 	pfns->batch_start_index = start_index;
1071 	pfns->batch_end_index = start_index;
1072 	pfns->last_index = last_index;
1073 	pfn_reader_user_init(&pfns->user, pages);
1074 	rc = batch_init(&pfns->batch, last_index - start_index + 1);
1077 	interval_tree_double_span_iter_first(&pfns->span, &pages->access_itree,
1091 static void pfn_reader_release_pins(struct pfn_reader *pfns)
1093 	struct iopt_pages *pages = pfns->pages;
1095 	if (pfns->user.upages_end > pfns->batch_end_index) {
1096 		size_t npages = pfns->user.upages_end - pfns->batch_end_index;
1099 		unpin_user_pages(pfns->user.upages + (pfns->batch_end_index -
1100 						      pfns->user.upages_start),
1103 		pfns->user.upages_end = pfns->batch_end_index;
1105 	if (pfns->batch_start_index != pfns->batch_end_index) {
1106 		pfn_reader_unpin(pfns);
1107 		pfns->batch_start_index = pfns->batch_end_index;
1111 static void pfn_reader_destroy(struct pfn_reader *pfns)
1113 	struct iopt_pages *pages = pfns->pages;
1115 	pfn_reader_release_pins(pfns);
1116 	pfn_reader_user_destroy(&pfns->user, pfns->pages);
1117 	batch_destroy(&pfns->batch, NULL);
1121 static int pfn_reader_first(struct pfn_reader *pfns, struct iopt_pages *pages,
1130 	rc = pfn_reader_init(pfns, pages, start_index, last_index);
1133 	rc = pfn_reader_next(pfns);
1135 		pfn_reader_destroy(pfns);
1226 		 * unmaps must always 'cut' at a place where the pfns are not
1229 		 * middle of the domains we need to keep reading pfns until we
1230 		 * find a cut point to do the unmap. The pfns we read are
1354  * Read the pfns from the area's underlying iopt_pages and map them into the
1360 	struct pfn_reader pfns;
1365 	rc = pfn_reader_first(&pfns, area->pages, iopt_area_index(area),
1370 	while (!pfn_reader_done(&pfns)) {
1371 		done_end_index = pfns.batch_start_index;
1372 		rc = batch_to_domain(&pfns.batch, domain, area,
1373 				     pfns.batch_start_index);
1376 		done_end_index = pfns.batch_end_index;
1378 		rc = pfn_reader_next(&pfns);
1383 	rc = pfn_reader_update_pinned(&pfns);
1389 	pfn_reader_release_pins(&pfns);
1393 	pfn_reader_destroy(&pfns);
1414 	struct pfn_reader pfns;
1424 	rc = pfn_reader_first(&pfns, pages, iopt_area_index(area),
1429 	while (!pfn_reader_done(&pfns)) {
1430 		done_first_end_index = pfns.batch_end_index;
1431 		done_all_end_index = pfns.batch_start_index;
1433 			rc = batch_to_domain(&pfns.batch, domain, area,
1434 					     pfns.batch_start_index);
1440 		rc = pfn_reader_next(&pfns);
1444 	rc = pfn_reader_update_pinned(&pfns);
1453 	pfn_reader_release_pins(&pfns);
1478 	pfn_reader_destroy(&pfns);
1749 	struct pfn_reader pfns;
1754 	rc = pfn_reader_first(&pfns, pages, start_index, last_index);
1758 	while (!pfn_reader_done(&pfns)) {
1761 		done = batch_rw(&pfns.batch, data, offset, length, flags);
1765 		pfn_reader_unpin(&pfns);
1767 		rc = pfn_reader_next(&pfns);
1774 	pfn_reader_destroy(&pfns);