iommu/iommufd/pages.c

193  * index is the number of PAGE_SIZE units from the start of the area's
194  * iopt_pages. If the iova is sub page-size then the area has an iova that
197 static unsigned long iopt_area_index_to_iova(struct iopt_area *area,
201 		WARN_ON(index < iopt_area_index(area) ||
202 			index > iopt_area_last_index(area));
203 	index -= iopt_area_index(area);
205 		return iopt_area_iova(area);
206 	return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE;
209 static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area,
213 		WARN_ON(index < iopt_area_index(area) ||
214 			index > iopt_area_last_index(area));
215 	if (index == iopt_area_last_index(area))
216 		return iopt_area_last_iova(area);
217 	return iopt_area_iova(area) - area->page_offset +
218 	       (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1;
237 static void iopt_area_unmap_domain_range(struct iopt_area *area,
242 	unsigned long start_iova = iopt_area_index_to_iova(area, start_index);
245 			   iopt_area_index_to_iova_last(area, last_index) -
377 			      struct iopt_area *area, unsigned long start_index,
384 	iova = iopt_area_index_to_iova(area, start_index);
385 	if (start_index == iopt_area_index(area))
386 		page_offset = area->page_offset;
403 					   struct iopt_area *area,
412 	iova = iopt_area_index_to_iova(area, start_index);
413 	if (start_index == iopt_area_index(area))
414 		page_offset = area->page_offset;
428 				       struct iopt_area *area,
435 	batch_from_domain(batch, domain, area, start_index, last_index);
441  * mode permits splitting a mapped area up, and then one of the splits is
476 			   struct iopt_area *area, unsigned long start_index)
478 	bool disable_large_pages = area->iopt->disable_large_pages;
479 	unsigned long last_iova = iopt_area_last_iova(area);
488 	if (start_index == iopt_area_index(area))
489 		page_offset = area->page_offset;
491 		iopt_area_index_to_iova(area, start_index);
500 				next_iova - iova, area->iommu_prot);
504 				       next_iova - iova, area->iommu_prot,
930  * - The iommu_domain under an area
980 	struct iopt_area *area;
997 		 * and we'll find another area.
999 		area = iopt_pages_find_domain_area(pfns->pages, start_index);
1000 		if (WARN_ON(!area))
1005 			&pfns->batch, area->storage_domain, area, start_index,
1006 			min(iopt_area_last_index(area), span->last_used));
1197 iopt_area_unpin_domain(struct pfn_batch *batch, struct iopt_area *area,
1215 			batch_from_domain(batch, domain, area, start,
1236 			batch_from_domain_continue(batch, domain, area,
1242 				area, domain, *unmapped_end_index,
1257 static void __iopt_area_unfill_domain(struct iopt_area *area,
1263 	unsigned long start_index = iopt_area_index(area);
1293 		iopt_area_unpin_domain(&batch, area, pages, domain,
1302 		iopt_area_unmap_domain_range(area, domain, unmapped_end_index,
1309 static void iopt_area_unfill_partial_domain(struct iopt_area *area,
1314 	if (end_index != iopt_area_index(area))
1315 		__iopt_area_unfill_domain(area, pages, domain, end_index - 1);
1320  * @area: The IOVA range to unmap
1326 void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain)
1328 	iommu_unmap_nofail(domain, iopt_area_iova(area),
1329 			   iopt_area_length(area));
1334  * @area: IOVA area to use
1335  * @pages: page supplier for the area (area->pages is NULL)
1342 void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages,
1345 	__iopt_area_unfill_domain(area, pages, domain,
1346 				  iopt_area_last_index(area));
1350  * iopt_area_fill_domain() - Map PFNs from the area into a domain
1351  * @area: IOVA area to use
1354  * Read the pfns from the area's underlying iopt_pages and map them into the
1357 int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain)
1363 	lockdep_assert_held(&area->pages->mutex);
1365 	rc = pfn_reader_first(&pfns, area->pages, iopt_area_index(area),
1366 			      iopt_area_last_index(area));
1372 		rc = batch_to_domain(&pfns.batch, domain, area,
1390 	iopt_area_unfill_partial_domain(area, area->pages, domain,
1398  * iopt_area_fill_domains() - Install PFNs into the area's domains
1399  * @area: The area to act on
1400  * @pages: The pages associated with the area (area->pages is NULL)
1402  * Called during area creation. The area is freshly created and not inserted in
1404  * area's io_pagetable and the area is installed in the domains_itree.
1408 int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages)
1418 	lockdep_assert_held(&area->iopt->domains_rwsem);
1420 	if (xa_empty(&area->iopt->domains))
1424 	rc = pfn_reader_first(&pfns, pages, iopt_area_index(area),
1425 			      iopt_area_last_index(area));
1432 		xa_for_each(&area->iopt->domains, index, domain) {
1433 			rc = batch_to_domain(&pfns.batch, domain, area,
1448 	area->storage_domain = xa_load(&area->iopt->domains, 0);
1449 	interval_tree_insert(&area->pages_node, &pages->domains_itree);
1454 	xa_for_each(&area->iopt->domains, unmap_index, domain) {
1463 		 * The area is not yet part of the domains_itree so we have to
1467 		if (unmap_index != area->iopt->next_domain_id - 1) {
1468 			if (end_index != iopt_area_index(area))
1470 					area, domain, iopt_area_index(area),
1473 			iopt_area_unfill_partial_domain(area, pages, domain,
1485  * iopt_area_unfill_domains() - unmap PFNs from the area's domains
1486  * @area: The area to act on
1487  * @pages: The pages associated with the area (area->pages is NULL)
1489  * Called during area destruction. This unmaps the iova's covered by all the
1490  * area's domains and releases the PFNs.
1492 void iopt_area_unfill_domains(struct iopt_area *area, struct iopt_pages *pages)
1494 	struct io_pagetable *iopt = area->iopt;
1501 	if (!area->storage_domain)
1505 		if (domain != area->storage_domain)
1507 				area, domain, iopt_area_index(area),
1508 				iopt_area_last_index(area));
1511 		WARN_ON(RB_EMPTY_NODE(&area->pages_node.rb));
1512 	interval_tree_remove(&area->pages_node, &pages->domains_itree);
1513 	iopt_area_unfill_domain(area, pages, area->storage_domain);
1514 	area->storage_domain = NULL;
1617 		struct iopt_area *area;
1619 		area = iopt_pages_find_domain_area(pages, start_index);
1620 		if (WARN_ON(!area))
1623 		domain_last = min(iopt_area_last_index(area), last_index);
1624 		out_pages = raw_pages_from_domain(area->storage_domain, area,
1904  * @area: The source of PFNs
1915 int iopt_area_add_access(struct iopt_area *area, unsigned long start_index,
1919 	struct iopt_pages *pages = area->pages;
1929 		area->num_accesses++;
1950 	area->num_accesses++;
1964  * @area: The source of PFNs
1971 void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index,
1974 	struct iopt_pages *pages = area->pages;
1982 	WARN_ON(area->num_accesses == 0 || access->users == 0);
1983 	area->num_accesses--;