iommu/iommufd/io_pagetable.c

25 	struct iopt_area *area;
40 	iter->area = iopt_area_iter_first(iopt, iova, iova);
41 	if (!iter->area)
43 	if (!iter->area->pages) {
44 		iter->area = NULL;
47 	return iter->area;
54 	if (!iter->area)
56 	last_iova = iopt_area_last_iova(iter->area);
61 	iter->area = iopt_area_iter_next(iter->area, iter->cur_iova,
63 	if (!iter->area)
65 	if (iter->cur_iova != iopt_area_iova(iter->area) ||
66 	    !iter->area->pages) {
67 		iter->area = NULL;
70 	return iter->area;
188  * The area takes a slice of the pages from start_bytes to start_byte + length
190 static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area,
200 	area->iommu_prot = iommu_prot;
201 	area->page_offset = start_byte % PAGE_SIZE;
202 	if (area->page_offset & (iopt->iova_alignment - 1))
205 	area->node.start = iova;
206 	if (check_add_overflow(iova, length - 1, &area->node.last))
209 	area->pages_node.start = start_byte / PAGE_SIZE;
210 	if (check_add_overflow(start_byte, length - 1, &area->pages_node.last))
212 	area->pages_node.last = area->pages_node.last / PAGE_SIZE;
213 	if (WARN_ON(area->pages_node.last >= pages->npages))
217 	 * The area is inserted with a NULL pages indicating it is not fully
220 	area->iopt = iopt;
221 	interval_tree_insert(&area->node, &iopt->area_itree);
227 	struct iopt_area *area;
229 	area = kzalloc(sizeof(*area), GFP_KERNEL_ACCOUNT);
230 	if (!area)
232 	RB_CLEAR_NODE(&area->node.rb);
233 	RB_CLEAR_NODE(&area->pages_node.rb);
234 	return area;
247 		elm->area = iopt_area_alloc();
248 		if (!elm->area)
284 		rc = iopt_insert_area(iopt, elm->area, elm->pages, iova,
296 static void iopt_abort_area(struct iopt_area *area)
299 		WARN_ON(area->pages);
300 	if (area->iopt) {
301 		down_write(&area->iopt->iova_rwsem);
302 		interval_tree_remove(&area->node, &area->iopt->area_itree);
303 		up_write(&area->iopt->iova_rwsem);
305 	kfree(area);
314 		if (elm->area)
315 			iopt_abort_area(elm->area);
330 		rc = iopt_area_fill_domains(elm->area, elm->pages);
340 		iopt_area_unfill_domains(undo_elm->area, undo_elm->pages);
365 		 * area->pages must be set inside the domains_rwsem to ensure
369 		elm->area->pages = elm->pages;
371 		elm->area = NULL;
419 		if (elm.area)
420 			iopt_abort_area(elm.area);
439 	struct iopt_area *area;
449 	iopt_for_each_contig_area(&iter, area, arg->iopt, iova, last_iova) {
450 		unsigned long last = min(last_iova, iopt_area_last_iova(area));
551 	struct iopt_area *area;
558 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
559 	     area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
560 		if (!area->pages)
563 		ret = ops->read_and_clear_dirty(domain, iopt_area_iova(area),
564 						iopt_area_length(area), 0,
604 	struct iopt_area *area;
613 	iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
615 		unsigned long last = min(last_iova, iopt_area_last_iova(area));
622 		elm->start_byte = iopt_area_start_byte(area, iter.cur_iova);
623 		elm->pages = area->pages;
643 	struct iopt_area *area;
649 	 * The domains_rwsem must be held in read mode any time any area->pages
651 	 * concurrently with cleaning up the area.
656 	while ((area = iopt_area_iter_first(iopt, start, last))) {
657 		unsigned long area_last = iopt_area_last_iova(area);
658 		unsigned long area_first = iopt_area_iova(area);
661 		/* Userspace should not race map/unmap's of the same area */
662 		if (!area->pages) {
680 		if (area->num_accesses) {
681 			size_t length = iopt_area_length(area);
684 			area->prevent_access = true;
696 		pages = area->pages;
697 		area->pages = NULL;
700 		iopt_area_unfill_domains(area, pages);
701 		iopt_abort_area(area);
864  * This is used when removing a domain from the iopt. Every area in the iopt
871 	struct iopt_area *area;
885 		for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
886 		     area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
887 			struct iopt_pages *pages = area->pages;
894 				WARN_ON(!area->storage_domain);
895 			if (area->storage_domain == domain)
896 				area->storage_domain = storage_domain;
899 			iopt_area_unmap_domain(area, domain);
904 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
905 	     area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
906 		struct iopt_pages *pages = area->pages;
912 		interval_tree_remove(&area->pages_node, &pages->domains_itree);
913 		WARN_ON(area->storage_domain != domain);
914 		area->storage_domain = NULL;
915 		iopt_area_unfill_domain(area, pages, domain);
925  * Fill the domain with PFNs from every area in the iopt. On failure the domain
932 	struct iopt_area *area;
938 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
939 	     area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
940 		struct iopt_pages *pages = area->pages;
946 		rc = iopt_area_fill_domain(area, domain);
951 		if (!area->storage_domain) {
953 			area->storage_domain = domain;
954 			interval_tree_insert(&area->pages_node,
962 	end_area = area;
963 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
964 	     area = iopt_area_iter_next(area, 0, ULONG_MAX)) {
965 		struct iopt_pages *pages = area->pages;
967 		if (area == end_area)
973 			interval_tree_remove(&area->pages_node,
975 			area->storage_domain = NULL;
977 		iopt_area_unfill_domain(area, pages, domain);
983 /* All existing area's conform to an increased page size */
988 	struct iopt_area *area;
993 	for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
994 	     area = iopt_area_iter_next(area, 0, ULONG_MAX))
995 		if ((iopt_area_iova(area) & align_mask) ||
996 		    (iopt_area_length(area) & align_mask) ||
997 		    (area->page_offset & align_mask))
1052 	/* No area exists that is outside the allowed domain aperture */
1160  * iopt_area_split - Split an area into two parts at iova
1161  * @area: The area to split
1162  * @iova: Becomes the last of a new area
1164  * This splits an area into two. It is part of the VFIO compatibility to allow
1168 static int iopt_area_split(struct iopt_area *area, unsigned long iova)
1170 	unsigned long alignment = area->iopt->iova_alignment;
1171 	unsigned long last_iova = iopt_area_last_iova(area);
1172 	unsigned long start_iova = iopt_area_iova(area);
1174 	struct io_pagetable *iopt = area->iopt;
1175 	struct iopt_pages *pages = area->pages;
1185 	if (!pages || area->prevent_access)
1189 	    iopt_area_start_byte(area, new_start) & (alignment - 1))
1207 	if (area->num_accesses) {
1216 	if (area->storage_domain && !iopt->disable_large_pages) {
1221 	interval_tree_remove(&area->node, &iopt->area_itree);
1222 	rc = iopt_insert_area(iopt, lhs, area->pages, start_iova,
1223 			      iopt_area_start_byte(area, start_iova),
1225 			      area->iommu_prot);
1229 	rc = iopt_insert_area(iopt, rhs, area->pages, new_start,
1230 			      iopt_area_start_byte(area, new_start),
1231 			      last_iova - new_start + 1, area->iommu_prot);
1236 	 * If the original area has filled a domain, domains_itree has to be
1239 	if (area->storage_domain) {
1240 		interval_tree_remove(&area->pages_node, &pages->domains_itree);
1245 	lhs->storage_domain = area->storage_domain;
1246 	lhs->pages = area->pages;
1247 	rhs->storage_domain = area->storage_domain;
1248 	rhs->pages = area->pages;
1250 	kfree(area);
1262 	interval_tree_insert(&area->node, &iopt->area_itree);
1279 		struct iopt_area *area;
1281 		area = iopt_area_iter_first(iopt, iovas[i], iovas[i]);
1282 		if (!area)
1284 		rc = iopt_area_split(area, iovas[i]);