drivers/iommu/iova.c

8 #include <linux/iova.h>
31 static struct iova *to_iova(struct rb_node *node)
33 	return rb_entry(node, struct iova, node);
71 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new)
80 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)
82 	struct iova *cached_iova;
135 /* Insert the iova into domain rbtree by holding writer lock */
137 iova_insert_rbtree(struct rb_root *root, struct iova *iova,
145 		struct iova *this = to_iova(*new);
149 		if (iova->pfn_lo < this->pfn_lo)
151 		else if (iova->pfn_lo > this->pfn_lo)
159 	rb_link_node(&iova->node, parent, new);
160 	rb_insert_color(&iova->node, root);
165 			struct iova *new, bool size_aligned)
168 	struct iova *curr_iova;
228 static struct iova *alloc_iova_mem(void)
233 static void free_iova_mem(struct iova *iova)
235 	if (iova->pfn_lo != IOVA_ANCHOR)
236 		kmem_cache_free(iova_cache, iova);
240  * alloc_iova - allocates an iova
241  * @iovad: - iova domain in question
245  * This function allocates an iova in the range iovad->start_pfn to limit_pfn,
247  * flag is set then the allocated address iova->pfn_lo will be naturally
250 struct iova *
255 	struct iova *new_iova;
274 static struct iova *
282 		struct iova *iova = to_iova(node);
284 		if (pfn < iova->pfn_lo)
286 		else if (pfn > iova->pfn_hi)
289 			return iova;	/* pfn falls within iova's range */
295 static void remove_iova(struct iova_domain *iovad, struct iova *iova)
298 	__cached_rbnode_delete_update(iovad, iova);
299 	rb_erase(&iova->node, &iovad->rbroot);
303  * find_iova - finds an iova for a given pfn
304  * @iovad: - iova domain in question.
306  * This function finds and returns an iova belonging to the
309 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)
312 	struct iova *iova;
316 	iova = private_find_iova(iovad, pfn);
318 	return iova;
323  * __free_iova - frees the given iova
324  * @iovad: iova domain in question.
325  * @iova: iova in question.
326  * Frees the given iova belonging to the giving domain
329 __free_iova(struct iova_domain *iovad, struct iova *iova)
334 	remove_iova(iovad, iova);
336 	free_iova_mem(iova);
341  * free_iova - finds and frees the iova for a given pfn
342  * @iovad: - iova domain in question.
344  * This functions finds an iova for a given pfn and then
345  * frees the iova from that domain.
351 	struct iova *iova;
354 	iova = private_find_iova(iovad, pfn);
355 	if (!iova) {
359 	remove_iova(iovad, iova);
361 	free_iova_mem(iova);
366  * alloc_iova_fast - allocates an iova from rcache
367  * @iovad: - iova domain in question
371  * This function tries to satisfy an iova allocation from the rcache,
380 	struct iova *new_iova;
416  * free_iova_fast - free iova pfn range into rcache
417  * @iovad: - iova domain in question.
420  * This functions frees an iova range by trying to put it into the rcache,
421  * falling back to regular iova deallocation via free_iova() if this fails.
441  * put_iova_domain - destroys the iova domain
442  * @iovad: - iova domain in question.
443  * All the iova's in that domain are destroyed.
447 	struct iova *iova, *tmp;
452 	rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
453 		free_iova_mem(iova);
461 	struct iova *iova = to_iova(node);
463 	if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo))
468 static inline struct iova *
471 	struct iova *iova;
473 	iova = alloc_iova_mem();
474 	if (iova) {
475 		iova->pfn_lo = pfn_lo;
476 		iova->pfn_hi = pfn_hi;
479 	return iova;
482 static struct iova *
486 	struct iova *iova;
488 	iova = alloc_and_init_iova(pfn_lo, pfn_hi);
489 	if (iova)
490 		iova_insert_rbtree(&iovad->rbroot, iova, NULL);
492 	return iova;
496 __adjust_overlap_range(struct iova *iova,
499 	if (*pfn_lo < iova->pfn_lo)
500 		iova->pfn_lo = *pfn_lo;
501 	if (*pfn_hi > iova->pfn_hi)
502 		*pfn_lo = iova->pfn_hi + 1;
506  * reserve_iova - reserves an iova in the given range
507  * @iovad: - iova domain pointer
513 struct iova *
519 	struct iova *iova;
529 			iova = to_iova(node);
530 			__adjust_overlap_range(iova, &pfn_lo, &pfn_hi);
531 			if ((pfn_lo >= iova->pfn_lo) &&
532 				(pfn_hi <= iova->pfn_hi))
543 	iova = __insert_new_range(iovad, pfn_lo, pfn_hi);
547 	return iova;
625 		struct iova *iova = private_find_iova(iovad, mag->pfns[i]);
627 		if (WARN_ON(!iova))
630 		remove_iova(iovad, iova);
631 		free_iova_mem(iova);
953 		iova_cache = kmem_cache_create("iommu_iova", sizeof(struct iova), 0,
964 		err = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead",