drivers/vfio/vfio_iommu_spapr_tce.c

209 		struct iommu_table *tbl = container->tables[i];
211 		if (tbl) {
212 			unsigned long entry = ioba >> tbl->it_page_shift;
213 			unsigned long start = tbl->it_offset;
214 			unsigned long end = start + tbl->it_size;
217 				*ptbl = tbl;
339 		struct iommu_table *tbl,
342 		struct iommu_table *tbl);
362 		struct iommu_table *tbl = container->tables[i];
364 		if (!tbl)
367 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
368 		tce_iommu_free_table(container, tbl);
411 		struct iommu_table *tbl, unsigned long entry)
416 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
422 			tbl->it_page_shift, &hpa, &mem);
433 		struct iommu_table *tbl,
442 		if (tbl->it_indirect_levels && tbl->it_userspace) {
451 			__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
455 				entry |= tbl->it_level_size - 1;
464 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
473 			tce_iommu_unuse_page_v2(container, tbl, entry);
480 	iommu_tce_kill(tbl, firstentry, pages);
501 		struct iommu_table *tbl,
510 		unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
517 				tbl->it_page_shift)) {
524 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
529 					__func__, entry << tbl->it_page_shift,
537 		tce += IOMMU_PAGE_SIZE(tbl);
541 		tce_iommu_clear(container, tbl, entry, i);
543 		iommu_tce_kill(tbl, entry, pages);
549 		struct iommu_table *tbl,
559 		__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
562 				tce, tbl->it_page_shift, &hpa, &mem);
567 				tbl->it_page_shift)) {
573 		hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
580 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
584 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
586 					__func__, entry << tbl->it_page_shift,
592 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
596 		tce += IOMMU_PAGE_SIZE(tbl);
600 		tce_iommu_clear(container, tbl, entry, i);
602 		iommu_tce_kill(tbl, entry, pages);
636 		struct iommu_table *tbl)
638 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
640 	iommu_tce_table_put(tbl);
650 	struct iommu_table *tbl = NULL;
674 			page_shift, window_size, levels, &tbl);
678 	BUG_ON(!tbl->it_ops->free);
687 		ret = table_group->ops->set_window(table_group, num, tbl);
692 	container->tables[num] = tbl;
695 	*start_addr = tbl->it_offset << tbl->it_page_shift;
704 	tce_iommu_free_table(container, tbl);
713 	struct iommu_table *tbl;
717 	num = tce_iommu_find_table(container, start_addr, &tbl);
721 	BUG_ON(!tbl->it_size);
741 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
742 	tce_iommu_free_table(container, tbl);
907 		struct iommu_table *tbl = NULL;
930 		num = tce_iommu_find_table(container, param.iova, &tbl);
934 		if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
935 				(param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
951 		ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
956 			ret = tce_iommu_build_v2(container, tbl,
957 					param.iova >> tbl->it_page_shift,
959 					param.size >> tbl->it_page_shift,
962 			ret = tce_iommu_build(container, tbl,
963 					param.iova >> tbl->it_page_shift,
965 					param.size >> tbl->it_page_shift,
968 		iommu_flush_tce(tbl);
974 		struct iommu_table *tbl = NULL;
997 		num = tce_iommu_find_table(container, param.iova, &tbl);
1001 		if (param.size & ~IOMMU_PAGE_MASK(tbl))
1004 		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
1005 				param.size >> tbl->it_page_shift);
1009 		ret = tce_iommu_clear(container, tbl,
1010 				param.iova >> tbl->it_page_shift,
1011 				param.size >> tbl->it_page_shift);
1012 		iommu_flush_tce(tbl);
1212 		struct iommu_table *tbl = container->tables[i];
1214 		if (!tbl)
1217 		ret = table_group->ops->set_window(table_group, i, tbl);