drivers/iommu/dma-iommu.c

613 static bool dev_use_sg_swiotlb(struct device *dev, struct scatterlist *sg,
627 	 * direction, check the individual lengths in the sg list. If any
631 		for_each_sg(sg, s, nents, i)
991 		struct scatterlist *sg;
994 		for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
995 			arch_dma_prep_coherent(sg_page(sg), sg->length);
1108 	struct scatterlist *sg;
1112 		for_each_sg(sgl, sg, nelems, i)
1113 			iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
1114 						      sg->length, dir);
1116 		for_each_sg(sgl, sg, nelems, i)
1117 			arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
1124 	struct scatterlist *sg;
1128 		for_each_sg(sgl, sg, nelems, i)
1130 							 sg_dma_address(sg),
1131 							 sg->length, dir);
1133 		for_each_sg(sgl, sg, nelems, i)
1134 			arch_sync_dma_for_device(sg_phys(sg), sg->length, dir);
1223 static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
1226 	struct scatterlist *s, *cur = sg;
1231 	for_each_sg(sg, s, nents, i) {
1291 static void __invalidate_sg(struct scatterlist *sg, int nents)
1296 	for_each_sg(sg, s, nents, i) {
1310 static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *sg,
1316 	for_each_sg(sg, s, nents, i)
1321 static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg,
1327 	sg_dma_mark_swiotlb(sg);
1329 	for_each_sg(sg, s, nents, i) {
1340 	iommu_dma_unmap_sg_swiotlb(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
1351 static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
1373 	if (dev_use_sg_swiotlb(dev, sg, nents, dir))
1374 		return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs);
1377 		iommu_dma_sync_sg_for_device(dev, sg, nents, dir);
1385 	for_each_sg(sg, s, nents, i) {
1443 		return __finalise_sg(dev, sg, nents, 0);
1455 	ret = iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC);
1459 	return __finalise_sg(dev, sg, nents, iova);
1464 	__invalidate_sg(sg, nents);
1471 static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
1478 	if (sg_dma_is_swiotlb(sg)) {
1479 		iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs);
1484 		iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);
1491 	for_each_sg(sg, tmp, nents, i) {