vdpa/vdpa_user/iova_domain.c

20 static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,
35 	ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last,
45 static void vduse_iotlb_del_range(struct vduse_iova_domain *domain,
51 	while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) {
55 		vhost_iotlb_map_free(domain->iotlb, map);
59 int vduse_domain_set_map(struct vduse_iova_domain *domain,
67 	spin_lock(&domain->iotlb_lock);
68 	vduse_iotlb_del_range(domain, start, last);
73 		ret = vduse_iotlb_add_range(domain, map->start, map->last,
80 	spin_unlock(&domain->iotlb_lock);
84 	vduse_iotlb_del_range(domain, start, last);
85 	spin_unlock(&domain->iotlb_lock);
89 void vduse_domain_clear_map(struct vduse_iova_domain *domain,
95 	spin_lock(&domain->iotlb_lock);
98 		vduse_iotlb_del_range(domain, map->start, map->last);
100 	spin_unlock(&domain->iotlb_lock);
103 static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,
110 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];
123 static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,
130 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];
160 static void vduse_domain_bounce(struct vduse_iova_domain *domain,
169 	if (iova >= domain->bounce_size)
173 		map = &domain->bounce_maps[iova >> PAGE_SHIFT];
190 vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova)
197 	spin_lock(&domain->iotlb_lock);
198 	map = vhost_iotlb_itree_first(domain->iotlb, start, last);
205 	spin_unlock(&domain->iotlb_lock);
211 vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)
216 	read_lock(&domain->bounce_lock);
217 	map = &domain->bounce_maps[iova >> PAGE_SHIFT];
218 	if (domain->user_bounce_pages || !map->bounce_page)
224 	read_unlock(&domain->bounce_lock);
230 vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
235 	bounce_pfns = domain->bounce_size >> PAGE_SHIFT;
238 		map = &domain->bounce_maps[pfn];
250 int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
257 	if (count != (domain->bounce_size >> PAGE_SHIFT))
260 	write_lock(&domain->bounce_lock);
262 	if (domain->user_bounce_pages)
266 		map = &domain->bounce_maps[i];
278 	domain->user_bounce_pages = true;
281 	write_unlock(&domain->bounce_lock);
286 void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)
291 	write_lock(&domain->bounce_lock);
292 	if (!domain->user_bounce_pages)
295 	count = domain->bounce_size >> PAGE_SHIFT;
299 		map = &domain->bounce_maps[i];
312 	domain->user_bounce_pages = false;
314 	write_unlock(&domain->bounce_lock);
317 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)
319 	if (!domain->bounce_map)
322 	spin_lock(&domain->iotlb_lock);
323 	if (!domain->bounce_map)
326 	vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1);
327 	domain->bounce_map = 0;
329 	spin_unlock(&domain->iotlb_lock);
332 static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)
336 	if (domain->bounce_map)
339 	spin_lock(&domain->iotlb_lock);
340 	if (domain->bounce_map)
343 	ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1,
344 				    0, VHOST_MAP_RW, domain->file, 0);
348 	domain->bounce_map = 1;
350 	spin_unlock(&domain->iotlb_lock);
376 void vduse_domain_sync_single_for_device(struct vduse_iova_domain *domain,
380 	read_lock(&domain->bounce_lock);
382 		vduse_domain_bounce(domain, dma_addr, size, DMA_TO_DEVICE);
383 	read_unlock(&domain->bounce_lock);
386 void vduse_domain_sync_single_for_cpu(struct vduse_iova_domain *domain,
390 	read_lock(&domain->bounce_lock);
392 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
393 	read_unlock(&domain->bounce_lock);
396 dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
401 	struct iova_domain *iovad = &domain->stream_iovad;
402 	unsigned long limit = domain->bounce_size - 1;
409 	if (vduse_domain_init_bounce_map(domain))
412 	read_lock(&domain->bounce_lock);
413 	if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))
418 		vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);
420 	read_unlock(&domain->bounce_lock);
424 	read_unlock(&domain->bounce_lock);
430 void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
434 	struct iova_domain *iovad = &domain->stream_iovad;
435 	read_lock(&domain->bounce_lock);
438 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
440 	vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
441 	read_unlock(&domain->bounce_lock);
445 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
449 	struct iova_domain *iovad = &domain->consistent_iovad;
450 	unsigned long limit = domain->iova_limit;
457 	spin_lock(&domain->iotlb_lock);
458 	if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,
460 				  domain->file, (u64)iova)) {
461 		spin_unlock(&domain->iotlb_lock);
464 	spin_unlock(&domain->iotlb_lock);
479 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
483 	struct iova_domain *iovad = &domain->consistent_iovad;
488 	spin_lock(&domain->iotlb_lock);
489 	map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,
492 		spin_unlock(&domain->iotlb_lock);
499 	vhost_iotlb_map_free(domain->iotlb, map);
500 	spin_unlock(&domain->iotlb_lock);
508 	struct vduse_iova_domain *domain = vmf->vma->vm_private_data;
512 	if (!domain)
515 	if (iova < domain->bounce_size)
516 		page = vduse_domain_get_bounce_page(domain, iova);
518 		page = vduse_domain_get_coherent_page(domain, iova);
534 	struct vduse_iova_domain *domain = file->private_data;
537 	vma->vm_private_data = domain;
545 	struct vduse_iova_domain *domain = file->private_data;
547 	spin_lock(&domain->iotlb_lock);
548 	vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
549 	vduse_domain_remove_user_bounce_pages(domain);
550 	vduse_domain_free_kernel_bounce_pages(domain);
551 	spin_unlock(&domain->iotlb_lock);
552 	put_iova_domain(&domain->stream_iovad);
553 	put_iova_domain(&domain->consistent_iovad);
554 	vhost_iotlb_free(domain->iotlb);
555 	vfree(domain->bounce_maps);
556 	kfree(domain);
567 void vduse_domain_destroy(struct vduse_iova_domain *domain)
569 	fput(domain->file);
575 	struct vduse_iova_domain *domain;
585 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
586 	if (!domain)
589 	domain->iotlb = vhost_iotlb_alloc(0, 0);
590 	if (!domain->iotlb)
593 	domain->iova_limit = iova_limit;
594 	domain->bounce_size = PAGE_ALIGN(bounce_size);
595 	domain->bounce_maps = vzalloc(bounce_pfns *
597 	if (!domain->bounce_maps)
601 		map = &domain->bounce_maps[pfn];
604 	file = anon_inode_getfile("[vduse-domain]", &vduse_domain_fops,
605 				domain, O_RDWR);
609 	domain->file = file;
610 	rwlock_init(&domain->bounce_lock);
611 	spin_lock_init(&domain->iotlb_lock);
612 	init_iova_domain(&domain->stream_iovad,
614 	ret = iova_domain_init_rcaches(&domain->stream_iovad);
617 	init_iova_domain(&domain->consistent_iovad,
619 	ret = iova_domain_init_rcaches(&domain->consistent_iovad);
623 	return domain;
625 	put_iova_domain(&domain->stream_iovad);
629 	vfree(domain->bounce_maps);
631 	vhost_iotlb_free(domain->iotlb);
633 	kfree(domain);