drm2/ttm/ttm_bo.c

62 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
64 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
92 		ttm_mem_type_debug(bo->bdev, mem_type);
113 	struct ttm_bo_device *bdev = bo->bdev;
132 	ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
161 	struct ttm_bo_device *bdev = bo->bdev;
170 		man = &bdev->man[bo->mem.mem_type];
372 	struct ttm_bo_device *bdev = bo->bdev;
380 	if (bdev->need_dma32)
388 		bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
394 		bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
417 	struct ttm_bo_device *bdev = bo->bdev;
418 	bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
419 	bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
420 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
421 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
456 			if (bdev->driver->move_notify)
457 				bdev->driver->move_notify(bo, mem);
464 	if (bdev->driver->move_notify)
465 		bdev->driver->move_notify(bo, mem);
470 	else if (bdev->driver->move)
471 		ret = bdev->driver->move(bo, evict, interruptible,
477 		if (bdev->driver->move_notify) {
481 			bdev->driver->move_notify(bo, mem);
491 		ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
499 		    bdev->man[bo->mem.mem_type].gpu_offset;
507 	new_man = &bdev->man[bo->mem.mem_type];
527 	if (bo->bdev->driver->move_notify)
528 		bo->bdev->driver->move_notify(bo, NULL);
552 	struct ttm_bo_device *bdev = bo->bdev;
554 	struct ttm_bo_driver *driver = bdev->driver;
562 	mtx_lock(&bdev->fence_lock);
565 		mtx_unlock(&bdev->fence_lock);
577 	mtx_unlock(&bdev->fence_lock);
585 	list_add_tail(&bo->ddestroy, &bdev->ddestroy);
592 	taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
612 	struct ttm_bo_device *bdev = bo->bdev;
613 	struct ttm_bo_driver *driver = bdev->driver;
618 	mtx_lock(&bdev->fence_lock);
630 		mtx_unlock(&bdev->fence_lock);
645 		mtx_lock(&bdev->fence_lock);
647 		mtx_unlock(&bdev->fence_lock);
667 		mtx_unlock(&bdev->fence_lock);
693 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
695 	struct ttm_bo_global *glob = bdev->glob;
700 	if (list_empty(&bdev->ddestroy))
703 	entry = list_first_entry(&bdev->ddestroy,
710 		if (entry->ddestroy.next != &bdev->ddestroy) {
750 	struct ttm_bo_device *bdev = arg;
752 	if (ttm_bo_delayed_delete(bdev, false)) {
753 		taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
760 	struct ttm_bo_device *bdev = bo->bdev;
761 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
763 	rw_wlock(&bdev->vm_lock);
766 		    &bdev->addr_space_rb, bo);
770 	rw_wunlock(&bdev->vm_lock);
788 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
792 	if (taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending))
793 		taskqueue_drain_timeout(taskqueue_thread, &bdev->wq);
797 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
800 		taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
808 	struct ttm_bo_device *bdev = bo->bdev;
813 	mtx_lock(&bdev->fence_lock);
815 	mtx_unlock(&bdev->fence_lock);
835 	bdev->driver->evict_flags(bo, &placement);
860 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
865 	struct ttm_bo_global *glob = bdev->glob;
866 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
909 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
926 	struct ttm_bo_device *bdev = bo->bdev;
927 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
936 		ret = ttm_mem_evict_first(bdev, mem_type,
1005 	struct ttm_bo_device *bdev = bo->bdev;
1020 		man = &bdev->man[mem_type];
1066 		man = &bdev->man[mem_type];
1113 	struct ttm_bo_device *bdev = bo->bdev;
1122 	mtx_lock(&bdev->fence_lock);
1124 	mtx_unlock(&bdev->fence_lock);
1217 int ttm_bo_init(struct ttm_bo_device *bdev,
1231 	struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1263 	bo->bdev = bdev;
1264 	bo->glob = bdev->glob;
1311 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1324 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1338 int ttm_bo_create(struct ttm_bo_device *bdev,
1352 	acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1353 	ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1362 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1365 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1366 	struct ttm_bo_global *glob = bdev->glob;
1376 		ret = ttm_mem_evict_first(bdev, mem_type, false, false);
1390 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1399 	man = &bdev->man[mem_type];
1412 		ttm_bo_force_list_clean(bdev, mem_type, false);
1420 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1422 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1434 	return ttm_bo_force_list_clean(bdev, mem_type, true);
1437 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1444 	man = &bdev->man[type];
1451 	ret = bdev->driver->init_mem_type(bdev, type, man);
1454 	man->bdev = bdev;
1535 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1540 	struct ttm_bo_global *glob = bdev->glob;
1543 		man = &bdev->man[i];
1546 			if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1556 	list_del(&bdev->device_list);
1559 	if (taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, NULL))
1560 		taskqueue_drain_timeout(taskqueue_thread, &bdev->wq);
1562 	while (ttm_bo_delayed_delete(bdev, true))
1566 	if (list_empty(&bdev->ddestroy))
1569 	if (list_empty(&bdev->man[0].lru))
1573 	MPASS(drm_mm_clean(&bdev->addr_space_mm));
1574 	rw_wlock(&bdev->vm_lock);
1575 	drm_mm_takedown(&bdev->addr_space_mm);
1576 	rw_wunlock(&bdev->vm_lock);
1581 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1589 	rw_init(&bdev->vm_lock, "ttmvml");
1590 	bdev->driver = driver;
1592 	memset(bdev->man, 0, sizeof(bdev->man));
1598 	ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1602 	RB_INIT(&bdev->addr_space_rb);
1603 	ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
1607 	TIMEOUT_TASK_INIT(taskqueue_thread, &bdev->wq, 0,
1608 	    ttm_bo_delayed_workqueue, bdev);
1609 	INIT_LIST_HEAD(&bdev->ddestroy);
1610 	bdev->dev_mapping = NULL;
1611 	bdev->glob = glob;
1612 	bdev->need_dma32 = need_dma32;
1613 	bdev->val_seq = 0;
1614 	mtx_init(&bdev->fence_lock, "ttmfence", NULL, MTX_DEF);
1616 	list_add_tail(&bdev->device_list, &glob->device_list);
1621 	ttm_bo_clean_mm(bdev, 0);
1630 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1632 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1656 	struct ttm_bo_device *bdev = bo->bdev;
1657 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1666 	struct ttm_bo_device *bdev = bo->bdev;
1668 	/* The caller acquired bdev->vm_lock. */
1669 	RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo);
1685 	struct ttm_bo_device *bdev = bo->bdev;
1689 	ret = drm_mm_pre_get(&bdev->addr_space_mm);
1693 	rw_wlock(&bdev->vm_lock);
1694 	bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm,
1706 		rw_wunlock(&bdev->vm_lock);
1711 	rw_wunlock(&bdev->vm_lock);
1716 	rw_wunlock(&bdev->vm_lock);
1723 	struct ttm_bo_driver *driver = bo->bdev->driver;
1724 	struct ttm_bo_device *bdev = bo->bdev;
1737 			mtx_unlock(&bdev->fence_lock);
1739 			mtx_lock(&bdev->fence_lock);
1747 		mtx_unlock(&bdev->fence_lock);
1752 			mtx_lock(&bdev->fence_lock);
1755 		mtx_lock(&bdev->fence_lock);
1761 			mtx_unlock(&bdev->fence_lock);
1764 			mtx_lock(&bdev->fence_lock);
1766 			mtx_unlock(&bdev->fence_lock);
1768 			mtx_lock(&bdev->fence_lock);
1776 	struct ttm_bo_device *bdev = bo->bdev;
1786 	mtx_lock(&bdev->fence_lock);
1788 	mtx_unlock(&bdev->fence_lock);
1844 	mtx_lock(&bo->bdev->fence_lock);
1846 	mtx_unlock(&bo->bdev->fence_lock);
1872 	if (bo->bdev->driver->swap_notify)
1873 		bo->bdev->driver->swap_notify(bo);
1891 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1893 	while (ttm_bo_swapout(&bdev->glob->shrink) == 0)