346void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) 347{ 348 ttm_bo_add_to_lru(bo); 349 atomic_set(&bo->reserved, 0); 350 wakeup(bo); 351} 352 353void ttm_bo_unreserve(struct ttm_buffer_object *bo) 354{ 355 struct ttm_bo_global *glob = bo->glob; 356 357 mtx_lock(&glob->lru_lock); 358 ttm_bo_unreserve_locked(bo); 359 mtx_unlock(&glob->lru_lock); 360} 361 362/* 363 * Call bo->mutex locked. 364 */ 365static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 366{ 367 struct ttm_bo_device *bdev = bo->bdev; 368 struct ttm_bo_global *glob = bo->glob; 369 int ret = 0; 370 uint32_t page_flags = 0; 371 372 TTM_ASSERT_LOCKED(&bo->mutex); 373 bo->ttm = NULL; 374 375 if (bdev->need_dma32) 376 page_flags |= TTM_PAGE_FLAG_DMA32; 377 378 switch (bo->type) { 379 case ttm_bo_type_device: 380 if (zero_alloc) 381 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 382 case ttm_bo_type_kernel: 383 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 384 page_flags, glob->dummy_read_page); 385 if (unlikely(bo->ttm == NULL)) 386 ret = -ENOMEM; 387 break; 388 case ttm_bo_type_sg: 389 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 390 page_flags | TTM_PAGE_FLAG_SG, 391 glob->dummy_read_page); 392 if (unlikely(bo->ttm == NULL)) { 393 ret = -ENOMEM; 394 break; 395 } 396 bo->ttm->sg = bo->sg; 397 break; 398 default: 399 printf("[TTM] Illegal buffer object type\n"); 400 ret = -EINVAL; 401 break; 402 } 403 404 return ret; 405} 406 407static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 408 struct ttm_mem_reg *mem, 409 bool evict, bool interruptible, 410 bool no_wait_gpu) 411{ 412 struct ttm_bo_device *bdev = bo->bdev; 413 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 414 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 415 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 416 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 417 int ret = 0; 418 419 if (old_is_pci || new_is_pci || 420 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 421 ret = ttm_mem_io_lock(old_man, true); 422 if (unlikely(ret != 0)) 423 goto out_err; 424 ttm_bo_unmap_virtual_locked(bo); 425 ttm_mem_io_unlock(old_man); 426 } 427 428 /* 429 * Create and bind a ttm if required. 430 */ 431 432 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 433 if (bo->ttm == NULL) { 434 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 435 ret = ttm_bo_add_ttm(bo, zero); 436 if (ret) 437 goto out_err; 438 } 439 440 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 441 if (ret) 442 goto out_err; 443 444 if (mem->mem_type != TTM_PL_SYSTEM) { 445 ret = ttm_tt_bind(bo->ttm, mem); 446 if (ret) 447 goto out_err; 448 } 449 450 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 451 if (bdev->driver->move_notify) 452 bdev->driver->move_notify(bo, mem); 453 bo->mem = *mem; 454 mem->mm_node = NULL; 455 goto moved; 456 } 457 } 458 459 if (bdev->driver->move_notify) 460 bdev->driver->move_notify(bo, mem); 461 462 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 463 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 464 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 465 else if (bdev->driver->move) 466 ret = bdev->driver->move(bo, evict, interruptible, 467 no_wait_gpu, mem); 468 else 469 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 470 471 if (ret) { 472 if (bdev->driver->move_notify) { 473 struct ttm_mem_reg tmp_mem = *mem; 474 *mem = bo->mem; 475 bo->mem = tmp_mem; 476 bdev->driver->move_notify(bo, mem); 477 bo->mem = *mem; 478 } 479 480 goto out_err; 481 } 482 483moved: 484 if (bo->evicted) { 485 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 486 if (ret) 487 printf("[TTM] Can not flush read caches\n"); 488 bo->evicted = false; 489 } 490 491 if (bo->mem.mm_node) { 492 bo->offset = (bo->mem.start << PAGE_SHIFT) + 493 bdev->man[bo->mem.mem_type].gpu_offset; 494 bo->cur_placement = bo->mem.placement; 495 } else 496 bo->offset = 0; 497 498 return 0; 499 500out_err: 501 new_man = &bdev->man[bo->mem.mem_type]; 502 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 503 ttm_tt_unbind(bo->ttm); 504 ttm_tt_destroy(bo->ttm); 505 bo->ttm = NULL; 506 } 507 508 return ret; 509} 510 511/** 512 * Call bo::reserved. 513 * Will release GPU memory type usage on destruction. 514 * This is the place to put in driver specific hooks to release 515 * driver private resources. 516 * Will release the bo::reserved lock. 517 */ 518 519static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 520{ 521 if (bo->bdev->driver->move_notify) 522 bo->bdev->driver->move_notify(bo, NULL); 523 524 if (bo->ttm) { 525 ttm_tt_unbind(bo->ttm); 526 ttm_tt_destroy(bo->ttm); 527 bo->ttm = NULL; 528 } 529 ttm_bo_mem_put(bo, &bo->mem); 530 531 atomic_set(&bo->reserved, 0); 532 wakeup(&bo); 533 534 /* 535 * Since the final reference to this bo may not be dropped by 536 * the current task we have to put a memory barrier here to make 537 * sure the changes done in this function are always visible. 538 * 539 * This function only needs protection against the final kref_put. 540 */ 541 mb(); 542} 543 544static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 545{ 546 struct ttm_bo_device *bdev = bo->bdev; 547 struct ttm_bo_global *glob = bo->glob; 548 struct ttm_bo_driver *driver = bdev->driver; 549 void *sync_obj = NULL; 550 int put_count; 551 int ret; 552 553 mtx_lock(&glob->lru_lock); 554 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 555 556 mtx_lock(&bdev->fence_lock); 557 (void) ttm_bo_wait(bo, false, false, true); 558 if (!ret && !bo->sync_obj) { 559 mtx_unlock(&bdev->fence_lock); 560 put_count = ttm_bo_del_from_lru(bo); 561 562 mtx_unlock(&glob->lru_lock); 563 ttm_bo_cleanup_memtype_use(bo); 564 565 ttm_bo_list_ref_sub(bo, put_count, true); 566 567 return; 568 } 569 if (bo->sync_obj) 570 sync_obj = driver->sync_obj_ref(bo->sync_obj); 571 mtx_unlock(&bdev->fence_lock); 572 573 if (!ret) { 574 atomic_set(&bo->reserved, 0); 575 wakeup(bo); 576 } 577 578 refcount_acquire(&bo->list_kref); 579 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 580 mtx_unlock(&glob->lru_lock); 581 582 if (sync_obj) { 583 driver->sync_obj_flush(sync_obj); 584 driver->sync_obj_unref(&sync_obj); 585 } 586 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 587 ((hz / 100) < 1) ? 1 : hz / 100); 588} 589 590/** 591 * function ttm_bo_cleanup_refs_and_unlock 592 * If bo idle, remove from delayed- and lru lists, and unref. 593 * If not idle, do nothing. 594 * 595 * Must be called with lru_lock and reservation held, this function 596 * will drop both before returning. 597 * 598 * @interruptible Any sleeps should occur interruptibly. 599 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 600 */ 601 602static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 603 bool interruptible, 604 bool no_wait_gpu) 605{ 606 struct ttm_bo_device *bdev = bo->bdev; 607 struct ttm_bo_driver *driver = bdev->driver; 608 struct ttm_bo_global *glob = bo->glob; 609 int put_count; 610 int ret; 611 612 mtx_lock(&bdev->fence_lock); 613 ret = ttm_bo_wait(bo, false, false, true); 614 615 if (ret && !no_wait_gpu) { 616 void *sync_obj; 617 618 /* 619 * Take a reference to the fence and unreserve, 620 * at this point the buffer should be dead, so 621 * no new sync objects can be attached. 622 */ 623 sync_obj = driver->sync_obj_ref(bo->sync_obj); 624 mtx_unlock(&bdev->fence_lock); 625 626 atomic_set(&bo->reserved, 0); 627 wakeup(bo); 628 mtx_unlock(&glob->lru_lock); 629 630 ret = driver->sync_obj_wait(sync_obj, false, interruptible); 631 driver->sync_obj_unref(&sync_obj); 632 if (ret) 633 return ret; 634 635 /* 636 * remove sync_obj with ttm_bo_wait, the wait should be 637 * finished, and no new wait object should have been added. 638 */ 639 mtx_lock(&bdev->fence_lock); 640 ret = ttm_bo_wait(bo, false, false, true); 641 mtx_unlock(&bdev->fence_lock); 642 if (ret) 643 return ret; 644 645 mtx_lock(&glob->lru_lock); 646 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 647 648 /* 649 * We raced, and lost, someone else holds the reservation now, 650 * and is probably busy in ttm_bo_cleanup_memtype_use. 651 * 652 * Even if it's not the case, because we finished waiting any 653 * delayed destruction would succeed, so just return success 654 * here. 655 */ 656 if (ret) { 657 mtx_unlock(&glob->lru_lock); 658 return 0; 659 } 660 } else 661 mtx_unlock(&bdev->fence_lock); 662 663 if (ret || unlikely(list_empty(&bo->ddestroy))) { 664 atomic_set(&bo->reserved, 0); 665 wakeup(bo); 666 mtx_unlock(&glob->lru_lock); 667 return ret; 668 } 669 670 put_count = ttm_bo_del_from_lru(bo); 671 list_del_init(&bo->ddestroy); 672 ++put_count; 673 674 mtx_unlock(&glob->lru_lock); 675 ttm_bo_cleanup_memtype_use(bo); 676 677 ttm_bo_list_ref_sub(bo, put_count, true); 678 679 return 0; 680} 681 682/** 683 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 684 * encountered buffers. 685 */ 686 687static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 688{ 689 struct ttm_bo_global *glob = bdev->glob; 690 struct ttm_buffer_object *entry = NULL; 691 int ret = 0; 692 693 mtx_lock(&glob->lru_lock); 694 if (list_empty(&bdev->ddestroy)) 695 goto out_unlock; 696 697 entry = list_first_entry(&bdev->ddestroy, 698 struct ttm_buffer_object, ddestroy); 699 refcount_acquire(&entry->list_kref); 700 701 for (;;) { 702 struct ttm_buffer_object *nentry = NULL; 703 704 if (entry->ddestroy.next != &bdev->ddestroy) { 705 nentry = list_first_entry(&entry->ddestroy, 706 struct ttm_buffer_object, ddestroy); 707 refcount_acquire(&nentry->list_kref); 708 } 709 710 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); 711 if (remove_all && ret) { 712 ret = ttm_bo_reserve_nolru(entry, false, false, 713 false, 0); 714 } 715 716 if (!ret) 717 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 718 !remove_all); 719 else 720 mtx_unlock(&glob->lru_lock); 721 722 if (refcount_release(&entry->list_kref)) 723 ttm_bo_release_list(entry); 724 entry = nentry; 725 726 if (ret || !entry) 727 goto out; 728 729 mtx_lock(&glob->lru_lock); 730 if (list_empty(&entry->ddestroy)) 731 break; 732 } 733 734out_unlock: 735 mtx_unlock(&glob->lru_lock); 736out: 737 if (entry && refcount_release(&entry->list_kref)) 738 ttm_bo_release_list(entry); 739 return ret; 740} 741 742static void ttm_bo_delayed_workqueue(void *arg, int pending __unused) 743{ 744 struct ttm_bo_device *bdev = arg; 745 746 if (ttm_bo_delayed_delete(bdev, false)) { 747 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 748 ((hz / 100) < 1) ? 1 : hz / 100); 749 } 750} 751 752static void ttm_bo_release(struct ttm_buffer_object *bo) 753{ 754 struct ttm_bo_device *bdev = bo->bdev; 755 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 756 757 rw_wlock(&bdev->vm_lock); 758 if (likely(bo->vm_node != NULL)) { 759 RB_REMOVE(ttm_bo_device_buffer_objects, 760 &bdev->addr_space_rb, bo); 761 drm_mm_put_block(bo->vm_node); 762 bo->vm_node = NULL; 763 } 764 rw_wunlock(&bdev->vm_lock); 765 ttm_mem_io_lock(man, false); 766 ttm_mem_io_free_vm(bo); 767 ttm_mem_io_unlock(man); 768 ttm_bo_cleanup_refs_or_queue(bo); 769 if (refcount_release(&bo->list_kref)) 770 ttm_bo_release_list(bo); 771} 772 773void ttm_bo_unref(struct ttm_buffer_object **p_bo) 774{ 775 struct ttm_buffer_object *bo = *p_bo; 776 777 *p_bo = NULL; 778 if (refcount_release(&bo->kref)) 779 ttm_bo_release(bo); 780} 781 782int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 783{ 784 int pending; 785 786 taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending); 787 if (pending) 788 taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); 789 return (pending); 790} 791 792void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 793{ 794 if (resched) { 795 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 796 ((hz / 100) < 1) ? 1 : hz / 100); 797 } 798} 799 800static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 801 bool no_wait_gpu) 802{ 803 struct ttm_bo_device *bdev = bo->bdev; 804 struct ttm_mem_reg evict_mem; 805 struct ttm_placement placement; 806 int ret = 0; 807 808 mtx_lock(&bdev->fence_lock); 809 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 810 mtx_unlock(&bdev->fence_lock); 811 812 if (unlikely(ret != 0)) { 813 if (ret != -ERESTART) { 814 printf("[TTM] Failed to expire sync object before buffer eviction\n"); 815 } 816 goto out; 817 } 818 819 MPASS(ttm_bo_is_reserved(bo)); 820 821 evict_mem = bo->mem; 822 evict_mem.mm_node = NULL; 823 evict_mem.bus.io_reserved_vm = false; 824 evict_mem.bus.io_reserved_count = 0; 825 826 placement.fpfn = 0; 827 placement.lpfn = 0; 828 placement.num_placement = 0; 829 placement.num_busy_placement = 0; 830 bdev->driver->evict_flags(bo, &placement); 831 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 832 no_wait_gpu); 833 if (ret) { 834 if (ret != -ERESTART) { 835 printf("[TTM] Failed to find memory space for buffer 0x%p eviction\n", 836 bo); 837 ttm_bo_mem_space_debug(bo, &placement); 838 } 839 goto out; 840 } 841 842 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 843 no_wait_gpu); 844 if (ret) { 845 if (ret != -ERESTART) 846 printf("[TTM] Buffer eviction failed\n"); 847 ttm_bo_mem_put(bo, &evict_mem); 848 goto out; 849 } 850 bo->evicted = true; 851out: 852 return ret; 853} 854 855static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 856 uint32_t mem_type, 857 bool interruptible, 858 bool no_wait_gpu) 859{ 860 struct ttm_bo_global *glob = bdev->glob; 861 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 862 struct ttm_buffer_object *bo; 863 int ret = -EBUSY, put_count; 864 865 mtx_lock(&glob->lru_lock); 866 list_for_each_entry(bo, &man->lru, lru) { 867 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 868 if (!ret) 869 break; 870 } 871 872 if (ret) { 873 mtx_unlock(&glob->lru_lock); 874 return ret; 875 } 876 877 refcount_acquire(&bo->list_kref); 878 879 if (!list_empty(&bo->ddestroy)) { 880 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 881 no_wait_gpu); 882 if (refcount_release(&bo->list_kref)) 883 ttm_bo_release_list(bo); 884 return ret; 885 } 886 887 put_count = ttm_bo_del_from_lru(bo); 888 mtx_unlock(&glob->lru_lock); 889 890 MPASS(ret == 0); 891 892 ttm_bo_list_ref_sub(bo, put_count, true); 893 894 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 895 ttm_bo_unreserve(bo); 896 897 if (refcount_release(&bo->list_kref)) 898 ttm_bo_release_list(bo); 899 return ret; 900} 901 902void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 903{ 904 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 905 906 if (mem->mm_node) 907 (*man->func->put_node)(man, mem); 908} 909 910/** 911 * Repeatedly evict memory from the LRU for @mem_type until we create enough 912 * space, or we've evicted everything and there isn't enough space. 913 */ 914static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 915 uint32_t mem_type, 916 struct ttm_placement *placement, 917 struct ttm_mem_reg *mem, 918 bool interruptible, 919 bool no_wait_gpu) 920{ 921 struct ttm_bo_device *bdev = bo->bdev; 922 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 923 int ret; 924 925 do { 926 ret = (*man->func->get_node)(man, bo, placement, mem); 927 if (unlikely(ret != 0)) 928 return ret; 929 if (mem->mm_node) 930 break; 931 ret = ttm_mem_evict_first(bdev, mem_type, 932 interruptible, no_wait_gpu); 933 if (unlikely(ret != 0)) 934 return ret; 935 } while (1); 936 if (mem->mm_node == NULL) 937 return -ENOMEM; 938 mem->mem_type = mem_type; 939 return 0; 940} 941 942static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 943 uint32_t cur_placement, 944 uint32_t proposed_placement) 945{ 946 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 947 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 948 949 /** 950 * Keep current caching if possible. 951 */ 952 953 if ((cur_placement & caching) != 0) 954 result |= (cur_placement & caching); 955 else if ((man->default_caching & caching) != 0) 956 result |= man->default_caching; 957 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 958 result |= TTM_PL_FLAG_CACHED; 959 else if ((TTM_PL_FLAG_WC & caching) != 0) 960 result |= TTM_PL_FLAG_WC; 961 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 962 result |= TTM_PL_FLAG_UNCACHED; 963 964 return result; 965} 966 967static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 968 uint32_t mem_type, 969 uint32_t proposed_placement, 970 uint32_t *masked_placement) 971{ 972 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 973 974 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 975 return false; 976 977 if ((proposed_placement & man->available_caching) == 0) 978 return false; 979 980 cur_flags |= (proposed_placement & man->available_caching); 981 982 *masked_placement = cur_flags; 983 return true; 984} 985 986/** 987 * Creates space for memory region @mem according to its type. 988 * 989 * This function first searches for free space in compatible memory types in 990 * the priority order defined by the driver. If free space isn't found, then 991 * ttm_bo_mem_force_space is attempted in priority order to evict and find 992 * space. 993 */ 994int ttm_bo_mem_space(struct ttm_buffer_object *bo, 995 struct ttm_placement *placement, 996 struct ttm_mem_reg *mem, 997 bool interruptible, 998 bool no_wait_gpu) 999{ 1000 struct ttm_bo_device *bdev = bo->bdev; 1001 struct ttm_mem_type_manager *man; 1002 uint32_t mem_type = TTM_PL_SYSTEM; 1003 uint32_t cur_flags = 0; 1004 bool type_found = false; 1005 bool type_ok = false; 1006 bool has_erestartsys = false; 1007 int i, ret; 1008 1009 mem->mm_node = NULL; 1010 for (i = 0; i < placement->num_placement; ++i) { 1011 ret = ttm_mem_type_from_flags(placement->placement[i], 1012 &mem_type); 1013 if (ret) 1014 return ret; 1015 man = &bdev->man[mem_type]; 1016 1017 type_ok = ttm_bo_mt_compatible(man, 1018 mem_type, 1019 placement->placement[i], 1020 &cur_flags); 1021 1022 if (!type_ok) 1023 continue; 1024 1025 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1026 cur_flags); 1027 /* 1028 * Use the access and other non-mapping-related flag bits from 1029 * the memory placement flags to the current flags 1030 */ 1031 ttm_flag_masked(&cur_flags, placement->placement[i], 1032 ~TTM_PL_MASK_MEMTYPE); 1033 1034 if (mem_type == TTM_PL_SYSTEM) 1035 break; 1036 1037 if (man->has_type && man->use_type) { 1038 type_found = true; 1039 ret = (*man->func->get_node)(man, bo, placement, mem); 1040 if (unlikely(ret)) 1041 return ret; 1042 } 1043 if (mem->mm_node) 1044 break; 1045 } 1046 1047 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 1048 mem->mem_type = mem_type; 1049 mem->placement = cur_flags; 1050 return 0; 1051 } 1052 1053 if (!type_found) 1054 return -EINVAL; 1055 1056 for (i = 0; i < placement->num_busy_placement; ++i) { 1057 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 1058 &mem_type); 1059 if (ret) 1060 return ret; 1061 man = &bdev->man[mem_type]; 1062 if (!man->has_type) 1063 continue; 1064 if (!ttm_bo_mt_compatible(man, 1065 mem_type, 1066 placement->busy_placement[i], 1067 &cur_flags)) 1068 continue; 1069 1070 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1071 cur_flags); 1072 /* 1073 * Use the access and other non-mapping-related flag bits from 1074 * the memory placement flags to the current flags 1075 */ 1076 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 1077 ~TTM_PL_MASK_MEMTYPE); 1078 1079 1080 if (mem_type == TTM_PL_SYSTEM) { 1081 mem->mem_type = mem_type; 1082 mem->placement = cur_flags; 1083 mem->mm_node = NULL; 1084 return 0; 1085 } 1086 1087 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 1088 interruptible, no_wait_gpu); 1089 if (ret == 0 && mem->mm_node) { 1090 mem->placement = cur_flags; 1091 return 0; 1092 } 1093 if (ret == -ERESTART) 1094 has_erestartsys = true; 1095 } 1096 ret = (has_erestartsys) ? -ERESTART : -ENOMEM; 1097 return ret; 1098} 1099 1100static 1101int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1102 struct ttm_placement *placement, 1103 bool interruptible, 1104 bool no_wait_gpu) 1105{ 1106 int ret = 0; 1107 struct ttm_mem_reg mem; 1108 struct ttm_bo_device *bdev = bo->bdev; 1109 1110 MPASS(ttm_bo_is_reserved(bo)); 1111 1112 /* 1113 * FIXME: It's possible to pipeline buffer moves. 1114 * Have the driver move function wait for idle when necessary, 1115 * instead of doing it here. 1116 */ 1117 mtx_lock(&bdev->fence_lock); 1118 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1119 mtx_unlock(&bdev->fence_lock); 1120 if (ret) 1121 return ret; 1122 mem.num_pages = bo->num_pages; 1123 mem.size = mem.num_pages << PAGE_SHIFT; 1124 mem.page_alignment = bo->mem.page_alignment; 1125 mem.bus.io_reserved_vm = false; 1126 mem.bus.io_reserved_count = 0; 1127 /* 1128 * Determine where to move the buffer. 1129 */ 1130 ret = ttm_bo_mem_space(bo, placement, &mem, 1131 interruptible, no_wait_gpu); 1132 if (ret) 1133 goto out_unlock; 1134 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1135 interruptible, no_wait_gpu); 1136out_unlock: 1137 if (ret && mem.mm_node) 1138 ttm_bo_mem_put(bo, &mem); 1139 return ret; 1140} 1141 1142static int ttm_bo_mem_compat(struct ttm_placement *placement, 1143 struct ttm_mem_reg *mem) 1144{ 1145 int i; 1146 1147 if (mem->mm_node && placement->lpfn != 0 && 1148 (mem->start < placement->fpfn || 1149 mem->start + mem->num_pages > placement->lpfn)) 1150 return -1; 1151 1152 for (i = 0; i < placement->num_placement; i++) { 1153 if ((placement->placement[i] & mem->placement & 1154 TTM_PL_MASK_CACHING) && 1155 (placement->placement[i] & mem->placement & 1156 TTM_PL_MASK_MEM)) 1157 return i; 1158 } 1159 return -1; 1160} 1161 1162int ttm_bo_validate(struct ttm_buffer_object *bo, 1163 struct ttm_placement *placement, 1164 bool interruptible, 1165 bool no_wait_gpu) 1166{ 1167 int ret; 1168 1169 MPASS(ttm_bo_is_reserved(bo)); 1170 /* Check that range is valid */ 1171 if (placement->lpfn || placement->fpfn) 1172 if (placement->fpfn > placement->lpfn || 1173 (placement->lpfn - placement->fpfn) < bo->num_pages) 1174 return -EINVAL; 1175 /* 1176 * Check whether we need to move buffer. 1177 */ 1178 ret = ttm_bo_mem_compat(placement, &bo->mem); 1179 if (ret < 0) { 1180 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1181 no_wait_gpu); 1182 if (ret) 1183 return ret; 1184 } else { 1185 /* 1186 * Use the access and other non-mapping-related flag bits from 1187 * the compatible memory placement flags to the active flags 1188 */ 1189 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1190 ~TTM_PL_MASK_MEMTYPE); 1191 } 1192 /* 1193 * We might need to add a TTM. 1194 */ 1195 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1196 ret = ttm_bo_add_ttm(bo, true); 1197 if (ret) 1198 return ret; 1199 } 1200 return 0; 1201} 1202 1203int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1204 struct ttm_placement *placement) 1205{ 1206 MPASS(!((placement->fpfn || placement->lpfn) && 1207 (bo->mem.num_pages > (placement->lpfn - placement->fpfn)))); 1208 1209 return 0; 1210} 1211 1212int ttm_bo_init(struct ttm_bo_device *bdev, 1213 struct ttm_buffer_object *bo, 1214 unsigned long size, 1215 enum ttm_bo_type type, 1216 struct ttm_placement *placement, 1217 uint32_t page_alignment, 1218 bool interruptible, 1219 struct vm_object *persistent_swap_storage, 1220 size_t acc_size, 1221 struct sg_table *sg, 1222 void (*destroy) (struct ttm_buffer_object *)) 1223{ 1224 int ret = 0; 1225 unsigned long num_pages; 1226 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1227 1228 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1229 if (ret) { 1230 printf("[TTM] Out of kernel memory\n"); 1231 if (destroy) 1232 (*destroy)(bo); 1233 else 1234 free(bo, M_TTM_BO); 1235 return -ENOMEM; 1236 } 1237 1238 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1239 if (num_pages == 0) { 1240 printf("[TTM] Illegal buffer object size\n"); 1241 if (destroy) 1242 (*destroy)(bo); 1243 else 1244 free(bo, M_TTM_BO); 1245 ttm_mem_global_free(mem_glob, acc_size); 1246 return -EINVAL; 1247 } 1248 bo->destroy = destroy; 1249 1250 refcount_init(&bo->kref, 1); 1251 refcount_init(&bo->list_kref, 1); 1252 atomic_set(&bo->cpu_writers, 0); 1253 atomic_set(&bo->reserved, 1); 1254 INIT_LIST_HEAD(&bo->lru); 1255 INIT_LIST_HEAD(&bo->ddestroy); 1256 INIT_LIST_HEAD(&bo->swap); 1257 INIT_LIST_HEAD(&bo->io_reserve_lru); 1258 bo->bdev = bdev; 1259 bo->glob = bdev->glob; 1260 bo->type = type; 1261 bo->num_pages = num_pages; 1262 bo->mem.size = num_pages << PAGE_SHIFT; 1263 bo->mem.mem_type = TTM_PL_SYSTEM; 1264 bo->mem.num_pages = bo->num_pages; 1265 bo->mem.mm_node = NULL; 1266 bo->mem.page_alignment = page_alignment; 1267 bo->mem.bus.io_reserved_vm = false; 1268 bo->mem.bus.io_reserved_count = 0; 1269 bo->priv_flags = 0; 1270 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1271 bo->seq_valid = false; 1272 bo->persistent_swap_storage = persistent_swap_storage; 1273 bo->acc_size = acc_size; 1274 bo->sg = sg; 1275 atomic_inc(&bo->glob->bo_count); 1276 1277 ret = ttm_bo_check_placement(bo, placement); 1278 if (unlikely(ret != 0)) 1279 goto out_err; 1280 1281 /* 1282 * For ttm_bo_type_device buffers, allocate 1283 * address space from the device. 1284 */ 1285 if (bo->type == ttm_bo_type_device || 1286 bo->type == ttm_bo_type_sg) { 1287 ret = ttm_bo_setup_vm(bo); 1288 if (ret) 1289 goto out_err; 1290 } 1291 1292 ret = ttm_bo_validate(bo, placement, interruptible, false); 1293 if (ret) 1294 goto out_err; 1295 1296 ttm_bo_unreserve(bo); 1297 return 0; 1298 1299out_err: 1300 ttm_bo_unreserve(bo); 1301 ttm_bo_unref(&bo); 1302 1303 return ret; 1304} 1305 1306size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1307 unsigned long bo_size, 1308 unsigned struct_size) 1309{ 1310 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1311 size_t size = 0; 1312 1313 size += ttm_round_pot(struct_size); 1314 size += PAGE_ALIGN(npages * sizeof(void *)); 1315 size += ttm_round_pot(sizeof(struct ttm_tt)); 1316 return size; 1317} 1318 1319size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1320 unsigned long bo_size, 1321 unsigned struct_size) 1322{ 1323 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1324 size_t size = 0; 1325 1326 size += ttm_round_pot(struct_size); 1327 size += PAGE_ALIGN(npages * sizeof(void *)); 1328 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1329 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1330 return size; 1331} 1332 1333int ttm_bo_create(struct ttm_bo_device *bdev, 1334 unsigned long size, 1335 enum ttm_bo_type type, 1336 struct ttm_placement *placement, 1337 uint32_t page_alignment, 1338 bool interruptible, 1339 struct vm_object *persistent_swap_storage, 1340 struct ttm_buffer_object **p_bo) 1341{ 1342 struct ttm_buffer_object *bo; 1343 size_t acc_size; 1344 int ret; 1345 1346 bo = malloc(sizeof(*bo), M_TTM_BO, M_WAITOK | M_ZERO); 1347 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1348 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1349 interruptible, persistent_swap_storage, acc_size, 1350 NULL, NULL); 1351 if (likely(ret == 0)) 1352 *p_bo = bo; 1353 1354 return ret; 1355} 1356 1357static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1358 unsigned mem_type, bool allow_errors) 1359{ 1360 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1361 struct ttm_bo_global *glob = bdev->glob; 1362 int ret; 1363 1364 /* 1365 * Can't use standard list traversal since we're unlocking. 1366 */ 1367 1368 mtx_lock(&glob->lru_lock); 1369 while (!list_empty(&man->lru)) { 1370 mtx_unlock(&glob->lru_lock); 1371 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1372 if (ret) { 1373 if (allow_errors) { 1374 return ret; 1375 } else { 1376 printf("[TTM] Cleanup eviction failed\n"); 1377 } 1378 } 1379 mtx_lock(&glob->lru_lock); 1380 } 1381 mtx_unlock(&glob->lru_lock); 1382 return 0; 1383} 1384 1385int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1386{ 1387 struct ttm_mem_type_manager *man; 1388 int ret = -EINVAL; 1389 1390 if (mem_type >= TTM_NUM_MEM_TYPES) { 1391 printf("[TTM] Illegal memory type %d\n", mem_type); 1392 return ret; 1393 } 1394 man = &bdev->man[mem_type]; 1395 1396 if (!man->has_type) { 1397 printf("[TTM] Trying to take down uninitialized memory manager type %u\n", 1398 mem_type); 1399 return ret; 1400 } 1401 1402 man->use_type = false; 1403 man->has_type = false; 1404 1405 ret = 0; 1406 if (mem_type > 0) { 1407 ttm_bo_force_list_clean(bdev, mem_type, false); 1408 1409 ret = (*man->func->takedown)(man); 1410 } 1411 1412 return ret; 1413} 1414 1415int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1416{ 1417 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1418 1419 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1420 printf("[TTM] Illegal memory manager memory type %u\n", mem_type); 1421 return -EINVAL; 1422 } 1423 1424 if (!man->has_type) { 1425 printf("[TTM] Memory type %u has not been initialized\n", mem_type); 1426 return 0; 1427 } 1428 1429 return ttm_bo_force_list_clean(bdev, mem_type, true); 1430} 1431 1432int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1433 unsigned long p_size) 1434{ 1435 int ret = -EINVAL; 1436 struct ttm_mem_type_manager *man; 1437 1438 MPASS(type < TTM_NUM_MEM_TYPES); 1439 man = &bdev->man[type]; 1440 MPASS(!man->has_type); 1441 man->io_reserve_fastpath = true; 1442 man->use_io_reserve_lru = false; 1443 sx_init(&man->io_reserve_mutex, "ttmman"); 1444 INIT_LIST_HEAD(&man->io_reserve_lru); 1445 1446 ret = bdev->driver->init_mem_type(bdev, type, man); 1447 if (ret) 1448 return ret; 1449 man->bdev = bdev; 1450 1451 ret = 0; 1452 if (type != TTM_PL_SYSTEM) { 1453 ret = (*man->func->init)(man, p_size); 1454 if (ret) 1455 return ret; 1456 } 1457 man->has_type = true; 1458 man->use_type = true; 1459 man->size = p_size; 1460 1461 INIT_LIST_HEAD(&man->lru); 1462 1463 return 0; 1464} 1465 1466static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob) 1467{ 1468 1469 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1470 vm_page_free(glob->dummy_read_page); 1471} 1472 1473void ttm_bo_global_release(struct drm_global_reference *ref) 1474{ 1475 struct ttm_bo_global *glob = ref->object; 1476 1477 if (refcount_release(&glob->kobj_ref)) 1478 ttm_bo_global_kobj_release(glob); 1479} 1480 1481int ttm_bo_global_init(struct drm_global_reference *ref) 1482{ 1483 struct ttm_bo_global_ref *bo_ref = 1484 container_of(ref, struct ttm_bo_global_ref, ref); 1485 struct ttm_bo_global *glob = ref->object; 1486 int ret; 1487 1488 sx_init(&glob->device_list_mutex, "ttmdlm"); 1489 mtx_init(&glob->lru_lock, "ttmlru", NULL, MTX_DEF); 1490 glob->mem_glob = bo_ref->mem_glob; 1491 glob->dummy_read_page = vm_page_alloc_contig(NULL, 0, 1492 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ, 1493 1, 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE); 1494 1495 if (unlikely(glob->dummy_read_page == NULL)) { 1496 ret = -ENOMEM; 1497 goto out_no_drp; 1498 } 1499 1500 INIT_LIST_HEAD(&glob->swap_lru); 1501 INIT_LIST_HEAD(&glob->device_list); 1502 1503 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1504 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1505 if (unlikely(ret != 0)) { 1506 printf("[TTM] Could not register buffer object swapout\n"); 1507 goto out_no_shrink; 1508 } 1509 1510 atomic_set(&glob->bo_count, 0); 1511 1512 refcount_init(&glob->kobj_ref, 1); 1513 return (0); 1514 1515out_no_shrink: 1516 vm_page_free(glob->dummy_read_page); 1517out_no_drp: 1518 free(glob, M_DRM_GLOBAL); 1519 return ret; 1520} 1521 1522int ttm_bo_device_release(struct ttm_bo_device *bdev) 1523{ 1524 int ret = 0; 1525 unsigned i = TTM_NUM_MEM_TYPES; 1526 struct ttm_mem_type_manager *man; 1527 struct ttm_bo_global *glob = bdev->glob; 1528 1529 while (i--) { 1530 man = &bdev->man[i]; 1531 if (man->has_type) { 1532 man->use_type = false; 1533 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1534 ret = -EBUSY; 1535 printf("[TTM] DRM memory manager type %d is not clean\n", 1536 i); 1537 } 1538 man->has_type = false; 1539 } 1540 } 1541 1542 sx_xlock(&glob->device_list_mutex); 1543 list_del(&bdev->device_list); 1544 sx_xunlock(&glob->device_list_mutex); 1545 1546 if (taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, NULL)) 1547 taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); 1548 1549 while (ttm_bo_delayed_delete(bdev, true)) 1550 ; 1551 1552 mtx_lock(&glob->lru_lock); 1553 if (list_empty(&bdev->ddestroy)) 1554 TTM_DEBUG("Delayed destroy list was clean\n"); 1555 1556 if (list_empty(&bdev->man[0].lru)) 1557 TTM_DEBUG("Swap list was clean\n"); 1558 mtx_unlock(&glob->lru_lock); 1559 1560 MPASS(drm_mm_clean(&bdev->addr_space_mm)); 1561 rw_wlock(&bdev->vm_lock); 1562 drm_mm_takedown(&bdev->addr_space_mm); 1563 rw_wunlock(&bdev->vm_lock); 1564 1565 return ret; 1566} 1567 1568int ttm_bo_device_init(struct ttm_bo_device *bdev, 1569 struct ttm_bo_global *glob, 1570 struct ttm_bo_driver *driver, 1571 uint64_t file_page_offset, 1572 bool need_dma32) 1573{ 1574 int ret = -EINVAL; 1575 1576 rw_init(&bdev->vm_lock, "ttmvml"); 1577 bdev->driver = driver; 1578 1579 memset(bdev->man, 0, sizeof(bdev->man)); 1580 1581 /* 1582 * Initialize the system memory buffer type. 1583 * Other types need to be driver / IOCTL initialized. 1584 */ 1585 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1586 if (unlikely(ret != 0)) 1587 goto out_no_sys; 1588 1589 RB_INIT(&bdev->addr_space_rb); 1590 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1591 if (unlikely(ret != 0)) 1592 goto out_no_addr_mm; 1593 1594 TIMEOUT_TASK_INIT(taskqueue_thread, &bdev->wq, 0, 1595 ttm_bo_delayed_workqueue, bdev); 1596 INIT_LIST_HEAD(&bdev->ddestroy); 1597 bdev->dev_mapping = NULL; 1598 bdev->glob = glob; 1599 bdev->need_dma32 = need_dma32; 1600 bdev->val_seq = 0; 1601 mtx_init(&bdev->fence_lock, "ttmfence", NULL, MTX_DEF); 1602 sx_xlock(&glob->device_list_mutex); 1603 list_add_tail(&bdev->device_list, &glob->device_list); 1604 sx_xunlock(&glob->device_list_mutex); 1605 1606 return 0; 1607out_no_addr_mm: 1608 ttm_bo_clean_mm(bdev, 0); 1609out_no_sys: 1610 return ret; 1611} 1612 1613/* 1614 * buffer object vm functions. 1615 */ 1616 1617bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1618{ 1619 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1620 1621 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1622 if (mem->mem_type == TTM_PL_SYSTEM) 1623 return false; 1624 1625 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1626 return false; 1627 1628 if (mem->placement & TTM_PL_FLAG_CACHED) 1629 return false; 1630 } 1631 return true; 1632} 1633 1634void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1635{ 1636 struct ttm_bo_device *bdev = bo->bdev; 1637 /* off_t offset = (off_t)bo->addr_space_offset;XXXKIB */ 1638 /* off_t holelen = ((off_t)bo->mem.num_pages) << PAGE_SHIFT;XXXKIB */ 1639 1640 if (!bdev->dev_mapping) 1641 return; 1642 /* unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); XXXKIB */ 1643 ttm_mem_io_free_vm(bo); 1644} 1645 1646void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1647{ 1648 struct ttm_bo_device *bdev = bo->bdev; 1649 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1650 1651 ttm_mem_io_lock(man, false); 1652 ttm_bo_unmap_virtual_locked(bo); 1653 ttm_mem_io_unlock(man); 1654} 1655 1656static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1657{ 1658 struct ttm_bo_device *bdev = bo->bdev; 1659 1660 /* The caller acquired bdev->vm_lock. */ 1661 RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo); 1662} 1663 1664/** 1665 * ttm_bo_setup_vm: 1666 * 1667 * @bo: the buffer to allocate address space for 1668 * 1669 * Allocate address space in the drm device so that applications 1670 * can mmap the buffer and access the contents. This only 1671 * applies to ttm_bo_type_device objects as others are not 1672 * placed in the drm device address space. 1673 */ 1674 1675static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1676{ 1677 struct ttm_bo_device *bdev = bo->bdev; 1678 int ret; 1679 1680retry_pre_get: 1681 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1682 if (unlikely(ret != 0)) 1683 return ret; 1684 1685 rw_wlock(&bdev->vm_lock); 1686 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1687 bo->mem.num_pages, 0, 0); 1688 1689 if (unlikely(bo->vm_node == NULL)) { 1690 ret = -ENOMEM; 1691 goto out_unlock; 1692 } 1693 1694 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1695 bo->mem.num_pages, 0); 1696 1697 if (unlikely(bo->vm_node == NULL)) { 1698 rw_wunlock(&bdev->vm_lock); 1699 goto retry_pre_get; 1700 } 1701 1702 ttm_bo_vm_insert_rb(bo); 1703 rw_wunlock(&bdev->vm_lock); 1704 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1705 1706 return 0; 1707out_unlock: 1708 rw_wunlock(&bdev->vm_lock); 1709 return ret; 1710} 1711 1712int ttm_bo_wait(struct ttm_buffer_object *bo, 1713 bool lazy, bool interruptible, bool no_wait) 1714{ 1715 struct ttm_bo_driver *driver = bo->bdev->driver; 1716 struct ttm_bo_device *bdev = bo->bdev; 1717 void *sync_obj; 1718 int ret = 0; 1719 1720 if (likely(bo->sync_obj == NULL)) 1721 return 0; 1722 1723 while (bo->sync_obj) { 1724 1725 if (driver->sync_obj_signaled(bo->sync_obj)) { 1726 void *tmp_obj = bo->sync_obj; 1727 bo->sync_obj = NULL; 1728 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1729 mtx_unlock(&bdev->fence_lock); 1730 driver->sync_obj_unref(&tmp_obj); 1731 mtx_lock(&bdev->fence_lock); 1732 continue; 1733 } 1734 1735 if (no_wait) 1736 return -EBUSY; 1737 1738 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1739 mtx_unlock(&bdev->fence_lock); 1740 ret = driver->sync_obj_wait(sync_obj, 1741 lazy, interruptible); 1742 if (unlikely(ret != 0)) { 1743 driver->sync_obj_unref(&sync_obj); 1744 mtx_lock(&bdev->fence_lock); 1745 return ret; 1746 } 1747 mtx_lock(&bdev->fence_lock); 1748 if (likely(bo->sync_obj == sync_obj)) { 1749 void *tmp_obj = bo->sync_obj; 1750 bo->sync_obj = NULL; 1751 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1752 &bo->priv_flags); 1753 mtx_unlock(&bdev->fence_lock); 1754 driver->sync_obj_unref(&sync_obj); 1755 driver->sync_obj_unref(&tmp_obj); 1756 mtx_lock(&bdev->fence_lock); 1757 } else { 1758 mtx_unlock(&bdev->fence_lock); 1759 driver->sync_obj_unref(&sync_obj); 1760 mtx_lock(&bdev->fence_lock); 1761 } 1762 } 1763 return 0; 1764} 1765 1766int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1767{ 1768 struct ttm_bo_device *bdev = bo->bdev; 1769 int ret = 0; 1770 1771 /* 1772 * Using ttm_bo_reserve makes sure the lru lists are updated. 1773 */ 1774 1775 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1776 if (unlikely(ret != 0)) 1777 return ret; 1778 mtx_lock(&bdev->fence_lock); 1779 ret = ttm_bo_wait(bo, false, true, no_wait); 1780 mtx_unlock(&bdev->fence_lock); 1781 if (likely(ret == 0)) 1782 atomic_inc(&bo->cpu_writers); 1783 ttm_bo_unreserve(bo); 1784 return ret; 1785} 1786 1787void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1788{ 1789 atomic_dec(&bo->cpu_writers); 1790} 1791 1792/** 1793 * A buffer object shrink method that tries to swap out the first 1794 * buffer object on the bo_global::swap_lru list. 1795 */ 1796 1797static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1798{ 1799 struct ttm_bo_global *glob = 1800 container_of(shrink, struct ttm_bo_global, shrink); 1801 struct ttm_buffer_object *bo; 1802 int ret = -EBUSY; 1803 int put_count; 1804 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1805 1806 mtx_lock(&glob->lru_lock); 1807 list_for_each_entry(bo, &glob->swap_lru, swap) { 1808 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 1809 if (!ret) 1810 break; 1811 } 1812 1813 if (ret) { 1814 mtx_unlock(&glob->lru_lock); 1815 return ret; 1816 } 1817 1818 refcount_acquire(&bo->list_kref); 1819 1820 if (!list_empty(&bo->ddestroy)) { 1821 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1822 if (refcount_release(&bo->list_kref)) 1823 ttm_bo_release_list(bo); 1824 return ret; 1825 } 1826 1827 put_count = ttm_bo_del_from_lru(bo); 1828 mtx_unlock(&glob->lru_lock); 1829 1830 ttm_bo_list_ref_sub(bo, put_count, true); 1831 1832 /** 1833 * Wait for GPU, then move to system cached. 1834 */ 1835 1836 mtx_lock(&bo->bdev->fence_lock); 1837 ret = ttm_bo_wait(bo, false, false, false); 1838 mtx_unlock(&bo->bdev->fence_lock); 1839 1840 if (unlikely(ret != 0)) 1841 goto out; 1842 1843 if ((bo->mem.placement & swap_placement) != swap_placement) { 1844 struct ttm_mem_reg evict_mem; 1845 1846 evict_mem = bo->mem; 1847 evict_mem.mm_node = NULL; 1848 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1849 evict_mem.mem_type = TTM_PL_SYSTEM; 1850 1851 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1852 false, false); 1853 if (unlikely(ret != 0)) 1854 goto out; 1855 } 1856 1857 ttm_bo_unmap_virtual(bo); 1858 1859 /** 1860 * Swap out. Buffer will be swapped in again as soon as 1861 * anyone tries to access a ttm page. 1862 */ 1863 1864 if (bo->bdev->driver->swap_notify) 1865 bo->bdev->driver->swap_notify(bo); 1866 1867 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1868out: 1869 1870 /** 1871 * 1872 * Unreserve without putting on LRU to avoid swapping out an 1873 * already swapped buffer. 1874 */ 1875 1876 atomic_set(&bo->reserved, 0); 1877 wakeup(bo); 1878 if (refcount_release(&bo->list_kref)) 1879 ttm_bo_release_list(bo); 1880 return ret; 1881} 1882 1883void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1884{ 1885 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1886 ; 1887}
|