296void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) 297{ 298 ttm_bo_add_to_lru(bo); 299 atomic_set(&bo->reserved, 0); 300 wakeup(bo); 301} 302 303void ttm_bo_unreserve(struct ttm_buffer_object *bo) 304{ 305 struct ttm_bo_global *glob = bo->glob; 306 307 mtx_lock(&glob->lru_lock); 308 ttm_bo_unreserve_locked(bo); 309 mtx_unlock(&glob->lru_lock); 310} 311 312/* 313 * Call bo->mutex locked. 314 */ 315static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 316{ 317 struct ttm_bo_device *bdev = bo->bdev; 318 struct ttm_bo_global *glob = bo->glob; 319 int ret = 0; 320 uint32_t page_flags = 0; 321 322 TTM_ASSERT_LOCKED(&bo->mutex); 323 bo->ttm = NULL; 324 325 if (bdev->need_dma32) 326 page_flags |= TTM_PAGE_FLAG_DMA32; 327 328 switch (bo->type) { 329 case ttm_bo_type_device: 330 if (zero_alloc) 331 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 332 case ttm_bo_type_kernel: 333 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 334 page_flags, glob->dummy_read_page); 335 if (unlikely(bo->ttm == NULL)) 336 ret = -ENOMEM; 337 break; 338 case ttm_bo_type_sg: 339 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 340 page_flags | TTM_PAGE_FLAG_SG, 341 glob->dummy_read_page); 342 if (unlikely(bo->ttm == NULL)) { 343 ret = -ENOMEM; 344 break; 345 } 346 bo->ttm->sg = bo->sg; 347 break; 348 default: 349 printf("[TTM] Illegal buffer object type\n"); 350 ret = -EINVAL; 351 break; 352 } 353 354 return ret; 355} 356 357static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 358 struct ttm_mem_reg *mem, 359 bool evict, bool interruptible, 360 bool no_wait_gpu) 361{ 362 struct ttm_bo_device *bdev = bo->bdev; 363 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 364 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 365 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 366 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 367 int ret = 0; 368 369 if (old_is_pci || new_is_pci || 370 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 371 ret = ttm_mem_io_lock(old_man, true); 372 if (unlikely(ret != 0)) 373 goto out_err; 374 ttm_bo_unmap_virtual_locked(bo); 375 ttm_mem_io_unlock(old_man); 376 } 377 378 /* 379 * Create and bind a ttm if required. 380 */ 381 382 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 383 if (bo->ttm == NULL) { 384 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 385 ret = ttm_bo_add_ttm(bo, zero); 386 if (ret) 387 goto out_err; 388 } 389 390 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 391 if (ret) 392 goto out_err; 393 394 if (mem->mem_type != TTM_PL_SYSTEM) { 395 ret = ttm_tt_bind(bo->ttm, mem); 396 if (ret) 397 goto out_err; 398 } 399 400 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 401 if (bdev->driver->move_notify) 402 bdev->driver->move_notify(bo, mem); 403 bo->mem = *mem; 404 mem->mm_node = NULL; 405 goto moved; 406 } 407 } 408 409 if (bdev->driver->move_notify) 410 bdev->driver->move_notify(bo, mem); 411 412 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 413 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 414 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 415 else if (bdev->driver->move) 416 ret = bdev->driver->move(bo, evict, interruptible, 417 no_wait_gpu, mem); 418 else 419 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 420 421 if (ret) { 422 if (bdev->driver->move_notify) { 423 struct ttm_mem_reg tmp_mem = *mem; 424 *mem = bo->mem; 425 bo->mem = tmp_mem; 426 bdev->driver->move_notify(bo, mem); 427 bo->mem = *mem; 428 } 429 430 goto out_err; 431 } 432 433moved: 434 if (bo->evicted) { 435 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 436 if (ret) 437 printf("[TTM] Can not flush read caches\n"); 438 bo->evicted = false; 439 } 440 441 if (bo->mem.mm_node) { 442 bo->offset = (bo->mem.start << PAGE_SHIFT) + 443 bdev->man[bo->mem.mem_type].gpu_offset; 444 bo->cur_placement = bo->mem.placement; 445 } else 446 bo->offset = 0; 447 448 return 0; 449 450out_err: 451 new_man = &bdev->man[bo->mem.mem_type]; 452 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 453 ttm_tt_unbind(bo->ttm); 454 ttm_tt_destroy(bo->ttm); 455 bo->ttm = NULL; 456 } 457 458 return ret; 459} 460 461/** 462 * Call bo::reserved. 463 * Will release GPU memory type usage on destruction. 464 * This is the place to put in driver specific hooks to release 465 * driver private resources. 466 * Will release the bo::reserved lock. 467 */ 468 469static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 470{ 471 if (bo->bdev->driver->move_notify) 472 bo->bdev->driver->move_notify(bo, NULL); 473 474 if (bo->ttm) { 475 ttm_tt_unbind(bo->ttm); 476 ttm_tt_destroy(bo->ttm); 477 bo->ttm = NULL; 478 } 479 ttm_bo_mem_put(bo, &bo->mem); 480 481 atomic_set(&bo->reserved, 0); 482 wakeup(&bo); 483 484 /* 485 * Since the final reference to this bo may not be dropped by 486 * the current task we have to put a memory barrier here to make 487 * sure the changes done in this function are always visible. 488 * 489 * This function only needs protection against the final kref_put. 490 */ 491 mb(); 492} 493 494static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 495{ 496 struct ttm_bo_device *bdev = bo->bdev; 497 struct ttm_bo_global *glob = bo->glob; 498 struct ttm_bo_driver *driver = bdev->driver; 499 void *sync_obj = NULL; 500 int put_count; 501 int ret; 502 503 mtx_lock(&glob->lru_lock); 504 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 505 506 mtx_lock(&bdev->fence_lock); 507 (void) ttm_bo_wait(bo, false, false, true); 508 if (!ret && !bo->sync_obj) { 509 mtx_unlock(&bdev->fence_lock); 510 put_count = ttm_bo_del_from_lru(bo); 511 512 mtx_unlock(&glob->lru_lock); 513 ttm_bo_cleanup_memtype_use(bo); 514 515 ttm_bo_list_ref_sub(bo, put_count, true); 516 517 return; 518 } 519 if (bo->sync_obj) 520 sync_obj = driver->sync_obj_ref(bo->sync_obj); 521 mtx_unlock(&bdev->fence_lock); 522 523 if (!ret) { 524 atomic_set(&bo->reserved, 0); 525 wakeup(bo); 526 } 527 528 refcount_acquire(&bo->list_kref); 529 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 530 mtx_unlock(&glob->lru_lock); 531 532 if (sync_obj) { 533 driver->sync_obj_flush(sync_obj); 534 driver->sync_obj_unref(&sync_obj); 535 } 536 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 537 ((hz / 100) < 1) ? 1 : hz / 100); 538} 539 540/** 541 * function ttm_bo_cleanup_refs_and_unlock 542 * If bo idle, remove from delayed- and lru lists, and unref. 543 * If not idle, do nothing. 544 * 545 * Must be called with lru_lock and reservation held, this function 546 * will drop both before returning. 547 * 548 * @interruptible Any sleeps should occur interruptibly. 549 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 550 */ 551 552static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 553 bool interruptible, 554 bool no_wait_gpu) 555{ 556 struct ttm_bo_device *bdev = bo->bdev; 557 struct ttm_bo_driver *driver = bdev->driver; 558 struct ttm_bo_global *glob = bo->glob; 559 int put_count; 560 int ret; 561 562 mtx_lock(&bdev->fence_lock); 563 ret = ttm_bo_wait(bo, false, false, true); 564 565 if (ret && !no_wait_gpu) { 566 void *sync_obj; 567 568 /* 569 * Take a reference to the fence and unreserve, 570 * at this point the buffer should be dead, so 571 * no new sync objects can be attached. 572 */ 573 sync_obj = driver->sync_obj_ref(bo->sync_obj); 574 mtx_unlock(&bdev->fence_lock); 575 576 atomic_set(&bo->reserved, 0); 577 wakeup(bo); 578 mtx_unlock(&glob->lru_lock); 579 580 ret = driver->sync_obj_wait(sync_obj, false, interruptible); 581 driver->sync_obj_unref(&sync_obj); 582 if (ret) 583 return ret; 584 585 /* 586 * remove sync_obj with ttm_bo_wait, the wait should be 587 * finished, and no new wait object should have been added. 588 */ 589 mtx_lock(&bdev->fence_lock); 590 ret = ttm_bo_wait(bo, false, false, true); 591 mtx_unlock(&bdev->fence_lock); 592 if (ret) 593 return ret; 594 595 mtx_lock(&glob->lru_lock); 596 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 597 598 /* 599 * We raced, and lost, someone else holds the reservation now, 600 * and is probably busy in ttm_bo_cleanup_memtype_use. 601 * 602 * Even if it's not the case, because we finished waiting any 603 * delayed destruction would succeed, so just return success 604 * here. 605 */ 606 if (ret) { 607 mtx_unlock(&glob->lru_lock); 608 return 0; 609 } 610 } else 611 mtx_unlock(&bdev->fence_lock); 612 613 if (ret || unlikely(list_empty(&bo->ddestroy))) { 614 atomic_set(&bo->reserved, 0); 615 wakeup(bo); 616 mtx_unlock(&glob->lru_lock); 617 return ret; 618 } 619 620 put_count = ttm_bo_del_from_lru(bo); 621 list_del_init(&bo->ddestroy); 622 ++put_count; 623 624 mtx_unlock(&glob->lru_lock); 625 ttm_bo_cleanup_memtype_use(bo); 626 627 ttm_bo_list_ref_sub(bo, put_count, true); 628 629 return 0; 630} 631 632/** 633 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 634 * encountered buffers. 635 */ 636 637static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 638{ 639 struct ttm_bo_global *glob = bdev->glob; 640 struct ttm_buffer_object *entry = NULL; 641 int ret = 0; 642 643 mtx_lock(&glob->lru_lock); 644 if (list_empty(&bdev->ddestroy)) 645 goto out_unlock; 646 647 entry = list_first_entry(&bdev->ddestroy, 648 struct ttm_buffer_object, ddestroy); 649 refcount_acquire(&entry->list_kref); 650 651 for (;;) { 652 struct ttm_buffer_object *nentry = NULL; 653 654 if (entry->ddestroy.next != &bdev->ddestroy) { 655 nentry = list_first_entry(&entry->ddestroy, 656 struct ttm_buffer_object, ddestroy); 657 refcount_acquire(&nentry->list_kref); 658 } 659 660 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); 661 if (remove_all && ret) { 662 ret = ttm_bo_reserve_nolru(entry, false, false, 663 false, 0); 664 } 665 666 if (!ret) 667 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 668 !remove_all); 669 else 670 mtx_unlock(&glob->lru_lock); 671 672 if (refcount_release(&entry->list_kref)) 673 ttm_bo_release_list(entry); 674 entry = nentry; 675 676 if (ret || !entry) 677 goto out; 678 679 mtx_lock(&glob->lru_lock); 680 if (list_empty(&entry->ddestroy)) 681 break; 682 } 683 684out_unlock: 685 mtx_unlock(&glob->lru_lock); 686out: 687 if (entry && refcount_release(&entry->list_kref)) 688 ttm_bo_release_list(entry); 689 return ret; 690} 691 692static void ttm_bo_delayed_workqueue(void *arg, int pending __unused) 693{ 694 struct ttm_bo_device *bdev = arg; 695 696 if (ttm_bo_delayed_delete(bdev, false)) { 697 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 698 ((hz / 100) < 1) ? 1 : hz / 100); 699 } 700} 701 702static void ttm_bo_release(struct ttm_buffer_object *bo) 703{ 704 struct ttm_bo_device *bdev = bo->bdev; 705 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 706 707 rw_wlock(&bdev->vm_lock); 708 if (likely(bo->vm_node != NULL)) { 709 RB_REMOVE(ttm_bo_device_buffer_objects, 710 &bdev->addr_space_rb, bo); 711 drm_mm_put_block(bo->vm_node); 712 bo->vm_node = NULL; 713 } 714 rw_wunlock(&bdev->vm_lock); 715 ttm_mem_io_lock(man, false); 716 ttm_mem_io_free_vm(bo); 717 ttm_mem_io_unlock(man); 718 ttm_bo_cleanup_refs_or_queue(bo); 719 if (refcount_release(&bo->list_kref)) 720 ttm_bo_release_list(bo); 721} 722 723void ttm_bo_unref(struct ttm_buffer_object **p_bo) 724{ 725 struct ttm_buffer_object *bo = *p_bo; 726 727 *p_bo = NULL; 728 if (refcount_release(&bo->kref)) 729 ttm_bo_release(bo); 730} 731 732int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 733{ 734 int pending; 735 736 taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending); 737 if (pending) 738 taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); 739 return (pending); 740} 741 742void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 743{ 744 if (resched) { 745 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 746 ((hz / 100) < 1) ? 1 : hz / 100); 747 } 748} 749 750static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 751 bool no_wait_gpu) 752{ 753 struct ttm_bo_device *bdev = bo->bdev; 754 struct ttm_mem_reg evict_mem; 755 struct ttm_placement placement; 756 int ret = 0; 757 758 mtx_lock(&bdev->fence_lock); 759 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 760 mtx_unlock(&bdev->fence_lock); 761 762 if (unlikely(ret != 0)) { 763 if (ret != -ERESTART) { 764 printf("[TTM] Failed to expire sync object before buffer eviction\n"); 765 } 766 goto out; 767 } 768 769 MPASS(ttm_bo_is_reserved(bo)); 770 771 evict_mem = bo->mem; 772 evict_mem.mm_node = NULL; 773 evict_mem.bus.io_reserved_vm = false; 774 evict_mem.bus.io_reserved_count = 0; 775 776 placement.fpfn = 0; 777 placement.lpfn = 0; 778 placement.num_placement = 0; 779 placement.num_busy_placement = 0; 780 bdev->driver->evict_flags(bo, &placement); 781 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 782 no_wait_gpu); 783 if (ret) { 784 if (ret != -ERESTART) { 785 printf("[TTM] Failed to find memory space for buffer 0x%p eviction\n", 786 bo); 787 ttm_bo_mem_space_debug(bo, &placement); 788 } 789 goto out; 790 } 791 792 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 793 no_wait_gpu); 794 if (ret) { 795 if (ret != -ERESTART) 796 printf("[TTM] Buffer eviction failed\n"); 797 ttm_bo_mem_put(bo, &evict_mem); 798 goto out; 799 } 800 bo->evicted = true; 801out: 802 return ret; 803} 804 805static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 806 uint32_t mem_type, 807 bool interruptible, 808 bool no_wait_gpu) 809{ 810 struct ttm_bo_global *glob = bdev->glob; 811 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 812 struct ttm_buffer_object *bo; 813 int ret = -EBUSY, put_count; 814 815 mtx_lock(&glob->lru_lock); 816 list_for_each_entry(bo, &man->lru, lru) { 817 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 818 if (!ret) 819 break; 820 } 821 822 if (ret) { 823 mtx_unlock(&glob->lru_lock); 824 return ret; 825 } 826 827 refcount_acquire(&bo->list_kref); 828 829 if (!list_empty(&bo->ddestroy)) { 830 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 831 no_wait_gpu); 832 if (refcount_release(&bo->list_kref)) 833 ttm_bo_release_list(bo); 834 return ret; 835 } 836 837 put_count = ttm_bo_del_from_lru(bo); 838 mtx_unlock(&glob->lru_lock); 839 840 MPASS(ret == 0); 841 842 ttm_bo_list_ref_sub(bo, put_count, true); 843 844 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 845 ttm_bo_unreserve(bo); 846 847 if (refcount_release(&bo->list_kref)) 848 ttm_bo_release_list(bo); 849 return ret; 850} 851 852void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 853{ 854 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 855 856 if (mem->mm_node) 857 (*man->func->put_node)(man, mem); 858} 859 860/** 861 * Repeatedly evict memory from the LRU for @mem_type until we create enough 862 * space, or we've evicted everything and there isn't enough space. 863 */ 864static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 865 uint32_t mem_type, 866 struct ttm_placement *placement, 867 struct ttm_mem_reg *mem, 868 bool interruptible, 869 bool no_wait_gpu) 870{ 871 struct ttm_bo_device *bdev = bo->bdev; 872 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 873 int ret; 874 875 do { 876 ret = (*man->func->get_node)(man, bo, placement, mem); 877 if (unlikely(ret != 0)) 878 return ret; 879 if (mem->mm_node) 880 break; 881 ret = ttm_mem_evict_first(bdev, mem_type, 882 interruptible, no_wait_gpu); 883 if (unlikely(ret != 0)) 884 return ret; 885 } while (1); 886 if (mem->mm_node == NULL) 887 return -ENOMEM; 888 mem->mem_type = mem_type; 889 return 0; 890} 891 892static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 893 uint32_t cur_placement, 894 uint32_t proposed_placement) 895{ 896 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 897 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 898 899 /** 900 * Keep current caching if possible. 901 */ 902 903 if ((cur_placement & caching) != 0) 904 result |= (cur_placement & caching); 905 else if ((man->default_caching & caching) != 0) 906 result |= man->default_caching; 907 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 908 result |= TTM_PL_FLAG_CACHED; 909 else if ((TTM_PL_FLAG_WC & caching) != 0) 910 result |= TTM_PL_FLAG_WC; 911 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 912 result |= TTM_PL_FLAG_UNCACHED; 913 914 return result; 915} 916 917static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 918 uint32_t mem_type, 919 uint32_t proposed_placement, 920 uint32_t *masked_placement) 921{ 922 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 923 924 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 925 return false; 926 927 if ((proposed_placement & man->available_caching) == 0) 928 return false; 929 930 cur_flags |= (proposed_placement & man->available_caching); 931 932 *masked_placement = cur_flags; 933 return true; 934} 935 936/** 937 * Creates space for memory region @mem according to its type. 938 * 939 * This function first searches for free space in compatible memory types in 940 * the priority order defined by the driver. If free space isn't found, then 941 * ttm_bo_mem_force_space is attempted in priority order to evict and find 942 * space. 943 */ 944int ttm_bo_mem_space(struct ttm_buffer_object *bo, 945 struct ttm_placement *placement, 946 struct ttm_mem_reg *mem, 947 bool interruptible, 948 bool no_wait_gpu) 949{ 950 struct ttm_bo_device *bdev = bo->bdev; 951 struct ttm_mem_type_manager *man; 952 uint32_t mem_type = TTM_PL_SYSTEM; 953 uint32_t cur_flags = 0; 954 bool type_found = false; 955 bool type_ok = false; 956 bool has_erestartsys = false; 957 int i, ret; 958 959 mem->mm_node = NULL; 960 for (i = 0; i < placement->num_placement; ++i) { 961 ret = ttm_mem_type_from_flags(placement->placement[i], 962 &mem_type); 963 if (ret) 964 return ret; 965 man = &bdev->man[mem_type]; 966 967 type_ok = ttm_bo_mt_compatible(man, 968 mem_type, 969 placement->placement[i], 970 &cur_flags); 971 972 if (!type_ok) 973 continue; 974 975 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 976 cur_flags); 977 /* 978 * Use the access and other non-mapping-related flag bits from 979 * the memory placement flags to the current flags 980 */ 981 ttm_flag_masked(&cur_flags, placement->placement[i], 982 ~TTM_PL_MASK_MEMTYPE); 983 984 if (mem_type == TTM_PL_SYSTEM) 985 break; 986 987 if (man->has_type && man->use_type) { 988 type_found = true; 989 ret = (*man->func->get_node)(man, bo, placement, mem); 990 if (unlikely(ret)) 991 return ret; 992 } 993 if (mem->mm_node) 994 break; 995 } 996 997 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 998 mem->mem_type = mem_type; 999 mem->placement = cur_flags; 1000 return 0; 1001 } 1002 1003 if (!type_found) 1004 return -EINVAL; 1005 1006 for (i = 0; i < placement->num_busy_placement; ++i) { 1007 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 1008 &mem_type); 1009 if (ret) 1010 return ret; 1011 man = &bdev->man[mem_type]; 1012 if (!man->has_type) 1013 continue; 1014 if (!ttm_bo_mt_compatible(man, 1015 mem_type, 1016 placement->busy_placement[i], 1017 &cur_flags)) 1018 continue; 1019 1020 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1021 cur_flags); 1022 /* 1023 * Use the access and other non-mapping-related flag bits from 1024 * the memory placement flags to the current flags 1025 */ 1026 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 1027 ~TTM_PL_MASK_MEMTYPE); 1028 1029 1030 if (mem_type == TTM_PL_SYSTEM) { 1031 mem->mem_type = mem_type; 1032 mem->placement = cur_flags; 1033 mem->mm_node = NULL; 1034 return 0; 1035 } 1036 1037 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 1038 interruptible, no_wait_gpu); 1039 if (ret == 0 && mem->mm_node) { 1040 mem->placement = cur_flags; 1041 return 0; 1042 } 1043 if (ret == -ERESTART) 1044 has_erestartsys = true; 1045 } 1046 ret = (has_erestartsys) ? -ERESTART : -ENOMEM; 1047 return ret; 1048} 1049 1050static 1051int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1052 struct ttm_placement *placement, 1053 bool interruptible, 1054 bool no_wait_gpu) 1055{ 1056 int ret = 0; 1057 struct ttm_mem_reg mem; 1058 struct ttm_bo_device *bdev = bo->bdev; 1059 1060 MPASS(ttm_bo_is_reserved(bo)); 1061 1062 /* 1063 * FIXME: It's possible to pipeline buffer moves. 1064 * Have the driver move function wait for idle when necessary, 1065 * instead of doing it here. 1066 */ 1067 mtx_lock(&bdev->fence_lock); 1068 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1069 mtx_unlock(&bdev->fence_lock); 1070 if (ret) 1071 return ret; 1072 mem.num_pages = bo->num_pages; 1073 mem.size = mem.num_pages << PAGE_SHIFT; 1074 mem.page_alignment = bo->mem.page_alignment; 1075 mem.bus.io_reserved_vm = false; 1076 mem.bus.io_reserved_count = 0; 1077 /* 1078 * Determine where to move the buffer. 1079 */ 1080 ret = ttm_bo_mem_space(bo, placement, &mem, 1081 interruptible, no_wait_gpu); 1082 if (ret) 1083 goto out_unlock; 1084 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1085 interruptible, no_wait_gpu); 1086out_unlock: 1087 if (ret && mem.mm_node) 1088 ttm_bo_mem_put(bo, &mem); 1089 return ret; 1090} 1091 1092static int ttm_bo_mem_compat(struct ttm_placement *placement, 1093 struct ttm_mem_reg *mem) 1094{ 1095 int i; 1096 1097 if (mem->mm_node && placement->lpfn != 0 && 1098 (mem->start < placement->fpfn || 1099 mem->start + mem->num_pages > placement->lpfn)) 1100 return -1; 1101 1102 for (i = 0; i < placement->num_placement; i++) { 1103 if ((placement->placement[i] & mem->placement & 1104 TTM_PL_MASK_CACHING) && 1105 (placement->placement[i] & mem->placement & 1106 TTM_PL_MASK_MEM)) 1107 return i; 1108 } 1109 return -1; 1110} 1111 1112int ttm_bo_validate(struct ttm_buffer_object *bo, 1113 struct ttm_placement *placement, 1114 bool interruptible, 1115 bool no_wait_gpu) 1116{ 1117 int ret; 1118 1119 MPASS(ttm_bo_is_reserved(bo)); 1120 /* Check that range is valid */ 1121 if (placement->lpfn || placement->fpfn) 1122 if (placement->fpfn > placement->lpfn || 1123 (placement->lpfn - placement->fpfn) < bo->num_pages) 1124 return -EINVAL; 1125 /* 1126 * Check whether we need to move buffer. 1127 */ 1128 ret = ttm_bo_mem_compat(placement, &bo->mem); 1129 if (ret < 0) { 1130 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1131 no_wait_gpu); 1132 if (ret) 1133 return ret; 1134 } else { 1135 /* 1136 * Use the access and other non-mapping-related flag bits from 1137 * the compatible memory placement flags to the active flags 1138 */ 1139 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1140 ~TTM_PL_MASK_MEMTYPE); 1141 } 1142 /* 1143 * We might need to add a TTM. 1144 */ 1145 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1146 ret = ttm_bo_add_ttm(bo, true); 1147 if (ret) 1148 return ret; 1149 } 1150 return 0; 1151} 1152 1153int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1154 struct ttm_placement *placement) 1155{ 1156 MPASS(!((placement->fpfn || placement->lpfn) && 1157 (bo->mem.num_pages > (placement->lpfn - placement->fpfn)))); 1158 1159 return 0; 1160} 1161 1162int ttm_bo_init(struct ttm_bo_device *bdev, 1163 struct ttm_buffer_object *bo, 1164 unsigned long size, 1165 enum ttm_bo_type type, 1166 struct ttm_placement *placement, 1167 uint32_t page_alignment, 1168 bool interruptible, 1169 struct vm_object *persistent_swap_storage, 1170 size_t acc_size, 1171 struct sg_table *sg, 1172 void (*destroy) (struct ttm_buffer_object *)) 1173{ 1174 int ret = 0; 1175 unsigned long num_pages; 1176 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1177 1178 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1179 if (ret) { 1180 printf("[TTM] Out of kernel memory\n"); 1181 if (destroy) 1182 (*destroy)(bo); 1183 else 1184 free(bo, M_TTM_BO); 1185 return -ENOMEM; 1186 } 1187 1188 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1189 if (num_pages == 0) { 1190 printf("[TTM] Illegal buffer object size\n"); 1191 if (destroy) 1192 (*destroy)(bo); 1193 else 1194 free(bo, M_TTM_BO); 1195 ttm_mem_global_free(mem_glob, acc_size); 1196 return -EINVAL; 1197 } 1198 bo->destroy = destroy; 1199 1200 refcount_init(&bo->kref, 1); 1201 refcount_init(&bo->list_kref, 1); 1202 atomic_set(&bo->cpu_writers, 0); 1203 atomic_set(&bo->reserved, 1); 1204 INIT_LIST_HEAD(&bo->lru); 1205 INIT_LIST_HEAD(&bo->ddestroy); 1206 INIT_LIST_HEAD(&bo->swap); 1207 INIT_LIST_HEAD(&bo->io_reserve_lru); 1208 bo->bdev = bdev; 1209 bo->glob = bdev->glob; 1210 bo->type = type; 1211 bo->num_pages = num_pages; 1212 bo->mem.size = num_pages << PAGE_SHIFT; 1213 bo->mem.mem_type = TTM_PL_SYSTEM; 1214 bo->mem.num_pages = bo->num_pages; 1215 bo->mem.mm_node = NULL; 1216 bo->mem.page_alignment = page_alignment; 1217 bo->mem.bus.io_reserved_vm = false; 1218 bo->mem.bus.io_reserved_count = 0; 1219 bo->priv_flags = 0; 1220 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1221 bo->seq_valid = false; 1222 bo->persistent_swap_storage = persistent_swap_storage; 1223 bo->acc_size = acc_size; 1224 bo->sg = sg; 1225 atomic_inc(&bo->glob->bo_count); 1226 1227 ret = ttm_bo_check_placement(bo, placement); 1228 if (unlikely(ret != 0)) 1229 goto out_err; 1230 1231 /* 1232 * For ttm_bo_type_device buffers, allocate 1233 * address space from the device. 1234 */ 1235 if (bo->type == ttm_bo_type_device || 1236 bo->type == ttm_bo_type_sg) { 1237 ret = ttm_bo_setup_vm(bo); 1238 if (ret) 1239 goto out_err; 1240 } 1241 1242 ret = ttm_bo_validate(bo, placement, interruptible, false); 1243 if (ret) 1244 goto out_err; 1245 1246 ttm_bo_unreserve(bo); 1247 return 0; 1248 1249out_err: 1250 ttm_bo_unreserve(bo); 1251 ttm_bo_unref(&bo); 1252 1253 return ret; 1254} 1255 1256size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1257 unsigned long bo_size, 1258 unsigned struct_size) 1259{ 1260 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1261 size_t size = 0; 1262 1263 size += ttm_round_pot(struct_size); 1264 size += PAGE_ALIGN(npages * sizeof(void *)); 1265 size += ttm_round_pot(sizeof(struct ttm_tt)); 1266 return size; 1267} 1268 1269size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1270 unsigned long bo_size, 1271 unsigned struct_size) 1272{ 1273 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1274 size_t size = 0; 1275 1276 size += ttm_round_pot(struct_size); 1277 size += PAGE_ALIGN(npages * sizeof(void *)); 1278 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1279 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1280 return size; 1281} 1282 1283int ttm_bo_create(struct ttm_bo_device *bdev, 1284 unsigned long size, 1285 enum ttm_bo_type type, 1286 struct ttm_placement *placement, 1287 uint32_t page_alignment, 1288 bool interruptible, 1289 struct vm_object *persistent_swap_storage, 1290 struct ttm_buffer_object **p_bo) 1291{ 1292 struct ttm_buffer_object *bo; 1293 size_t acc_size; 1294 int ret; 1295 1296 bo = malloc(sizeof(*bo), M_TTM_BO, M_WAITOK | M_ZERO); 1297 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1298 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1299 interruptible, persistent_swap_storage, acc_size, 1300 NULL, NULL); 1301 if (likely(ret == 0)) 1302 *p_bo = bo; 1303 1304 return ret; 1305} 1306 1307static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1308 unsigned mem_type, bool allow_errors) 1309{ 1310 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1311 struct ttm_bo_global *glob = bdev->glob; 1312 int ret; 1313 1314 /* 1315 * Can't use standard list traversal since we're unlocking. 1316 */ 1317 1318 mtx_lock(&glob->lru_lock); 1319 while (!list_empty(&man->lru)) { 1320 mtx_unlock(&glob->lru_lock); 1321 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1322 if (ret) { 1323 if (allow_errors) { 1324 return ret; 1325 } else { 1326 printf("[TTM] Cleanup eviction failed\n"); 1327 } 1328 } 1329 mtx_lock(&glob->lru_lock); 1330 } 1331 mtx_unlock(&glob->lru_lock); 1332 return 0; 1333} 1334 1335int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1336{ 1337 struct ttm_mem_type_manager *man; 1338 int ret = -EINVAL; 1339 1340 if (mem_type >= TTM_NUM_MEM_TYPES) { 1341 printf("[TTM] Illegal memory type %d\n", mem_type); 1342 return ret; 1343 } 1344 man = &bdev->man[mem_type]; 1345 1346 if (!man->has_type) { 1347 printf("[TTM] Trying to take down uninitialized memory manager type %u\n", 1348 mem_type); 1349 return ret; 1350 } 1351 1352 man->use_type = false; 1353 man->has_type = false; 1354 1355 ret = 0; 1356 if (mem_type > 0) { 1357 ttm_bo_force_list_clean(bdev, mem_type, false); 1358 1359 ret = (*man->func->takedown)(man); 1360 } 1361 1362 return ret; 1363} 1364 1365int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1366{ 1367 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1368 1369 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1370 printf("[TTM] Illegal memory manager memory type %u\n", mem_type); 1371 return -EINVAL; 1372 } 1373 1374 if (!man->has_type) { 1375 printf("[TTM] Memory type %u has not been initialized\n", mem_type); 1376 return 0; 1377 } 1378 1379 return ttm_bo_force_list_clean(bdev, mem_type, true); 1380} 1381 1382int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1383 unsigned long p_size) 1384{ 1385 int ret = -EINVAL; 1386 struct ttm_mem_type_manager *man; 1387 1388 MPASS(type < TTM_NUM_MEM_TYPES); 1389 man = &bdev->man[type]; 1390 MPASS(!man->has_type); 1391 man->io_reserve_fastpath = true; 1392 man->use_io_reserve_lru = false; 1393 sx_init(&man->io_reserve_mutex, "ttmman"); 1394 INIT_LIST_HEAD(&man->io_reserve_lru); 1395 1396 ret = bdev->driver->init_mem_type(bdev, type, man); 1397 if (ret) 1398 return ret; 1399 man->bdev = bdev; 1400 1401 ret = 0; 1402 if (type != TTM_PL_SYSTEM) { 1403 ret = (*man->func->init)(man, p_size); 1404 if (ret) 1405 return ret; 1406 } 1407 man->has_type = true; 1408 man->use_type = true; 1409 man->size = p_size; 1410 1411 INIT_LIST_HEAD(&man->lru); 1412 1413 return 0; 1414} 1415 1416static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob) 1417{ 1418 1419 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1420 vm_page_free(glob->dummy_read_page); 1421} 1422 1423void ttm_bo_global_release(struct drm_global_reference *ref) 1424{ 1425 struct ttm_bo_global *glob = ref->object; 1426 1427 if (refcount_release(&glob->kobj_ref)) 1428 ttm_bo_global_kobj_release(glob); 1429} 1430 1431int ttm_bo_global_init(struct drm_global_reference *ref) 1432{ 1433 struct ttm_bo_global_ref *bo_ref = 1434 container_of(ref, struct ttm_bo_global_ref, ref); 1435 struct ttm_bo_global *glob = ref->object; 1436 int ret; 1437 1438 sx_init(&glob->device_list_mutex, "ttmdlm"); 1439 mtx_init(&glob->lru_lock, "ttmlru", NULL, MTX_DEF); 1440 glob->mem_glob = bo_ref->mem_glob; 1441 glob->dummy_read_page = vm_page_alloc_contig(NULL, 0, 1442 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ, 1443 1, 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE); 1444 1445 if (unlikely(glob->dummy_read_page == NULL)) { 1446 ret = -ENOMEM; 1447 goto out_no_drp; 1448 } 1449 1450 INIT_LIST_HEAD(&glob->swap_lru); 1451 INIT_LIST_HEAD(&glob->device_list); 1452 1453 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1454 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1455 if (unlikely(ret != 0)) { 1456 printf("[TTM] Could not register buffer object swapout\n"); 1457 goto out_no_shrink; 1458 } 1459 1460 atomic_set(&glob->bo_count, 0); 1461 1462 refcount_init(&glob->kobj_ref, 1); 1463 return (0); 1464 1465out_no_shrink: 1466 vm_page_free(glob->dummy_read_page); 1467out_no_drp: 1468 free(glob, M_DRM_GLOBAL); 1469 return ret; 1470} 1471 1472int ttm_bo_device_release(struct ttm_bo_device *bdev) 1473{ 1474 int ret = 0; 1475 unsigned i = TTM_NUM_MEM_TYPES; 1476 struct ttm_mem_type_manager *man; 1477 struct ttm_bo_global *glob = bdev->glob; 1478 1479 while (i--) { 1480 man = &bdev->man[i]; 1481 if (man->has_type) { 1482 man->use_type = false; 1483 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1484 ret = -EBUSY; 1485 printf("[TTM] DRM memory manager type %d is not clean\n", 1486 i); 1487 } 1488 man->has_type = false; 1489 } 1490 } 1491 1492 sx_xlock(&glob->device_list_mutex); 1493 list_del(&bdev->device_list); 1494 sx_xunlock(&glob->device_list_mutex); 1495 1496 if (taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, NULL)) 1497 taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); 1498 1499 while (ttm_bo_delayed_delete(bdev, true)) 1500 ; 1501 1502 mtx_lock(&glob->lru_lock); 1503 if (list_empty(&bdev->ddestroy)) 1504 TTM_DEBUG("Delayed destroy list was clean\n"); 1505 1506 if (list_empty(&bdev->man[0].lru)) 1507 TTM_DEBUG("Swap list was clean\n"); 1508 mtx_unlock(&glob->lru_lock); 1509 1510 MPASS(drm_mm_clean(&bdev->addr_space_mm)); 1511 rw_wlock(&bdev->vm_lock); 1512 drm_mm_takedown(&bdev->addr_space_mm); 1513 rw_wunlock(&bdev->vm_lock); 1514 1515 return ret; 1516} 1517 1518int ttm_bo_device_init(struct ttm_bo_device *bdev, 1519 struct ttm_bo_global *glob, 1520 struct ttm_bo_driver *driver, 1521 uint64_t file_page_offset, 1522 bool need_dma32) 1523{ 1524 int ret = -EINVAL; 1525 1526 rw_init(&bdev->vm_lock, "ttmvml"); 1527 bdev->driver = driver; 1528 1529 memset(bdev->man, 0, sizeof(bdev->man)); 1530 1531 /* 1532 * Initialize the system memory buffer type. 1533 * Other types need to be driver / IOCTL initialized. 1534 */ 1535 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1536 if (unlikely(ret != 0)) 1537 goto out_no_sys; 1538 1539 RB_INIT(&bdev->addr_space_rb); 1540 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1541 if (unlikely(ret != 0)) 1542 goto out_no_addr_mm; 1543 1544 TIMEOUT_TASK_INIT(taskqueue_thread, &bdev->wq, 0, 1545 ttm_bo_delayed_workqueue, bdev); 1546 INIT_LIST_HEAD(&bdev->ddestroy); 1547 bdev->dev_mapping = NULL; 1548 bdev->glob = glob; 1549 bdev->need_dma32 = need_dma32; 1550 bdev->val_seq = 0; 1551 mtx_init(&bdev->fence_lock, "ttmfence", NULL, MTX_DEF); 1552 sx_xlock(&glob->device_list_mutex); 1553 list_add_tail(&bdev->device_list, &glob->device_list); 1554 sx_xunlock(&glob->device_list_mutex); 1555 1556 return 0; 1557out_no_addr_mm: 1558 ttm_bo_clean_mm(bdev, 0); 1559out_no_sys: 1560 return ret; 1561} 1562 1563/* 1564 * buffer object vm functions. 1565 */ 1566 1567bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1568{ 1569 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1570 1571 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1572 if (mem->mem_type == TTM_PL_SYSTEM) 1573 return false; 1574 1575 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1576 return false; 1577 1578 if (mem->placement & TTM_PL_FLAG_CACHED) 1579 return false; 1580 } 1581 return true; 1582} 1583 1584void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1585{ 1586 struct ttm_bo_device *bdev = bo->bdev; 1587 /* off_t offset = (off_t)bo->addr_space_offset;XXXKIB */ 1588 /* off_t holelen = ((off_t)bo->mem.num_pages) << PAGE_SHIFT;XXXKIB */ 1589 1590 if (!bdev->dev_mapping) 1591 return; 1592 /* unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); XXXKIB */ 1593 ttm_mem_io_free_vm(bo); 1594} 1595 1596void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1597{ 1598 struct ttm_bo_device *bdev = bo->bdev; 1599 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1600 1601 ttm_mem_io_lock(man, false); 1602 ttm_bo_unmap_virtual_locked(bo); 1603 ttm_mem_io_unlock(man); 1604} 1605 1606static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1607{ 1608 struct ttm_bo_device *bdev = bo->bdev; 1609 1610 /* The caller acquired bdev->vm_lock. */ 1611 RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo); 1612} 1613 1614/** 1615 * ttm_bo_setup_vm: 1616 * 1617 * @bo: the buffer to allocate address space for 1618 * 1619 * Allocate address space in the drm device so that applications 1620 * can mmap the buffer and access the contents. This only 1621 * applies to ttm_bo_type_device objects as others are not 1622 * placed in the drm device address space. 1623 */ 1624 1625static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1626{ 1627 struct ttm_bo_device *bdev = bo->bdev; 1628 int ret; 1629 1630retry_pre_get: 1631 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1632 if (unlikely(ret != 0)) 1633 return ret; 1634 1635 rw_wlock(&bdev->vm_lock); 1636 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1637 bo->mem.num_pages, 0, 0); 1638 1639 if (unlikely(bo->vm_node == NULL)) { 1640 ret = -ENOMEM; 1641 goto out_unlock; 1642 } 1643 1644 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1645 bo->mem.num_pages, 0); 1646 1647 if (unlikely(bo->vm_node == NULL)) { 1648 rw_wunlock(&bdev->vm_lock); 1649 goto retry_pre_get; 1650 } 1651 1652 ttm_bo_vm_insert_rb(bo); 1653 rw_wunlock(&bdev->vm_lock); 1654 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1655 1656 return 0; 1657out_unlock: 1658 rw_wunlock(&bdev->vm_lock); 1659 return ret; 1660} 1661 1662int ttm_bo_wait(struct ttm_buffer_object *bo, 1663 bool lazy, bool interruptible, bool no_wait) 1664{ 1665 struct ttm_bo_driver *driver = bo->bdev->driver; 1666 struct ttm_bo_device *bdev = bo->bdev; 1667 void *sync_obj; 1668 int ret = 0; 1669 1670 if (likely(bo->sync_obj == NULL)) 1671 return 0; 1672 1673 while (bo->sync_obj) { 1674 1675 if (driver->sync_obj_signaled(bo->sync_obj)) { 1676 void *tmp_obj = bo->sync_obj; 1677 bo->sync_obj = NULL; 1678 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1679 mtx_unlock(&bdev->fence_lock); 1680 driver->sync_obj_unref(&tmp_obj); 1681 mtx_lock(&bdev->fence_lock); 1682 continue; 1683 } 1684 1685 if (no_wait) 1686 return -EBUSY; 1687 1688 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1689 mtx_unlock(&bdev->fence_lock); 1690 ret = driver->sync_obj_wait(sync_obj, 1691 lazy, interruptible); 1692 if (unlikely(ret != 0)) { 1693 driver->sync_obj_unref(&sync_obj); 1694 mtx_lock(&bdev->fence_lock); 1695 return ret; 1696 } 1697 mtx_lock(&bdev->fence_lock); 1698 if (likely(bo->sync_obj == sync_obj)) { 1699 void *tmp_obj = bo->sync_obj; 1700 bo->sync_obj = NULL; 1701 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1702 &bo->priv_flags); 1703 mtx_unlock(&bdev->fence_lock); 1704 driver->sync_obj_unref(&sync_obj); 1705 driver->sync_obj_unref(&tmp_obj); 1706 mtx_lock(&bdev->fence_lock); 1707 } else { 1708 mtx_unlock(&bdev->fence_lock); 1709 driver->sync_obj_unref(&sync_obj); 1710 mtx_lock(&bdev->fence_lock); 1711 } 1712 } 1713 return 0; 1714} 1715 1716int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1717{ 1718 struct ttm_bo_device *bdev = bo->bdev; 1719 int ret = 0; 1720 1721 /* 1722 * Using ttm_bo_reserve makes sure the lru lists are updated. 1723 */ 1724 1725 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1726 if (unlikely(ret != 0)) 1727 return ret; 1728 mtx_lock(&bdev->fence_lock); 1729 ret = ttm_bo_wait(bo, false, true, no_wait); 1730 mtx_unlock(&bdev->fence_lock); 1731 if (likely(ret == 0)) 1732 atomic_inc(&bo->cpu_writers); 1733 ttm_bo_unreserve(bo); 1734 return ret; 1735} 1736 1737void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1738{ 1739 atomic_dec(&bo->cpu_writers); 1740} 1741 1742/** 1743 * A buffer object shrink method that tries to swap out the first 1744 * buffer object on the bo_global::swap_lru list. 1745 */ 1746 1747static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1748{ 1749 struct ttm_bo_global *glob = 1750 container_of(shrink, struct ttm_bo_global, shrink); 1751 struct ttm_buffer_object *bo; 1752 int ret = -EBUSY; 1753 int put_count; 1754 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1755 1756 mtx_lock(&glob->lru_lock); 1757 list_for_each_entry(bo, &glob->swap_lru, swap) { 1758 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 1759 if (!ret) 1760 break; 1761 } 1762 1763 if (ret) { 1764 mtx_unlock(&glob->lru_lock); 1765 return ret; 1766 } 1767 1768 refcount_acquire(&bo->list_kref); 1769 1770 if (!list_empty(&bo->ddestroy)) { 1771 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1772 if (refcount_release(&bo->list_kref)) 1773 ttm_bo_release_list(bo); 1774 return ret; 1775 } 1776 1777 put_count = ttm_bo_del_from_lru(bo); 1778 mtx_unlock(&glob->lru_lock); 1779 1780 ttm_bo_list_ref_sub(bo, put_count, true); 1781 1782 /** 1783 * Wait for GPU, then move to system cached. 1784 */ 1785 1786 mtx_lock(&bo->bdev->fence_lock); 1787 ret = ttm_bo_wait(bo, false, false, false); 1788 mtx_unlock(&bo->bdev->fence_lock); 1789 1790 if (unlikely(ret != 0)) 1791 goto out; 1792 1793 if ((bo->mem.placement & swap_placement) != swap_placement) { 1794 struct ttm_mem_reg evict_mem; 1795 1796 evict_mem = bo->mem; 1797 evict_mem.mm_node = NULL; 1798 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1799 evict_mem.mem_type = TTM_PL_SYSTEM; 1800 1801 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1802 false, false); 1803 if (unlikely(ret != 0)) 1804 goto out; 1805 } 1806 1807 ttm_bo_unmap_virtual(bo); 1808 1809 /** 1810 * Swap out. Buffer will be swapped in again as soon as 1811 * anyone tries to access a ttm page. 1812 */ 1813 1814 if (bo->bdev->driver->swap_notify) 1815 bo->bdev->driver->swap_notify(bo); 1816 1817 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1818out: 1819 1820 /** 1821 * 1822 * Unreserve without putting on LRU to avoid swapping out an 1823 * already swapped buffer. 1824 */ 1825 1826 atomic_set(&bo->reserved, 0); 1827 wakeup(bo); 1828 if (refcount_release(&bo->list_kref)) 1829 ttm_bo_release_list(bo); 1830 return ret; 1831} 1832 1833void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1834{ 1835 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1836 ; 1837}
| 346void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) 347{ 348 ttm_bo_add_to_lru(bo); 349 atomic_set(&bo->reserved, 0); 350 wakeup(bo); 351} 352 353void ttm_bo_unreserve(struct ttm_buffer_object *bo) 354{ 355 struct ttm_bo_global *glob = bo->glob; 356 357 mtx_lock(&glob->lru_lock); 358 ttm_bo_unreserve_locked(bo); 359 mtx_unlock(&glob->lru_lock); 360} 361 362/* 363 * Call bo->mutex locked. 364 */ 365static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 366{ 367 struct ttm_bo_device *bdev = bo->bdev; 368 struct ttm_bo_global *glob = bo->glob; 369 int ret = 0; 370 uint32_t page_flags = 0; 371 372 TTM_ASSERT_LOCKED(&bo->mutex); 373 bo->ttm = NULL; 374 375 if (bdev->need_dma32) 376 page_flags |= TTM_PAGE_FLAG_DMA32; 377 378 switch (bo->type) { 379 case ttm_bo_type_device: 380 if (zero_alloc) 381 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 382 case ttm_bo_type_kernel: 383 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 384 page_flags, glob->dummy_read_page); 385 if (unlikely(bo->ttm == NULL)) 386 ret = -ENOMEM; 387 break; 388 case ttm_bo_type_sg: 389 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 390 page_flags | TTM_PAGE_FLAG_SG, 391 glob->dummy_read_page); 392 if (unlikely(bo->ttm == NULL)) { 393 ret = -ENOMEM; 394 break; 395 } 396 bo->ttm->sg = bo->sg; 397 break; 398 default: 399 printf("[TTM] Illegal buffer object type\n"); 400 ret = -EINVAL; 401 break; 402 } 403 404 return ret; 405} 406 407static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 408 struct ttm_mem_reg *mem, 409 bool evict, bool interruptible, 410 bool no_wait_gpu) 411{ 412 struct ttm_bo_device *bdev = bo->bdev; 413 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 414 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 415 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 416 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 417 int ret = 0; 418 419 if (old_is_pci || new_is_pci || 420 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 421 ret = ttm_mem_io_lock(old_man, true); 422 if (unlikely(ret != 0)) 423 goto out_err; 424 ttm_bo_unmap_virtual_locked(bo); 425 ttm_mem_io_unlock(old_man); 426 } 427 428 /* 429 * Create and bind a ttm if required. 430 */ 431 432 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 433 if (bo->ttm == NULL) { 434 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 435 ret = ttm_bo_add_ttm(bo, zero); 436 if (ret) 437 goto out_err; 438 } 439 440 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 441 if (ret) 442 goto out_err; 443 444 if (mem->mem_type != TTM_PL_SYSTEM) { 445 ret = ttm_tt_bind(bo->ttm, mem); 446 if (ret) 447 goto out_err; 448 } 449 450 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 451 if (bdev->driver->move_notify) 452 bdev->driver->move_notify(bo, mem); 453 bo->mem = *mem; 454 mem->mm_node = NULL; 455 goto moved; 456 } 457 } 458 459 if (bdev->driver->move_notify) 460 bdev->driver->move_notify(bo, mem); 461 462 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 463 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 464 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 465 else if (bdev->driver->move) 466 ret = bdev->driver->move(bo, evict, interruptible, 467 no_wait_gpu, mem); 468 else 469 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 470 471 if (ret) { 472 if (bdev->driver->move_notify) { 473 struct ttm_mem_reg tmp_mem = *mem; 474 *mem = bo->mem; 475 bo->mem = tmp_mem; 476 bdev->driver->move_notify(bo, mem); 477 bo->mem = *mem; 478 } 479 480 goto out_err; 481 } 482 483moved: 484 if (bo->evicted) { 485 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 486 if (ret) 487 printf("[TTM] Can not flush read caches\n"); 488 bo->evicted = false; 489 } 490 491 if (bo->mem.mm_node) { 492 bo->offset = (bo->mem.start << PAGE_SHIFT) + 493 bdev->man[bo->mem.mem_type].gpu_offset; 494 bo->cur_placement = bo->mem.placement; 495 } else 496 bo->offset = 0; 497 498 return 0; 499 500out_err: 501 new_man = &bdev->man[bo->mem.mem_type]; 502 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 503 ttm_tt_unbind(bo->ttm); 504 ttm_tt_destroy(bo->ttm); 505 bo->ttm = NULL; 506 } 507 508 return ret; 509} 510 511/** 512 * Call bo::reserved. 513 * Will release GPU memory type usage on destruction. 514 * This is the place to put in driver specific hooks to release 515 * driver private resources. 516 * Will release the bo::reserved lock. 517 */ 518 519static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 520{ 521 if (bo->bdev->driver->move_notify) 522 bo->bdev->driver->move_notify(bo, NULL); 523 524 if (bo->ttm) { 525 ttm_tt_unbind(bo->ttm); 526 ttm_tt_destroy(bo->ttm); 527 bo->ttm = NULL; 528 } 529 ttm_bo_mem_put(bo, &bo->mem); 530 531 atomic_set(&bo->reserved, 0); 532 wakeup(&bo); 533 534 /* 535 * Since the final reference to this bo may not be dropped by 536 * the current task we have to put a memory barrier here to make 537 * sure the changes done in this function are always visible. 538 * 539 * This function only needs protection against the final kref_put. 540 */ 541 mb(); 542} 543 544static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 545{ 546 struct ttm_bo_device *bdev = bo->bdev; 547 struct ttm_bo_global *glob = bo->glob; 548 struct ttm_bo_driver *driver = bdev->driver; 549 void *sync_obj = NULL; 550 int put_count; 551 int ret; 552 553 mtx_lock(&glob->lru_lock); 554 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 555 556 mtx_lock(&bdev->fence_lock); 557 (void) ttm_bo_wait(bo, false, false, true); 558 if (!ret && !bo->sync_obj) { 559 mtx_unlock(&bdev->fence_lock); 560 put_count = ttm_bo_del_from_lru(bo); 561 562 mtx_unlock(&glob->lru_lock); 563 ttm_bo_cleanup_memtype_use(bo); 564 565 ttm_bo_list_ref_sub(bo, put_count, true); 566 567 return; 568 } 569 if (bo->sync_obj) 570 sync_obj = driver->sync_obj_ref(bo->sync_obj); 571 mtx_unlock(&bdev->fence_lock); 572 573 if (!ret) { 574 atomic_set(&bo->reserved, 0); 575 wakeup(bo); 576 } 577 578 refcount_acquire(&bo->list_kref); 579 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 580 mtx_unlock(&glob->lru_lock); 581 582 if (sync_obj) { 583 driver->sync_obj_flush(sync_obj); 584 driver->sync_obj_unref(&sync_obj); 585 } 586 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 587 ((hz / 100) < 1) ? 1 : hz / 100); 588} 589 590/** 591 * function ttm_bo_cleanup_refs_and_unlock 592 * If bo idle, remove from delayed- and lru lists, and unref. 593 * If not idle, do nothing. 594 * 595 * Must be called with lru_lock and reservation held, this function 596 * will drop both before returning. 597 * 598 * @interruptible Any sleeps should occur interruptibly. 599 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 600 */ 601 602static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 603 bool interruptible, 604 bool no_wait_gpu) 605{ 606 struct ttm_bo_device *bdev = bo->bdev; 607 struct ttm_bo_driver *driver = bdev->driver; 608 struct ttm_bo_global *glob = bo->glob; 609 int put_count; 610 int ret; 611 612 mtx_lock(&bdev->fence_lock); 613 ret = ttm_bo_wait(bo, false, false, true); 614 615 if (ret && !no_wait_gpu) { 616 void *sync_obj; 617 618 /* 619 * Take a reference to the fence and unreserve, 620 * at this point the buffer should be dead, so 621 * no new sync objects can be attached. 622 */ 623 sync_obj = driver->sync_obj_ref(bo->sync_obj); 624 mtx_unlock(&bdev->fence_lock); 625 626 atomic_set(&bo->reserved, 0); 627 wakeup(bo); 628 mtx_unlock(&glob->lru_lock); 629 630 ret = driver->sync_obj_wait(sync_obj, false, interruptible); 631 driver->sync_obj_unref(&sync_obj); 632 if (ret) 633 return ret; 634 635 /* 636 * remove sync_obj with ttm_bo_wait, the wait should be 637 * finished, and no new wait object should have been added. 638 */ 639 mtx_lock(&bdev->fence_lock); 640 ret = ttm_bo_wait(bo, false, false, true); 641 mtx_unlock(&bdev->fence_lock); 642 if (ret) 643 return ret; 644 645 mtx_lock(&glob->lru_lock); 646 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 647 648 /* 649 * We raced, and lost, someone else holds the reservation now, 650 * and is probably busy in ttm_bo_cleanup_memtype_use. 651 * 652 * Even if it's not the case, because we finished waiting any 653 * delayed destruction would succeed, so just return success 654 * here. 655 */ 656 if (ret) { 657 mtx_unlock(&glob->lru_lock); 658 return 0; 659 } 660 } else 661 mtx_unlock(&bdev->fence_lock); 662 663 if (ret || unlikely(list_empty(&bo->ddestroy))) { 664 atomic_set(&bo->reserved, 0); 665 wakeup(bo); 666 mtx_unlock(&glob->lru_lock); 667 return ret; 668 } 669 670 put_count = ttm_bo_del_from_lru(bo); 671 list_del_init(&bo->ddestroy); 672 ++put_count; 673 674 mtx_unlock(&glob->lru_lock); 675 ttm_bo_cleanup_memtype_use(bo); 676 677 ttm_bo_list_ref_sub(bo, put_count, true); 678 679 return 0; 680} 681 682/** 683 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 684 * encountered buffers. 685 */ 686 687static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 688{ 689 struct ttm_bo_global *glob = bdev->glob; 690 struct ttm_buffer_object *entry = NULL; 691 int ret = 0; 692 693 mtx_lock(&glob->lru_lock); 694 if (list_empty(&bdev->ddestroy)) 695 goto out_unlock; 696 697 entry = list_first_entry(&bdev->ddestroy, 698 struct ttm_buffer_object, ddestroy); 699 refcount_acquire(&entry->list_kref); 700 701 for (;;) { 702 struct ttm_buffer_object *nentry = NULL; 703 704 if (entry->ddestroy.next != &bdev->ddestroy) { 705 nentry = list_first_entry(&entry->ddestroy, 706 struct ttm_buffer_object, ddestroy); 707 refcount_acquire(&nentry->list_kref); 708 } 709 710 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); 711 if (remove_all && ret) { 712 ret = ttm_bo_reserve_nolru(entry, false, false, 713 false, 0); 714 } 715 716 if (!ret) 717 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 718 !remove_all); 719 else 720 mtx_unlock(&glob->lru_lock); 721 722 if (refcount_release(&entry->list_kref)) 723 ttm_bo_release_list(entry); 724 entry = nentry; 725 726 if (ret || !entry) 727 goto out; 728 729 mtx_lock(&glob->lru_lock); 730 if (list_empty(&entry->ddestroy)) 731 break; 732 } 733 734out_unlock: 735 mtx_unlock(&glob->lru_lock); 736out: 737 if (entry && refcount_release(&entry->list_kref)) 738 ttm_bo_release_list(entry); 739 return ret; 740} 741 742static void ttm_bo_delayed_workqueue(void *arg, int pending __unused) 743{ 744 struct ttm_bo_device *bdev = arg; 745 746 if (ttm_bo_delayed_delete(bdev, false)) { 747 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 748 ((hz / 100) < 1) ? 1 : hz / 100); 749 } 750} 751 752static void ttm_bo_release(struct ttm_buffer_object *bo) 753{ 754 struct ttm_bo_device *bdev = bo->bdev; 755 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 756 757 rw_wlock(&bdev->vm_lock); 758 if (likely(bo->vm_node != NULL)) { 759 RB_REMOVE(ttm_bo_device_buffer_objects, 760 &bdev->addr_space_rb, bo); 761 drm_mm_put_block(bo->vm_node); 762 bo->vm_node = NULL; 763 } 764 rw_wunlock(&bdev->vm_lock); 765 ttm_mem_io_lock(man, false); 766 ttm_mem_io_free_vm(bo); 767 ttm_mem_io_unlock(man); 768 ttm_bo_cleanup_refs_or_queue(bo); 769 if (refcount_release(&bo->list_kref)) 770 ttm_bo_release_list(bo); 771} 772 773void ttm_bo_unref(struct ttm_buffer_object **p_bo) 774{ 775 struct ttm_buffer_object *bo = *p_bo; 776 777 *p_bo = NULL; 778 if (refcount_release(&bo->kref)) 779 ttm_bo_release(bo); 780} 781 782int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 783{ 784 int pending; 785 786 taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending); 787 if (pending) 788 taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); 789 return (pending); 790} 791 792void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 793{ 794 if (resched) { 795 taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq, 796 ((hz / 100) < 1) ? 1 : hz / 100); 797 } 798} 799 800static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 801 bool no_wait_gpu) 802{ 803 struct ttm_bo_device *bdev = bo->bdev; 804 struct ttm_mem_reg evict_mem; 805 struct ttm_placement placement; 806 int ret = 0; 807 808 mtx_lock(&bdev->fence_lock); 809 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 810 mtx_unlock(&bdev->fence_lock); 811 812 if (unlikely(ret != 0)) { 813 if (ret != -ERESTART) { 814 printf("[TTM] Failed to expire sync object before buffer eviction\n"); 815 } 816 goto out; 817 } 818 819 MPASS(ttm_bo_is_reserved(bo)); 820 821 evict_mem = bo->mem; 822 evict_mem.mm_node = NULL; 823 evict_mem.bus.io_reserved_vm = false; 824 evict_mem.bus.io_reserved_count = 0; 825 826 placement.fpfn = 0; 827 placement.lpfn = 0; 828 placement.num_placement = 0; 829 placement.num_busy_placement = 0; 830 bdev->driver->evict_flags(bo, &placement); 831 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 832 no_wait_gpu); 833 if (ret) { 834 if (ret != -ERESTART) { 835 printf("[TTM] Failed to find memory space for buffer 0x%p eviction\n", 836 bo); 837 ttm_bo_mem_space_debug(bo, &placement); 838 } 839 goto out; 840 } 841 842 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 843 no_wait_gpu); 844 if (ret) { 845 if (ret != -ERESTART) 846 printf("[TTM] Buffer eviction failed\n"); 847 ttm_bo_mem_put(bo, &evict_mem); 848 goto out; 849 } 850 bo->evicted = true; 851out: 852 return ret; 853} 854 855static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 856 uint32_t mem_type, 857 bool interruptible, 858 bool no_wait_gpu) 859{ 860 struct ttm_bo_global *glob = bdev->glob; 861 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 862 struct ttm_buffer_object *bo; 863 int ret = -EBUSY, put_count; 864 865 mtx_lock(&glob->lru_lock); 866 list_for_each_entry(bo, &man->lru, lru) { 867 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 868 if (!ret) 869 break; 870 } 871 872 if (ret) { 873 mtx_unlock(&glob->lru_lock); 874 return ret; 875 } 876 877 refcount_acquire(&bo->list_kref); 878 879 if (!list_empty(&bo->ddestroy)) { 880 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 881 no_wait_gpu); 882 if (refcount_release(&bo->list_kref)) 883 ttm_bo_release_list(bo); 884 return ret; 885 } 886 887 put_count = ttm_bo_del_from_lru(bo); 888 mtx_unlock(&glob->lru_lock); 889 890 MPASS(ret == 0); 891 892 ttm_bo_list_ref_sub(bo, put_count, true); 893 894 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 895 ttm_bo_unreserve(bo); 896 897 if (refcount_release(&bo->list_kref)) 898 ttm_bo_release_list(bo); 899 return ret; 900} 901 902void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 903{ 904 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 905 906 if (mem->mm_node) 907 (*man->func->put_node)(man, mem); 908} 909 910/** 911 * Repeatedly evict memory from the LRU for @mem_type until we create enough 912 * space, or we've evicted everything and there isn't enough space. 913 */ 914static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 915 uint32_t mem_type, 916 struct ttm_placement *placement, 917 struct ttm_mem_reg *mem, 918 bool interruptible, 919 bool no_wait_gpu) 920{ 921 struct ttm_bo_device *bdev = bo->bdev; 922 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 923 int ret; 924 925 do { 926 ret = (*man->func->get_node)(man, bo, placement, mem); 927 if (unlikely(ret != 0)) 928 return ret; 929 if (mem->mm_node) 930 break; 931 ret = ttm_mem_evict_first(bdev, mem_type, 932 interruptible, no_wait_gpu); 933 if (unlikely(ret != 0)) 934 return ret; 935 } while (1); 936 if (mem->mm_node == NULL) 937 return -ENOMEM; 938 mem->mem_type = mem_type; 939 return 0; 940} 941 942static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 943 uint32_t cur_placement, 944 uint32_t proposed_placement) 945{ 946 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 947 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 948 949 /** 950 * Keep current caching if possible. 951 */ 952 953 if ((cur_placement & caching) != 0) 954 result |= (cur_placement & caching); 955 else if ((man->default_caching & caching) != 0) 956 result |= man->default_caching; 957 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 958 result |= TTM_PL_FLAG_CACHED; 959 else if ((TTM_PL_FLAG_WC & caching) != 0) 960 result |= TTM_PL_FLAG_WC; 961 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 962 result |= TTM_PL_FLAG_UNCACHED; 963 964 return result; 965} 966 967static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 968 uint32_t mem_type, 969 uint32_t proposed_placement, 970 uint32_t *masked_placement) 971{ 972 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 973 974 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 975 return false; 976 977 if ((proposed_placement & man->available_caching) == 0) 978 return false; 979 980 cur_flags |= (proposed_placement & man->available_caching); 981 982 *masked_placement = cur_flags; 983 return true; 984} 985 986/** 987 * Creates space for memory region @mem according to its type. 988 * 989 * This function first searches for free space in compatible memory types in 990 * the priority order defined by the driver. If free space isn't found, then 991 * ttm_bo_mem_force_space is attempted in priority order to evict and find 992 * space. 993 */ 994int ttm_bo_mem_space(struct ttm_buffer_object *bo, 995 struct ttm_placement *placement, 996 struct ttm_mem_reg *mem, 997 bool interruptible, 998 bool no_wait_gpu) 999{ 1000 struct ttm_bo_device *bdev = bo->bdev; 1001 struct ttm_mem_type_manager *man; 1002 uint32_t mem_type = TTM_PL_SYSTEM; 1003 uint32_t cur_flags = 0; 1004 bool type_found = false; 1005 bool type_ok = false; 1006 bool has_erestartsys = false; 1007 int i, ret; 1008 1009 mem->mm_node = NULL; 1010 for (i = 0; i < placement->num_placement; ++i) { 1011 ret = ttm_mem_type_from_flags(placement->placement[i], 1012 &mem_type); 1013 if (ret) 1014 return ret; 1015 man = &bdev->man[mem_type]; 1016 1017 type_ok = ttm_bo_mt_compatible(man, 1018 mem_type, 1019 placement->placement[i], 1020 &cur_flags); 1021 1022 if (!type_ok) 1023 continue; 1024 1025 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1026 cur_flags); 1027 /* 1028 * Use the access and other non-mapping-related flag bits from 1029 * the memory placement flags to the current flags 1030 */ 1031 ttm_flag_masked(&cur_flags, placement->placement[i], 1032 ~TTM_PL_MASK_MEMTYPE); 1033 1034 if (mem_type == TTM_PL_SYSTEM) 1035 break; 1036 1037 if (man->has_type && man->use_type) { 1038 type_found = true; 1039 ret = (*man->func->get_node)(man, bo, placement, mem); 1040 if (unlikely(ret)) 1041 return ret; 1042 } 1043 if (mem->mm_node) 1044 break; 1045 } 1046 1047 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 1048 mem->mem_type = mem_type; 1049 mem->placement = cur_flags; 1050 return 0; 1051 } 1052 1053 if (!type_found) 1054 return -EINVAL; 1055 1056 for (i = 0; i < placement->num_busy_placement; ++i) { 1057 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 1058 &mem_type); 1059 if (ret) 1060 return ret; 1061 man = &bdev->man[mem_type]; 1062 if (!man->has_type) 1063 continue; 1064 if (!ttm_bo_mt_compatible(man, 1065 mem_type, 1066 placement->busy_placement[i], 1067 &cur_flags)) 1068 continue; 1069 1070 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1071 cur_flags); 1072 /* 1073 * Use the access and other non-mapping-related flag bits from 1074 * the memory placement flags to the current flags 1075 */ 1076 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 1077 ~TTM_PL_MASK_MEMTYPE); 1078 1079 1080 if (mem_type == TTM_PL_SYSTEM) { 1081 mem->mem_type = mem_type; 1082 mem->placement = cur_flags; 1083 mem->mm_node = NULL; 1084 return 0; 1085 } 1086 1087 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 1088 interruptible, no_wait_gpu); 1089 if (ret == 0 && mem->mm_node) { 1090 mem->placement = cur_flags; 1091 return 0; 1092 } 1093 if (ret == -ERESTART) 1094 has_erestartsys = true; 1095 } 1096 ret = (has_erestartsys) ? -ERESTART : -ENOMEM; 1097 return ret; 1098} 1099 1100static 1101int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1102 struct ttm_placement *placement, 1103 bool interruptible, 1104 bool no_wait_gpu) 1105{ 1106 int ret = 0; 1107 struct ttm_mem_reg mem; 1108 struct ttm_bo_device *bdev = bo->bdev; 1109 1110 MPASS(ttm_bo_is_reserved(bo)); 1111 1112 /* 1113 * FIXME: It's possible to pipeline buffer moves. 1114 * Have the driver move function wait for idle when necessary, 1115 * instead of doing it here. 1116 */ 1117 mtx_lock(&bdev->fence_lock); 1118 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1119 mtx_unlock(&bdev->fence_lock); 1120 if (ret) 1121 return ret; 1122 mem.num_pages = bo->num_pages; 1123 mem.size = mem.num_pages << PAGE_SHIFT; 1124 mem.page_alignment = bo->mem.page_alignment; 1125 mem.bus.io_reserved_vm = false; 1126 mem.bus.io_reserved_count = 0; 1127 /* 1128 * Determine where to move the buffer. 1129 */ 1130 ret = ttm_bo_mem_space(bo, placement, &mem, 1131 interruptible, no_wait_gpu); 1132 if (ret) 1133 goto out_unlock; 1134 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1135 interruptible, no_wait_gpu); 1136out_unlock: 1137 if (ret && mem.mm_node) 1138 ttm_bo_mem_put(bo, &mem); 1139 return ret; 1140} 1141 1142static int ttm_bo_mem_compat(struct ttm_placement *placement, 1143 struct ttm_mem_reg *mem) 1144{ 1145 int i; 1146 1147 if (mem->mm_node && placement->lpfn != 0 && 1148 (mem->start < placement->fpfn || 1149 mem->start + mem->num_pages > placement->lpfn)) 1150 return -1; 1151 1152 for (i = 0; i < placement->num_placement; i++) { 1153 if ((placement->placement[i] & mem->placement & 1154 TTM_PL_MASK_CACHING) && 1155 (placement->placement[i] & mem->placement & 1156 TTM_PL_MASK_MEM)) 1157 return i; 1158 } 1159 return -1; 1160} 1161 1162int ttm_bo_validate(struct ttm_buffer_object *bo, 1163 struct ttm_placement *placement, 1164 bool interruptible, 1165 bool no_wait_gpu) 1166{ 1167 int ret; 1168 1169 MPASS(ttm_bo_is_reserved(bo)); 1170 /* Check that range is valid */ 1171 if (placement->lpfn || placement->fpfn) 1172 if (placement->fpfn > placement->lpfn || 1173 (placement->lpfn - placement->fpfn) < bo->num_pages) 1174 return -EINVAL; 1175 /* 1176 * Check whether we need to move buffer. 1177 */ 1178 ret = ttm_bo_mem_compat(placement, &bo->mem); 1179 if (ret < 0) { 1180 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1181 no_wait_gpu); 1182 if (ret) 1183 return ret; 1184 } else { 1185 /* 1186 * Use the access and other non-mapping-related flag bits from 1187 * the compatible memory placement flags to the active flags 1188 */ 1189 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1190 ~TTM_PL_MASK_MEMTYPE); 1191 } 1192 /* 1193 * We might need to add a TTM. 1194 */ 1195 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1196 ret = ttm_bo_add_ttm(bo, true); 1197 if (ret) 1198 return ret; 1199 } 1200 return 0; 1201} 1202 1203int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1204 struct ttm_placement *placement) 1205{ 1206 MPASS(!((placement->fpfn || placement->lpfn) && 1207 (bo->mem.num_pages > (placement->lpfn - placement->fpfn)))); 1208 1209 return 0; 1210} 1211 1212int ttm_bo_init(struct ttm_bo_device *bdev, 1213 struct ttm_buffer_object *bo, 1214 unsigned long size, 1215 enum ttm_bo_type type, 1216 struct ttm_placement *placement, 1217 uint32_t page_alignment, 1218 bool interruptible, 1219 struct vm_object *persistent_swap_storage, 1220 size_t acc_size, 1221 struct sg_table *sg, 1222 void (*destroy) (struct ttm_buffer_object *)) 1223{ 1224 int ret = 0; 1225 unsigned long num_pages; 1226 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1227 1228 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1229 if (ret) { 1230 printf("[TTM] Out of kernel memory\n"); 1231 if (destroy) 1232 (*destroy)(bo); 1233 else 1234 free(bo, M_TTM_BO); 1235 return -ENOMEM; 1236 } 1237 1238 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1239 if (num_pages == 0) { 1240 printf("[TTM] Illegal buffer object size\n"); 1241 if (destroy) 1242 (*destroy)(bo); 1243 else 1244 free(bo, M_TTM_BO); 1245 ttm_mem_global_free(mem_glob, acc_size); 1246 return -EINVAL; 1247 } 1248 bo->destroy = destroy; 1249 1250 refcount_init(&bo->kref, 1); 1251 refcount_init(&bo->list_kref, 1); 1252 atomic_set(&bo->cpu_writers, 0); 1253 atomic_set(&bo->reserved, 1); 1254 INIT_LIST_HEAD(&bo->lru); 1255 INIT_LIST_HEAD(&bo->ddestroy); 1256 INIT_LIST_HEAD(&bo->swap); 1257 INIT_LIST_HEAD(&bo->io_reserve_lru); 1258 bo->bdev = bdev; 1259 bo->glob = bdev->glob; 1260 bo->type = type; 1261 bo->num_pages = num_pages; 1262 bo->mem.size = num_pages << PAGE_SHIFT; 1263 bo->mem.mem_type = TTM_PL_SYSTEM; 1264 bo->mem.num_pages = bo->num_pages; 1265 bo->mem.mm_node = NULL; 1266 bo->mem.page_alignment = page_alignment; 1267 bo->mem.bus.io_reserved_vm = false; 1268 bo->mem.bus.io_reserved_count = 0; 1269 bo->priv_flags = 0; 1270 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1271 bo->seq_valid = false; 1272 bo->persistent_swap_storage = persistent_swap_storage; 1273 bo->acc_size = acc_size; 1274 bo->sg = sg; 1275 atomic_inc(&bo->glob->bo_count); 1276 1277 ret = ttm_bo_check_placement(bo, placement); 1278 if (unlikely(ret != 0)) 1279 goto out_err; 1280 1281 /* 1282 * For ttm_bo_type_device buffers, allocate 1283 * address space from the device. 1284 */ 1285 if (bo->type == ttm_bo_type_device || 1286 bo->type == ttm_bo_type_sg) { 1287 ret = ttm_bo_setup_vm(bo); 1288 if (ret) 1289 goto out_err; 1290 } 1291 1292 ret = ttm_bo_validate(bo, placement, interruptible, false); 1293 if (ret) 1294 goto out_err; 1295 1296 ttm_bo_unreserve(bo); 1297 return 0; 1298 1299out_err: 1300 ttm_bo_unreserve(bo); 1301 ttm_bo_unref(&bo); 1302 1303 return ret; 1304} 1305 1306size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1307 unsigned long bo_size, 1308 unsigned struct_size) 1309{ 1310 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1311 size_t size = 0; 1312 1313 size += ttm_round_pot(struct_size); 1314 size += PAGE_ALIGN(npages * sizeof(void *)); 1315 size += ttm_round_pot(sizeof(struct ttm_tt)); 1316 return size; 1317} 1318 1319size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1320 unsigned long bo_size, 1321 unsigned struct_size) 1322{ 1323 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1324 size_t size = 0; 1325 1326 size += ttm_round_pot(struct_size); 1327 size += PAGE_ALIGN(npages * sizeof(void *)); 1328 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1329 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1330 return size; 1331} 1332 1333int ttm_bo_create(struct ttm_bo_device *bdev, 1334 unsigned long size, 1335 enum ttm_bo_type type, 1336 struct ttm_placement *placement, 1337 uint32_t page_alignment, 1338 bool interruptible, 1339 struct vm_object *persistent_swap_storage, 1340 struct ttm_buffer_object **p_bo) 1341{ 1342 struct ttm_buffer_object *bo; 1343 size_t acc_size; 1344 int ret; 1345 1346 bo = malloc(sizeof(*bo), M_TTM_BO, M_WAITOK | M_ZERO); 1347 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1348 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1349 interruptible, persistent_swap_storage, acc_size, 1350 NULL, NULL); 1351 if (likely(ret == 0)) 1352 *p_bo = bo; 1353 1354 return ret; 1355} 1356 1357static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1358 unsigned mem_type, bool allow_errors) 1359{ 1360 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1361 struct ttm_bo_global *glob = bdev->glob; 1362 int ret; 1363 1364 /* 1365 * Can't use standard list traversal since we're unlocking. 1366 */ 1367 1368 mtx_lock(&glob->lru_lock); 1369 while (!list_empty(&man->lru)) { 1370 mtx_unlock(&glob->lru_lock); 1371 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1372 if (ret) { 1373 if (allow_errors) { 1374 return ret; 1375 } else { 1376 printf("[TTM] Cleanup eviction failed\n"); 1377 } 1378 } 1379 mtx_lock(&glob->lru_lock); 1380 } 1381 mtx_unlock(&glob->lru_lock); 1382 return 0; 1383} 1384 1385int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1386{ 1387 struct ttm_mem_type_manager *man; 1388 int ret = -EINVAL; 1389 1390 if (mem_type >= TTM_NUM_MEM_TYPES) { 1391 printf("[TTM] Illegal memory type %d\n", mem_type); 1392 return ret; 1393 } 1394 man = &bdev->man[mem_type]; 1395 1396 if (!man->has_type) { 1397 printf("[TTM] Trying to take down uninitialized memory manager type %u\n", 1398 mem_type); 1399 return ret; 1400 } 1401 1402 man->use_type = false; 1403 man->has_type = false; 1404 1405 ret = 0; 1406 if (mem_type > 0) { 1407 ttm_bo_force_list_clean(bdev, mem_type, false); 1408 1409 ret = (*man->func->takedown)(man); 1410 } 1411 1412 return ret; 1413} 1414 1415int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1416{ 1417 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1418 1419 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1420 printf("[TTM] Illegal memory manager memory type %u\n", mem_type); 1421 return -EINVAL; 1422 } 1423 1424 if (!man->has_type) { 1425 printf("[TTM] Memory type %u has not been initialized\n", mem_type); 1426 return 0; 1427 } 1428 1429 return ttm_bo_force_list_clean(bdev, mem_type, true); 1430} 1431 1432int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1433 unsigned long p_size) 1434{ 1435 int ret = -EINVAL; 1436 struct ttm_mem_type_manager *man; 1437 1438 MPASS(type < TTM_NUM_MEM_TYPES); 1439 man = &bdev->man[type]; 1440 MPASS(!man->has_type); 1441 man->io_reserve_fastpath = true; 1442 man->use_io_reserve_lru = false; 1443 sx_init(&man->io_reserve_mutex, "ttmman"); 1444 INIT_LIST_HEAD(&man->io_reserve_lru); 1445 1446 ret = bdev->driver->init_mem_type(bdev, type, man); 1447 if (ret) 1448 return ret; 1449 man->bdev = bdev; 1450 1451 ret = 0; 1452 if (type != TTM_PL_SYSTEM) { 1453 ret = (*man->func->init)(man, p_size); 1454 if (ret) 1455 return ret; 1456 } 1457 man->has_type = true; 1458 man->use_type = true; 1459 man->size = p_size; 1460 1461 INIT_LIST_HEAD(&man->lru); 1462 1463 return 0; 1464} 1465 1466static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob) 1467{ 1468 1469 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1470 vm_page_free(glob->dummy_read_page); 1471} 1472 1473void ttm_bo_global_release(struct drm_global_reference *ref) 1474{ 1475 struct ttm_bo_global *glob = ref->object; 1476 1477 if (refcount_release(&glob->kobj_ref)) 1478 ttm_bo_global_kobj_release(glob); 1479} 1480 1481int ttm_bo_global_init(struct drm_global_reference *ref) 1482{ 1483 struct ttm_bo_global_ref *bo_ref = 1484 container_of(ref, struct ttm_bo_global_ref, ref); 1485 struct ttm_bo_global *glob = ref->object; 1486 int ret; 1487 1488 sx_init(&glob->device_list_mutex, "ttmdlm"); 1489 mtx_init(&glob->lru_lock, "ttmlru", NULL, MTX_DEF); 1490 glob->mem_glob = bo_ref->mem_glob; 1491 glob->dummy_read_page = vm_page_alloc_contig(NULL, 0, 1492 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ, 1493 1, 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE); 1494 1495 if (unlikely(glob->dummy_read_page == NULL)) { 1496 ret = -ENOMEM; 1497 goto out_no_drp; 1498 } 1499 1500 INIT_LIST_HEAD(&glob->swap_lru); 1501 INIT_LIST_HEAD(&glob->device_list); 1502 1503 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1504 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1505 if (unlikely(ret != 0)) { 1506 printf("[TTM] Could not register buffer object swapout\n"); 1507 goto out_no_shrink; 1508 } 1509 1510 atomic_set(&glob->bo_count, 0); 1511 1512 refcount_init(&glob->kobj_ref, 1); 1513 return (0); 1514 1515out_no_shrink: 1516 vm_page_free(glob->dummy_read_page); 1517out_no_drp: 1518 free(glob, M_DRM_GLOBAL); 1519 return ret; 1520} 1521 1522int ttm_bo_device_release(struct ttm_bo_device *bdev) 1523{ 1524 int ret = 0; 1525 unsigned i = TTM_NUM_MEM_TYPES; 1526 struct ttm_mem_type_manager *man; 1527 struct ttm_bo_global *glob = bdev->glob; 1528 1529 while (i--) { 1530 man = &bdev->man[i]; 1531 if (man->has_type) { 1532 man->use_type = false; 1533 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1534 ret = -EBUSY; 1535 printf("[TTM] DRM memory manager type %d is not clean\n", 1536 i); 1537 } 1538 man->has_type = false; 1539 } 1540 } 1541 1542 sx_xlock(&glob->device_list_mutex); 1543 list_del(&bdev->device_list); 1544 sx_xunlock(&glob->device_list_mutex); 1545 1546 if (taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, NULL)) 1547 taskqueue_drain_timeout(taskqueue_thread, &bdev->wq); 1548 1549 while (ttm_bo_delayed_delete(bdev, true)) 1550 ; 1551 1552 mtx_lock(&glob->lru_lock); 1553 if (list_empty(&bdev->ddestroy)) 1554 TTM_DEBUG("Delayed destroy list was clean\n"); 1555 1556 if (list_empty(&bdev->man[0].lru)) 1557 TTM_DEBUG("Swap list was clean\n"); 1558 mtx_unlock(&glob->lru_lock); 1559 1560 MPASS(drm_mm_clean(&bdev->addr_space_mm)); 1561 rw_wlock(&bdev->vm_lock); 1562 drm_mm_takedown(&bdev->addr_space_mm); 1563 rw_wunlock(&bdev->vm_lock); 1564 1565 return ret; 1566} 1567 1568int ttm_bo_device_init(struct ttm_bo_device *bdev, 1569 struct ttm_bo_global *glob, 1570 struct ttm_bo_driver *driver, 1571 uint64_t file_page_offset, 1572 bool need_dma32) 1573{ 1574 int ret = -EINVAL; 1575 1576 rw_init(&bdev->vm_lock, "ttmvml"); 1577 bdev->driver = driver; 1578 1579 memset(bdev->man, 0, sizeof(bdev->man)); 1580 1581 /* 1582 * Initialize the system memory buffer type. 1583 * Other types need to be driver / IOCTL initialized. 1584 */ 1585 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1586 if (unlikely(ret != 0)) 1587 goto out_no_sys; 1588 1589 RB_INIT(&bdev->addr_space_rb); 1590 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1591 if (unlikely(ret != 0)) 1592 goto out_no_addr_mm; 1593 1594 TIMEOUT_TASK_INIT(taskqueue_thread, &bdev->wq, 0, 1595 ttm_bo_delayed_workqueue, bdev); 1596 INIT_LIST_HEAD(&bdev->ddestroy); 1597 bdev->dev_mapping = NULL; 1598 bdev->glob = glob; 1599 bdev->need_dma32 = need_dma32; 1600 bdev->val_seq = 0; 1601 mtx_init(&bdev->fence_lock, "ttmfence", NULL, MTX_DEF); 1602 sx_xlock(&glob->device_list_mutex); 1603 list_add_tail(&bdev->device_list, &glob->device_list); 1604 sx_xunlock(&glob->device_list_mutex); 1605 1606 return 0; 1607out_no_addr_mm: 1608 ttm_bo_clean_mm(bdev, 0); 1609out_no_sys: 1610 return ret; 1611} 1612 1613/* 1614 * buffer object vm functions. 1615 */ 1616 1617bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1618{ 1619 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1620 1621 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1622 if (mem->mem_type == TTM_PL_SYSTEM) 1623 return false; 1624 1625 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1626 return false; 1627 1628 if (mem->placement & TTM_PL_FLAG_CACHED) 1629 return false; 1630 } 1631 return true; 1632} 1633 1634void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1635{ 1636 struct ttm_bo_device *bdev = bo->bdev; 1637 /* off_t offset = (off_t)bo->addr_space_offset;XXXKIB */ 1638 /* off_t holelen = ((off_t)bo->mem.num_pages) << PAGE_SHIFT;XXXKIB */ 1639 1640 if (!bdev->dev_mapping) 1641 return; 1642 /* unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); XXXKIB */ 1643 ttm_mem_io_free_vm(bo); 1644} 1645 1646void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1647{ 1648 struct ttm_bo_device *bdev = bo->bdev; 1649 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1650 1651 ttm_mem_io_lock(man, false); 1652 ttm_bo_unmap_virtual_locked(bo); 1653 ttm_mem_io_unlock(man); 1654} 1655 1656static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1657{ 1658 struct ttm_bo_device *bdev = bo->bdev; 1659 1660 /* The caller acquired bdev->vm_lock. */ 1661 RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo); 1662} 1663 1664/** 1665 * ttm_bo_setup_vm: 1666 * 1667 * @bo: the buffer to allocate address space for 1668 * 1669 * Allocate address space in the drm device so that applications 1670 * can mmap the buffer and access the contents. This only 1671 * applies to ttm_bo_type_device objects as others are not 1672 * placed in the drm device address space. 1673 */ 1674 1675static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1676{ 1677 struct ttm_bo_device *bdev = bo->bdev; 1678 int ret; 1679 1680retry_pre_get: 1681 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1682 if (unlikely(ret != 0)) 1683 return ret; 1684 1685 rw_wlock(&bdev->vm_lock); 1686 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1687 bo->mem.num_pages, 0, 0); 1688 1689 if (unlikely(bo->vm_node == NULL)) { 1690 ret = -ENOMEM; 1691 goto out_unlock; 1692 } 1693 1694 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1695 bo->mem.num_pages, 0); 1696 1697 if (unlikely(bo->vm_node == NULL)) { 1698 rw_wunlock(&bdev->vm_lock); 1699 goto retry_pre_get; 1700 } 1701 1702 ttm_bo_vm_insert_rb(bo); 1703 rw_wunlock(&bdev->vm_lock); 1704 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1705 1706 return 0; 1707out_unlock: 1708 rw_wunlock(&bdev->vm_lock); 1709 return ret; 1710} 1711 1712int ttm_bo_wait(struct ttm_buffer_object *bo, 1713 bool lazy, bool interruptible, bool no_wait) 1714{ 1715 struct ttm_bo_driver *driver = bo->bdev->driver; 1716 struct ttm_bo_device *bdev = bo->bdev; 1717 void *sync_obj; 1718 int ret = 0; 1719 1720 if (likely(bo->sync_obj == NULL)) 1721 return 0; 1722 1723 while (bo->sync_obj) { 1724 1725 if (driver->sync_obj_signaled(bo->sync_obj)) { 1726 void *tmp_obj = bo->sync_obj; 1727 bo->sync_obj = NULL; 1728 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1729 mtx_unlock(&bdev->fence_lock); 1730 driver->sync_obj_unref(&tmp_obj); 1731 mtx_lock(&bdev->fence_lock); 1732 continue; 1733 } 1734 1735 if (no_wait) 1736 return -EBUSY; 1737 1738 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1739 mtx_unlock(&bdev->fence_lock); 1740 ret = driver->sync_obj_wait(sync_obj, 1741 lazy, interruptible); 1742 if (unlikely(ret != 0)) { 1743 driver->sync_obj_unref(&sync_obj); 1744 mtx_lock(&bdev->fence_lock); 1745 return ret; 1746 } 1747 mtx_lock(&bdev->fence_lock); 1748 if (likely(bo->sync_obj == sync_obj)) { 1749 void *tmp_obj = bo->sync_obj; 1750 bo->sync_obj = NULL; 1751 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1752 &bo->priv_flags); 1753 mtx_unlock(&bdev->fence_lock); 1754 driver->sync_obj_unref(&sync_obj); 1755 driver->sync_obj_unref(&tmp_obj); 1756 mtx_lock(&bdev->fence_lock); 1757 } else { 1758 mtx_unlock(&bdev->fence_lock); 1759 driver->sync_obj_unref(&sync_obj); 1760 mtx_lock(&bdev->fence_lock); 1761 } 1762 } 1763 return 0; 1764} 1765 1766int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1767{ 1768 struct ttm_bo_device *bdev = bo->bdev; 1769 int ret = 0; 1770 1771 /* 1772 * Using ttm_bo_reserve makes sure the lru lists are updated. 1773 */ 1774 1775 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1776 if (unlikely(ret != 0)) 1777 return ret; 1778 mtx_lock(&bdev->fence_lock); 1779 ret = ttm_bo_wait(bo, false, true, no_wait); 1780 mtx_unlock(&bdev->fence_lock); 1781 if (likely(ret == 0)) 1782 atomic_inc(&bo->cpu_writers); 1783 ttm_bo_unreserve(bo); 1784 return ret; 1785} 1786 1787void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1788{ 1789 atomic_dec(&bo->cpu_writers); 1790} 1791 1792/** 1793 * A buffer object shrink method that tries to swap out the first 1794 * buffer object on the bo_global::swap_lru list. 1795 */ 1796 1797static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1798{ 1799 struct ttm_bo_global *glob = 1800 container_of(shrink, struct ttm_bo_global, shrink); 1801 struct ttm_buffer_object *bo; 1802 int ret = -EBUSY; 1803 int put_count; 1804 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1805 1806 mtx_lock(&glob->lru_lock); 1807 list_for_each_entry(bo, &glob->swap_lru, swap) { 1808 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 1809 if (!ret) 1810 break; 1811 } 1812 1813 if (ret) { 1814 mtx_unlock(&glob->lru_lock); 1815 return ret; 1816 } 1817 1818 refcount_acquire(&bo->list_kref); 1819 1820 if (!list_empty(&bo->ddestroy)) { 1821 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1822 if (refcount_release(&bo->list_kref)) 1823 ttm_bo_release_list(bo); 1824 return ret; 1825 } 1826 1827 put_count = ttm_bo_del_from_lru(bo); 1828 mtx_unlock(&glob->lru_lock); 1829 1830 ttm_bo_list_ref_sub(bo, put_count, true); 1831 1832 /** 1833 * Wait for GPU, then move to system cached. 1834 */ 1835 1836 mtx_lock(&bo->bdev->fence_lock); 1837 ret = ttm_bo_wait(bo, false, false, false); 1838 mtx_unlock(&bo->bdev->fence_lock); 1839 1840 if (unlikely(ret != 0)) 1841 goto out; 1842 1843 if ((bo->mem.placement & swap_placement) != swap_placement) { 1844 struct ttm_mem_reg evict_mem; 1845 1846 evict_mem = bo->mem; 1847 evict_mem.mm_node = NULL; 1848 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1849 evict_mem.mem_type = TTM_PL_SYSTEM; 1850 1851 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1852 false, false); 1853 if (unlikely(ret != 0)) 1854 goto out; 1855 } 1856 1857 ttm_bo_unmap_virtual(bo); 1858 1859 /** 1860 * Swap out. Buffer will be swapped in again as soon as 1861 * anyone tries to access a ttm page. 1862 */ 1863 1864 if (bo->bdev->driver->swap_notify) 1865 bo->bdev->driver->swap_notify(bo); 1866 1867 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1868out: 1869 1870 /** 1871 * 1872 * Unreserve without putting on LRU to avoid swapping out an 1873 * already swapped buffer. 1874 */ 1875 1876 atomic_set(&bo->reserved, 0); 1877 wakeup(bo); 1878 if (refcount_release(&bo->list_kref)) 1879 ttm_bo_release_list(bo); 1880 return ret; 1881} 1882 1883void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1884{ 1885 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1886 ; 1887}
|