1/* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2/************************************************************************** 3 * 4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28/* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32#define pr_fmt(fmt) "[TTM] " fmt 33 34#include <drm/ttm/ttm_bo.h> 35#include <drm/ttm/ttm_placement.h> 36#include <drm/ttm/ttm_tt.h> 37 38#include <linux/jiffies.h> 39#include <linux/slab.h> 40#include <linux/sched.h> 41#include <linux/mm.h> 42#include <linux/file.h> 43#include <linux/module.h> 44#include <linux/atomic.h> 45#include <linux/dma-resv.h> 46 47#include "ttm_module.h" 48 49static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 50 struct ttm_placement *placement) 51{ 52 struct drm_printer p = drm_debug_printer(TTM_PFX); 53 struct ttm_resource_manager *man; 54 int i, mem_type; 55 56 for (i = 0; i < placement->num_placement; i++) { 57 mem_type = placement->placement[i].mem_type; 58 drm_printf(&p, " placement[%d]=0x%08X (%d)\n", 59 i, placement->placement[i].flags, mem_type); 60 man = ttm_manager_type(bo->bdev, mem_type); 61 ttm_resource_manager_debug(man, &p); 62 } 63} 64 65/** 66 * ttm_bo_move_to_lru_tail 67 * 68 * @bo: The buffer object. 69 * 70 * Move this BO to the tail of all lru lists used to lookup and reserve an 71 * object. This function must be called with struct ttm_global::lru_lock 72 * held, and is used to make a BO less likely to be considered for eviction. 73 */ 74void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) 75{ 76 dma_resv_assert_held(bo->base.resv); 77 78 if (bo->resource) 79 ttm_resource_move_to_lru_tail(bo->resource); 80} 81EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 82 83/** 84 * ttm_bo_set_bulk_move - update BOs bulk move object 85 * 86 * @bo: The buffer object. 87 * @bulk: bulk move structure 88 * 89 * Update the BOs bulk move object, making sure that resources are added/removed 90 * as well. A bulk move allows to move many resource on the LRU at once, 91 * resulting in much less overhead of maintaining the LRU. 92 * The only requirement is that the resources stay together on the LRU and are 93 * never separated. This is enforces by setting the bulk_move structure on a BO. 94 * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of 95 * their LRU list. 96 */ 97void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, 98 struct ttm_lru_bulk_move *bulk) 99{ 100 dma_resv_assert_held(bo->base.resv); 101 102 if (bo->bulk_move == bulk) 103 return; 104 105 spin_lock(&bo->bdev->lru_lock); 106 if (bo->resource) 107 ttm_resource_del_bulk_move(bo->resource, bo); 108 bo->bulk_move = bulk; 109 if (bo->resource) 110 ttm_resource_add_bulk_move(bo->resource, bo); 111 spin_unlock(&bo->bdev->lru_lock); 112} 113EXPORT_SYMBOL(ttm_bo_set_bulk_move); 114 115static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 116 struct ttm_resource *mem, bool evict, 117 struct ttm_operation_ctx *ctx, 118 struct ttm_place *hop) 119{ 120 struct ttm_device *bdev = bo->bdev; 121 bool old_use_tt, new_use_tt; 122 int ret; 123 124 old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt; 125 new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt; 126 127 ttm_bo_unmap_virtual(bo); 128 129 /* 130 * Create and bind a ttm if required. 131 */ 132 133 if (new_use_tt) { 134 /* Zero init the new TTM structure if the old location should 135 * have used one as well. 136 */ 137 ret = ttm_tt_create(bo, old_use_tt); 138 if (ret) 139 goto out_err; 140 141 if (mem->mem_type != TTM_PL_SYSTEM) { 142 ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx); 143 if (ret) 144 goto out_err; 145 } 146 } 147 148 ret = dma_resv_reserve_fences(bo->base.resv, 1); 149 if (ret) 150 goto out_err; 151 152 ret = bdev->funcs->move(bo, evict, ctx, mem, hop); 153 if (ret) { 154 if (ret == -EMULTIHOP) 155 return ret; 156 goto out_err; 157 } 158 159 ctx->bytes_moved += bo->base.size; 160 return 0; 161 162out_err: 163 if (!old_use_tt) 164 ttm_bo_tt_destroy(bo); 165 166 return ret; 167} 168 169/* 170 * Call bo::reserved. 171 * Will release GPU memory type usage on destruction. 172 * This is the place to put in driver specific hooks to release 173 * driver private resources. 174 * Will release the bo::reserved lock. 175 */ 176 177static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 178{ 179 if (bo->bdev->funcs->delete_mem_notify) 180 bo->bdev->funcs->delete_mem_notify(bo); 181 182 ttm_bo_tt_destroy(bo); 183 ttm_resource_free(bo, &bo->resource); 184} 185 186static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) 187{ 188 int r; 189 190 if (bo->base.resv == &bo->base._resv) 191 return 0; 192 193 BUG_ON(!dma_resv_trylock(&bo->base._resv)); 194 195 r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); 196 dma_resv_unlock(&bo->base._resv); 197 if (r) 198 return r; 199 200 if (bo->type != ttm_bo_type_sg) { 201 /* This works because the BO is about to be destroyed and nobody 202 * reference it any more. The only tricky case is the trylock on 203 * the resv object while holding the lru_lock. 204 */ 205 spin_lock(&bo->bdev->lru_lock); 206 bo->base.resv = &bo->base._resv; 207 spin_unlock(&bo->bdev->lru_lock); 208 } 209 210 return r; 211} 212 213static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 214{ 215 struct dma_resv *resv = &bo->base._resv; 216 struct dma_resv_iter cursor; 217 struct dma_fence *fence; 218 219 dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); 220 dma_resv_for_each_fence_unlocked(&cursor, fence) { 221 if (!fence->ops->signaled) 222 dma_fence_enable_sw_signaling(fence); 223 } 224 dma_resv_iter_end(&cursor); 225} 226 227/** 228 * ttm_bo_cleanup_refs 229 * If bo idle, remove from lru lists, and unref. 230 * If not idle, block if possible. 231 * 232 * Must be called with lru_lock and reservation held, this function 233 * will drop the lru lock and optionally the reservation lock before returning. 234 * 235 * @bo: The buffer object to clean-up 236 * @interruptible: Any sleeps should occur interruptibly. 237 * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. 238 * @unlock_resv: Unlock the reservation lock as well. 239 */ 240 241static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 242 bool interruptible, bool no_wait_gpu, 243 bool unlock_resv) 244{ 245 struct dma_resv *resv = &bo->base._resv; 246 int ret; 247 248 if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP)) 249 ret = 0; 250 else 251 ret = -EBUSY; 252 253 if (ret && !no_wait_gpu) { 254 long lret; 255 256 if (unlock_resv) 257 dma_resv_unlock(bo->base.resv); 258 spin_unlock(&bo->bdev->lru_lock); 259 260 lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP, 261 interruptible, 262 30 * HZ); 263 264 if (lret < 0) 265 return lret; 266 else if (lret == 0) 267 return -EBUSY; 268 269 spin_lock(&bo->bdev->lru_lock); 270 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { 271 /* 272 * We raced, and lost, someone else holds the reservation now, 273 * and is probably busy in ttm_bo_cleanup_memtype_use. 274 * 275 * Even if it's not the case, because we finished waiting any 276 * delayed destruction would succeed, so just return success 277 * here. 278 */ 279 spin_unlock(&bo->bdev->lru_lock); 280 return 0; 281 } 282 ret = 0; 283 } 284 285 if (ret) { 286 if (unlock_resv) 287 dma_resv_unlock(bo->base.resv); 288 spin_unlock(&bo->bdev->lru_lock); 289 return ret; 290 } 291 292 spin_unlock(&bo->bdev->lru_lock); 293 ttm_bo_cleanup_memtype_use(bo); 294 295 if (unlock_resv) 296 dma_resv_unlock(bo->base.resv); 297 298 return 0; 299} 300 301/* 302 * Block for the dma_resv object to become idle, lock the buffer and clean up 303 * the resource and tt object. 304 */ 305static void ttm_bo_delayed_delete(struct work_struct *work) 306{ 307 struct ttm_buffer_object *bo; 308 309 bo = container_of(work, typeof(*bo), delayed_delete); 310 311 dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false, 312 MAX_SCHEDULE_TIMEOUT); 313 dma_resv_lock(bo->base.resv, NULL); 314 ttm_bo_cleanup_memtype_use(bo); 315 dma_resv_unlock(bo->base.resv); 316 ttm_bo_put(bo); 317} 318 319static void ttm_bo_release(struct kref *kref) 320{ 321 struct ttm_buffer_object *bo = 322 container_of(kref, struct ttm_buffer_object, kref); 323 struct ttm_device *bdev = bo->bdev; 324 int ret; 325 326 WARN_ON_ONCE(bo->pin_count); 327 WARN_ON_ONCE(bo->bulk_move); 328 329 if (!bo->deleted) { 330 ret = ttm_bo_individualize_resv(bo); 331 if (ret) { 332 /* Last resort, if we fail to allocate memory for the 333 * fences block for the BO to become idle 334 */ 335 dma_resv_wait_timeout(bo->base.resv, 336 DMA_RESV_USAGE_BOOKKEEP, false, 337 30 * HZ); 338 } 339 340 if (bo->bdev->funcs->release_notify) 341 bo->bdev->funcs->release_notify(bo); 342 343 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); 344 ttm_mem_io_free(bdev, bo->resource); 345 346 if (!dma_resv_test_signaled(bo->base.resv, 347 DMA_RESV_USAGE_BOOKKEEP) || 348 (want_init_on_free() && (bo->ttm != NULL)) || 349 !dma_resv_trylock(bo->base.resv)) { 350 /* The BO is not idle, resurrect it for delayed destroy */ 351 ttm_bo_flush_all_fences(bo); 352 bo->deleted = true; 353 354 spin_lock(&bo->bdev->lru_lock); 355 356 /* 357 * Make pinned bos immediately available to 358 * shrinkers, now that they are queued for 359 * destruction. 360 * 361 * FIXME: QXL is triggering this. Can be removed when the 362 * driver is fixed. 363 */ 364 if (bo->pin_count) { 365 bo->pin_count = 0; 366 ttm_resource_move_to_lru_tail(bo->resource); 367 } 368 369 kref_init(&bo->kref); 370 spin_unlock(&bo->bdev->lru_lock); 371 372 INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); 373 queue_work(bdev->wq, &bo->delayed_delete); 374 return; 375 } 376 377 ttm_bo_cleanup_memtype_use(bo); 378 dma_resv_unlock(bo->base.resv); 379 } 380 381 atomic_dec(&ttm_glob.bo_count); 382 bo->destroy(bo); 383} 384 385/** 386 * ttm_bo_put 387 * 388 * @bo: The buffer object. 389 * 390 * Unreference a buffer object. 391 */ 392void ttm_bo_put(struct ttm_buffer_object *bo) 393{ 394 kref_put(&bo->kref, ttm_bo_release); 395} 396EXPORT_SYMBOL(ttm_bo_put); 397 398static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, 399 struct ttm_resource **mem, 400 struct ttm_operation_ctx *ctx, 401 struct ttm_place *hop) 402{ 403 struct ttm_placement hop_placement; 404 struct ttm_resource *hop_mem; 405 int ret; 406 407 hop_placement.num_placement = hop_placement.num_busy_placement = 1; 408 hop_placement.placement = hop_placement.busy_placement = hop; 409 410 /* find space in the bounce domain */ 411 ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); 412 if (ret) 413 return ret; 414 /* move to the bounce domain */ 415 ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); 416 if (ret) { 417 ttm_resource_free(bo, &hop_mem); 418 return ret; 419 } 420 return 0; 421} 422 423static int ttm_bo_evict(struct ttm_buffer_object *bo, 424 struct ttm_operation_ctx *ctx) 425{ 426 struct ttm_device *bdev = bo->bdev; 427 struct ttm_resource *evict_mem; 428 struct ttm_placement placement; 429 struct ttm_place hop; 430 int ret = 0; 431 432 memset(&hop, 0, sizeof(hop)); 433 434 dma_resv_assert_held(bo->base.resv); 435 436 placement.num_placement = 0; 437 placement.num_busy_placement = 0; 438 bdev->funcs->evict_flags(bo, &placement); 439 440 if (!placement.num_placement && !placement.num_busy_placement) { 441 ret = ttm_bo_wait_ctx(bo, ctx); 442 if (ret) 443 return ret; 444 445 /* 446 * Since we've already synced, this frees backing store 447 * immediately. 448 */ 449 return ttm_bo_pipeline_gutting(bo); 450 } 451 452 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); 453 if (ret) { 454 if (ret != -ERESTARTSYS) { 455 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 456 bo); 457 ttm_bo_mem_space_debug(bo, &placement); 458 } 459 goto out; 460 } 461 462 do { 463 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); 464 if (ret != -EMULTIHOP) 465 break; 466 467 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop); 468 } while (!ret); 469 470 if (ret) { 471 ttm_resource_free(bo, &evict_mem); 472 if (ret != -ERESTARTSYS && ret != -EINTR) 473 pr_err("Buffer eviction failed\n"); 474 } 475out: 476 return ret; 477} 478 479/** 480 * ttm_bo_eviction_valuable 481 * 482 * @bo: The buffer object to evict 483 * @place: the placement we need to make room for 484 * 485 * Check if it is valuable to evict the BO to make room for the given placement. 486 */ 487bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 488 const struct ttm_place *place) 489{ 490 struct ttm_resource *res = bo->resource; 491 struct ttm_device *bdev = bo->bdev; 492 493 dma_resv_assert_held(bo->base.resv); 494 if (bo->resource->mem_type == TTM_PL_SYSTEM) 495 return true; 496 497 /* Don't evict this BO if it's outside of the 498 * requested placement range 499 */ 500 return ttm_resource_intersects(bdev, res, place, bo->base.size); 501} 502EXPORT_SYMBOL(ttm_bo_eviction_valuable); 503 504/* 505 * Check the target bo is allowable to be evicted or swapout, including cases: 506 * 507 * a. if share same reservation object with ctx->resv, have assumption 508 * reservation objects should already be locked, so not lock again and 509 * return true directly when either the opreation allow_reserved_eviction 510 * or the target bo already is in delayed free list; 511 * 512 * b. Otherwise, trylock it. 513 */ 514static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, 515 struct ttm_operation_ctx *ctx, 516 const struct ttm_place *place, 517 bool *locked, bool *busy) 518{ 519 bool ret = false; 520 521 if (bo->pin_count) { 522 *locked = false; 523 if (busy) 524 *busy = false; 525 return false; 526 } 527 528 if (bo->base.resv == ctx->resv) { 529 dma_resv_assert_held(bo->base.resv); 530 if (ctx->allow_res_evict) 531 ret = true; 532 *locked = false; 533 if (busy) 534 *busy = false; 535 } else { 536 ret = dma_resv_trylock(bo->base.resv); 537 *locked = ret; 538 if (busy) 539 *busy = !ret; 540 } 541 542 if (ret && place && (bo->resource->mem_type != place->mem_type || 543 !bo->bdev->funcs->eviction_valuable(bo, place))) { 544 ret = false; 545 if (*locked) { 546 dma_resv_unlock(bo->base.resv); 547 *locked = false; 548 } 549 } 550 551 return ret; 552} 553 554/** 555 * ttm_mem_evict_wait_busy - wait for a busy BO to become available 556 * 557 * @busy_bo: BO which couldn't be locked with trylock 558 * @ctx: operation context 559 * @ticket: acquire ticket 560 * 561 * Try to lock a busy buffer object to avoid failing eviction. 562 */ 563static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, 564 struct ttm_operation_ctx *ctx, 565 struct ww_acquire_ctx *ticket) 566{ 567 int r; 568 569 if (!busy_bo || !ticket) 570 return -EBUSY; 571 572 if (ctx->interruptible) 573 r = dma_resv_lock_interruptible(busy_bo->base.resv, 574 ticket); 575 else 576 r = dma_resv_lock(busy_bo->base.resv, ticket); 577 578 /* 579 * TODO: It would be better to keep the BO locked until allocation is at 580 * least tried one more time, but that would mean a much larger rework 581 * of TTM. 582 */ 583 if (!r) 584 dma_resv_unlock(busy_bo->base.resv); 585 586 return r == -EDEADLK ? -EBUSY : r; 587} 588 589int ttm_mem_evict_first(struct ttm_device *bdev, 590 struct ttm_resource_manager *man, 591 const struct ttm_place *place, 592 struct ttm_operation_ctx *ctx, 593 struct ww_acquire_ctx *ticket) 594{ 595 struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; 596 struct ttm_resource_cursor cursor; 597 struct ttm_resource *res; 598 bool locked = false; 599 int ret; 600 601 spin_lock(&bdev->lru_lock); 602 ttm_resource_manager_for_each_res(man, &cursor, res) { 603 bool busy; 604 605 if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place, 606 &locked, &busy)) { 607 if (busy && !busy_bo && ticket != 608 dma_resv_locking_ctx(res->bo->base.resv)) 609 busy_bo = res->bo; 610 continue; 611 } 612 613 if (ttm_bo_get_unless_zero(res->bo)) { 614 bo = res->bo; 615 break; 616 } 617 if (locked) 618 dma_resv_unlock(res->bo->base.resv); 619 } 620 621 if (!bo) { 622 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) 623 busy_bo = NULL; 624 spin_unlock(&bdev->lru_lock); 625 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); 626 if (busy_bo) 627 ttm_bo_put(busy_bo); 628 return ret; 629 } 630 631 if (bo->deleted) { 632 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, 633 ctx->no_wait_gpu, locked); 634 ttm_bo_put(bo); 635 return ret; 636 } 637 638 spin_unlock(&bdev->lru_lock); 639 640 ret = ttm_bo_evict(bo, ctx); 641 if (locked) 642 ttm_bo_unreserve(bo); 643 else 644 ttm_bo_move_to_lru_tail_unlocked(bo); 645 646 ttm_bo_put(bo); 647 return ret; 648} 649 650/** 651 * ttm_bo_pin - Pin the buffer object. 652 * @bo: The buffer object to pin 653 * 654 * Make sure the buffer is not evicted any more during memory pressure. 655 * @bo must be unpinned again by calling ttm_bo_unpin(). 656 */ 657void ttm_bo_pin(struct ttm_buffer_object *bo) 658{ 659 dma_resv_assert_held(bo->base.resv); 660 WARN_ON_ONCE(!kref_read(&bo->kref)); 661 spin_lock(&bo->bdev->lru_lock); 662 if (bo->resource) 663 ttm_resource_del_bulk_move(bo->resource, bo); 664 ++bo->pin_count; 665 spin_unlock(&bo->bdev->lru_lock); 666} 667EXPORT_SYMBOL(ttm_bo_pin); 668 669/** 670 * ttm_bo_unpin - Unpin the buffer object. 671 * @bo: The buffer object to unpin 672 * 673 * Allows the buffer object to be evicted again during memory pressure. 674 */ 675void ttm_bo_unpin(struct ttm_buffer_object *bo) 676{ 677 dma_resv_assert_held(bo->base.resv); 678 WARN_ON_ONCE(!kref_read(&bo->kref)); 679 if (WARN_ON_ONCE(!bo->pin_count)) 680 return; 681 682 spin_lock(&bo->bdev->lru_lock); 683 --bo->pin_count; 684 if (bo->resource) 685 ttm_resource_add_bulk_move(bo->resource, bo); 686 spin_unlock(&bo->bdev->lru_lock); 687} 688EXPORT_SYMBOL(ttm_bo_unpin); 689 690/* 691 * Add the last move fence to the BO as kernel dependency and reserve a new 692 * fence slot. 693 */ 694static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 695 struct ttm_resource_manager *man, 696 struct ttm_resource *mem, 697 bool no_wait_gpu) 698{ 699 struct dma_fence *fence; 700 int ret; 701 702 spin_lock(&man->move_lock); 703 fence = dma_fence_get(man->move); 704 spin_unlock(&man->move_lock); 705 706 if (!fence) 707 return 0; 708 709 if (no_wait_gpu) { 710 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; 711 dma_fence_put(fence); 712 return ret; 713 } 714 715 dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); 716 717 ret = dma_resv_reserve_fences(bo->base.resv, 1); 718 dma_fence_put(fence); 719 return ret; 720} 721 722/* 723 * Repeatedly evict memory from the LRU for @mem_type until we create enough 724 * space, or we've evicted everything and there isn't enough space. 725 */ 726static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 727 const struct ttm_place *place, 728 struct ttm_resource **mem, 729 struct ttm_operation_ctx *ctx) 730{ 731 struct ttm_device *bdev = bo->bdev; 732 struct ttm_resource_manager *man; 733 struct ww_acquire_ctx *ticket; 734 int ret; 735 736 man = ttm_manager_type(bdev, place->mem_type); 737 ticket = dma_resv_locking_ctx(bo->base.resv); 738 do { 739 ret = ttm_resource_alloc(bo, place, mem); 740 if (likely(!ret)) 741 break; 742 if (unlikely(ret != -ENOSPC)) 743 return ret; 744 ret = ttm_mem_evict_first(bdev, man, place, ctx, 745 ticket); 746 if (unlikely(ret != 0)) 747 return ret; 748 } while (1); 749 750 return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); 751} 752 753/** 754 * ttm_bo_mem_space 755 * 756 * @bo: Pointer to a struct ttm_buffer_object. the data of which 757 * we want to allocate space for. 758 * @placement: Proposed new placement for the buffer object. 759 * @mem: A struct ttm_resource. 760 * @ctx: if and how to sleep, lock buffers and alloc memory 761 * 762 * Allocate memory space for the buffer object pointed to by @bo, using 763 * the placement flags in @placement, potentially evicting other idle buffer objects. 764 * This function may sleep while waiting for space to become available. 765 * Returns: 766 * -EBUSY: No space available (only if no_wait == 1). 767 * -ENOSPC: Could not allocate space for the buffer object, either due to 768 * fragmentation or concurrent allocators. 769 * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. 770 */ 771int ttm_bo_mem_space(struct ttm_buffer_object *bo, 772 struct ttm_placement *placement, 773 struct ttm_resource **mem, 774 struct ttm_operation_ctx *ctx) 775{ 776 struct ttm_device *bdev = bo->bdev; 777 bool type_found = false; 778 int i, ret; 779 780 ret = dma_resv_reserve_fences(bo->base.resv, 1); 781 if (unlikely(ret)) 782 return ret; 783 784 for (i = 0; i < placement->num_placement; ++i) { 785 const struct ttm_place *place = &placement->placement[i]; 786 struct ttm_resource_manager *man; 787 788 man = ttm_manager_type(bdev, place->mem_type); 789 if (!man || !ttm_resource_manager_used(man)) 790 continue; 791 792 type_found = true; 793 ret = ttm_resource_alloc(bo, place, mem); 794 if (ret == -ENOSPC) 795 continue; 796 if (unlikely(ret)) 797 goto error; 798 799 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); 800 if (unlikely(ret)) { 801 ttm_resource_free(bo, mem); 802 if (ret == -EBUSY) 803 continue; 804 805 goto error; 806 } 807 return 0; 808 } 809 810 for (i = 0; i < placement->num_busy_placement; ++i) { 811 const struct ttm_place *place = &placement->busy_placement[i]; 812 struct ttm_resource_manager *man; 813 814 man = ttm_manager_type(bdev, place->mem_type); 815 if (!man || !ttm_resource_manager_used(man)) 816 continue; 817 818 type_found = true; 819 ret = ttm_bo_mem_force_space(bo, place, mem, ctx); 820 if (likely(!ret)) 821 return 0; 822 823 if (ret && ret != -EBUSY) 824 goto error; 825 } 826 827 ret = -ENOSPC; 828 if (!type_found) { 829 pr_err(TTM_PFX "No compatible memory type found\n"); 830 ret = -EINVAL; 831 } 832 833error: 834 return ret; 835} 836EXPORT_SYMBOL(ttm_bo_mem_space); 837 838static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 839 struct ttm_placement *placement, 840 struct ttm_operation_ctx *ctx) 841{ 842 struct ttm_resource *mem; 843 struct ttm_place hop; 844 int ret; 845 846 dma_resv_assert_held(bo->base.resv); 847 848 /* 849 * Determine where to move the buffer. 850 * 851 * If driver determines move is going to need 852 * an extra step then it will return -EMULTIHOP 853 * and the buffer will be moved to the temporary 854 * stop and the driver will be called to make 855 * the second hop. 856 */ 857 ret = ttm_bo_mem_space(bo, placement, &mem, ctx); 858 if (ret) 859 return ret; 860bounce: 861 ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop); 862 if (ret == -EMULTIHOP) { 863 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); 864 if (ret) 865 goto out; 866 /* try and move to final place now. */ 867 goto bounce; 868 } 869out: 870 if (ret) 871 ttm_resource_free(bo, &mem); 872 return ret; 873} 874 875/** 876 * ttm_bo_validate 877 * 878 * @bo: The buffer object. 879 * @placement: Proposed placement for the buffer object. 880 * @ctx: validation parameters. 881 * 882 * Changes placement and caching policy of the buffer object 883 * according proposed placement. 884 * Returns 885 * -EINVAL on invalid proposed placement. 886 * -ENOMEM on out-of-memory condition. 887 * -EBUSY if no_wait is true and buffer busy. 888 * -ERESTARTSYS if interrupted by a signal. 889 */ 890int ttm_bo_validate(struct ttm_buffer_object *bo, 891 struct ttm_placement *placement, 892 struct ttm_operation_ctx *ctx) 893{ 894 int ret; 895 896 dma_resv_assert_held(bo->base.resv); 897 898 /* 899 * Remove the backing store if no placement is given. 900 */ 901 if (!placement->num_placement && !placement->num_busy_placement) 902 return ttm_bo_pipeline_gutting(bo); 903 904 /* Check whether we need to move buffer. */ 905 if (bo->resource && ttm_resource_compat(bo->resource, placement)) 906 return 0; 907 908 /* Moving of pinned BOs is forbidden */ 909 if (bo->pin_count) 910 return -EINVAL; 911 912 ret = ttm_bo_move_buffer(bo, placement, ctx); 913 /* For backward compatibility with userspace */ 914 if (ret == -ENOSPC) 915 return -ENOMEM; 916 if (ret) 917 return ret; 918 919 /* 920 * We might need to add a TTM. 921 */ 922 if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { 923 ret = ttm_tt_create(bo, true); 924 if (ret) 925 return ret; 926 } 927 return 0; 928} 929EXPORT_SYMBOL(ttm_bo_validate); 930 931/** 932 * ttm_bo_init_reserved 933 * 934 * @bdev: Pointer to a ttm_device struct. 935 * @bo: Pointer to a ttm_buffer_object to be initialized. 936 * @type: Requested type of buffer object. 937 * @placement: Initial placement for buffer object. 938 * @alignment: Data alignment in pages. 939 * @ctx: TTM operation context for memory allocation. 940 * @sg: Scatter-gather table. 941 * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. 942 * @destroy: Destroy function. Use NULL for kfree(). 943 * 944 * This function initializes a pre-allocated struct ttm_buffer_object. 945 * As this object may be part of a larger structure, this function, 946 * together with the @destroy function, enables driver-specific objects 947 * derived from a ttm_buffer_object. 948 * 949 * On successful return, the caller owns an object kref to @bo. The kref and 950 * list_kref are usually set to 1, but note that in some situations, other 951 * tasks may already be holding references to @bo as well. 952 * Furthermore, if resv == NULL, the buffer's reservation lock will be held, 953 * and it is the caller's responsibility to call ttm_bo_unreserve. 954 * 955 * If a failure occurs, the function will call the @destroy function. Thus, 956 * after a failure, dereferencing @bo is illegal and will likely cause memory 957 * corruption. 958 * 959 * Returns 960 * -ENOMEM: Out of memory. 961 * -EINVAL: Invalid placement flags. 962 * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. 963 */ 964int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, 965 enum ttm_bo_type type, struct ttm_placement *placement, 966 uint32_t alignment, struct ttm_operation_ctx *ctx, 967 struct sg_table *sg, struct dma_resv *resv, 968 void (*destroy) (struct ttm_buffer_object *)) 969{ 970 int ret; 971 972 kref_init(&bo->kref); 973 bo->bdev = bdev; 974 bo->type = type; 975 bo->page_alignment = alignment; 976 bo->destroy = destroy; 977 bo->pin_count = 0; 978 bo->sg = sg; 979 bo->bulk_move = NULL; 980 if (resv) 981 bo->base.resv = resv; 982 else 983 bo->base.resv = &bo->base._resv; 984 atomic_inc(&ttm_glob.bo_count); 985 986 /* 987 * For ttm_bo_type_device buffers, allocate 988 * address space from the device. 989 */ 990 if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { 991 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, 992 PFN_UP(bo->base.size)); 993 if (ret) 994 goto err_put; 995 } 996 997 /* passed reservation objects should already be locked, 998 * since otherwise lockdep will be angered in radeon. 999 */ 1000 if (!resv) 1001 WARN_ON(!dma_resv_trylock(bo->base.resv)); 1002 else 1003 dma_resv_assert_held(resv); 1004 1005 ret = ttm_bo_validate(bo, placement, ctx); 1006 if (unlikely(ret)) 1007 goto err_unlock; 1008 1009 return 0; 1010 1011err_unlock: 1012 if (!resv) 1013 dma_resv_unlock(bo->base.resv); 1014 1015err_put: 1016 ttm_bo_put(bo); 1017 return ret; 1018} 1019EXPORT_SYMBOL(ttm_bo_init_reserved); 1020 1021/** 1022 * ttm_bo_init_validate 1023 * 1024 * @bdev: Pointer to a ttm_device struct. 1025 * @bo: Pointer to a ttm_buffer_object to be initialized. 1026 * @type: Requested type of buffer object. 1027 * @placement: Initial placement for buffer object. 1028 * @alignment: Data alignment in pages. 1029 * @interruptible: If needing to sleep to wait for GPU resources, 1030 * sleep interruptible. 1031 * pinned in physical memory. If this behaviour is not desired, this member 1032 * holds a pointer to a persistent shmem object. Typically, this would 1033 * point to the shmem object backing a GEM object if TTM is used to back a 1034 * GEM user interface. 1035 * @sg: Scatter-gather table. 1036 * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. 1037 * @destroy: Destroy function. Use NULL for kfree(). 1038 * 1039 * This function initializes a pre-allocated struct ttm_buffer_object. 1040 * As this object may be part of a larger structure, this function, 1041 * together with the @destroy function, 1042 * enables driver-specific objects derived from a ttm_buffer_object. 1043 * 1044 * On successful return, the caller owns an object kref to @bo. The kref and 1045 * list_kref are usually set to 1, but note that in some situations, other 1046 * tasks may already be holding references to @bo as well. 1047 * 1048 * If a failure occurs, the function will call the @destroy function, Thus, 1049 * after a failure, dereferencing @bo is illegal and will likely cause memory 1050 * corruption. 1051 * 1052 * Returns 1053 * -ENOMEM: Out of memory. 1054 * -EINVAL: Invalid placement flags. 1055 * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. 1056 */ 1057int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, 1058 enum ttm_bo_type type, struct ttm_placement *placement, 1059 uint32_t alignment, bool interruptible, 1060 struct sg_table *sg, struct dma_resv *resv, 1061 void (*destroy) (struct ttm_buffer_object *)) 1062{ 1063 struct ttm_operation_ctx ctx = { interruptible, false }; 1064 int ret; 1065 1066 ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, 1067 sg, resv, destroy); 1068 if (ret) 1069 return ret; 1070 1071 if (!resv) 1072 ttm_bo_unreserve(bo); 1073 1074 return 0; 1075} 1076EXPORT_SYMBOL(ttm_bo_init_validate); 1077 1078/* 1079 * buffer object vm functions. 1080 */ 1081 1082/** 1083 * ttm_bo_unmap_virtual 1084 * 1085 * @bo: tear down the virtual mappings for this BO 1086 */ 1087void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1088{ 1089 struct ttm_device *bdev = bo->bdev; 1090 1091#ifdef __linux__ 1092 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); 1093#else 1094 if (drm_mm_node_allocated(&bo->base.vma_node.vm_node)) { 1095 struct vm_page *pg; 1096 bus_addr_t addr; 1097 paddr_t paddr; 1098 unsigned i; 1099 1100 if (bo->resource && bo->resource->bus.is_iomem) { 1101 addr = bo->resource->bus.offset; 1102 paddr = bus_space_mmap(bdev->memt, addr, 0, 0, 0); 1103 for (i = 0; i < PFN_UP(bo->base.size); i++) { 1104 pg = PHYS_TO_VM_PAGE(paddr); 1105 if (pg) 1106 pmap_page_protect(pg, PROT_NONE); 1107 paddr += PAGE_SIZE; 1108 } 1109 } else if (bo->ttm) { 1110 for (i = 0; i < bo->ttm->num_pages; i++) { 1111 pg = bo->ttm->pages[i]; 1112 if (pg) 1113 pmap_page_protect(pg, PROT_NONE); 1114 } 1115 } 1116 } 1117#endif 1118 ttm_mem_io_free(bdev, bo->resource); 1119} 1120EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1121 1122/** 1123 * ttm_bo_wait_ctx - wait for buffer idle. 1124 * 1125 * @bo: The buffer object. 1126 * @ctx: defines how to wait 1127 * 1128 * Waits for the buffer to be idle. Used timeout depends on the context. 1129 * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or 1130 * zero on success. 1131 */ 1132int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) 1133{ 1134 long ret; 1135 1136 if (ctx->no_wait_gpu) { 1137 if (dma_resv_test_signaled(bo->base.resv, 1138 DMA_RESV_USAGE_BOOKKEEP)) 1139 return 0; 1140 else 1141 return -EBUSY; 1142 } 1143 1144 ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, 1145 ctx->interruptible, 15 * HZ); 1146 if (unlikely(ret < 0)) 1147 return ret; 1148 if (unlikely(ret == 0)) 1149 return -EBUSY; 1150 return 0; 1151} 1152EXPORT_SYMBOL(ttm_bo_wait_ctx); 1153 1154int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, 1155 gfp_t gfp_flags) 1156{ 1157 struct ttm_place place; 1158 bool locked; 1159 long ret; 1160 1161 /* 1162 * While the bo may already reside in SYSTEM placement, set 1163 * SYSTEM as new placement to cover also the move further below. 1164 * The driver may use the fact that we're moving from SYSTEM 1165 * as an indication that we're about to swap out. 1166 */ 1167 memset(&place, 0, sizeof(place)); 1168 place.mem_type = bo->resource->mem_type; 1169 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL)) 1170 return -EBUSY; 1171 1172 if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || 1173 bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || 1174 bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED || 1175 !ttm_bo_get_unless_zero(bo)) { 1176 if (locked) 1177 dma_resv_unlock(bo->base.resv); 1178 return -EBUSY; 1179 } 1180 1181 if (bo->deleted) { 1182 ret = ttm_bo_cleanup_refs(bo, false, false, locked); 1183 ttm_bo_put(bo); 1184 return ret == -EBUSY ? -ENOSPC : ret; 1185 } 1186 1187 /* TODO: Cleanup the locking */ 1188 spin_unlock(&bo->bdev->lru_lock); 1189 1190 /* 1191 * Move to system cached 1192 */ 1193 if (bo->resource->mem_type != TTM_PL_SYSTEM) { 1194 struct ttm_resource *evict_mem; 1195 struct ttm_place hop; 1196 1197 memset(&hop, 0, sizeof(hop)); 1198 place.mem_type = TTM_PL_SYSTEM; 1199 ret = ttm_resource_alloc(bo, &place, &evict_mem); 1200 if (unlikely(ret)) 1201 goto out; 1202 1203 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); 1204 if (unlikely(ret != 0)) { 1205 WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n"); 1206 ttm_resource_free(bo, &evict_mem); 1207 goto out; 1208 } 1209 } 1210 1211 /* 1212 * Make sure BO is idle. 1213 */ 1214 ret = ttm_bo_wait_ctx(bo, ctx); 1215 if (unlikely(ret != 0)) 1216 goto out; 1217 1218 ttm_bo_unmap_virtual(bo); 1219 1220 /* 1221 * Swap out. Buffer will be swapped in again as soon as 1222 * anyone tries to access a ttm page. 1223 */ 1224 if (bo->bdev->funcs->swap_notify) 1225 bo->bdev->funcs->swap_notify(bo); 1226 1227 if (ttm_tt_is_populated(bo->ttm)) 1228 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); 1229out: 1230 1231 /* 1232 * Unreserve without putting on LRU to avoid swapping out an 1233 * already swapped buffer. 1234 */ 1235 if (locked) 1236 dma_resv_unlock(bo->base.resv); 1237 ttm_bo_put(bo); 1238 return ret == -EBUSY ? -ENOSPC : ret; 1239} 1240 1241void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) 1242{ 1243 if (bo->ttm == NULL) 1244 return; 1245 1246 ttm_tt_unpopulate(bo->bdev, bo->ttm); 1247 ttm_tt_destroy(bo->bdev, bo->ttm); 1248 bo->ttm = NULL; 1249} 1250