1/* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 29#include <linux/dma-fence-array.h> 30#include <linux/interval_tree_generic.h> 31#include <linux/idr.h> 32#include <linux/dma-buf.h> 33 34#include <drm/amdgpu_drm.h> 35#include <drm/drm_drv.h> 36#include <drm/ttm/ttm_tt.h> 37#include <drm/drm_exec.h> 38#include "amdgpu.h" 39#include "amdgpu_trace.h" 40#include "amdgpu_amdkfd.h" 41#include "amdgpu_gmc.h" 42#include "amdgpu_xgmi.h" 43#include "amdgpu_dma_buf.h" 44#include "amdgpu_res_cursor.h" 45#include "../amdkfd/kfd_svm.h" 46 47/** 48 * DOC: GPUVM 49 * 50 * GPUVM is the MMU functionality provided on the GPU. 51 * GPUVM is similar to the legacy GART on older asics, however 52 * rather than there being a single global GART table 53 * for the entire GPU, there can be multiple GPUVM page tables active 54 * at any given time. The GPUVM page tables can contain a mix 55 * VRAM pages and system pages (both memory and MMIO) and system pages 56 * can be mapped as snooped (cached system pages) or unsnooped 57 * (uncached system pages). 58 * 59 * Each active GPUVM has an ID associated with it and there is a page table 60 * linked with each VMID. When executing a command buffer, 61 * the kernel tells the engine what VMID to use for that command 62 * buffer. VMIDs are allocated dynamically as commands are submitted. 63 * The userspace drivers maintain their own address space and the kernel 64 * sets up their pages tables accordingly when they submit their 65 * command buffers and a VMID is assigned. 66 * The hardware supports up to 16 active GPUVMs at any given time. 67 * 68 * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending 69 * on the ASIC family. GPUVM supports RWX attributes on each page as well 70 * as other features such as encryption and caching attributes. 71 * 72 * VMID 0 is special. It is the GPUVM used for the kernel driver. In 73 * addition to an aperture managed by a page table, VMID 0 also has 74 * several other apertures. There is an aperture for direct access to VRAM 75 * and there is a legacy AGP aperture which just forwards accesses directly 76 * to the matching system physical addresses (or IOVAs when an IOMMU is 77 * present). These apertures provide direct access to these memories without 78 * incurring the overhead of a page table. VMID 0 is used by the kernel 79 * driver for tasks like memory management. 80 * 81 * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory. 82 * For user applications, each application can have their own unique GPUVM 83 * address space. The application manages the address space and the kernel 84 * driver manages the GPUVM page tables for each process. If an GPU client 85 * accesses an invalid page, it will generate a GPU page fault, similar to 86 * accessing an invalid page on a CPU. 87 */ 88 89#define START(node) ((node)->start) 90#define LAST(node) ((node)->last) 91 92#ifdef __linux__ 93INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, 94 START, LAST, static, amdgpu_vm_it) 95#else 96static struct amdgpu_bo_va_mapping * 97amdgpu_vm_it_iter_first(struct rb_root_cached *root, uint64_t start, 98 uint64_t last) 99{ 100 struct amdgpu_bo_va_mapping *node; 101 struct rb_node *rb; 102 103 for (rb = rb_first_cached(root); rb; rb = rb_next(rb)) { 104 node = rb_entry(rb, typeof(*node), rb); 105 if (LAST(node) >= start && START(node) <= last) 106 return node; 107 } 108 return NULL; 109} 110 111static struct amdgpu_bo_va_mapping * 112amdgpu_vm_it_iter_next(struct amdgpu_bo_va_mapping *node, uint64_t start, 113 uint64_t last) 114{ 115 struct rb_node *rb = &node->rb; 116 117 for (rb = rb_next(rb); rb; rb = rb_next(rb)) { 118 node = rb_entry(rb, typeof(*node), rb); 119 if (LAST(node) >= start && START(node) <= last) 120 return node; 121 } 122 return NULL; 123} 124 125static void 126amdgpu_vm_it_remove(struct amdgpu_bo_va_mapping *node, 127 struct rb_root_cached *root) 128{ 129 rb_erase_cached(&node->rb, root); 130} 131 132static void 133amdgpu_vm_it_insert(struct amdgpu_bo_va_mapping *node, 134 struct rb_root_cached *root) 135{ 136 struct rb_node **iter = &root->rb_root.rb_node; 137 struct rb_node *parent = NULL; 138 struct amdgpu_bo_va_mapping *iter_node; 139 140 while (*iter) { 141 parent = *iter; 142 iter_node = rb_entry(*iter, struct amdgpu_bo_va_mapping, rb); 143 144 if (node->start < iter_node->start) 145 iter = &(*iter)->rb_left; 146 else 147 iter = &(*iter)->rb_right; 148 } 149 150 rb_link_node(&node->rb, parent, iter); 151 rb_insert_color_cached(&node->rb, root, false); 152} 153#endif 154 155#undef START 156#undef LAST 157 158/** 159 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback 160 */ 161struct amdgpu_prt_cb { 162 163 /** 164 * @adev: amdgpu device 165 */ 166 struct amdgpu_device *adev; 167 168 /** 169 * @cb: callback 170 */ 171 struct dma_fence_cb cb; 172}; 173 174/** 175 * struct amdgpu_vm_tlb_seq_struct - Helper to increment the TLB flush sequence 176 */ 177struct amdgpu_vm_tlb_seq_struct { 178 /** 179 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on 180 */ 181 struct amdgpu_vm *vm; 182 183 /** 184 * @cb: callback 185 */ 186 struct dma_fence_cb cb; 187}; 188 189/** 190 * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping 191 * 192 * @adev: amdgpu_device pointer 193 * @vm: amdgpu_vm pointer 194 * @pasid: the pasid the VM is using on this GPU 195 * 196 * Set the pasid this VM is using on this GPU, can also be used to remove the 197 * pasid by passing in zero. 198 * 199 */ 200int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm, 201 u32 pasid) 202{ 203 int r; 204 205 if (vm->pasid == pasid) 206 return 0; 207 208 if (vm->pasid) { 209 r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid)); 210 if (r < 0) 211 return r; 212 213 vm->pasid = 0; 214 } 215 216 if (pasid) { 217 r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm, 218 GFP_KERNEL)); 219 if (r < 0) 220 return r; 221 222 vm->pasid = pasid; 223 } 224 225 226 return 0; 227} 228 229/** 230 * amdgpu_vm_bo_evicted - vm_bo is evicted 231 * 232 * @vm_bo: vm_bo which is evicted 233 * 234 * State for PDs/PTs and per VM BOs which are not at the location they should 235 * be. 236 */ 237static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) 238{ 239 struct amdgpu_vm *vm = vm_bo->vm; 240 struct amdgpu_bo *bo = vm_bo->bo; 241 242 vm_bo->moved = true; 243 spin_lock(&vm_bo->vm->status_lock); 244 if (bo->tbo.type == ttm_bo_type_kernel) 245 list_move(&vm_bo->vm_status, &vm->evicted); 246 else 247 list_move_tail(&vm_bo->vm_status, &vm->evicted); 248 spin_unlock(&vm_bo->vm->status_lock); 249} 250/** 251 * amdgpu_vm_bo_moved - vm_bo is moved 252 * 253 * @vm_bo: vm_bo which is moved 254 * 255 * State for per VM BOs which are moved, but that change is not yet reflected 256 * in the page tables. 257 */ 258static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) 259{ 260 spin_lock(&vm_bo->vm->status_lock); 261 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 262 spin_unlock(&vm_bo->vm->status_lock); 263} 264 265/** 266 * amdgpu_vm_bo_idle - vm_bo is idle 267 * 268 * @vm_bo: vm_bo which is now idle 269 * 270 * State for PDs/PTs and per VM BOs which have gone through the state machine 271 * and are now idle. 272 */ 273static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) 274{ 275 spin_lock(&vm_bo->vm->status_lock); 276 list_move(&vm_bo->vm_status, &vm_bo->vm->idle); 277 spin_unlock(&vm_bo->vm->status_lock); 278 vm_bo->moved = false; 279} 280 281/** 282 * amdgpu_vm_bo_invalidated - vm_bo is invalidated 283 * 284 * @vm_bo: vm_bo which is now invalidated 285 * 286 * State for normal BOs which are invalidated and that change not yet reflected 287 * in the PTs. 288 */ 289static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) 290{ 291 spin_lock(&vm_bo->vm->status_lock); 292 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); 293 spin_unlock(&vm_bo->vm->status_lock); 294} 295 296/** 297 * amdgpu_vm_bo_relocated - vm_bo is reloacted 298 * 299 * @vm_bo: vm_bo which is relocated 300 * 301 * State for PDs/PTs which needs to update their parent PD. 302 * For the root PD, just move to idle state. 303 */ 304static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) 305{ 306 if (vm_bo->bo->parent) { 307 spin_lock(&vm_bo->vm->status_lock); 308 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 309 spin_unlock(&vm_bo->vm->status_lock); 310 } else { 311 amdgpu_vm_bo_idle(vm_bo); 312 } 313} 314 315/** 316 * amdgpu_vm_bo_done - vm_bo is done 317 * 318 * @vm_bo: vm_bo which is now done 319 * 320 * State for normal BOs which are invalidated and that change has been updated 321 * in the PTs. 322 */ 323static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) 324{ 325 spin_lock(&vm_bo->vm->status_lock); 326 list_move(&vm_bo->vm_status, &vm_bo->vm->done); 327 spin_unlock(&vm_bo->vm->status_lock); 328} 329 330/** 331 * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine 332 * @vm: the VM which state machine to reset 333 * 334 * Move all vm_bo object in the VM into a state where they will be updated 335 * again during validation. 336 */ 337static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm) 338{ 339 struct amdgpu_vm_bo_base *vm_bo, *tmp; 340 341 spin_lock(&vm->status_lock); 342 list_splice_init(&vm->done, &vm->invalidated); 343 list_for_each_entry(vm_bo, &vm->invalidated, vm_status) 344 vm_bo->moved = true; 345 list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) { 346 struct amdgpu_bo *bo = vm_bo->bo; 347 348 vm_bo->moved = true; 349 if (!bo || bo->tbo.type != ttm_bo_type_kernel) 350 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 351 else if (bo->parent) 352 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 353 } 354 spin_unlock(&vm->status_lock); 355} 356 357/** 358 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm 359 * 360 * @base: base structure for tracking BO usage in a VM 361 * @vm: vm to which bo is to be added 362 * @bo: amdgpu buffer object 363 * 364 * Initialize a bo_va_base structure and add it to the appropriate lists 365 * 366 */ 367void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 368 struct amdgpu_vm *vm, struct amdgpu_bo *bo) 369{ 370 base->vm = vm; 371 base->bo = bo; 372 base->next = NULL; 373 INIT_LIST_HEAD(&base->vm_status); 374 375 if (!bo) 376 return; 377 base->next = bo->vm_bo; 378 bo->vm_bo = base; 379 380 if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv) 381 return; 382 383 dma_resv_assert_held(vm->root.bo->tbo.base.resv); 384 385 ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move); 386 if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) 387 amdgpu_vm_bo_relocated(base); 388 else 389 amdgpu_vm_bo_idle(base); 390 391 if (bo->preferred_domains & 392 amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) 393 return; 394 395 /* 396 * we checked all the prerequisites, but it looks like this per vm bo 397 * is currently evicted. add the bo to the evicted list to make sure it 398 * is validated on next vm use to avoid fault. 399 * */ 400 amdgpu_vm_bo_evicted(base); 401} 402 403/** 404 * amdgpu_vm_lock_pd - lock PD in drm_exec 405 * 406 * @vm: vm providing the BOs 407 * @exec: drm execution context 408 * @num_fences: number of extra fences to reserve 409 * 410 * Lock the VM root PD in the DRM execution context. 411 */ 412int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec, 413 unsigned int num_fences) 414{ 415 /* We need at least two fences for the VM PD/PT updates */ 416 return drm_exec_prepare_obj(exec, &vm->root.bo->tbo.base, 417 2 + num_fences); 418} 419 420/** 421 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU 422 * 423 * @adev: amdgpu device pointer 424 * @vm: vm providing the BOs 425 * 426 * Move all BOs to the end of LRU and remember their positions to put them 427 * together. 428 */ 429void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, 430 struct amdgpu_vm *vm) 431{ 432 spin_lock(&adev->mman.bdev.lru_lock); 433 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 434 spin_unlock(&adev->mman.bdev.lru_lock); 435} 436 437/* Create scheduler entities for page table updates */ 438static int amdgpu_vm_init_entities(struct amdgpu_device *adev, 439 struct amdgpu_vm *vm) 440{ 441 int r; 442 443 r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL, 444 adev->vm_manager.vm_pte_scheds, 445 adev->vm_manager.vm_pte_num_scheds, NULL); 446 if (r) 447 goto error; 448 449 return drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL, 450 adev->vm_manager.vm_pte_scheds, 451 adev->vm_manager.vm_pte_num_scheds, NULL); 452 453error: 454 drm_sched_entity_destroy(&vm->immediate); 455 return r; 456} 457 458/* Destroy the entities for page table updates again */ 459static void amdgpu_vm_fini_entities(struct amdgpu_vm *vm) 460{ 461 drm_sched_entity_destroy(&vm->immediate); 462 drm_sched_entity_destroy(&vm->delayed); 463} 464 465/** 466 * amdgpu_vm_generation - return the page table re-generation counter 467 * @adev: the amdgpu_device 468 * @vm: optional VM to check, might be NULL 469 * 470 * Returns a page table re-generation token to allow checking if submissions 471 * are still valid to use this VM. The VM parameter might be NULL in which case 472 * just the VRAM lost counter will be used. 473 */ 474uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm) 475{ 476 uint64_t result = (u64)atomic_read(&adev->vram_lost_counter) << 32; 477 478 if (!vm) 479 return result; 480 481 result += vm->generation; 482 /* Add one if the page tables will be re-generated on next CS */ 483 if (drm_sched_entity_error(&vm->delayed)) 484 ++result; 485 486 return result; 487} 488 489/** 490 * amdgpu_vm_validate_pt_bos - validate the page table BOs 491 * 492 * @adev: amdgpu device pointer 493 * @vm: vm providing the BOs 494 * @validate: callback to do the validation 495 * @param: parameter for the validation callback 496 * 497 * Validate the page table BOs on command submission if neccessary. 498 * 499 * Returns: 500 * Validation result. 501 */ 502int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, 503 int (*validate)(void *p, struct amdgpu_bo *bo), 504 void *param) 505{ 506 struct amdgpu_vm_bo_base *bo_base; 507 struct amdgpu_bo *shadow; 508 struct amdgpu_bo *bo; 509 int r; 510 511 if (drm_sched_entity_error(&vm->delayed)) { 512 ++vm->generation; 513 amdgpu_vm_bo_reset_state_machine(vm); 514 amdgpu_vm_fini_entities(vm); 515 r = amdgpu_vm_init_entities(adev, vm); 516 if (r) 517 return r; 518 } 519 520 spin_lock(&vm->status_lock); 521 while (!list_empty(&vm->evicted)) { 522 bo_base = list_first_entry(&vm->evicted, 523 struct amdgpu_vm_bo_base, 524 vm_status); 525 spin_unlock(&vm->status_lock); 526 527 bo = bo_base->bo; 528 shadow = amdgpu_bo_shadowed(bo); 529 530 r = validate(param, bo); 531 if (r) 532 return r; 533 if (shadow) { 534 r = validate(param, shadow); 535 if (r) 536 return r; 537 } 538 539 if (bo->tbo.type != ttm_bo_type_kernel) { 540 amdgpu_vm_bo_moved(bo_base); 541 } else { 542 vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); 543 amdgpu_vm_bo_relocated(bo_base); 544 } 545 spin_lock(&vm->status_lock); 546 } 547 spin_unlock(&vm->status_lock); 548 549 amdgpu_vm_eviction_lock(vm); 550 vm->evicting = false; 551 amdgpu_vm_eviction_unlock(vm); 552 553 return 0; 554} 555 556/** 557 * amdgpu_vm_ready - check VM is ready for updates 558 * 559 * @vm: VM to check 560 * 561 * Check if all VM PDs/PTs are ready for updates 562 * 563 * Returns: 564 * True if VM is not evicting. 565 */ 566bool amdgpu_vm_ready(struct amdgpu_vm *vm) 567{ 568 bool empty; 569 bool ret; 570 571 amdgpu_vm_eviction_lock(vm); 572 ret = !vm->evicting; 573 amdgpu_vm_eviction_unlock(vm); 574 575 spin_lock(&vm->status_lock); 576 empty = list_empty(&vm->evicted); 577 spin_unlock(&vm->status_lock); 578 579 return ret && empty; 580} 581 582/** 583 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug 584 * 585 * @adev: amdgpu_device pointer 586 */ 587void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) 588{ 589 const struct amdgpu_ip_block *ip_block; 590 bool has_compute_vm_bug; 591 struct amdgpu_ring *ring; 592 int i; 593 594 has_compute_vm_bug = false; 595 596 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 597 if (ip_block) { 598 /* Compute has a VM bug for GFX version < 7. 599 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ 600 if (ip_block->version->major <= 7) 601 has_compute_vm_bug = true; 602 else if (ip_block->version->major == 8) 603 if (adev->gfx.mec_fw_version < 673) 604 has_compute_vm_bug = true; 605 } 606 607 for (i = 0; i < adev->num_rings; i++) { 608 ring = adev->rings[i]; 609 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 610 /* only compute rings */ 611 ring->has_compute_vm_bug = has_compute_vm_bug; 612 else 613 ring->has_compute_vm_bug = false; 614 } 615} 616 617/** 618 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. 619 * 620 * @ring: ring on which the job will be submitted 621 * @job: job to submit 622 * 623 * Returns: 624 * True if sync is needed. 625 */ 626bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, 627 struct amdgpu_job *job) 628{ 629 struct amdgpu_device *adev = ring->adev; 630 unsigned vmhub = ring->vm_hub; 631 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 632 633 if (job->vmid == 0) 634 return false; 635 636 if (job->vm_needs_flush || ring->has_compute_vm_bug) 637 return true; 638 639 if (ring->funcs->emit_gds_switch && job->gds_switch_needed) 640 return true; 641 642 if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid])) 643 return true; 644 645 return false; 646} 647 648/** 649 * amdgpu_vm_flush - hardware flush the vm 650 * 651 * @ring: ring to use for flush 652 * @job: related job 653 * @need_pipe_sync: is pipe sync needed 654 * 655 * Emit a VM flush when it is necessary. 656 * 657 * Returns: 658 * 0 on success, errno otherwise. 659 */ 660int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, 661 bool need_pipe_sync) 662{ 663 struct amdgpu_device *adev = ring->adev; 664 unsigned vmhub = ring->vm_hub; 665 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 666 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; 667 bool spm_update_needed = job->spm_update_needed; 668 bool gds_switch_needed = ring->funcs->emit_gds_switch && 669 job->gds_switch_needed; 670 bool vm_flush_needed = job->vm_needs_flush; 671 struct dma_fence *fence = NULL; 672 bool pasid_mapping_needed = false; 673 unsigned patch_offset = 0; 674 int r; 675 676 if (amdgpu_vmid_had_gpu_reset(adev, id)) { 677 gds_switch_needed = true; 678 vm_flush_needed = true; 679 pasid_mapping_needed = true; 680 spm_update_needed = true; 681 } 682 683 mutex_lock(&id_mgr->lock); 684 if (id->pasid != job->pasid || !id->pasid_mapping || 685 !dma_fence_is_signaled(id->pasid_mapping)) 686 pasid_mapping_needed = true; 687 mutex_unlock(&id_mgr->lock); 688 689 gds_switch_needed &= !!ring->funcs->emit_gds_switch; 690 vm_flush_needed &= !!ring->funcs->emit_vm_flush && 691 job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET; 692 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && 693 ring->funcs->emit_wreg; 694 695 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync) 696 return 0; 697 698 amdgpu_ring_ib_begin(ring); 699 if (ring->funcs->init_cond_exec) 700 patch_offset = amdgpu_ring_init_cond_exec(ring); 701 702 if (need_pipe_sync) 703 amdgpu_ring_emit_pipeline_sync(ring); 704 705 if (vm_flush_needed) { 706 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr); 707 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr); 708 } 709 710 if (pasid_mapping_needed) 711 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); 712 713 if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid) 714 adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid); 715 716 if (!ring->is_mes_queue && ring->funcs->emit_gds_switch && 717 gds_switch_needed) { 718 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, 719 job->gds_size, job->gws_base, 720 job->gws_size, job->oa_base, 721 job->oa_size); 722 } 723 724 if (vm_flush_needed || pasid_mapping_needed) { 725 r = amdgpu_fence_emit(ring, &fence, NULL, 0); 726 if (r) 727 return r; 728 } 729 730 if (vm_flush_needed) { 731 mutex_lock(&id_mgr->lock); 732 dma_fence_put(id->last_flush); 733 id->last_flush = dma_fence_get(fence); 734 id->current_gpu_reset_count = 735 atomic_read(&adev->gpu_reset_counter); 736 mutex_unlock(&id_mgr->lock); 737 } 738 739 if (pasid_mapping_needed) { 740 mutex_lock(&id_mgr->lock); 741 id->pasid = job->pasid; 742 dma_fence_put(id->pasid_mapping); 743 id->pasid_mapping = dma_fence_get(fence); 744 mutex_unlock(&id_mgr->lock); 745 } 746 dma_fence_put(fence); 747 748 if (ring->funcs->patch_cond_exec) 749 amdgpu_ring_patch_cond_exec(ring, patch_offset); 750 751 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ 752 if (ring->funcs->emit_switch_buffer) { 753 amdgpu_ring_emit_switch_buffer(ring); 754 amdgpu_ring_emit_switch_buffer(ring); 755 } 756 amdgpu_ring_ib_end(ring); 757 return 0; 758} 759 760/** 761 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 762 * 763 * @vm: requested vm 764 * @bo: requested buffer object 765 * 766 * Find @bo inside the requested vm. 767 * Search inside the @bos vm list for the requested vm 768 * Returns the found bo_va or NULL if none is found 769 * 770 * Object has to be reserved! 771 * 772 * Returns: 773 * Found bo_va or NULL. 774 */ 775struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 776 struct amdgpu_bo *bo) 777{ 778 struct amdgpu_vm_bo_base *base; 779 780 for (base = bo->vm_bo; base; base = base->next) { 781 if (base->vm != vm) 782 continue; 783 784 return container_of(base, struct amdgpu_bo_va, base); 785 } 786 return NULL; 787} 788 789/** 790 * amdgpu_vm_map_gart - Resolve gart mapping of addr 791 * 792 * @pages_addr: optional DMA address to use for lookup 793 * @addr: the unmapped addr 794 * 795 * Look up the physical address of the page that the pte resolves 796 * to. 797 * 798 * Returns: 799 * The pointer for the page table entry. 800 */ 801uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) 802{ 803 uint64_t result; 804 805 /* page table offset */ 806 result = pages_addr[addr >> PAGE_SHIFT]; 807 808 /* in case cpu page size != gpu page size*/ 809 result |= addr & (~LINUX_PAGE_MASK); 810 811 result &= 0xFFFFFFFFFFFFF000ULL; 812 813 return result; 814} 815 816/** 817 * amdgpu_vm_update_pdes - make sure that all directories are valid 818 * 819 * @adev: amdgpu_device pointer 820 * @vm: requested vm 821 * @immediate: submit immediately to the paging queue 822 * 823 * Makes sure all directories are up to date. 824 * 825 * Returns: 826 * 0 for success, error for failure. 827 */ 828int amdgpu_vm_update_pdes(struct amdgpu_device *adev, 829 struct amdgpu_vm *vm, bool immediate) 830{ 831 struct amdgpu_vm_update_params params; 832 struct amdgpu_vm_bo_base *entry; 833 bool flush_tlb_needed = false; 834 DRM_LIST_HEAD(relocated); 835 int r, idx; 836 837 spin_lock(&vm->status_lock); 838 list_splice_init(&vm->relocated, &relocated); 839 spin_unlock(&vm->status_lock); 840 841 if (list_empty(&relocated)) 842 return 0; 843 844 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 845 return -ENODEV; 846 847 memset(¶ms, 0, sizeof(params)); 848 params.adev = adev; 849 params.vm = vm; 850 params.immediate = immediate; 851 852 r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); 853 if (r) 854 goto error; 855 856 list_for_each_entry(entry, &relocated, vm_status) { 857 /* vm_flush_needed after updating moved PDEs */ 858 flush_tlb_needed |= entry->moved; 859 860 r = amdgpu_vm_pde_update(¶ms, entry); 861 if (r) 862 goto error; 863 } 864 865 r = vm->update_funcs->commit(¶ms, &vm->last_update); 866 if (r) 867 goto error; 868 869 if (flush_tlb_needed) 870 atomic64_inc(&vm->tlb_seq); 871 872 while (!list_empty(&relocated)) { 873 entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base, 874 vm_status); 875 amdgpu_vm_bo_idle(entry); 876 } 877 878error: 879 drm_dev_exit(idx); 880 return r; 881} 882 883/** 884 * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence 885 * @fence: unused 886 * @cb: the callback structure 887 * 888 * Increments the tlb sequence to make sure that future CS execute a VM flush. 889 */ 890static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence, 891 struct dma_fence_cb *cb) 892{ 893 struct amdgpu_vm_tlb_seq_struct *tlb_cb; 894 895 tlb_cb = container_of(cb, typeof(*tlb_cb), cb); 896 atomic64_inc(&tlb_cb->vm->tlb_seq); 897 kfree(tlb_cb); 898} 899 900/** 901 * amdgpu_vm_update_range - update a range in the vm page table 902 * 903 * @adev: amdgpu_device pointer to use for commands 904 * @vm: the VM to update the range 905 * @immediate: immediate submission in a page fault 906 * @unlocked: unlocked invalidation during MM callback 907 * @flush_tlb: trigger tlb invalidation after update completed 908 * @resv: fences we need to sync to 909 * @start: start of mapped range 910 * @last: last mapped entry 911 * @flags: flags for the entries 912 * @offset: offset into nodes and pages_addr 913 * @vram_base: base for vram mappings 914 * @res: ttm_resource to map 915 * @pages_addr: DMA addresses to use for mapping 916 * @fence: optional resulting fence 917 * 918 * Fill in the page table entries between @start and @last. 919 * 920 * Returns: 921 * 0 for success, negative erro code for failure. 922 */ 923int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, 924 bool immediate, bool unlocked, bool flush_tlb, 925 struct dma_resv *resv, uint64_t start, uint64_t last, 926 uint64_t flags, uint64_t offset, uint64_t vram_base, 927 struct ttm_resource *res, dma_addr_t *pages_addr, 928 struct dma_fence **fence) 929{ 930 struct amdgpu_vm_update_params params; 931 struct amdgpu_vm_tlb_seq_struct *tlb_cb; 932 struct amdgpu_res_cursor cursor; 933 enum amdgpu_sync_mode sync_mode; 934 int r, idx; 935 936 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 937 return -ENODEV; 938 939 tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL); 940 if (!tlb_cb) { 941 r = -ENOMEM; 942 goto error_unlock; 943 } 944 945 /* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache, 946 * heavy-weight flush TLB unconditionally. 947 */ 948 flush_tlb |= adev->gmc.xgmi.num_physical_nodes && 949 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0); 950 951 /* 952 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB 953 */ 954 flush_tlb |= adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 0, 0); 955 956 memset(¶ms, 0, sizeof(params)); 957 params.adev = adev; 958 params.vm = vm; 959 params.immediate = immediate; 960 params.pages_addr = pages_addr; 961 params.unlocked = unlocked; 962 963 /* Implicitly sync to command submissions in the same VM before 964 * unmapping. Sync to moving fences before mapping. 965 */ 966 if (!(flags & AMDGPU_PTE_VALID)) 967 sync_mode = AMDGPU_SYNC_EQ_OWNER; 968 else 969 sync_mode = AMDGPU_SYNC_EXPLICIT; 970 971 amdgpu_vm_eviction_lock(vm); 972 if (vm->evicting) { 973 r = -EBUSY; 974 goto error_free; 975 } 976 977 if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) { 978 struct dma_fence *tmp = dma_fence_get_stub(); 979 980 amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true); 981 swap(vm->last_unlocked, tmp); 982 dma_fence_put(tmp); 983 } 984 985 r = vm->update_funcs->prepare(¶ms, resv, sync_mode); 986 if (r) 987 goto error_free; 988 989 amdgpu_res_first(pages_addr ? NULL : res, offset, 990 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor); 991 while (cursor.remaining) { 992 uint64_t tmp, num_entries, addr; 993 994 num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT; 995 if (pages_addr) { 996 bool contiguous = true; 997 998 if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) { 999 uint64_t pfn = cursor.start >> PAGE_SHIFT; 1000 uint64_t count; 1001 1002 contiguous = pages_addr[pfn + 1] == 1003 pages_addr[pfn] + PAGE_SIZE; 1004 1005 tmp = num_entries / 1006 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1007 for (count = 2; count < tmp; ++count) { 1008 uint64_t idx = pfn + count; 1009 1010 if (contiguous != (pages_addr[idx] == 1011 pages_addr[idx - 1] + PAGE_SIZE)) 1012 break; 1013 } 1014 if (!contiguous) 1015 count--; 1016 num_entries = count * 1017 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1018 } 1019 1020 if (!contiguous) { 1021 addr = cursor.start; 1022 params.pages_addr = pages_addr; 1023 } else { 1024 addr = pages_addr[cursor.start >> PAGE_SHIFT]; 1025 params.pages_addr = NULL; 1026 } 1027 1028 } else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) { 1029 addr = vram_base + cursor.start; 1030 } else { 1031 addr = 0; 1032 } 1033 1034 tmp = start + num_entries; 1035 r = amdgpu_vm_ptes_update(¶ms, start, tmp, addr, flags); 1036 if (r) 1037 goto error_free; 1038 1039 amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE); 1040 start = tmp; 1041 } 1042 1043 r = vm->update_funcs->commit(¶ms, fence); 1044 1045 if (flush_tlb || params.table_freed) { 1046 tlb_cb->vm = vm; 1047 if (fence && *fence && 1048 !dma_fence_add_callback(*fence, &tlb_cb->cb, 1049 amdgpu_vm_tlb_seq_cb)) { 1050 dma_fence_put(vm->last_tlb_flush); 1051 vm->last_tlb_flush = dma_fence_get(*fence); 1052 } else { 1053 amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb); 1054 } 1055 tlb_cb = NULL; 1056 } 1057 1058error_free: 1059 kfree(tlb_cb); 1060 1061error_unlock: 1062 amdgpu_vm_eviction_unlock(vm); 1063 drm_dev_exit(idx); 1064 return r; 1065} 1066 1067static void amdgpu_vm_bo_get_memory(struct amdgpu_bo_va *bo_va, 1068 struct amdgpu_mem_stats *stats) 1069{ 1070 struct amdgpu_vm *vm = bo_va->base.vm; 1071 struct amdgpu_bo *bo = bo_va->base.bo; 1072 1073 if (!bo) 1074 return; 1075 1076 /* 1077 * For now ignore BOs which are currently locked and potentially 1078 * changing their location. 1079 */ 1080 if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv && 1081 !dma_resv_trylock(bo->tbo.base.resv)) 1082 return; 1083 1084 amdgpu_bo_get_memory(bo, stats); 1085 if (bo->tbo.base.resv != vm->root.bo->tbo.base.resv) 1086 dma_resv_unlock(bo->tbo.base.resv); 1087} 1088 1089void amdgpu_vm_get_memory(struct amdgpu_vm *vm, 1090 struct amdgpu_mem_stats *stats) 1091{ 1092 struct amdgpu_bo_va *bo_va, *tmp; 1093 1094 spin_lock(&vm->status_lock); 1095 list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) 1096 amdgpu_vm_bo_get_memory(bo_va, stats); 1097 1098 list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) 1099 amdgpu_vm_bo_get_memory(bo_va, stats); 1100 1101 list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) 1102 amdgpu_vm_bo_get_memory(bo_va, stats); 1103 1104 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) 1105 amdgpu_vm_bo_get_memory(bo_va, stats); 1106 1107 list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) 1108 amdgpu_vm_bo_get_memory(bo_va, stats); 1109 1110 list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) 1111 amdgpu_vm_bo_get_memory(bo_va, stats); 1112 spin_unlock(&vm->status_lock); 1113} 1114 1115/** 1116 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 1117 * 1118 * @adev: amdgpu_device pointer 1119 * @bo_va: requested BO and VM object 1120 * @clear: if true clear the entries 1121 * 1122 * Fill in the page table entries for @bo_va. 1123 * 1124 * Returns: 1125 * 0 for success, -EINVAL for failure. 1126 */ 1127int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, 1128 bool clear) 1129{ 1130 struct amdgpu_bo *bo = bo_va->base.bo; 1131 struct amdgpu_vm *vm = bo_va->base.vm; 1132 struct amdgpu_bo_va_mapping *mapping; 1133 dma_addr_t *pages_addr = NULL; 1134 struct ttm_resource *mem; 1135 struct dma_fence **last_update; 1136 bool flush_tlb = clear; 1137 struct dma_resv *resv; 1138 uint64_t vram_base; 1139 uint64_t flags; 1140 int r; 1141 1142 if (clear || !bo) { 1143 mem = NULL; 1144 resv = vm->root.bo->tbo.base.resv; 1145 } else { 1146 struct drm_gem_object *obj = &bo->tbo.base; 1147 1148 resv = bo->tbo.base.resv; 1149#ifdef notyet 1150 if (obj->import_attach && bo_va->is_xgmi) { 1151 struct dma_buf *dma_buf = obj->import_attach->dmabuf; 1152 struct drm_gem_object *gobj = dma_buf->priv; 1153 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); 1154 1155 if (abo->tbo.resource && 1156 abo->tbo.resource->mem_type == TTM_PL_VRAM) 1157 bo = gem_to_amdgpu_bo(gobj); 1158 } 1159#endif 1160 mem = bo->tbo.resource; 1161 if (mem && (mem->mem_type == TTM_PL_TT || 1162 mem->mem_type == AMDGPU_PL_PREEMPT)) 1163 pages_addr = bo->tbo.ttm->dma_address; 1164 } 1165 1166 if (bo) { 1167 struct amdgpu_device *bo_adev; 1168 1169 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem); 1170 1171 if (amdgpu_bo_encrypted(bo)) 1172 flags |= AMDGPU_PTE_TMZ; 1173 1174 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 1175 vram_base = bo_adev->vm_manager.vram_base_offset; 1176 } else { 1177 flags = 0x0; 1178 vram_base = 0; 1179 } 1180 1181 if (clear || (bo && bo->tbo.base.resv == 1182 vm->root.bo->tbo.base.resv)) 1183 last_update = &vm->last_update; 1184 else 1185 last_update = &bo_va->last_pt_update; 1186 1187 if (!clear && bo_va->base.moved) { 1188 flush_tlb = true; 1189 list_splice_init(&bo_va->valids, &bo_va->invalids); 1190 1191 } else if (bo_va->cleared != clear) { 1192 list_splice_init(&bo_va->valids, &bo_va->invalids); 1193 } 1194 1195 list_for_each_entry(mapping, &bo_va->invalids, list) { 1196 uint64_t update_flags = flags; 1197 1198 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 1199 * but in case of something, we filter the flags in first place 1200 */ 1201 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 1202 update_flags &= ~AMDGPU_PTE_READABLE; 1203 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 1204 update_flags &= ~AMDGPU_PTE_WRITEABLE; 1205 1206 /* Apply ASIC specific mapping flags */ 1207 amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags); 1208 1209 trace_amdgpu_vm_bo_update(mapping); 1210 1211 r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb, 1212 resv, mapping->start, mapping->last, 1213 update_flags, mapping->offset, 1214 vram_base, mem, pages_addr, 1215 last_update); 1216 if (r) 1217 return r; 1218 } 1219 1220 /* If the BO is not in its preferred location add it back to 1221 * the evicted list so that it gets validated again on the 1222 * next command submission. 1223 */ 1224 if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) { 1225 uint32_t mem_type = bo->tbo.resource->mem_type; 1226 1227 if (!(bo->preferred_domains & 1228 amdgpu_mem_type_to_domain(mem_type))) 1229 amdgpu_vm_bo_evicted(&bo_va->base); 1230 else 1231 amdgpu_vm_bo_idle(&bo_va->base); 1232 } else { 1233 amdgpu_vm_bo_done(&bo_va->base); 1234 } 1235 1236 list_splice_init(&bo_va->invalids, &bo_va->valids); 1237 bo_va->cleared = clear; 1238 bo_va->base.moved = false; 1239 1240 if (trace_amdgpu_vm_bo_mapping_enabled()) { 1241 list_for_each_entry(mapping, &bo_va->valids, list) 1242 trace_amdgpu_vm_bo_mapping(mapping); 1243 } 1244 1245 return 0; 1246} 1247 1248/** 1249 * amdgpu_vm_update_prt_state - update the global PRT state 1250 * 1251 * @adev: amdgpu_device pointer 1252 */ 1253static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev) 1254{ 1255 unsigned long flags; 1256 bool enable; 1257 1258 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags); 1259 enable = !!atomic_read(&adev->vm_manager.num_prt_users); 1260 adev->gmc.gmc_funcs->set_prt(adev, enable); 1261 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags); 1262} 1263 1264/** 1265 * amdgpu_vm_prt_get - add a PRT user 1266 * 1267 * @adev: amdgpu_device pointer 1268 */ 1269static void amdgpu_vm_prt_get(struct amdgpu_device *adev) 1270{ 1271 if (!adev->gmc.gmc_funcs->set_prt) 1272 return; 1273 1274 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1) 1275 amdgpu_vm_update_prt_state(adev); 1276} 1277 1278/** 1279 * amdgpu_vm_prt_put - drop a PRT user 1280 * 1281 * @adev: amdgpu_device pointer 1282 */ 1283static void amdgpu_vm_prt_put(struct amdgpu_device *adev) 1284{ 1285 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0) 1286 amdgpu_vm_update_prt_state(adev); 1287} 1288 1289/** 1290 * amdgpu_vm_prt_cb - callback for updating the PRT status 1291 * 1292 * @fence: fence for the callback 1293 * @_cb: the callback function 1294 */ 1295static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb) 1296{ 1297 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb); 1298 1299 amdgpu_vm_prt_put(cb->adev); 1300 kfree(cb); 1301} 1302 1303/** 1304 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status 1305 * 1306 * @adev: amdgpu_device pointer 1307 * @fence: fence for the callback 1308 */ 1309static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev, 1310 struct dma_fence *fence) 1311{ 1312 struct amdgpu_prt_cb *cb; 1313 1314 if (!adev->gmc.gmc_funcs->set_prt) 1315 return; 1316 1317 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL); 1318 if (!cb) { 1319 /* Last resort when we are OOM */ 1320 if (fence) 1321 dma_fence_wait(fence, false); 1322 1323 amdgpu_vm_prt_put(adev); 1324 } else { 1325 cb->adev = adev; 1326 if (!fence || dma_fence_add_callback(fence, &cb->cb, 1327 amdgpu_vm_prt_cb)) 1328 amdgpu_vm_prt_cb(fence, &cb->cb); 1329 } 1330} 1331 1332/** 1333 * amdgpu_vm_free_mapping - free a mapping 1334 * 1335 * @adev: amdgpu_device pointer 1336 * @vm: requested vm 1337 * @mapping: mapping to be freed 1338 * @fence: fence of the unmap operation 1339 * 1340 * Free a mapping and make sure we decrease the PRT usage count if applicable. 1341 */ 1342static void amdgpu_vm_free_mapping(struct amdgpu_device *adev, 1343 struct amdgpu_vm *vm, 1344 struct amdgpu_bo_va_mapping *mapping, 1345 struct dma_fence *fence) 1346{ 1347 if (mapping->flags & AMDGPU_PTE_PRT) 1348 amdgpu_vm_add_prt_cb(adev, fence); 1349 kfree(mapping); 1350} 1351 1352/** 1353 * amdgpu_vm_prt_fini - finish all prt mappings 1354 * 1355 * @adev: amdgpu_device pointer 1356 * @vm: requested vm 1357 * 1358 * Register a cleanup callback to disable PRT support after VM dies. 1359 */ 1360static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1361{ 1362 struct dma_resv *resv = vm->root.bo->tbo.base.resv; 1363 struct dma_resv_iter cursor; 1364 struct dma_fence *fence; 1365 1366 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) { 1367 /* Add a callback for each fence in the reservation object */ 1368 amdgpu_vm_prt_get(adev); 1369 amdgpu_vm_add_prt_cb(adev, fence); 1370 } 1371} 1372 1373/** 1374 * amdgpu_vm_clear_freed - clear freed BOs in the PT 1375 * 1376 * @adev: amdgpu_device pointer 1377 * @vm: requested vm 1378 * @fence: optional resulting fence (unchanged if no work needed to be done 1379 * or if an error occurred) 1380 * 1381 * Make sure all freed BOs are cleared in the PT. 1382 * PTs have to be reserved and mutex must be locked! 1383 * 1384 * Returns: 1385 * 0 for success. 1386 * 1387 */ 1388int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 1389 struct amdgpu_vm *vm, 1390 struct dma_fence **fence) 1391{ 1392 struct dma_resv *resv = vm->root.bo->tbo.base.resv; 1393 struct amdgpu_bo_va_mapping *mapping; 1394 uint64_t init_pte_value = 0; 1395 struct dma_fence *f = NULL; 1396 int r; 1397 1398 while (!list_empty(&vm->freed)) { 1399 mapping = list_first_entry(&vm->freed, 1400 struct amdgpu_bo_va_mapping, list); 1401 list_del(&mapping->list); 1402 1403 if (vm->pte_support_ats && 1404 mapping->start < AMDGPU_GMC_HOLE_START) 1405 init_pte_value = AMDGPU_PTE_DEFAULT_ATC; 1406 1407 r = amdgpu_vm_update_range(adev, vm, false, false, true, resv, 1408 mapping->start, mapping->last, 1409 init_pte_value, 0, 0, NULL, NULL, 1410 &f); 1411 amdgpu_vm_free_mapping(adev, vm, mapping, f); 1412 if (r) { 1413 dma_fence_put(f); 1414 return r; 1415 } 1416 } 1417 1418 if (fence && f) { 1419 dma_fence_put(*fence); 1420 *fence = f; 1421 } else { 1422 dma_fence_put(f); 1423 } 1424 1425 return 0; 1426 1427} 1428 1429/** 1430 * amdgpu_vm_handle_moved - handle moved BOs in the PT 1431 * 1432 * @adev: amdgpu_device pointer 1433 * @vm: requested vm 1434 * 1435 * Make sure all BOs which are moved are updated in the PTs. 1436 * 1437 * Returns: 1438 * 0 for success. 1439 * 1440 * PTs have to be reserved! 1441 */ 1442int amdgpu_vm_handle_moved(struct amdgpu_device *adev, 1443 struct amdgpu_vm *vm) 1444{ 1445 struct amdgpu_bo_va *bo_va; 1446 struct dma_resv *resv; 1447 bool clear; 1448 int r; 1449 1450 spin_lock(&vm->status_lock); 1451 while (!list_empty(&vm->moved)) { 1452 bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, 1453 base.vm_status); 1454 spin_unlock(&vm->status_lock); 1455 1456 /* Per VM BOs never need to bo cleared in the page tables */ 1457 r = amdgpu_vm_bo_update(adev, bo_va, false); 1458 if (r) 1459 return r; 1460 spin_lock(&vm->status_lock); 1461 } 1462 1463 while (!list_empty(&vm->invalidated)) { 1464 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, 1465 base.vm_status); 1466 resv = bo_va->base.bo->tbo.base.resv; 1467 spin_unlock(&vm->status_lock); 1468 1469 /* Try to reserve the BO to avoid clearing its ptes */ 1470 if (!amdgpu_vm_debug && dma_resv_trylock(resv)) 1471 clear = false; 1472 /* Somebody else is using the BO right now */ 1473 else 1474 clear = true; 1475 1476 r = amdgpu_vm_bo_update(adev, bo_va, clear); 1477 if (r) 1478 return r; 1479 1480 if (!clear) 1481 dma_resv_unlock(resv); 1482 spin_lock(&vm->status_lock); 1483 } 1484 spin_unlock(&vm->status_lock); 1485 1486 return 0; 1487} 1488 1489/** 1490 * amdgpu_vm_bo_add - add a bo to a specific vm 1491 * 1492 * @adev: amdgpu_device pointer 1493 * @vm: requested vm 1494 * @bo: amdgpu buffer object 1495 * 1496 * Add @bo into the requested vm. 1497 * Add @bo to the list of bos associated with the vm 1498 * 1499 * Returns: 1500 * Newly added bo_va or NULL for failure 1501 * 1502 * Object has to be reserved! 1503 */ 1504struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 1505 struct amdgpu_vm *vm, 1506 struct amdgpu_bo *bo) 1507{ 1508 struct amdgpu_bo_va *bo_va; 1509 1510 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 1511 if (bo_va == NULL) { 1512 return NULL; 1513 } 1514 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); 1515 1516 bo_va->ref_count = 1; 1517 bo_va->last_pt_update = dma_fence_get_stub(); 1518 INIT_LIST_HEAD(&bo_va->valids); 1519 INIT_LIST_HEAD(&bo_va->invalids); 1520 1521 if (!bo) 1522 return bo_va; 1523 1524 dma_resv_assert_held(bo->tbo.base.resv); 1525 if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) { 1526 bo_va->is_xgmi = true; 1527 /* Power up XGMI if it can be potentially used */ 1528 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20); 1529 } 1530 1531 return bo_va; 1532} 1533 1534 1535/** 1536 * amdgpu_vm_bo_insert_map - insert a new mapping 1537 * 1538 * @adev: amdgpu_device pointer 1539 * @bo_va: bo_va to store the address 1540 * @mapping: the mapping to insert 1541 * 1542 * Insert a new mapping into all structures. 1543 */ 1544static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, 1545 struct amdgpu_bo_va *bo_va, 1546 struct amdgpu_bo_va_mapping *mapping) 1547{ 1548 struct amdgpu_vm *vm = bo_va->base.vm; 1549 struct amdgpu_bo *bo = bo_va->base.bo; 1550 1551 mapping->bo_va = bo_va; 1552 list_add(&mapping->list, &bo_va->invalids); 1553 amdgpu_vm_it_insert(mapping, &vm->va); 1554 1555 if (mapping->flags & AMDGPU_PTE_PRT) 1556 amdgpu_vm_prt_get(adev); 1557 1558 if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv && 1559 !bo_va->base.moved) { 1560 amdgpu_vm_bo_moved(&bo_va->base); 1561 } 1562 trace_amdgpu_vm_bo_map(bo_va, mapping); 1563} 1564 1565/* Validate operation parameters to prevent potential abuse */ 1566static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev, 1567 struct amdgpu_bo *bo, 1568 uint64_t saddr, 1569 uint64_t offset, 1570 uint64_t size) 1571{ 1572 uint64_t tmp, lpfn; 1573 1574 if (saddr & AMDGPU_GPU_PAGE_MASK 1575 || offset & AMDGPU_GPU_PAGE_MASK 1576 || size & AMDGPU_GPU_PAGE_MASK) 1577 return -EINVAL; 1578 1579 if (check_add_overflow(saddr, size, &tmp) 1580 || check_add_overflow(offset, size, &tmp) 1581 || size == 0 /* which also leads to end < begin */) 1582 return -EINVAL; 1583 1584 /* make sure object fit at this offset */ 1585 if (bo && offset + size > amdgpu_bo_size(bo)) 1586 return -EINVAL; 1587 1588 /* Ensure last pfn not exceed max_pfn */ 1589 lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT; 1590 if (lpfn >= adev->vm_manager.max_pfn) 1591 return -EINVAL; 1592 1593 return 0; 1594} 1595 1596/** 1597 * amdgpu_vm_bo_map - map bo inside a vm 1598 * 1599 * @adev: amdgpu_device pointer 1600 * @bo_va: bo_va to store the address 1601 * @saddr: where to map the BO 1602 * @offset: requested offset in the BO 1603 * @size: BO size in bytes 1604 * @flags: attributes of pages (read/write/valid/etc.) 1605 * 1606 * Add a mapping of the BO at the specefied addr into the VM. 1607 * 1608 * Returns: 1609 * 0 for success, error for failure. 1610 * 1611 * Object has to be reserved and unreserved outside! 1612 */ 1613int amdgpu_vm_bo_map(struct amdgpu_device *adev, 1614 struct amdgpu_bo_va *bo_va, 1615 uint64_t saddr, uint64_t offset, 1616 uint64_t size, uint64_t flags) 1617{ 1618 struct amdgpu_bo_va_mapping *mapping, *tmp; 1619 struct amdgpu_bo *bo = bo_va->base.bo; 1620 struct amdgpu_vm *vm = bo_va->base.vm; 1621 uint64_t eaddr; 1622 int r; 1623 1624 r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size); 1625 if (r) 1626 return r; 1627 1628 saddr /= AMDGPU_GPU_PAGE_SIZE; 1629 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE; 1630 1631 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 1632 if (tmp) { 1633 /* bo and tmp overlap, invalid addr */ 1634 dev_err(adev->dev, "bo %p va 0x%010llx-0x%010llx conflict with " 1635 "0x%010llx-0x%010llx\n", bo, saddr, eaddr, 1636 tmp->start, tmp->last + 1); 1637 return -EINVAL; 1638 } 1639 1640 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1641 if (!mapping) 1642 return -ENOMEM; 1643 1644 mapping->start = saddr; 1645 mapping->last = eaddr; 1646 mapping->offset = offset; 1647 mapping->flags = flags; 1648 1649 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 1650 1651 return 0; 1652} 1653 1654/** 1655 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings 1656 * 1657 * @adev: amdgpu_device pointer 1658 * @bo_va: bo_va to store the address 1659 * @saddr: where to map the BO 1660 * @offset: requested offset in the BO 1661 * @size: BO size in bytes 1662 * @flags: attributes of pages (read/write/valid/etc.) 1663 * 1664 * Add a mapping of the BO at the specefied addr into the VM. Replace existing 1665 * mappings as we do so. 1666 * 1667 * Returns: 1668 * 0 for success, error for failure. 1669 * 1670 * Object has to be reserved and unreserved outside! 1671 */ 1672int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, 1673 struct amdgpu_bo_va *bo_va, 1674 uint64_t saddr, uint64_t offset, 1675 uint64_t size, uint64_t flags) 1676{ 1677 struct amdgpu_bo_va_mapping *mapping; 1678 struct amdgpu_bo *bo = bo_va->base.bo; 1679 uint64_t eaddr; 1680 int r; 1681 1682 r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size); 1683 if (r) 1684 return r; 1685 1686 /* Allocate all the needed memory */ 1687 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1688 if (!mapping) 1689 return -ENOMEM; 1690 1691 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); 1692 if (r) { 1693 kfree(mapping); 1694 return r; 1695 } 1696 1697 saddr /= AMDGPU_GPU_PAGE_SIZE; 1698 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE; 1699 1700 mapping->start = saddr; 1701 mapping->last = eaddr; 1702 mapping->offset = offset; 1703 mapping->flags = flags; 1704 1705 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 1706 1707 return 0; 1708} 1709 1710/** 1711 * amdgpu_vm_bo_unmap - remove bo mapping from vm 1712 * 1713 * @adev: amdgpu_device pointer 1714 * @bo_va: bo_va to remove the address from 1715 * @saddr: where to the BO is mapped 1716 * 1717 * Remove a mapping of the BO at the specefied addr from the VM. 1718 * 1719 * Returns: 1720 * 0 for success, error for failure. 1721 * 1722 * Object has to be reserved and unreserved outside! 1723 */ 1724int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 1725 struct amdgpu_bo_va *bo_va, 1726 uint64_t saddr) 1727{ 1728 struct amdgpu_bo_va_mapping *mapping; 1729 struct amdgpu_vm *vm = bo_va->base.vm; 1730 bool valid = true; 1731 1732 saddr /= AMDGPU_GPU_PAGE_SIZE; 1733 1734 list_for_each_entry(mapping, &bo_va->valids, list) { 1735 if (mapping->start == saddr) 1736 break; 1737 } 1738 1739 if (&mapping->list == &bo_va->valids) { 1740 valid = false; 1741 1742 list_for_each_entry(mapping, &bo_va->invalids, list) { 1743 if (mapping->start == saddr) 1744 break; 1745 } 1746 1747 if (&mapping->list == &bo_va->invalids) 1748 return -ENOENT; 1749 } 1750 1751 list_del(&mapping->list); 1752 amdgpu_vm_it_remove(mapping, &vm->va); 1753 mapping->bo_va = NULL; 1754 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1755 1756 if (valid) 1757 list_add(&mapping->list, &vm->freed); 1758 else 1759 amdgpu_vm_free_mapping(adev, vm, mapping, 1760 bo_va->last_pt_update); 1761 1762 return 0; 1763} 1764 1765/** 1766 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range 1767 * 1768 * @adev: amdgpu_device pointer 1769 * @vm: VM structure to use 1770 * @saddr: start of the range 1771 * @size: size of the range 1772 * 1773 * Remove all mappings in a range, split them as appropriate. 1774 * 1775 * Returns: 1776 * 0 for success, error for failure. 1777 */ 1778int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, 1779 struct amdgpu_vm *vm, 1780 uint64_t saddr, uint64_t size) 1781{ 1782 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; 1783 DRM_LIST_HEAD(removed); 1784 uint64_t eaddr; 1785 int r; 1786 1787 r = amdgpu_vm_verify_parameters(adev, NULL, saddr, 0, size); 1788 if (r) 1789 return r; 1790 1791 saddr /= AMDGPU_GPU_PAGE_SIZE; 1792 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE; 1793 1794 /* Allocate all the needed memory */ 1795 before = kzalloc(sizeof(*before), GFP_KERNEL); 1796 if (!before) 1797 return -ENOMEM; 1798 INIT_LIST_HEAD(&before->list); 1799 1800 after = kzalloc(sizeof(*after), GFP_KERNEL); 1801 if (!after) { 1802 kfree(before); 1803 return -ENOMEM; 1804 } 1805 INIT_LIST_HEAD(&after->list); 1806 1807 /* Now gather all removed mappings */ 1808 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 1809 while (tmp) { 1810 /* Remember mapping split at the start */ 1811 if (tmp->start < saddr) { 1812 before->start = tmp->start; 1813 before->last = saddr - 1; 1814 before->offset = tmp->offset; 1815 before->flags = tmp->flags; 1816 before->bo_va = tmp->bo_va; 1817 list_add(&before->list, &tmp->bo_va->invalids); 1818 } 1819 1820 /* Remember mapping split at the end */ 1821 if (tmp->last > eaddr) { 1822 after->start = eaddr + 1; 1823 after->last = tmp->last; 1824 after->offset = tmp->offset; 1825 after->offset += (after->start - tmp->start) << PAGE_SHIFT; 1826 after->flags = tmp->flags; 1827 after->bo_va = tmp->bo_va; 1828 list_add(&after->list, &tmp->bo_va->invalids); 1829 } 1830 1831 list_del(&tmp->list); 1832 list_add(&tmp->list, &removed); 1833 1834 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr); 1835 } 1836 1837 /* And free them up */ 1838 list_for_each_entry_safe(tmp, next, &removed, list) { 1839 amdgpu_vm_it_remove(tmp, &vm->va); 1840 list_del(&tmp->list); 1841 1842 if (tmp->start < saddr) 1843 tmp->start = saddr; 1844 if (tmp->last > eaddr) 1845 tmp->last = eaddr; 1846 1847 tmp->bo_va = NULL; 1848 list_add(&tmp->list, &vm->freed); 1849 trace_amdgpu_vm_bo_unmap(NULL, tmp); 1850 } 1851 1852 /* Insert partial mapping before the range */ 1853 if (!list_empty(&before->list)) { 1854 struct amdgpu_bo *bo = before->bo_va->base.bo; 1855 1856 amdgpu_vm_it_insert(before, &vm->va); 1857 if (before->flags & AMDGPU_PTE_PRT) 1858 amdgpu_vm_prt_get(adev); 1859 1860 if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv && 1861 !before->bo_va->base.moved) 1862 amdgpu_vm_bo_moved(&before->bo_va->base); 1863 } else { 1864 kfree(before); 1865 } 1866 1867 /* Insert partial mapping after the range */ 1868 if (!list_empty(&after->list)) { 1869 struct amdgpu_bo *bo = after->bo_va->base.bo; 1870 1871 amdgpu_vm_it_insert(after, &vm->va); 1872 if (after->flags & AMDGPU_PTE_PRT) 1873 amdgpu_vm_prt_get(adev); 1874 1875 if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv && 1876 !after->bo_va->base.moved) 1877 amdgpu_vm_bo_moved(&after->bo_va->base); 1878 } else { 1879 kfree(after); 1880 } 1881 1882 return 0; 1883} 1884 1885/** 1886 * amdgpu_vm_bo_lookup_mapping - find mapping by address 1887 * 1888 * @vm: the requested VM 1889 * @addr: the address 1890 * 1891 * Find a mapping by it's address. 1892 * 1893 * Returns: 1894 * The amdgpu_bo_va_mapping matching for addr or NULL 1895 * 1896 */ 1897struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, 1898 uint64_t addr) 1899{ 1900 return amdgpu_vm_it_iter_first(&vm->va, addr, addr); 1901} 1902 1903/** 1904 * amdgpu_vm_bo_trace_cs - trace all reserved mappings 1905 * 1906 * @vm: the requested vm 1907 * @ticket: CS ticket 1908 * 1909 * Trace all mappings of BOs reserved during a command submission. 1910 */ 1911void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) 1912{ 1913 struct amdgpu_bo_va_mapping *mapping; 1914 1915 if (!trace_amdgpu_vm_bo_cs_enabled()) 1916 return; 1917 1918 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping; 1919 mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) { 1920 if (mapping->bo_va && mapping->bo_va->base.bo) { 1921 struct amdgpu_bo *bo; 1922 1923 bo = mapping->bo_va->base.bo; 1924 if (dma_resv_locking_ctx(bo->tbo.base.resv) != 1925 ticket) 1926 continue; 1927 } 1928 1929 trace_amdgpu_vm_bo_cs(mapping); 1930 } 1931} 1932 1933/** 1934 * amdgpu_vm_bo_del - remove a bo from a specific vm 1935 * 1936 * @adev: amdgpu_device pointer 1937 * @bo_va: requested bo_va 1938 * 1939 * Remove @bo_va->bo from the requested vm. 1940 * 1941 * Object have to be reserved! 1942 */ 1943void amdgpu_vm_bo_del(struct amdgpu_device *adev, 1944 struct amdgpu_bo_va *bo_va) 1945{ 1946 struct amdgpu_bo_va_mapping *mapping, *next; 1947 struct amdgpu_bo *bo = bo_va->base.bo; 1948 struct amdgpu_vm *vm = bo_va->base.vm; 1949 struct amdgpu_vm_bo_base **base; 1950 1951 dma_resv_assert_held(vm->root.bo->tbo.base.resv); 1952 1953 if (bo) { 1954 dma_resv_assert_held(bo->tbo.base.resv); 1955 if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv) 1956 ttm_bo_set_bulk_move(&bo->tbo, NULL); 1957 1958 for (base = &bo_va->base.bo->vm_bo; *base; 1959 base = &(*base)->next) { 1960 if (*base != &bo_va->base) 1961 continue; 1962 1963 *base = bo_va->base.next; 1964 break; 1965 } 1966 } 1967 1968 spin_lock(&vm->status_lock); 1969 list_del(&bo_va->base.vm_status); 1970 spin_unlock(&vm->status_lock); 1971 1972 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 1973 list_del(&mapping->list); 1974 amdgpu_vm_it_remove(mapping, &vm->va); 1975 mapping->bo_va = NULL; 1976 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1977 list_add(&mapping->list, &vm->freed); 1978 } 1979 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 1980 list_del(&mapping->list); 1981 amdgpu_vm_it_remove(mapping, &vm->va); 1982 amdgpu_vm_free_mapping(adev, vm, mapping, 1983 bo_va->last_pt_update); 1984 } 1985 1986 dma_fence_put(bo_va->last_pt_update); 1987 1988 if (bo && bo_va->is_xgmi) 1989 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN); 1990 1991 kfree(bo_va); 1992} 1993 1994/** 1995 * amdgpu_vm_evictable - check if we can evict a VM 1996 * 1997 * @bo: A page table of the VM. 1998 * 1999 * Check if it is possible to evict a VM. 2000 */ 2001bool amdgpu_vm_evictable(struct amdgpu_bo *bo) 2002{ 2003 struct amdgpu_vm_bo_base *bo_base = bo->vm_bo; 2004 2005 /* Page tables of a destroyed VM can go away immediately */ 2006 if (!bo_base || !bo_base->vm) 2007 return true; 2008 2009 /* Don't evict VM page tables while they are busy */ 2010 if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP)) 2011 return false; 2012 2013 /* Try to block ongoing updates */ 2014 if (!amdgpu_vm_eviction_trylock(bo_base->vm)) 2015 return false; 2016 2017 /* Don't evict VM page tables while they are updated */ 2018 if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) { 2019 amdgpu_vm_eviction_unlock(bo_base->vm); 2020 return false; 2021 } 2022 2023 bo_base->vm->evicting = true; 2024 amdgpu_vm_eviction_unlock(bo_base->vm); 2025 return true; 2026} 2027 2028/** 2029 * amdgpu_vm_bo_invalidate - mark the bo as invalid 2030 * 2031 * @adev: amdgpu_device pointer 2032 * @bo: amdgpu buffer object 2033 * @evicted: is the BO evicted 2034 * 2035 * Mark @bo as invalid. 2036 */ 2037void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 2038 struct amdgpu_bo *bo, bool evicted) 2039{ 2040 struct amdgpu_vm_bo_base *bo_base; 2041 2042 /* shadow bo doesn't have bo base, its validation needs its parent */ 2043 if (bo->parent && (amdgpu_bo_shadowed(bo->parent) == bo)) 2044 bo = bo->parent; 2045 2046 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 2047 struct amdgpu_vm *vm = bo_base->vm; 2048 2049 if (evicted && bo->tbo.base.resv == vm->root.bo->tbo.base.resv) { 2050 amdgpu_vm_bo_evicted(bo_base); 2051 continue; 2052 } 2053 2054 if (bo_base->moved) 2055 continue; 2056 bo_base->moved = true; 2057 2058 if (bo->tbo.type == ttm_bo_type_kernel) 2059 amdgpu_vm_bo_relocated(bo_base); 2060 else if (bo->tbo.base.resv == vm->root.bo->tbo.base.resv) 2061 amdgpu_vm_bo_moved(bo_base); 2062 else 2063 amdgpu_vm_bo_invalidated(bo_base); 2064 } 2065} 2066 2067/** 2068 * amdgpu_vm_get_block_size - calculate VM page table size as power of two 2069 * 2070 * @vm_size: VM size 2071 * 2072 * Returns: 2073 * VM page table as power of two 2074 */ 2075static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size) 2076{ 2077 /* Total bits covered by PD + PTs */ 2078 unsigned bits = ilog2(vm_size) + 18; 2079 2080 /* Make sure the PD is 4K in size up to 8GB address space. 2081 Above that split equal between PD and PTs */ 2082 if (vm_size <= 8) 2083 return (bits - 9); 2084 else 2085 return ((bits + 3) / 2); 2086} 2087 2088/** 2089 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size 2090 * 2091 * @adev: amdgpu_device pointer 2092 * @min_vm_size: the minimum vm size in GB if it's set auto 2093 * @fragment_size_default: Default PTE fragment size 2094 * @max_level: max VMPT level 2095 * @max_bits: max address space size in bits 2096 * 2097 */ 2098void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, 2099 uint32_t fragment_size_default, unsigned max_level, 2100 unsigned max_bits) 2101{ 2102 unsigned int max_size = 1 << (max_bits - 30); 2103 unsigned int vm_size; 2104 uint64_t tmp; 2105 2106 /* adjust vm size first */ 2107 if (amdgpu_vm_size != -1) { 2108 vm_size = amdgpu_vm_size; 2109 if (vm_size > max_size) { 2110 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", 2111 amdgpu_vm_size, max_size); 2112 vm_size = max_size; 2113 } 2114 } else { 2115#ifdef __linux__ 2116 struct sysinfo si; 2117#endif 2118 unsigned int phys_ram_gb; 2119 2120 /* Optimal VM size depends on the amount of physical 2121 * RAM available. Underlying requirements and 2122 * assumptions: 2123 * 2124 * - Need to map system memory and VRAM from all GPUs 2125 * - VRAM from other GPUs not known here 2126 * - Assume VRAM <= system memory 2127 * - On GFX8 and older, VM space can be segmented for 2128 * different MTYPEs 2129 * - Need to allow room for fragmentation, guard pages etc. 2130 * 2131 * This adds up to a rough guess of system memory x3. 2132 * Round up to power of two to maximize the available 2133 * VM size with the given page table size. 2134 */ 2135#ifdef __linux__ 2136 si_meminfo(&si); 2137 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit + 2138 (1 << 30) - 1) >> 30; 2139#else 2140 phys_ram_gb = ((uint64_t)ptoa(physmem) + 2141 (1 << 30) - 1) >> 30; 2142#endif 2143 vm_size = roundup_pow_of_two( 2144 min(max(phys_ram_gb * 3, min_vm_size), max_size)); 2145 } 2146 2147 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18; 2148 2149 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn); 2150 if (amdgpu_vm_block_size != -1) 2151 tmp >>= amdgpu_vm_block_size - 9; 2152 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; 2153 adev->vm_manager.num_level = min(max_level, (unsigned)tmp); 2154 switch (adev->vm_manager.num_level) { 2155 case 3: 2156 adev->vm_manager.root_level = AMDGPU_VM_PDB2; 2157 break; 2158 case 2: 2159 adev->vm_manager.root_level = AMDGPU_VM_PDB1; 2160 break; 2161 case 1: 2162 adev->vm_manager.root_level = AMDGPU_VM_PDB0; 2163 break; 2164 default: 2165 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n"); 2166 } 2167 /* block size depends on vm size and hw setup*/ 2168 if (amdgpu_vm_block_size != -1) 2169 adev->vm_manager.block_size = 2170 min((unsigned)amdgpu_vm_block_size, max_bits 2171 - AMDGPU_GPU_PAGE_SHIFT 2172 - 9 * adev->vm_manager.num_level); 2173 else if (adev->vm_manager.num_level > 1) 2174 adev->vm_manager.block_size = 9; 2175 else 2176 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp); 2177 2178 if (amdgpu_vm_fragment_size == -1) 2179 adev->vm_manager.fragment_size = fragment_size_default; 2180 else 2181 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size; 2182 2183 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", 2184 vm_size, adev->vm_manager.num_level + 1, 2185 adev->vm_manager.block_size, 2186 adev->vm_manager.fragment_size); 2187} 2188 2189/** 2190 * amdgpu_vm_wait_idle - wait for the VM to become idle 2191 * 2192 * @vm: VM object to wait for 2193 * @timeout: timeout to wait for VM to become idle 2194 */ 2195long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) 2196{ 2197 timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv, 2198 DMA_RESV_USAGE_BOOKKEEP, 2199 true, timeout); 2200 if (timeout <= 0) 2201 return timeout; 2202 2203 return dma_fence_wait_timeout(vm->last_unlocked, true, timeout); 2204} 2205 2206/** 2207 * amdgpu_vm_init - initialize a vm instance 2208 * 2209 * @adev: amdgpu_device pointer 2210 * @vm: requested vm 2211 * @xcp_id: GPU partition selection id 2212 * 2213 * Init @vm fields. 2214 * 2215 * Returns: 2216 * 0 for success, error for failure. 2217 */ 2218int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 2219 int32_t xcp_id) 2220{ 2221 struct amdgpu_bo *root_bo; 2222 struct amdgpu_bo_vm *root; 2223 int r, i; 2224 2225 vm->va = RB_ROOT_CACHED; 2226 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2227 vm->reserved_vmid[i] = NULL; 2228 INIT_LIST_HEAD(&vm->evicted); 2229 INIT_LIST_HEAD(&vm->relocated); 2230 INIT_LIST_HEAD(&vm->moved); 2231 INIT_LIST_HEAD(&vm->idle); 2232 INIT_LIST_HEAD(&vm->invalidated); 2233 mtx_init(&vm->status_lock, IPL_NONE); 2234 INIT_LIST_HEAD(&vm->freed); 2235 INIT_LIST_HEAD(&vm->done); 2236 INIT_LIST_HEAD(&vm->pt_freed); 2237 INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work); 2238#ifdef __linux__ 2239 INIT_KFIFO(vm->faults); 2240#else 2241 SIMPLEQ_INIT(&vm->faults); 2242#endif 2243 2244 r = amdgpu_vm_init_entities(adev, vm); 2245 if (r) 2246 return r; 2247 2248 vm->pte_support_ats = false; 2249 vm->is_compute_context = false; 2250 2251 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2252 AMDGPU_VM_USE_CPU_FOR_GFX); 2253 2254 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2255 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2256 WARN_ONCE((vm->use_cpu_for_update && 2257 !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2258 "CPU update of VM recommended only for large BAR system\n"); 2259 2260 if (vm->use_cpu_for_update) 2261 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2262 else 2263 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2264 2265 vm->last_update = dma_fence_get_stub(); 2266 vm->last_unlocked = dma_fence_get_stub(); 2267 vm->last_tlb_flush = dma_fence_get_stub(); 2268 vm->generation = 0; 2269 2270 rw_init(&vm->eviction_lock, "avmev"); 2271 vm->evicting = false; 2272 2273 r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level, 2274 false, &root, xcp_id); 2275 if (r) 2276 goto error_free_delayed; 2277 2278 root_bo = amdgpu_bo_ref(&root->bo); 2279 r = amdgpu_bo_reserve(root_bo, true); 2280 if (r) { 2281 amdgpu_bo_unref(&root->shadow); 2282 amdgpu_bo_unref(&root_bo); 2283 goto error_free_delayed; 2284 } 2285 2286 amdgpu_vm_bo_base_init(&vm->root, vm, root_bo); 2287 r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1); 2288 if (r) 2289 goto error_free_root; 2290 2291 r = amdgpu_vm_pt_clear(adev, vm, root, false); 2292 if (r) 2293 goto error_free_root; 2294 2295 amdgpu_bo_unreserve(vm->root.bo); 2296 amdgpu_bo_unref(&root_bo); 2297 2298 return 0; 2299 2300error_free_root: 2301 amdgpu_vm_pt_free_root(adev, vm); 2302 amdgpu_bo_unreserve(vm->root.bo); 2303 amdgpu_bo_unref(&root_bo); 2304 2305error_free_delayed: 2306 dma_fence_put(vm->last_tlb_flush); 2307 dma_fence_put(vm->last_unlocked); 2308 amdgpu_vm_fini_entities(vm); 2309 2310 return r; 2311} 2312 2313/** 2314 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM 2315 * 2316 * @adev: amdgpu_device pointer 2317 * @vm: requested vm 2318 * 2319 * This only works on GFX VMs that don't have any BOs added and no 2320 * page tables allocated yet. 2321 * 2322 * Changes the following VM parameters: 2323 * - use_cpu_for_update 2324 * - pte_supports_ats 2325 * 2326 * Reinitializes the page directory to reflect the changed ATS 2327 * setting. 2328 * 2329 * Returns: 2330 * 0 for success, -errno for errors. 2331 */ 2332int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2333{ 2334 bool pte_support_ats = (adev->asic_type == CHIP_RAVEN); 2335 int r; 2336 2337 r = amdgpu_bo_reserve(vm->root.bo, true); 2338 if (r) 2339 return r; 2340 2341 /* Check if PD needs to be reinitialized and do it before 2342 * changing any other state, in case it fails. 2343 */ 2344 if (pte_support_ats != vm->pte_support_ats) { 2345 /* Sanity checks */ 2346 if (!amdgpu_vm_pt_is_root_clean(adev, vm)) { 2347 r = -EINVAL; 2348 goto unreserve_bo; 2349 } 2350 2351 vm->pte_support_ats = pte_support_ats; 2352 r = amdgpu_vm_pt_clear(adev, vm, to_amdgpu_bo_vm(vm->root.bo), 2353 false); 2354 if (r) 2355 goto unreserve_bo; 2356 } 2357 2358 /* Update VM state */ 2359 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2360 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2361 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2362 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2363 WARN_ONCE((vm->use_cpu_for_update && 2364 !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2365 "CPU update of VM recommended only for large BAR system\n"); 2366 2367 if (vm->use_cpu_for_update) { 2368 /* Sync with last SDMA update/clear before switching to CPU */ 2369 r = amdgpu_bo_sync_wait(vm->root.bo, 2370 AMDGPU_FENCE_OWNER_UNDEFINED, true); 2371 if (r) 2372 goto unreserve_bo; 2373 2374 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2375 r = amdgpu_vm_pt_map_tables(adev, vm); 2376 if (r) 2377 goto unreserve_bo; 2378 2379 } else { 2380 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2381 } 2382 2383 dma_fence_put(vm->last_update); 2384 vm->last_update = dma_fence_get_stub(); 2385 vm->is_compute_context = true; 2386 2387 /* Free the shadow bo for compute VM */ 2388 amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.bo)->shadow); 2389 2390 goto unreserve_bo; 2391 2392unreserve_bo: 2393 amdgpu_bo_unreserve(vm->root.bo); 2394 return r; 2395} 2396 2397/** 2398 * amdgpu_vm_release_compute - release a compute vm 2399 * @adev: amdgpu_device pointer 2400 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute 2401 * 2402 * This is a correspondant of amdgpu_vm_make_compute. It decouples compute 2403 * pasid from vm. Compute should stop use of vm after this call. 2404 */ 2405void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2406{ 2407 amdgpu_vm_set_pasid(adev, vm, 0); 2408 vm->is_compute_context = false; 2409} 2410 2411/** 2412 * amdgpu_vm_fini - tear down a vm instance 2413 * 2414 * @adev: amdgpu_device pointer 2415 * @vm: requested vm 2416 * 2417 * Tear down @vm. 2418 * Unbind the VM and remove all bos from the vm bo list 2419 */ 2420void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2421{ 2422 struct amdgpu_bo_va_mapping *mapping, *tmp; 2423 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt; 2424 struct amdgpu_bo *root; 2425 unsigned long flags; 2426 int i; 2427 2428 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); 2429 2430 flush_work(&vm->pt_free_work); 2431 2432 root = amdgpu_bo_ref(vm->root.bo); 2433 amdgpu_bo_reserve(root, true); 2434 amdgpu_vm_set_pasid(adev, vm, 0); 2435 dma_fence_wait(vm->last_unlocked, false); 2436 dma_fence_put(vm->last_unlocked); 2437 dma_fence_wait(vm->last_tlb_flush, false); 2438 /* Make sure that all fence callbacks have completed */ 2439 spin_lock_irqsave(vm->last_tlb_flush->lock, flags); 2440 spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags); 2441 dma_fence_put(vm->last_tlb_flush); 2442 2443 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 2444 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) { 2445 amdgpu_vm_prt_fini(adev, vm); 2446 prt_fini_needed = false; 2447 } 2448 2449 list_del(&mapping->list); 2450 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); 2451 } 2452 2453 amdgpu_vm_pt_free_root(adev, vm); 2454 amdgpu_bo_unreserve(root); 2455 amdgpu_bo_unref(&root); 2456 WARN_ON(vm->root.bo); 2457 2458 amdgpu_vm_fini_entities(vm); 2459 2460 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { 2461 dev_err(adev->dev, "still active bo inside vm\n"); 2462 } 2463 rbtree_postorder_for_each_entry_safe(mapping, tmp, 2464 &vm->va.rb_root, rb) { 2465 /* Don't remove the mapping here, we don't want to trigger a 2466 * rebalance and the tree is about to be destroyed anyway. 2467 */ 2468 list_del(&mapping->list); 2469 kfree(mapping); 2470 } 2471 2472 dma_fence_put(vm->last_update); 2473 2474 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) { 2475 if (vm->reserved_vmid[i]) { 2476 amdgpu_vmid_free_reserved(adev, i); 2477 vm->reserved_vmid[i] = false; 2478 } 2479 } 2480 2481} 2482 2483/** 2484 * amdgpu_vm_manager_init - init the VM manager 2485 * 2486 * @adev: amdgpu_device pointer 2487 * 2488 * Initialize the VM manager structures 2489 */ 2490void amdgpu_vm_manager_init(struct amdgpu_device *adev) 2491{ 2492 unsigned i; 2493 2494 /* Concurrent flushes are only possible starting with Vega10 and 2495 * are broken on Navi10 and Navi14. 2496 */ 2497 adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 || 2498 adev->asic_type == CHIP_NAVI10 || 2499 adev->asic_type == CHIP_NAVI14); 2500 amdgpu_vmid_mgr_init(adev); 2501 2502 adev->vm_manager.fence_context = 2503 dma_fence_context_alloc(AMDGPU_MAX_RINGS); 2504 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 2505 adev->vm_manager.seqno[i] = 0; 2506 2507 mtx_init(&adev->vm_manager.prt_lock, IPL_TTY); 2508 atomic_set(&adev->vm_manager.num_prt_users, 0); 2509 2510 /* If not overridden by the user, by default, only in large BAR systems 2511 * Compute VM tables will be updated by CPU 2512 */ 2513#ifdef CONFIG_X86_64 2514 if (amdgpu_vm_update_mode == -1) { 2515 /* For asic with VF MMIO access protection 2516 * avoid using CPU for VM table updates 2517 */ 2518 if (amdgpu_gmc_vram_full_visible(&adev->gmc) && 2519 !amdgpu_sriov_vf_mmio_access_protection(adev)) 2520 adev->vm_manager.vm_update_mode = 2521 AMDGPU_VM_USE_CPU_FOR_COMPUTE; 2522 else 2523 adev->vm_manager.vm_update_mode = 0; 2524 } else 2525 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode; 2526#else 2527 adev->vm_manager.vm_update_mode = 0; 2528#endif 2529 2530 xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ); 2531} 2532 2533/** 2534 * amdgpu_vm_manager_fini - cleanup VM manager 2535 * 2536 * @adev: amdgpu_device pointer 2537 * 2538 * Cleanup the VM manager and free resources. 2539 */ 2540void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 2541{ 2542 WARN_ON(!xa_empty(&adev->vm_manager.pasids)); 2543 xa_destroy(&adev->vm_manager.pasids); 2544 2545 amdgpu_vmid_mgr_fini(adev); 2546} 2547 2548/** 2549 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs. 2550 * 2551 * @dev: drm device pointer 2552 * @data: drm_amdgpu_vm 2553 * @filp: drm file pointer 2554 * 2555 * Returns: 2556 * 0 for success, -errno for errors. 2557 */ 2558int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 2559{ 2560 union drm_amdgpu_vm *args = data; 2561 struct amdgpu_device *adev = drm_to_adev(dev); 2562 struct amdgpu_fpriv *fpriv = filp->driver_priv; 2563 2564 /* No valid flags defined yet */ 2565 if (args->in.flags) 2566 return -EINVAL; 2567 2568 switch (args->in.op) { 2569 case AMDGPU_VM_OP_RESERVE_VMID: 2570 /* We only have requirement to reserve vmid from gfxhub */ 2571 if (!fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) { 2572 amdgpu_vmid_alloc_reserved(adev, AMDGPU_GFXHUB(0)); 2573 fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = true; 2574 } 2575 2576 break; 2577 case AMDGPU_VM_OP_UNRESERVE_VMID: 2578 if (fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) { 2579 amdgpu_vmid_free_reserved(adev, AMDGPU_GFXHUB(0)); 2580 fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = false; 2581 } 2582 break; 2583 default: 2584 return -EINVAL; 2585 } 2586 2587 return 0; 2588} 2589 2590/** 2591 * amdgpu_vm_get_task_info - Extracts task info for a PASID. 2592 * 2593 * @adev: drm device pointer 2594 * @pasid: PASID identifier for VM 2595 * @task_info: task_info to fill. 2596 */ 2597void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid, 2598 struct amdgpu_task_info *task_info) 2599{ 2600 struct amdgpu_vm *vm; 2601 unsigned long flags; 2602 2603 xa_lock_irqsave(&adev->vm_manager.pasids, flags); 2604 2605 vm = xa_load(&adev->vm_manager.pasids, pasid); 2606 if (vm) 2607 *task_info = vm->task_info; 2608 2609 xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); 2610} 2611 2612/** 2613 * amdgpu_vm_set_task_info - Sets VMs task info. 2614 * 2615 * @vm: vm for which to set the info 2616 */ 2617void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) 2618{ 2619 if (vm->task_info.pid) 2620 return; 2621 2622#ifdef __linux__ 2623 vm->task_info.pid = current->pid; 2624 get_task_comm(vm->task_info.task_name, current); 2625 2626 if (current->group_leader->mm != current->mm) 2627 return; 2628 2629 vm->task_info.tgid = current->group_leader->pid; 2630 get_task_comm(vm->task_info.process_name, current->group_leader); 2631#else 2632 /* thread */ 2633 vm->task_info.pid = curproc->p_tid; 2634 strlcpy(vm->task_info.task_name, curproc->p_p->ps_comm, 2635 sizeof(vm->task_info.task_name)); 2636 2637 /* process */ 2638 vm->task_info.tgid = curproc->p_p->ps_pid; 2639 strlcpy(vm->task_info.process_name, curproc->p_p->ps_comm, 2640 sizeof(vm->task_info.process_name)); 2641#endif 2642} 2643 2644/** 2645 * amdgpu_vm_handle_fault - graceful handling of VM faults. 2646 * @adev: amdgpu device pointer 2647 * @pasid: PASID of the VM 2648 * @vmid: VMID, only used for GFX 9.4.3. 2649 * @node_id: Node_id received in IH cookie. Only applicable for 2650 * GFX 9.4.3. 2651 * @addr: Address of the fault 2652 * @write_fault: true is write fault, false is read fault 2653 * 2654 * Try to gracefully handle a VM fault. Return true if the fault was handled and 2655 * shouldn't be reported any more. 2656 */ 2657bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, 2658 u32 vmid, u32 node_id, uint64_t addr, 2659 bool write_fault) 2660{ 2661 bool is_compute_context = false; 2662 struct amdgpu_bo *root; 2663 unsigned long irqflags; 2664 uint64_t value, flags; 2665 struct amdgpu_vm *vm; 2666 int r; 2667 2668 xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); 2669 vm = xa_load(&adev->vm_manager.pasids, pasid); 2670 if (vm) { 2671 root = amdgpu_bo_ref(vm->root.bo); 2672 is_compute_context = vm->is_compute_context; 2673 } else { 2674 root = NULL; 2675 } 2676 xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); 2677 2678 if (!root) 2679 return false; 2680 2681 addr /= AMDGPU_GPU_PAGE_SIZE; 2682 2683 if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid, 2684 node_id, addr, write_fault)) { 2685 amdgpu_bo_unref(&root); 2686 return true; 2687 } 2688 2689 r = amdgpu_bo_reserve(root, true); 2690 if (r) 2691 goto error_unref; 2692 2693 /* Double check that the VM still exists */ 2694 xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); 2695 vm = xa_load(&adev->vm_manager.pasids, pasid); 2696 if (vm && vm->root.bo != root) 2697 vm = NULL; 2698 xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); 2699 if (!vm) 2700 goto error_unlock; 2701 2702 flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED | 2703 AMDGPU_PTE_SYSTEM; 2704 2705 if (is_compute_context) { 2706 /* Intentionally setting invalid PTE flag 2707 * combination to force a no-retry-fault 2708 */ 2709 flags = AMDGPU_VM_NORETRY_FLAGS; 2710 value = 0; 2711 } else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) { 2712 /* Redirect the access to the dummy page */ 2713 value = adev->dummy_page_addr; 2714 flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE | 2715 AMDGPU_PTE_WRITEABLE; 2716 2717 } else { 2718 /* Let the hw retry silently on the PTE */ 2719 value = 0; 2720 } 2721 2722 r = dma_resv_reserve_fences(root->tbo.base.resv, 1); 2723 if (r) { 2724 pr_debug("failed %d to reserve fence slot\n", r); 2725 goto error_unlock; 2726 } 2727 2728 r = amdgpu_vm_update_range(adev, vm, true, false, false, NULL, addr, 2729 addr, flags, value, 0, NULL, NULL, NULL); 2730 if (r) 2731 goto error_unlock; 2732 2733 r = amdgpu_vm_update_pdes(adev, vm, true); 2734 2735error_unlock: 2736 amdgpu_bo_unreserve(root); 2737 if (r < 0) 2738 DRM_ERROR("Can't handle page fault (%d)\n", r); 2739 2740error_unref: 2741 amdgpu_bo_unref(&root); 2742 2743 return false; 2744} 2745 2746#if defined(CONFIG_DEBUG_FS) 2747/** 2748 * amdgpu_debugfs_vm_bo_info - print BO info for the VM 2749 * 2750 * @vm: Requested VM for printing BO info 2751 * @m: debugfs file 2752 * 2753 * Print BO information in debugfs file for the VM 2754 */ 2755void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) 2756{ 2757 struct amdgpu_bo_va *bo_va, *tmp; 2758 u64 total_idle = 0; 2759 u64 total_evicted = 0; 2760 u64 total_relocated = 0; 2761 u64 total_moved = 0; 2762 u64 total_invalidated = 0; 2763 u64 total_done = 0; 2764 unsigned int total_idle_objs = 0; 2765 unsigned int total_evicted_objs = 0; 2766 unsigned int total_relocated_objs = 0; 2767 unsigned int total_moved_objs = 0; 2768 unsigned int total_invalidated_objs = 0; 2769 unsigned int total_done_objs = 0; 2770 unsigned int id = 0; 2771 2772 spin_lock(&vm->status_lock); 2773 seq_puts(m, "\tIdle BOs:\n"); 2774 list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { 2775 if (!bo_va->base.bo) 2776 continue; 2777 total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 2778 } 2779 total_idle_objs = id; 2780 id = 0; 2781 2782 seq_puts(m, "\tEvicted BOs:\n"); 2783 list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) { 2784 if (!bo_va->base.bo) 2785 continue; 2786 total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 2787 } 2788 total_evicted_objs = id; 2789 id = 0; 2790 2791 seq_puts(m, "\tRelocated BOs:\n"); 2792 list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) { 2793 if (!bo_va->base.bo) 2794 continue; 2795 total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 2796 } 2797 total_relocated_objs = id; 2798 id = 0; 2799 2800 seq_puts(m, "\tMoved BOs:\n"); 2801 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { 2802 if (!bo_va->base.bo) 2803 continue; 2804 total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 2805 } 2806 total_moved_objs = id; 2807 id = 0; 2808 2809 seq_puts(m, "\tInvalidated BOs:\n"); 2810 list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { 2811 if (!bo_va->base.bo) 2812 continue; 2813 total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 2814 } 2815 total_invalidated_objs = id; 2816 id = 0; 2817 2818 seq_puts(m, "\tDone BOs:\n"); 2819 list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) { 2820 if (!bo_va->base.bo) 2821 continue; 2822 total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 2823 } 2824 spin_unlock(&vm->status_lock); 2825 total_done_objs = id; 2826 2827 seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle, 2828 total_idle_objs); 2829 seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted, 2830 total_evicted_objs); 2831 seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated, 2832 total_relocated_objs); 2833 seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved, 2834 total_moved_objs); 2835 seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated, 2836 total_invalidated_objs); 2837 seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done, 2838 total_done_objs); 2839} 2840#endif 2841