1/* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD$"); 31 32#include <dev/drm2/drmP.h> 33#include <dev/drm2/radeon/radeon_drm.h> 34#include "radeon.h" 35#include "radeon_reg.h" 36 37/* 38 * GART 39 * The GART (Graphics Aperture Remapping Table) is an aperture 40 * in the GPU's address space. System pages can be mapped into 41 * the aperture and look like contiguous pages from the GPU's 42 * perspective. A page table maps the pages in the aperture 43 * to the actual backing pages in system memory. 44 * 45 * Radeon GPUs support both an internal GART, as described above, 46 * and AGP. AGP works similarly, but the GART table is configured 47 * and maintained by the northbridge rather than the driver. 48 * Radeon hw has a separate AGP aperture that is programmed to 49 * point to the AGP aperture provided by the northbridge and the 50 * requests are passed through to the northbridge aperture. 51 * Both AGP and internal GART can be used at the same time, however 52 * that is not currently supported by the driver. 53 * 54 * This file handles the common internal GART management. 55 */ 56 57/* 58 * Common GART table functions. 59 */ 60/** 61 * radeon_gart_table_ram_alloc - allocate system ram for gart page table 62 * 63 * @rdev: radeon_device pointer 64 * 65 * Allocate system memory for GART page table 66 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 67 * gart table to be in system memory. 68 * Returns 0 for success, -ENOMEM for failure. 69 */ 70int radeon_gart_table_ram_alloc(struct radeon_device *rdev) 71{ 72 drm_dma_handle_t *dmah; 73 74 dmah = drm_pci_alloc(rdev->ddev, rdev->gart.table_size, 75 PAGE_SIZE, BUS_SPACE_MAXADDR); 76 if (dmah == NULL) { 77 return -ENOMEM; 78 } 79 rdev->gart.dmah = dmah; 80 rdev->gart.ptr = dmah->vaddr; 81#ifdef CONFIG_X86 82 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 83 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 84 pmap_change_attr((vm_offset_t)rdev->gart.ptr, 85 rdev->gart.table_size >> PAGE_SHIFT, PAT_UNCACHED); 86 } 87#endif 88 rdev->gart.table_addr = dmah->busaddr; 89 memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size); 90 return 0; 91} 92 93/** 94 * radeon_gart_table_ram_free - free system ram for gart page table 95 * 96 * @rdev: radeon_device pointer 97 * 98 * Free system memory for GART page table 99 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 100 * gart table to be in system memory. 101 */ 102void radeon_gart_table_ram_free(struct radeon_device *rdev) 103{ 104 if (rdev->gart.ptr == NULL) { 105 return; 106 } 107#ifdef CONFIG_X86 108 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 109 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 110 pmap_change_attr((vm_offset_t)rdev->gart.ptr, 111 rdev->gart.table_size >> PAGE_SHIFT, PAT_WRITE_COMBINING); 112 } 113#endif 114 drm_pci_free(rdev->ddev, rdev->gart.dmah); 115 rdev->gart.dmah = NULL; 116 rdev->gart.ptr = NULL; 117 rdev->gart.table_addr = 0; 118} 119 120/** 121 * radeon_gart_table_vram_alloc - allocate vram for gart page table 122 * 123 * @rdev: radeon_device pointer 124 * 125 * Allocate video memory for GART page table 126 * (pcie r4xx, r5xx+). These asics require the 127 * gart table to be in video memory. 128 * Returns 0 for success, error for failure. 129 */ 130int radeon_gart_table_vram_alloc(struct radeon_device *rdev) 131{ 132 int r; 133 134 if (rdev->gart.robj == NULL) { 135 r = radeon_bo_create(rdev, rdev->gart.table_size, 136 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, 137 NULL, &rdev->gart.robj); 138 if (r) { 139 return r; 140 } 141 } 142 return 0; 143} 144 145/** 146 * radeon_gart_table_vram_pin - pin gart page table in vram 147 * 148 * @rdev: radeon_device pointer 149 * 150 * Pin the GART page table in vram so it will not be moved 151 * by the memory manager (pcie r4xx, r5xx+). These asics require the 152 * gart table to be in video memory. 153 * Returns 0 for success, error for failure. 154 */ 155int radeon_gart_table_vram_pin(struct radeon_device *rdev) 156{ 157 uint64_t gpu_addr; 158 int r; 159 160 r = radeon_bo_reserve(rdev->gart.robj, false); 161 if (unlikely(r != 0)) 162 return r; 163 r = radeon_bo_pin(rdev->gart.robj, 164 RADEON_GEM_DOMAIN_VRAM, &gpu_addr); 165 if (r) { 166 radeon_bo_unreserve(rdev->gart.robj); 167 return r; 168 } 169 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr); 170 if (r) 171 radeon_bo_unpin(rdev->gart.robj); 172 radeon_bo_unreserve(rdev->gart.robj); 173 rdev->gart.table_addr = gpu_addr; 174 return r; 175} 176 177/** 178 * radeon_gart_table_vram_unpin - unpin gart page table in vram 179 * 180 * @rdev: radeon_device pointer 181 * 182 * Unpin the GART page table in vram (pcie r4xx, r5xx+). 183 * These asics require the gart table to be in video memory. 184 */ 185void radeon_gart_table_vram_unpin(struct radeon_device *rdev) 186{ 187 int r; 188 189 if (rdev->gart.robj == NULL) { 190 return; 191 } 192 r = radeon_bo_reserve(rdev->gart.robj, false); 193 if (likely(r == 0)) { 194 radeon_bo_kunmap(rdev->gart.robj); 195 radeon_bo_unpin(rdev->gart.robj); 196 radeon_bo_unreserve(rdev->gart.robj); 197 rdev->gart.ptr = NULL; 198 } 199} 200 201/** 202 * radeon_gart_table_vram_free - free gart page table vram 203 * 204 * @rdev: radeon_device pointer 205 * 206 * Free the video memory used for the GART page table 207 * (pcie r4xx, r5xx+). These asics require the gart table to 208 * be in video memory. 209 */ 210void radeon_gart_table_vram_free(struct radeon_device *rdev) 211{ 212 if (rdev->gart.robj == NULL) { 213 return; 214 } 215 radeon_gart_table_vram_unpin(rdev); 216 radeon_bo_unref(&rdev->gart.robj); 217} 218 219/* 220 * Common gart functions. 221 */ 222/** 223 * radeon_gart_unbind - unbind pages from the gart page table 224 * 225 * @rdev: radeon_device pointer 226 * @offset: offset into the GPU's gart aperture 227 * @pages: number of pages to unbind 228 * 229 * Unbinds the requested pages from the gart page table and 230 * replaces them with the dummy page (all asics). 231 */ 232void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, 233 int pages) 234{ 235 unsigned t; 236 unsigned p; 237 int i, j; 238 u64 page_base; 239 240 if (!rdev->gart.ready) { 241 DRM_ERROR("trying to unbind memory from uninitialized GART !\n"); 242 return; 243 } 244 t = offset / RADEON_GPU_PAGE_SIZE; 245 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 246 for (i = 0; i < pages; i++, p++) { 247 if (rdev->gart.pages[p]) { 248 rdev->gart.pages[p] = NULL; 249 rdev->gart.pages_addr[p] = rdev->dummy_page.addr; 250 page_base = rdev->gart.pages_addr[p]; 251 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 252 if (rdev->gart.ptr) { 253 radeon_gart_set_page(rdev, t, page_base); 254 } 255 page_base += RADEON_GPU_PAGE_SIZE; 256 } 257 } 258 } 259 mb(); 260 radeon_gart_tlb_flush(rdev); 261} 262 263/** 264 * radeon_gart_bind - bind pages into the gart page table 265 * 266 * @rdev: radeon_device pointer 267 * @offset: offset into the GPU's gart aperture 268 * @pages: number of pages to bind 269 * @pagelist: pages to bind 270 * @dma_addr: DMA addresses of pages 271 * 272 * Binds the requested pages to the gart page table 273 * (all asics). 274 * Returns 0 for success, -EINVAL for failure. 275 */ 276int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, 277 int pages, vm_page_t *pagelist, dma_addr_t *dma_addr) 278{ 279 unsigned t; 280 unsigned p; 281 uint64_t page_base; 282 int i, j; 283 284 if (!rdev->gart.ready) { 285 DRM_ERROR("trying to bind memory to uninitialized GART !\n"); 286 return -EINVAL; 287 } 288 t = offset / RADEON_GPU_PAGE_SIZE; 289 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 290 291 for (i = 0; i < pages; i++, p++) { 292 rdev->gart.pages_addr[p] = dma_addr[i]; 293 rdev->gart.pages[p] = pagelist[i]; 294 if (rdev->gart.ptr) { 295 page_base = rdev->gart.pages_addr[p]; 296 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 297 radeon_gart_set_page(rdev, t, page_base); 298 page_base += RADEON_GPU_PAGE_SIZE; 299 } 300 } 301 } 302 mb(); 303 radeon_gart_tlb_flush(rdev); 304 return 0; 305} 306 307/** 308 * radeon_gart_restore - bind all pages in the gart page table 309 * 310 * @rdev: radeon_device pointer 311 * 312 * Binds all pages in the gart page table (all asics). 313 * Used to rebuild the gart table on device startup or resume. 314 */ 315void radeon_gart_restore(struct radeon_device *rdev) 316{ 317 int i, j, t; 318 u64 page_base; 319 320 if (!rdev->gart.ptr) { 321 return; 322 } 323 for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) { 324 page_base = rdev->gart.pages_addr[i]; 325 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 326 radeon_gart_set_page(rdev, t, page_base); 327 page_base += RADEON_GPU_PAGE_SIZE; 328 } 329 } 330 mb(); 331 radeon_gart_tlb_flush(rdev); 332} 333 334/** 335 * radeon_gart_init - init the driver info for managing the gart 336 * 337 * @rdev: radeon_device pointer 338 * 339 * Allocate the dummy page and init the gart driver info (all asics). 340 * Returns 0 for success, error for failure. 341 */ 342int radeon_gart_init(struct radeon_device *rdev) 343{ 344 int r, i; 345 346 if (rdev->gart.pages) { 347 return 0; 348 } 349 /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */ 350 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) { 351 DRM_ERROR("Page size is smaller than GPU page size!\n"); 352 return -EINVAL; 353 } 354 r = radeon_dummy_page_init(rdev); 355 if (r) 356 return r; 357 /* Compute table size */ 358 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE; 359 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE; 360 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", 361 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); 362 /* Allocate pages table */ 363 rdev->gart.pages = malloc(sizeof(void *) * rdev->gart.num_cpu_pages, 364 DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 365 if (rdev->gart.pages == NULL) { 366 radeon_gart_fini(rdev); 367 return -ENOMEM; 368 } 369 rdev->gart.pages_addr = malloc(sizeof(dma_addr_t) * 370 rdev->gart.num_cpu_pages, 371 DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 372 if (rdev->gart.pages_addr == NULL) { 373 radeon_gart_fini(rdev); 374 return -ENOMEM; 375 } 376 /* set GART entry to point to the dummy page by default */ 377 for (i = 0; i < rdev->gart.num_cpu_pages; i++) { 378 rdev->gart.pages_addr[i] = rdev->dummy_page.addr; 379 } 380 return 0; 381} 382 383/** 384 * radeon_gart_fini - tear down the driver info for managing the gart 385 * 386 * @rdev: radeon_device pointer 387 * 388 * Tear down the gart driver info and free the dummy page (all asics). 389 */ 390void radeon_gart_fini(struct radeon_device *rdev) 391{ 392 if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) { 393 /* unbind pages */ 394 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); 395 } 396 rdev->gart.ready = false; 397 free(rdev->gart.pages, DRM_MEM_DRIVER); 398 free(rdev->gart.pages_addr, DRM_MEM_DRIVER); 399 rdev->gart.pages = NULL; 400 rdev->gart.pages_addr = NULL; 401 402 radeon_dummy_page_fini(rdev); 403} 404 405/* 406 * GPUVM 407 * GPUVM is similar to the legacy gart on older asics, however 408 * rather than there being a single global gart table 409 * for the entire GPU, there are multiple VM page tables active 410 * at any given time. The VM page tables can contain a mix 411 * vram pages and system memory pages and system memory pages 412 * can be mapped as snooped (cached system pages) or unsnooped 413 * (uncached system pages). 414 * Each VM has an ID associated with it and there is a page table 415 * associated with each VMID. When execting a command buffer, 416 * the kernel tells the ring what VMID to use for that command 417 * buffer. VMIDs are allocated dynamically as commands are submitted. 418 * The userspace drivers maintain their own address space and the kernel 419 * sets up their pages tables accordingly when they submit their 420 * command buffers and a VMID is assigned. 421 * Cayman/Trinity support up to 8 active VMs at any given time; 422 * SI supports 16. 423 */ 424 425/* 426 * vm helpers 427 * 428 * TODO bind a default page at vm initialization for default address 429 */ 430 431/** 432 * radeon_vm_num_pde - return the number of page directory entries 433 * 434 * @rdev: radeon_device pointer 435 * 436 * Calculate the number of page directory entries (cayman+). 437 */ 438static unsigned radeon_vm_num_pdes(struct radeon_device *rdev) 439{ 440 return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE; 441} 442 443/** 444 * radeon_vm_directory_size - returns the size of the page directory in bytes 445 * 446 * @rdev: radeon_device pointer 447 * 448 * Calculate the size of the page directory in bytes (cayman+). 449 */ 450static unsigned radeon_vm_directory_size(struct radeon_device *rdev) 451{ 452 return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8); 453} 454 455/** 456 * radeon_vm_manager_init - init the vm manager 457 * 458 * @rdev: radeon_device pointer 459 * 460 * Init the vm manager (cayman+). 461 * Returns 0 for success, error for failure. 462 */ 463int radeon_vm_manager_init(struct radeon_device *rdev) 464{ 465 struct radeon_vm *vm; 466 struct radeon_bo_va *bo_va; 467 int r; 468 unsigned size; 469 470 if (!rdev->vm_manager.enabled) { 471 /* allocate enough for 2 full VM pts */ 472 size = radeon_vm_directory_size(rdev); 473 size += rdev->vm_manager.max_pfn * 8; 474 size *= 2; 475 r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager, 476 RADEON_GPU_PAGE_ALIGN(size), 477 RADEON_GEM_DOMAIN_VRAM); 478 if (r) { 479 dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n", 480 (rdev->vm_manager.max_pfn * 8) >> 10); 481 return r; 482 } 483 484 r = radeon_asic_vm_init(rdev); 485 if (r) 486 return r; 487 488 rdev->vm_manager.enabled = true; 489 490 r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager); 491 if (r) 492 return r; 493 } 494 495 /* restore page table */ 496 list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) { 497 if (vm->page_directory == NULL) 498 continue; 499 500 list_for_each_entry(bo_va, &vm->va, vm_list) { 501 bo_va->valid = false; 502 } 503 } 504 return 0; 505} 506 507/** 508 * radeon_vm_free_pt - free the page table for a specific vm 509 * 510 * @rdev: radeon_device pointer 511 * @vm: vm to unbind 512 * 513 * Free the page table of a specific vm (cayman+). 514 * 515 * Global and local mutex must be lock! 516 */ 517static void radeon_vm_free_pt(struct radeon_device *rdev, 518 struct radeon_vm *vm) 519{ 520 struct radeon_bo_va *bo_va; 521 int i; 522 523 if (!vm->page_directory) 524 return; 525 526 list_del_init(&vm->list); 527 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 528 529 list_for_each_entry(bo_va, &vm->va, vm_list) { 530 bo_va->valid = false; 531 } 532 533 if (vm->page_tables == NULL) 534 return; 535 536 for (i = 0; i < radeon_vm_num_pdes(rdev); i++) 537 radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence); 538 539 free(vm->page_tables, DRM_MEM_DRIVER); 540} 541 542/** 543 * radeon_vm_manager_fini - tear down the vm manager 544 * 545 * @rdev: radeon_device pointer 546 * 547 * Tear down the VM manager (cayman+). 548 */ 549void radeon_vm_manager_fini(struct radeon_device *rdev) 550{ 551 struct radeon_vm *vm, *tmp; 552 int i; 553 554 if (!rdev->vm_manager.enabled) 555 return; 556 557 sx_xlock(&rdev->vm_manager.lock); 558 /* free all allocated page tables */ 559 list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) { 560 sx_xlock(&vm->mutex); 561 radeon_vm_free_pt(rdev, vm); 562 sx_xunlock(&vm->mutex); 563 } 564 for (i = 0; i < RADEON_NUM_VM; ++i) { 565 radeon_fence_unref(&rdev->vm_manager.active[i]); 566 } 567 radeon_asic_vm_fini(rdev); 568 sx_xunlock(&rdev->vm_manager.lock); 569 570 radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager); 571 radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager); 572 rdev->vm_manager.enabled = false; 573} 574 575/** 576 * radeon_vm_evict - evict page table to make room for new one 577 * 578 * @rdev: radeon_device pointer 579 * @vm: VM we want to allocate something for 580 * 581 * Evict a VM from the lru, making sure that it isn't @vm. (cayman+). 582 * Returns 0 for success, -ENOMEM for failure. 583 * 584 * Global and local mutex must be locked! 585 */ 586static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm) 587{ 588 struct radeon_vm *vm_evict; 589 590 if (list_empty(&rdev->vm_manager.lru_vm)) 591 return -ENOMEM; 592 593 vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, 594 struct radeon_vm, list); 595 if (vm_evict == vm) 596 return -ENOMEM; 597 598 sx_xlock(&vm_evict->mutex); 599 radeon_vm_free_pt(rdev, vm_evict); 600 sx_xunlock(&vm_evict->mutex); 601 return 0; 602} 603 604/** 605 * radeon_vm_alloc_pt - allocates a page table for a VM 606 * 607 * @rdev: radeon_device pointer 608 * @vm: vm to bind 609 * 610 * Allocate a page table for the requested vm (cayman+). 611 * Returns 0 for success, error for failure. 612 * 613 * Global and local mutex must be locked! 614 */ 615int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm) 616{ 617 unsigned pd_size, pts_size; 618 u64 *pd_addr; 619 int r; 620 621 if (vm == NULL) { 622 return -EINVAL; 623 } 624 625 if (vm->page_directory != NULL) { 626 return 0; 627 } 628 629retry: 630 pd_size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev)); 631 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 632 &vm->page_directory, pd_size, 633 RADEON_GPU_PAGE_SIZE, false); 634 if (r == -ENOMEM) { 635 r = radeon_vm_evict(rdev, vm); 636 if (r) 637 return r; 638 goto retry; 639 640 } else if (r) { 641 return r; 642 } 643 644 vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory); 645 646 /* Initially clear the page directory */ 647 pd_addr = radeon_sa_bo_cpu_addr(vm->page_directory); 648 memset(pd_addr, 0, pd_size); 649 650 pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *); 651 vm->page_tables = malloc(pts_size, DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 652 653 if (vm->page_tables == NULL) { 654 DRM_ERROR("Cannot allocate memory for page table array\n"); 655 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 656 return -ENOMEM; 657 } 658 659 return 0; 660} 661 662/** 663 * radeon_vm_add_to_lru - add VMs page table to LRU list 664 * 665 * @rdev: radeon_device pointer 666 * @vm: vm to add to LRU 667 * 668 * Add the allocated page table to the LRU list (cayman+). 669 * 670 * Global mutex must be locked! 671 */ 672void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm) 673{ 674 list_del_init(&vm->list); 675 list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); 676} 677 678/** 679 * radeon_vm_grab_id - allocate the next free VMID 680 * 681 * @rdev: radeon_device pointer 682 * @vm: vm to allocate id for 683 * @ring: ring we want to submit job to 684 * 685 * Allocate an id for the vm (cayman+). 686 * Returns the fence we need to sync to (if any). 687 * 688 * Global and local mutex must be locked! 689 */ 690struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, 691 struct radeon_vm *vm, int ring) 692{ 693 struct radeon_fence *best[RADEON_NUM_RINGS] = {}; 694 unsigned choices[2] = {}; 695 unsigned i; 696 697 /* check if the id is still valid */ 698 if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id]) 699 return NULL; 700 701 /* we definitely need to flush */ 702 radeon_fence_unref(&vm->last_flush); 703 704 /* skip over VMID 0, since it is the system VM */ 705 for (i = 1; i < rdev->vm_manager.nvm; ++i) { 706 struct radeon_fence *fence = rdev->vm_manager.active[i]; 707 708 if (fence == NULL) { 709 /* found a free one */ 710 vm->id = i; 711 return NULL; 712 } 713 714 if (radeon_fence_is_earlier(fence, best[fence->ring])) { 715 best[fence->ring] = fence; 716 choices[fence->ring == ring ? 0 : 1] = i; 717 } 718 } 719 720 for (i = 0; i < 2; ++i) { 721 if (choices[i]) { 722 vm->id = choices[i]; 723 return rdev->vm_manager.active[choices[i]]; 724 } 725 } 726 727 /* should never happen */ 728 panic("%s: failed to allocate next VMID", __func__); 729 return NULL; 730} 731 732/** 733 * radeon_vm_fence - remember fence for vm 734 * 735 * @rdev: radeon_device pointer 736 * @vm: vm we want to fence 737 * @fence: fence to remember 738 * 739 * Fence the vm (cayman+). 740 * Set the fence used to protect page table and id. 741 * 742 * Global and local mutex must be locked! 743 */ 744void radeon_vm_fence(struct radeon_device *rdev, 745 struct radeon_vm *vm, 746 struct radeon_fence *fence) 747{ 748 radeon_fence_unref(&rdev->vm_manager.active[vm->id]); 749 rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); 750 751 radeon_fence_unref(&vm->fence); 752 vm->fence = radeon_fence_ref(fence); 753} 754 755/** 756 * radeon_vm_bo_find - find the bo_va for a specific vm & bo 757 * 758 * @vm: requested vm 759 * @bo: requested buffer object 760 * 761 * Find @bo inside the requested vm (cayman+). 762 * Search inside the @bos vm list for the requested vm 763 * Returns the found bo_va or NULL if none is found 764 * 765 * Object has to be reserved! 766 */ 767struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, 768 struct radeon_bo *bo) 769{ 770 struct radeon_bo_va *bo_va; 771 772 list_for_each_entry(bo_va, &bo->va, bo_list) { 773 if (bo_va->vm == vm) { 774 return bo_va; 775 } 776 } 777 return NULL; 778} 779 780/** 781 * radeon_vm_bo_add - add a bo to a specific vm 782 * 783 * @rdev: radeon_device pointer 784 * @vm: requested vm 785 * @bo: radeon buffer object 786 * 787 * Add @bo into the requested vm (cayman+). 788 * Add @bo to the list of bos associated with the vm 789 * Returns newly added bo_va or NULL for failure 790 * 791 * Object has to be reserved! 792 */ 793struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev, 794 struct radeon_vm *vm, 795 struct radeon_bo *bo) 796{ 797 struct radeon_bo_va *bo_va; 798 799 bo_va = malloc(sizeof(struct radeon_bo_va), 800 DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 801 if (bo_va == NULL) { 802 return NULL; 803 } 804 bo_va->vm = vm; 805 bo_va->bo = bo; 806 bo_va->soffset = 0; 807 bo_va->eoffset = 0; 808 bo_va->flags = 0; 809 bo_va->valid = false; 810 bo_va->ref_count = 1; 811 INIT_LIST_HEAD(&bo_va->bo_list); 812 INIT_LIST_HEAD(&bo_va->vm_list); 813 814 sx_xlock(&vm->mutex); 815 list_add(&bo_va->vm_list, &vm->va); 816 list_add_tail(&bo_va->bo_list, &bo->va); 817 sx_xunlock(&vm->mutex); 818 819 return bo_va; 820} 821 822/** 823 * radeon_vm_bo_set_addr - set bos virtual address inside a vm 824 * 825 * @rdev: radeon_device pointer 826 * @bo_va: bo_va to store the address 827 * @soffset: requested offset of the buffer in the VM address space 828 * @flags: attributes of pages (read/write/valid/etc.) 829 * 830 * Set offset of @bo_va (cayman+). 831 * Validate and set the offset requested within the vm address space. 832 * Returns 0 for success, error for failure. 833 * 834 * Object has to be reserved! 835 */ 836int radeon_vm_bo_set_addr(struct radeon_device *rdev, 837 struct radeon_bo_va *bo_va, 838 uint64_t soffset, 839 uint32_t flags) 840{ 841 uint64_t size = radeon_bo_size(bo_va->bo); 842 uint64_t eoffset, last_offset = 0; 843 struct radeon_vm *vm = bo_va->vm; 844 struct radeon_bo_va *tmp; 845 struct list_head *head; 846 unsigned last_pfn; 847 848 if (soffset) { 849 /* make sure object fit at this offset */ 850 eoffset = soffset + size; 851 if (soffset >= eoffset) { 852 return -EINVAL; 853 } 854 855 last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; 856 if (last_pfn > rdev->vm_manager.max_pfn) { 857 dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", 858 last_pfn, rdev->vm_manager.max_pfn); 859 return -EINVAL; 860 } 861 862 } else { 863 eoffset = last_pfn = 0; 864 } 865 866 sx_xlock(&vm->mutex); 867 head = &vm->va; 868 last_offset = 0; 869 list_for_each_entry(tmp, &vm->va, vm_list) { 870 if (bo_va == tmp) { 871 /* skip over currently modified bo */ 872 continue; 873 } 874 875 if (soffset >= last_offset && eoffset <= tmp->soffset) { 876 /* bo can be added before this one */ 877 break; 878 } 879 if (eoffset > tmp->soffset && soffset < tmp->eoffset) { 880 /* bo and tmp overlap, invalid offset */ 881 dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", 882 bo_va->bo, (unsigned)bo_va->soffset, tmp->bo, 883 (unsigned)tmp->soffset, (unsigned)tmp->eoffset); 884 sx_xunlock(&vm->mutex); 885 return -EINVAL; 886 } 887 last_offset = tmp->eoffset; 888 head = &tmp->vm_list; 889 } 890 891 bo_va->soffset = soffset; 892 bo_va->eoffset = eoffset; 893 bo_va->flags = flags; 894 bo_va->valid = false; 895 list_move(&bo_va->vm_list, head); 896 897 sx_xunlock(&vm->mutex); 898 return 0; 899} 900 901/** 902 * radeon_vm_map_gart - get the physical address of a gart page 903 * 904 * @rdev: radeon_device pointer 905 * @addr: the unmapped addr 906 * 907 * Look up the physical address of the page that the pte resolves 908 * to (cayman+). 909 * Returns the physical address of the page. 910 */ 911uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) 912{ 913 uint64_t result; 914 915 /* page table offset */ 916 result = rdev->gart.pages_addr[addr >> PAGE_SHIFT]; 917 918 /* in case cpu page size != gpu page size*/ 919 /* 920 * FreeBSD port note: FreeBSD's PAGE_MASK is the inverse of 921 * Linux's one. That's why the test below doesn't inverse the 922 * constant. 923 */ 924 result |= addr & (PAGE_MASK); 925 926 return result; 927} 928 929/** 930 * radeon_vm_update_pdes - make sure that page directory is valid 931 * 932 * @rdev: radeon_device pointer 933 * @vm: requested vm 934 * @start: start of GPU address range 935 * @end: end of GPU address range 936 * 937 * Allocates new page tables if necessary 938 * and updates the page directory (cayman+). 939 * Returns 0 for success, error for failure. 940 * 941 * Global and local mutex must be locked! 942 */ 943static int radeon_vm_update_pdes(struct radeon_device *rdev, 944 struct radeon_vm *vm, 945 uint64_t start, uint64_t end) 946{ 947 static const uint32_t incr = RADEON_VM_PTE_COUNT * 8; 948 949 uint64_t last_pde = ~0, last_pt = ~0; 950 unsigned count = 0; 951 uint64_t pt_idx; 952 int r; 953 954 start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 955 end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 956 957 /* walk over the address space and update the page directory */ 958 for (pt_idx = start; pt_idx <= end; ++pt_idx) { 959 uint64_t pde, pt; 960 961 if (vm->page_tables[pt_idx]) 962 continue; 963 964retry: 965 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 966 &vm->page_tables[pt_idx], 967 RADEON_VM_PTE_COUNT * 8, 968 RADEON_GPU_PAGE_SIZE, false); 969 970 if (r == -ENOMEM) { 971 r = radeon_vm_evict(rdev, vm); 972 if (r) 973 return r; 974 goto retry; 975 } else if (r) { 976 return r; 977 } 978 979 pde = vm->pd_gpu_addr + pt_idx * 8; 980 981 pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 982 983 if (((last_pde + 8 * count) != pde) || 984 ((last_pt + incr * count) != pt)) { 985 986 if (count) { 987 radeon_asic_vm_set_page(rdev, last_pde, 988 last_pt, count, incr, 989 RADEON_VM_PAGE_VALID); 990 } 991 992 count = 1; 993 last_pde = pde; 994 last_pt = pt; 995 } else { 996 ++count; 997 } 998 } 999 1000 if (count) { 1001 radeon_asic_vm_set_page(rdev, last_pde, last_pt, count, 1002 incr, RADEON_VM_PAGE_VALID); 1003 1004 } 1005 1006 return 0; 1007} 1008 1009/** 1010 * radeon_vm_update_ptes - make sure that page tables are valid 1011 * 1012 * @rdev: radeon_device pointer 1013 * @vm: requested vm 1014 * @start: start of GPU address range 1015 * @end: end of GPU address range 1016 * @dst: destination address to map to 1017 * @flags: mapping flags 1018 * 1019 * Update the page tables in the range @start - @end (cayman+). 1020 * 1021 * Global and local mutex must be locked! 1022 */ 1023static void radeon_vm_update_ptes(struct radeon_device *rdev, 1024 struct radeon_vm *vm, 1025 uint64_t start, uint64_t end, 1026 uint64_t dst, uint32_t flags) 1027{ 1028 static const uint64_t mask = RADEON_VM_PTE_COUNT - 1; 1029 1030 uint64_t last_pte = ~0, last_dst = ~0; 1031 unsigned count = 0; 1032 uint64_t addr; 1033 1034 start = start / RADEON_GPU_PAGE_SIZE; 1035 end = end / RADEON_GPU_PAGE_SIZE; 1036 1037 /* walk over the address space and update the page tables */ 1038 for (addr = start; addr < end; ) { 1039 uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE; 1040 unsigned nptes; 1041 uint64_t pte; 1042 1043 if ((addr & ~mask) == (end & ~mask)) 1044 nptes = end - addr; 1045 else 1046 nptes = RADEON_VM_PTE_COUNT - (addr & mask); 1047 1048 pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 1049 pte += (addr & mask) * 8; 1050 1051 if ((last_pte + 8 * count) != pte) { 1052 1053 if (count) { 1054 radeon_asic_vm_set_page(rdev, last_pte, 1055 last_dst, count, 1056 RADEON_GPU_PAGE_SIZE, 1057 flags); 1058 } 1059 1060 count = nptes; 1061 last_pte = pte; 1062 last_dst = dst; 1063 } else { 1064 count += nptes; 1065 } 1066 1067 addr += nptes; 1068 dst += nptes * RADEON_GPU_PAGE_SIZE; 1069 } 1070 1071 if (count) { 1072 radeon_asic_vm_set_page(rdev, last_pte, last_dst, count, 1073 RADEON_GPU_PAGE_SIZE, flags); 1074 } 1075} 1076 1077/** 1078 * radeon_vm_bo_update_pte - map a bo into the vm page table 1079 * 1080 * @rdev: radeon_device pointer 1081 * @vm: requested vm 1082 * @bo: radeon buffer object 1083 * @mem: ttm mem 1084 * 1085 * Fill in the page table entries for @bo (cayman+). 1086 * Returns 0 for success, -EINVAL for failure. 1087 * 1088 * Object have to be reserved & global and local mutex must be locked! 1089 */ 1090int radeon_vm_bo_update_pte(struct radeon_device *rdev, 1091 struct radeon_vm *vm, 1092 struct radeon_bo *bo, 1093 struct ttm_mem_reg *mem) 1094{ 1095 unsigned ridx = rdev->asic->vm.pt_ring_index; 1096 struct radeon_ring *ring = &rdev->ring[ridx]; 1097 struct radeon_semaphore *sem = NULL; 1098 struct radeon_bo_va *bo_va; 1099 unsigned nptes, npdes, ndw; 1100 uint64_t addr; 1101 int r; 1102 1103 /* nothing to do if vm isn't bound */ 1104 if (vm->page_directory == NULL) 1105 return 0; 1106 1107 bo_va = radeon_vm_bo_find(vm, bo); 1108 if (bo_va == NULL) { 1109 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); 1110 return -EINVAL; 1111 } 1112 1113 if (!bo_va->soffset) { 1114 dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n", 1115 bo, vm); 1116 return -EINVAL; 1117 } 1118 1119 if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL)) 1120 return 0; 1121 1122 bo_va->flags &= ~RADEON_VM_PAGE_VALID; 1123 bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; 1124 if (mem) { 1125 addr = mem->start << PAGE_SHIFT; 1126 if (mem->mem_type != TTM_PL_SYSTEM) { 1127 bo_va->flags |= RADEON_VM_PAGE_VALID; 1128 bo_va->valid = true; 1129 } 1130 if (mem->mem_type == TTM_PL_TT) { 1131 bo_va->flags |= RADEON_VM_PAGE_SYSTEM; 1132 } else { 1133 addr += rdev->vm_manager.vram_base_offset; 1134 } 1135 } else { 1136 addr = 0; 1137 bo_va->valid = false; 1138 } 1139 1140 if (vm->fence && radeon_fence_signaled(vm->fence)) { 1141 radeon_fence_unref(&vm->fence); 1142 } 1143 1144 if (vm->fence && vm->fence->ring != ridx) { 1145 r = radeon_semaphore_create(rdev, &sem); 1146 if (r) { 1147 return r; 1148 } 1149 } 1150 1151 nptes = radeon_bo_ngpu_pages(bo); 1152 1153 /* assume two extra pdes in case the mapping overlaps the borders */ 1154 npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2; 1155 1156 /* estimate number of dw needed */ 1157 /* semaphore, fence and padding */ 1158 ndw = 32; 1159 1160 if (RADEON_VM_BLOCK_SIZE > 11) 1161 /* reserve space for one header for every 2k dwords */ 1162 ndw += (nptes >> 11) * 4; 1163 else 1164 /* reserve space for one header for 1165 every (1 << BLOCK_SIZE) entries */ 1166 ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4; 1167 1168 /* reserve space for pte addresses */ 1169 ndw += nptes * 2; 1170 1171 /* reserve space for one header for every 2k dwords */ 1172 ndw += (npdes >> 11) * 4; 1173 1174 /* reserve space for pde addresses */ 1175 ndw += npdes * 2; 1176 1177 r = radeon_ring_lock(rdev, ring, ndw); 1178 if (r) { 1179 return r; 1180 } 1181 1182 if (sem && radeon_fence_need_sync(vm->fence, ridx)) { 1183 radeon_semaphore_sync_rings(rdev, sem, vm->fence->ring, ridx); 1184 radeon_fence_note_sync(vm->fence, ridx); 1185 } 1186 1187 r = radeon_vm_update_pdes(rdev, vm, bo_va->soffset, bo_va->eoffset); 1188 if (r) { 1189 radeon_ring_unlock_undo(rdev, ring); 1190 return r; 1191 } 1192 1193 radeon_vm_update_ptes(rdev, vm, bo_va->soffset, bo_va->eoffset, 1194 addr, bo_va->flags); 1195 1196 radeon_fence_unref(&vm->fence); 1197 r = radeon_fence_emit(rdev, &vm->fence, ridx); 1198 if (r) { 1199 radeon_ring_unlock_undo(rdev, ring); 1200 return r; 1201 } 1202 radeon_ring_unlock_commit(rdev, ring); 1203 radeon_semaphore_free(rdev, &sem, vm->fence); 1204 radeon_fence_unref(&vm->last_flush); 1205 1206 return 0; 1207} 1208 1209/** 1210 * radeon_vm_bo_rmv - remove a bo to a specific vm 1211 * 1212 * @rdev: radeon_device pointer 1213 * @bo_va: requested bo_va 1214 * 1215 * Remove @bo_va->bo from the requested vm (cayman+). 1216 * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and 1217 * remove the ptes for @bo_va in the page table. 1218 * Returns 0 for success. 1219 * 1220 * Object have to be reserved! 1221 */ 1222int radeon_vm_bo_rmv(struct radeon_device *rdev, 1223 struct radeon_bo_va *bo_va) 1224{ 1225 int r; 1226 1227 sx_xlock(&rdev->vm_manager.lock); 1228 sx_xlock(&bo_va->vm->mutex); 1229 r = radeon_vm_bo_update_pte(rdev, bo_va->vm, bo_va->bo, NULL); 1230 sx_xunlock(&rdev->vm_manager.lock); 1231 list_del(&bo_va->vm_list); 1232 sx_xunlock(&bo_va->vm->mutex); 1233 list_del(&bo_va->bo_list); 1234 1235 free(bo_va, DRM_MEM_DRIVER); 1236 return r; 1237} 1238 1239/** 1240 * radeon_vm_bo_invalidate - mark the bo as invalid 1241 * 1242 * @rdev: radeon_device pointer 1243 * @vm: requested vm 1244 * @bo: radeon buffer object 1245 * 1246 * Mark @bo as invalid (cayman+). 1247 */ 1248void radeon_vm_bo_invalidate(struct radeon_device *rdev, 1249 struct radeon_bo *bo) 1250{ 1251 struct radeon_bo_va *bo_va; 1252 1253 list_for_each_entry(bo_va, &bo->va, bo_list) { 1254 bo_va->valid = false; 1255 } 1256} 1257 1258/** 1259 * radeon_vm_init - initialize a vm instance 1260 * 1261 * @rdev: radeon_device pointer 1262 * @vm: requested vm 1263 * 1264 * Init @vm fields (cayman+). 1265 */ 1266void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) 1267{ 1268 vm->id = 0; 1269 vm->fence = NULL; 1270 sx_init(&vm->mutex, "drm__radeon_vm__mutex"); 1271 INIT_LIST_HEAD(&vm->list); 1272 INIT_LIST_HEAD(&vm->va); 1273} 1274 1275/** 1276 * radeon_vm_fini - tear down a vm instance 1277 * 1278 * @rdev: radeon_device pointer 1279 * @vm: requested vm 1280 * 1281 * Tear down @vm (cayman+). 1282 * Unbind the VM and remove all bos from the vm bo list 1283 */ 1284void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) 1285{ 1286 struct radeon_bo_va *bo_va, *tmp; 1287 int r; 1288 1289 sx_xlock(&rdev->vm_manager.lock); 1290 sx_xlock(&vm->mutex); 1291 radeon_vm_free_pt(rdev, vm); 1292 sx_xunlock(&rdev->vm_manager.lock); 1293 1294 if (!list_empty(&vm->va)) { 1295 dev_err(rdev->dev, "still active bo inside vm\n"); 1296 } 1297 list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) { 1298 list_del_init(&bo_va->vm_list); 1299 r = radeon_bo_reserve(bo_va->bo, false); 1300 if (!r) { 1301 list_del_init(&bo_va->bo_list); 1302 radeon_bo_unreserve(bo_va->bo); 1303 free(bo_va, DRM_MEM_DRIVER); 1304 } 1305 } 1306 radeon_fence_unref(&vm->fence); 1307 radeon_fence_unref(&vm->last_flush); 1308 sx_xunlock(&vm->mutex); 1309} 1310