radeon_gart.c revision 285830
117658Sjulian/* 217658Sjulian * Copyright 2008 Advanced Micro Devices, Inc. 317658Sjulian * Copyright 2008 Red Hat Inc. 417658Sjulian * Copyright 2009 Jerome Glisse. 517658Sjulian * 617658Sjulian * Permission is hereby granted, free of charge, to any person obtaining a 717658Sjulian * copy of this software and associated documentation files (the "Software"), 817658Sjulian * to deal in the Software without restriction, including without limitation 917658Sjulian * the rights to use, copy, modify, merge, publish, distribute, sublicense, 1017658Sjulian * and/or sell copies of the Software, and to permit persons to whom the 1117658Sjulian * Software is furnished to do so, subject to the following conditions: 1217658Sjulian * 1317658Sjulian * The above copyright notice and this permission notice shall be included in 1417658Sjulian * all copies or substantial portions of the Software. 1517658Sjulian * 1617658Sjulian * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1717658Sjulian * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1817658Sjulian * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1917658Sjulian * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 2017658Sjulian * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 2117658Sjulian * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 2217658Sjulian * OTHER DEALINGS IN THE SOFTWARE. 2317658Sjulian * 2417658Sjulian * Authors: Dave Airlie 2517658Sjulian * Alex Deucher 2617658Sjulian * Jerome Glisse 2717658Sjulian */ 2817658Sjulian 2917658Sjulian#include <sys/cdefs.h> 3017658Sjulian__FBSDID("$FreeBSD: releng/10.2/sys/dev/drm2/radeon/radeon_gart.c 282199 2015-04-28 19:35:05Z dumbbell $"); 3117658Sjulian 3217658Sjulian#include <dev/drm2/drmP.h> 3317658Sjulian#include <dev/drm2/radeon/radeon_drm.h> 3417658Sjulian#include "radeon.h" 3517658Sjulian#include "radeon_reg.h" 3617658Sjulian 37116182Sobrien/* 38116182Sobrien * GART 39116182Sobrien * The GART (Graphics Aperture Remapping Table) is an aperture 40174921Srwatson * in the GPU's address space. System pages can be mapped into 41131927Smarcel * the aperture and look like contiguous pages from the GPU's 4228976Sbde * perspective. A page table maps the pages in the aperture 43134649Sscottl * to the actual backing pages in system memory. 44221173Sattilio * 4517658Sjulian * Radeon GPUs support both an internal GART, as described above, 4617658Sjulian * and AGP. AGP works similarly, but the GART table is configured 4717658Sjulian * and maintained by the northbridge rather than the driver. 4860041Sphk * Radeon hw has a separate AGP aperture that is programmed to 4931275Sbde * point to the AGP aperture provided by the northbridge and the 5078767Sjhb * requests are passed through to the northbridge aperture. 5178767Sjhb * Both AGP and internal GART can be used at the same time, however 5278767Sjhb * that is not currently supported by the driver. 53193066Sjamie * 54131927Smarcel * This file handles the common internal GART management. 5517658Sjulian */ 56183527Speter 5755539Sluoqi/* 58243980Salfred * Common GART table functions. 5989601Ssobomax */ 6021776Sbde/** 61164033Srwatson * radeon_gart_table_ram_alloc - allocate system ram for gart page table 6278767Sjhb * 6378767Sjhb * @rdev: radeon_device pointer 6478767Sjhb * 65248084Sattilio * Allocate system memory for GART page table 66137263Speter * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 67206878Sattilio * gart table to be in system memory. 6817658Sjulian * Returns 0 for success, -ENOMEM for failure. 6917658Sjulian */ 70225448Sattilioint radeon_gart_table_ram_alloc(struct radeon_device *rdev) 71221173Sattilio{ 7217658Sjulian drm_dma_handle_t *dmah; 73174921Srwatson 74174921Srwatson dmah = drm_pci_alloc(rdev->ddev, rdev->gart.table_size, 75118990Smarcel PAGE_SIZE, BUS_SPACE_MAXADDR); 7694169Sphk if (dmah == NULL) { 7791778Sjake return -ENOMEM; 7817658Sjulian } 79163606Srwatson rdev->gart.dmah = dmah; 80163606Srwatson rdev->gart.ptr = dmah->vaddr; 81157628Spjd#ifdef CONFIG_X86 82157628Spjd if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 83157628Spjd rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 84157628Spjd pmap_change_attr((vm_offset_t)rdev->gart.ptr, 85157628Spjd rdev->gart.table_size >> PAGE_SHIFT, PAT_UNCACHED); 86157628Spjd } 8717658Sjulian#endif 8817658Sjulian rdev->gart.table_addr = dmah->busaddr; 8917658Sjulian memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size); 9017658Sjulian return 0; 9117658Sjulian} 92258956Scperciva 93267992Shselasky/** 94258893Scperciva * radeon_gart_table_ram_free - free system ram for gart page table 95258893Scperciva * 9617658Sjulian * @rdev: radeon_device pointer 9717658Sjulian * 9817658Sjulian * Free system memory for GART page table 9917658Sjulian * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the 10017658Sjulian * gart table to be in system memory. 10117658Sjulian */ 10217658Sjulianvoid radeon_gart_table_ram_free(struct radeon_device *rdev) 103131927Smarcel{ 104131927Smarcel if (rdev->gart.ptr == NULL) { 10542135Smsmith return; 10617658Sjulian } 10742135Smsmith#ifdef CONFIG_X86 10817658Sjulian if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || 109228475Sobrien rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { 110267992Shselasky pmap_change_attr((vm_offset_t)rdev->gart.ptr, 111228487Sobrien rdev->gart.table_size >> PAGE_SHIFT, PAT_WRITE_COMBINING); 112103647Sjhb } 113131927Smarcel#endif 114213322Savg drm_pci_free(rdev->ddev, rdev->gart.dmah); 115103647Sjhb rdev->gart.dmah = NULL; 116213322Savg rdev->gart.ptr = NULL; 11717658Sjulian rdev->gart.table_addr = 0; 118228475Sobrien} 119267992Shselasky 120228487Sobrien/** 121131927Smarcel * radeon_gart_table_vram_alloc - allocate vram for gart page table 12217658Sjulian * 123213322Savg * @rdev: radeon_device pointer 124267992Shselasky * 12585202Speter * Allocate video memory for GART page table 12685202Speter * (pcie r4xx, r5xx+). These asics require the 127227309Sed * gart table to be in video memory. 128227309Sed * Returns 0 for success, error for failure. 12943436Smsmith */ 130225448Sattilioint radeon_gart_table_vram_alloc(struct radeon_device *rdev) 131225448Sattilio{ 132225448Sattilio int r; 133225448Sattilio 134225448Sattilio if (rdev->gart.robj == NULL) { 135225448Sattilio r = radeon_bo_create(rdev, rdev->gart.table_size, 136225448Sattilio PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, 137225448Sattilio NULL, &rdev->gart.robj); 13817658Sjulian if (r) { 13917658Sjulian return r; 14017658Sjulian } 14117658Sjulian } 14217658Sjulian return 0; 14317658Sjulian} 14493496Sphk 145155383Sjeff/** 14693496Sphk * radeon_gart_table_vram_pin - pin gart page table in vram 14767093Sps * 148131927Smarcel * @rdev: radeon_device pointer 149131927Smarcel * 150235777Sharti * Pin the GART page table in vram so it will not be moved 151131927Smarcel * by the memory manager (pcie r4xx, r5xx+). These asics require the 15265395Speter * gart table to be in video memory. 15365395Speter * Returns 0 for success, error for failure. 15465395Speter */ 15565395Speterint radeon_gart_table_vram_pin(struct radeon_device *rdev) 156243980Salfred{ 15717658Sjulian uint64_t gpu_addr; 15850107Smsmith int r; 159110859Salfred 16050107Smsmith r = radeon_bo_reserve(rdev->gart.robj, false); 16150107Smsmith if (unlikely(r != 0)) 162110859Salfred return r; 163110859Salfred r = radeon_bo_pin(rdev->gart.robj, 164214279Sbrucec RADEON_GEM_DOMAIN_VRAM, &gpu_addr); 165110859Salfred if (r) { 166110859Salfred radeon_bo_unreserve(rdev->gart.robj); 167110859Salfred return r; 168110859Salfred } 169110859Salfred r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr); 170110859Salfred if (r) 17150107Smsmith radeon_bo_unpin(rdev->gart.robj); 17248868Sphk radeon_bo_unreserve(rdev->gart.robj); 173177253Srwatson rdev->gart.table_addr = gpu_addr; 17450107Smsmith return r; 17517658Sjulian} 176167211Srwatson 17717658Sjulian/** 17882749Sdillon * radeon_gart_table_vram_unpin - unpin gart page table in vram 17917658Sjulian * 180225617Skmacy * @rdev: radeon_device pointer 18117658Sjulian * 18217658Sjulian * Unpin the GART page table in vram (pcie r4xx, r5xx+). 18317658Sjulian * These asics require the gart table to be in video memory. 184106024Srwatson */ 185106024Srwatsonvoid radeon_gart_table_vram_unpin(struct radeon_device *rdev) 186172930Srwatson{ 187106024Srwatson int r; 188106024Srwatson 189164033Srwatson if (rdev->gart.robj == NULL) { 190106024Srwatson return; 191106024Srwatson } 192214004Smarcel r = radeon_bo_reserve(rdev->gart.robj, false); 193106024Srwatson if (likely(r == 0)) { 194106024Srwatson radeon_bo_kunmap(rdev->gart.robj); 19582749Sdillon radeon_bo_unpin(rdev->gart.robj); 19617658Sjulian radeon_bo_unreserve(rdev->gart.robj); 19717658Sjulian rdev->gart.ptr = NULL; 19817658Sjulian } 19917658Sjulian} 20017658Sjulian 20117658Sjulian/** 20265268Smsmith * radeon_gart_table_vram_free - free gart page table vram 20317658Sjulian * 204110859Salfred * @rdev: radeon_device pointer 20517658Sjulian * 206264237Sed * Free the video memory used for the GART page table 20773913Sjhb * (pcie r4xx, r5xx+). These asics require the gart table to 208264237Sed * be in video memory. 209264237Sed */ 210264237Sedvoid radeon_gart_table_vram_free(struct radeon_device *rdev) 211264237Sed{ 212264237Sed if (rdev->gart.robj == NULL) { 213264237Sed return; 21473913Sjhb } 21517658Sjulian radeon_gart_table_vram_unpin(rdev); 216264237Sed radeon_bo_unref(&rdev->gart.robj); 217264240Sed} 21817658Sjulian 21917658Sjulian/* 22017658Sjulian * Common gart functions. 22154233Sphk */ 22265395Speter/** 22354233Sphk * radeon_gart_unbind - unbind pages from the gart page table 22454233Sphk * 22554233Sphk * @rdev: radeon_device pointer 22654233Sphk * @offset: offset into the GPU's gart aperture 22754233Sphk * @pages: number of pages to unbind 22854233Sphk * 22954233Sphk * Unbinds the requested pages from the gart page table and 23054233Sphk * replaces them with the dummy page (all asics). 23165764Sjhb */ 23254233Sphkvoid radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, 23354233Sphk int pages) 23454233Sphk{ 23554233Sphk unsigned t; 23665764Sjhb unsigned p; 23754233Sphk int i, j; 23854233Sphk u64 page_base; 23954233Sphk 24054233Sphk if (!rdev->gart.ready) { 24165764Sjhb DRM_ERROR("trying to unbind memory from uninitialized GART !\n"); 24254233Sphk return; 24354233Sphk } 24454233Sphk t = offset / RADEON_GPU_PAGE_SIZE; 24565764Sjhb p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 24654233Sphk for (i = 0; i < pages; i++, p++) { 24754233Sphk if (rdev->gart.pages[p]) { 248222801Smarcel rdev->gart.pages[p] = NULL; 249222801Smarcel rdev->gart.pages_addr[p] = rdev->dummy_page.addr; 25094169Sphk page_base = rdev->gart.pages_addr[p]; 251222801Smarcel for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 252269105Sgavin if (rdev->gart.ptr) { 253110859Salfred radeon_gart_set_page(rdev, t, page_base); 254269105Sgavin } 255222801Smarcel page_base += RADEON_GPU_PAGE_SIZE; 256222801Smarcel } 257222801Smarcel } 258222801Smarcel } 259132412Sjulian mb(); 26094169Sphk radeon_gart_tlb_flush(rdev); 261131927Smarcel} 26294169Sphk 263222801Smarcel/** 264222801Smarcel * radeon_gart_bind - bind pages into the gart page table 265174921Srwatson * 266222801Smarcel * @rdev: radeon_device pointer 267222801Smarcel * @offset: offset into the GPU's gart aperture 268174921Srwatson * @pages: number of pages to bind 269222801Smarcel * @pagelist: pages to bind 270174921Srwatson * @dma_addr: DMA addresses of pages 271222801Smarcel * 272269105Sgavin * Binds the requested pages to the gart page table 273222801Smarcel * (all asics). 274176788Sru * Returns 0 for success, -EINVAL for failure. 275269105Sgavin */ 27694169Sphkint radeon_gart_bind(struct radeon_device *rdev, unsigned offset, 27794169Sphk int pages, vm_page_t *pagelist, dma_addr_t *dma_addr) 278149875Struckman{ 279149875Struckman unsigned t; 280149875Struckman unsigned p; 281149875Struckman uint64_t page_base; 282175486Sattilio int i, j; 283149875Struckman 284149875Struckman if (!rdev->gart.ready) { 285149875Struckman DRM_ERROR("trying to bind memory to uninitialized GART !\n"); 286149875Struckman return -EINVAL; 287149875Struckman } 28817658Sjulian t = offset / RADEON_GPU_PAGE_SIZE; 289137329Snjl p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 29017658Sjulian 291214004Smarcel for (i = 0; i < pages; i++, p++) { 292214004Smarcel rdev->gart.pages_addr[p] = dma_addr[i]; 29317658Sjulian rdev->gart.pages[p] = pagelist[i]; 294133763Struckman if (rdev->gart.ptr) { 295264237Sed page_base = rdev->gart.pages_addr[p]; 29617658Sjulian for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 297137375Smarcel radeon_gart_set_page(rdev, t, page_base); 298137329Snjl page_base += RADEON_GPU_PAGE_SIZE; 299137329Snjl } 300137329Snjl } 301137329Snjl } 302137329Snjl mb(); 303228424Savg radeon_gart_tlb_flush(rdev); 304228424Savg return 0; 305228424Savg} 306228424Savg 307228424Savg/** 308228424Savg * radeon_gart_restore - bind all pages in the gart page table 309137263Speter * 310155383Sjeff * @rdev: radeon_device pointer 311155383Sjeff * 312137263Speter * Binds all pages in the gart page table (all asics). 31382119Sjhb * Used to rebuild the gart table on device startup or resume. 314131927Smarcel */ 31582119Sjhbvoid radeon_gart_restore(struct radeon_device *rdev) 31627997Sjulian{ 31727997Sjulian int i, j, t; 31827997Sjulian u64 page_base; 31950107Smsmith 32027997Sjulian if (!rdev->gart.ptr) { 32127997Sjulian return; 32227997Sjulian } 32327997Sjulian for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) { 32417658Sjulian page_base = rdev->gart.pages_addr[i]; 32517658Sjulian for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { 32665707Sjasone radeon_gart_set_page(rdev, t, page_base); 327131481Sjhb page_base += RADEON_GPU_PAGE_SIZE; 32865707Sjasone } 329131481Sjhb } 33017658Sjulian mb(); 33117658Sjulian radeon_gart_tlb_flush(rdev); 33217658Sjulian} 333221173Sattilio 334225617Skmacy/** 33517658Sjulian * radeon_gart_init - init the driver info for managing the gart 33634266Sjulian * 33734266Sjulian * @rdev: radeon_device pointer 33834266Sjulian * 33934266Sjulian * Allocate the dummy page and init the gart driver info (all asics). 34034266Sjulian * Returns 0 for success, error for failure. 34165707Sjasone */ 34217658Sjulianint radeon_gart_init(struct radeon_device *rdev) 343149875Struckman{ 344149875Struckman int r, i; 34517658Sjulian 346133763Struckman if (rdev->gart.pages) { 347133763Struckman return 0; 348136115Sphk } 34917658Sjulian /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */ 350133763Struckman if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) { 351133763Struckman DRM_ERROR("Page size is smaller than GPU page size!\n"); 352133763Struckman return -EINVAL; 353133763Struckman } 354133763Struckman r = radeon_dummy_page_init(rdev); 35517658Sjulian if (r) 35665707Sjasone return r; 35765707Sjasone /* Compute table size */ 35865707Sjasone rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE; 359236503Savg rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE; 360221173Sattilio DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", 361225617Skmacy rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); 362131481Sjhb /* Allocate pages table */ 363131481Sjhb rdev->gart.pages = malloc(sizeof(void *) * rdev->gart.num_cpu_pages, 364131481Sjhb DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 365131481Sjhb if (rdev->gart.pages == NULL) { 366131481Sjhb radeon_gart_fini(rdev); 367131481Sjhb return -ENOMEM; 368131481Sjhb } 36934266Sjulian rdev->gart.pages_addr = malloc(sizeof(dma_addr_t) * 370131481Sjhb rdev->gart.num_cpu_pages, 371131481Sjhb DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 372131481Sjhb if (rdev->gart.pages_addr == NULL) { 373131481Sjhb radeon_gart_fini(rdev); 374131481Sjhb return -ENOMEM; 375131481Sjhb } 376131481Sjhb /* set GART entry to point to the dummy page by default */ 377131481Sjhb for (i = 0; i < rdev->gart.num_cpu_pages; i++) { 378170307Sjeff rdev->gart.pages_addr[i] = rdev->dummy_page.addr; 379131481Sjhb } 380170307Sjeff return 0; 381131481Sjhb} 382131481Sjhb 383131481Sjhb/** 384131481Sjhb * radeon_gart_fini - tear down the driver info for managing the gart 38517658Sjulian * 386133418Snjl * @rdev: radeon_device pointer 38741137Smsmith * 38841137Smsmith * Tear down the gart driver info and free the dummy page (all asics). 38941137Smsmith */ 39041137Smsmithvoid radeon_gart_fini(struct radeon_device *rdev) 39141137Smsmith{ 39241137Smsmith if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) { 393149875Struckman /* unbind pages */ 394137186Sphk radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); 395137186Sphk } 396130640Sphk rdev->gart.ready = false; 39753452Sphk free(rdev->gart.pages, DRM_MEM_DRIVER); 39848225Smckusick free(rdev->gart.pages_addr, DRM_MEM_DRIVER); 39953023Sphk rdev->gart.pages = NULL; 40053023Sphk rdev->gart.pages_addr = NULL; 401137186Sphk 40253023Sphk radeon_dummy_page_fini(rdev); 403225448Sattilio} 404225448Sattilio 405225448Sattilio/* 406225448Sattilio * GPUVM 407225448Sattilio * GPUVM is similar to the legacy gart on older asics, however 408225448Sattilio * rather than there being a single global gart table 409225448Sattilio * for the entire GPU, there are multiple VM page tables active 410225448Sattilio * at any given time. The VM page tables can contain a mix 411225448Sattilio * vram pages and system memory pages and system memory pages 412225448Sattilio * can be mapped as snooped (cached system pages) or unsnooped 413225448Sattilio * (uncached system pages). 41446568Speter * Each VM has an ID associated with it and there is a page table 41541137Smsmith * associated with each VMID. When execting a command buffer, 41617658Sjulian * the kernel tells the the ring what VMID to use for that command 41717658Sjulian * buffer. VMIDs are allocated dynamically as commands are submitted. 41817658Sjulian * The userspace drivers maintain their own address space and the kernel 41917658Sjulian * sets up their pages tables accordingly when they submit their 42017658Sjulian * command buffers and a VMID is assigned. 421133763Struckman * Cayman/Trinity support up to 8 active VMs at any given time; 42217658Sjulian * SI supports 16. 42317658Sjulian */ 424133763Struckman 425133763Struckman/* 42617658Sjulian * vm helpers 42717658Sjulian * 42817658Sjulian * TODO bind a default page at vm initialization for default address 42917658Sjulian */ 43017658Sjulian 43117658Sjulian/** 432157628Spjd * radeon_vm_num_pde - return the number of page directory entries 43339237Sgibbs * 43417658Sjulian * @rdev: radeon_device pointer 43527997Sjulian * 43654233Sphk * Calculate the number of page directory entries (cayman+). 43754233Sphk */ 438228632Savgstatic unsigned radeon_vm_num_pdes(struct radeon_device *rdev) 439228632Savg{ 44027997Sjulian return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE; 44127997Sjulian} 44227997Sjulian 44327997Sjulian/** 44450107Smsmith * radeon_vm_directory_size - returns the size of the page directory in bytes 445137329Snjl * 446132412Sjulian * @rdev: radeon_device pointer 447222801Smarcel * 44839237Sgibbs * Calculate the size of the page directory in bytes (cayman+). 44939237Sgibbs */ 45050107Smsmithstatic unsigned radeon_vm_directory_size(struct radeon_device *rdev) 45139237Sgibbs{ 45250107Smsmith return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8); 45350107Smsmith} 45450107Smsmith 45550107Smsmith/** 45650107Smsmith * radeon_vm_manager_init - init the vm manager 45750107Smsmith * 45850107Smsmith * @rdev: radeon_device pointer 45950107Smsmith * 46050107Smsmith * Init the vm manager (cayman+). 46150107Smsmith * Returns 0 for success, error for failure. 462110859Salfred */ 46317658Sjulianint radeon_vm_manager_init(struct radeon_device *rdev) 46417658Sjulian{ 46517658Sjulian struct radeon_vm *vm; 46617658Sjulian struct radeon_bo_va *bo_va; 46719274Sjulian int r; 46819274Sjulian unsigned size; 46919274Sjulian 47019274Sjulian if (!rdev->vm_manager.enabled) { 47119274Sjulian /* allocate enough for 2 full VM pts */ 47239237Sgibbs size = radeon_vm_directory_size(rdev); 47319274Sjulian size += rdev->vm_manager.max_pfn * 8; 47419274Sjulian size *= 2; 47550107Smsmith r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager, 47650107Smsmith RADEON_GPU_PAGE_ALIGN(size), 47717658Sjulian RADEON_GEM_DOMAIN_VRAM); 47850107Smsmith if (r) { 47950107Smsmith dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n", 48050107Smsmith (rdev->vm_manager.max_pfn * 8) >> 10); 48150107Smsmith return r; 48250107Smsmith } 48350107Smsmith 48450107Smsmith r = radeon_asic_vm_init(rdev); 48550107Smsmith if (r) 48650107Smsmith return r; 48750107Smsmith 488258893Scperciva rdev->vm_manager.enabled = true; 489258893Scperciva 49039237Sgibbs r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager); 49139237Sgibbs if (r) 492258893Scperciva return r; 493258893Scperciva } 49439237Sgibbs 49539237Sgibbs /* restore page table */ 49639237Sgibbs list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) { 49739237Sgibbs if (vm->page_directory == NULL) 49839237Sgibbs continue; 49917658Sjulian 50039237Sgibbs list_for_each_entry(bo_va, &vm->va, vm_list) { 50150107Smsmith bo_va->valid = false; 50217658Sjulian } 50339237Sgibbs } 50450107Smsmith return 0; 50517658Sjulian} 50689522Snik 50789522Snik/** 50839237Sgibbs * radeon_vm_free_pt - free the page table for a specific vm 50917658Sjulian * 51050107Smsmith * @rdev: radeon_device pointer 51150107Smsmith * @vm: vm to unbind 51250107Smsmith * 51350107Smsmith * Free the page table of a specific vm (cayman+). 51450107Smsmith * 51550107Smsmith * Global and local mutex must be lock! 51650107Smsmith */ 51750107Smsmithstatic void radeon_vm_free_pt(struct radeon_device *rdev, 518110859Salfred struct radeon_vm *vm) 519206878Sattilio{ 520206878Sattilio struct radeon_bo_va *bo_va; 521206878Sattilio int i; 522196196Sattilio 523206878Sattilio if (!vm->page_directory) 524206878Sattilio return; 525206878Sattilio 526206878Sattilio list_del_init(&vm->list); 527206878Sattilio radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 528206878Sattilio 529206878Sattilio list_for_each_entry(bo_va, &vm->va, vm_list) { 530206897Sattilio bo_va->valid = false; 531206897Sattilio } 532196196Sattilio 533206897Sattilio if (vm->page_tables == NULL) 534206878Sattilio return; 535206897Sattilio 536206897Sattilio for (i = 0; i < radeon_vm_num_pdes(rdev); i++) 537206897Sattilio radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence); 538196196Sattilio 53917677Sjulian free(vm->page_tables, DRM_MEM_DRIVER); 54017658Sjulian} 54150107Smsmith 54217658Sjulian/** 54317658Sjulian * radeon_vm_manager_fini - tear down the vm manager 544244105Salfred * 545243980Salfred * @rdev: radeon_device pointer 546244099Salfred * 547244099Salfred * Tear down the VM manager (cayman+). 548244099Salfred */ 549243980Salfredvoid radeon_vm_manager_fini(struct radeon_device *rdev) 550243980Salfred{ 551243980Salfred struct radeon_vm *vm, *tmp; 552243980Salfred int i; 553243980Salfred 554243980Salfred if (!rdev->vm_manager.enabled) 555243980Salfred return; 556243980Salfred 557243980Salfred sx_xlock(&rdev->vm_manager.lock); 558243980Salfred /* free all allocated page tables */ 559243980Salfred list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) { 560267992Shselasky sx_xlock(&vm->mutex); 561243980Salfred radeon_vm_free_pt(rdev, vm); 562243980Salfred sx_xunlock(&vm->mutex); 563243980Salfred } 564244099Salfred for (i = 0; i < RADEON_NUM_VM; ++i) { 565267992Shselasky radeon_fence_unref(&rdev->vm_manager.active[i]); 566244099Salfred } 567244099Salfred radeon_asic_vm_fini(rdev); 568244099Salfred sx_xunlock(&rdev->vm_manager.lock); 569243980Salfred 570267992Shselasky radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager); 571243980Salfred radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager); 572243980Salfred rdev->vm_manager.enabled = false; 573243980Salfred} 574243980Salfred 575267992Shselasky/** 576243980Salfred * radeon_vm_evict - evict page table to make room for new one 577243980Salfred * 578267992Shselasky * @rdev: radeon_device pointer 579243980Salfred * @vm: VM we want to allocate something for 580243980Salfred * 581267992Shselasky * Evict a VM from the lru, making sure that it isn't @vm. (cayman+). 582243980Salfred * Returns 0 for success, -ENOMEM for failure. 583243980Salfred * 584267992Shselasky * Global and local mutex must be locked! 585243980Salfred */ 586243980Salfredstatic int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm) 587267992Shselasky{ 588243980Salfred struct radeon_vm *vm_evict; 589243980Salfred 590243980Salfred if (list_empty(&rdev->vm_manager.lru_vm)) 591243980Salfred return -ENOMEM; 592243980Salfred 593243980Salfred vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, 594243980Salfred struct radeon_vm, list); 595243980Salfred if (vm_evict == vm) 596243980Salfred return -ENOMEM; 597243980Salfred 598243980Salfred sx_xlock(&vm_evict->mutex); 599243980Salfred radeon_vm_free_pt(rdev, vm_evict); 600243980Salfred sx_xunlock(&vm_evict->mutex); 601243980Salfred return 0; 602243980Salfred} 603243980Salfred 604243980Salfred/** 605243980Salfred * radeon_vm_alloc_pt - allocates a page table for a VM 606243980Salfred * 607243980Salfred * @rdev: radeon_device pointer 608243980Salfred * @vm: vm to bind 609243980Salfred * 610243980Salfred * Allocate a page table for the requested vm (cayman+). 611243980Salfred * Returns 0 for success, error for failure. 61217658Sjulian * 613243980Salfred * Global and local mutex must be locked! 614243980Salfred */ 615243980Salfredint radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm) 616243980Salfred{ 617243980Salfred unsigned pd_size, pts_size; 618243980Salfred u64 *pd_addr; 619243980Salfred int r; 620243980Salfred 621243980Salfred if (vm == NULL) { 622243980Salfred return -EINVAL; 623243980Salfred } 624243980Salfred 625243980Salfred if (vm->page_directory != NULL) { 626243980Salfred return 0; 627243980Salfred } 628243980Salfred 629243980Salfredretry: 630243980Salfred pd_size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev)); 631243980Salfred r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 632243980Salfred &vm->page_directory, pd_size, 633243980Salfred RADEON_GPU_PAGE_SIZE, false); 634243980Salfred if (r == -ENOMEM) { 635243980Salfred r = radeon_vm_evict(rdev, vm); 636243980Salfred if (r) 637243980Salfred return r; 638243980Salfred goto retry; 639243980Salfred 640243980Salfred } else if (r) { 641243980Salfred return r; 642243980Salfred } 643243980Salfred 644243980Salfred vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory); 645243980Salfred 646243980Salfred /* Initially clear the page directory */ 647243980Salfred pd_addr = radeon_sa_bo_cpu_addr(vm->page_directory); 648243980Salfred memset(pd_addr, 0, pd_size); 649243980Salfred 650243980Salfred pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *); 651243980Salfred vm->page_tables = malloc(pts_size, DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 652243980Salfred 653243980Salfred if (vm->page_tables == NULL) { 654243980Salfred DRM_ERROR("Cannot allocate memory for page table array\n"); 655244099Salfred radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence); 656244099Salfred return -ENOMEM; 657244099Salfred } 658244099Salfred 659244099Salfred return 0; 660243980Salfred} 661243980Salfred 662243980Salfred/** 663243980Salfred * radeon_vm_add_to_lru - add VMs page table to LRU list 664243980Salfred * 66517658Sjulian * @rdev: radeon_device pointer 66617658Sjulian * @vm: vm to add to LRU 66717658Sjulian * 66817658Sjulian * Add the allocated page table to the LRU list (cayman+). 66917658Sjulian * 670130164Sphk * Global mutex must be locked! 67117658Sjulian */ 672243980Salfredvoid radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm) 673243980Salfred{ 674243980Salfred list_del_init(&vm->list); 675243980Salfred list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); 676243980Salfred} 677243980Salfred 678243980Salfred/** 679243980Salfred * radeon_vm_grab_id - allocate the next free VMID 680243980Salfred * 681213648Savg * @rdev: radeon_device pointer 682228424Savg * @vm: vm to allocate id for 683213648Savg * @ring: ring we want to submit job to 684100209Sgallatin * 685103647Sjhb * Allocate an id for the vm (cayman+). 68638874Sache * Returns the fence we need to sync to (if any). 68717658Sjulian * 688243515Savg * Global and local mutex must be locked! 689228424Savg */ 69065557Sjasonestruct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, 69182115Sjhb struct radeon_vm *vm, int ring) 692243515Savg{ 693243515Savg struct radeon_fence *best[RADEON_NUM_RINGS] = {}; 694243515Savg unsigned choices[2] = {}; 69582115Sjhb unsigned i; 696243515Savg 697243515Savg /* check if the id is still valid */ 698243515Savg if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id]) 699243515Savg return NULL; 700243515Savg 701228424Savg /* we definately need to flush */ 702243515Savg radeon_fence_unref(&vm->last_flush); 703243515Savg 704243515Savg /* skip over VMID 0, since it is the system VM */ 705243515Savg for (i = 1; i < rdev->vm_manager.nvm; ++i) { 706243515Savg struct radeon_fence *fence = rdev->vm_manager.active[i]; 707243515Savg 708243515Savg if (fence == NULL) { 70965557Sjasone /* found a free one */ 71065557Sjasone vm->id = i; 711222865Sattilio return NULL; 712103647Sjhb } 71317658Sjulian 71417658Sjulian if (radeon_fence_is_earlier(fence, best[fence->ring])) { 715103647Sjhb best[fence->ring] = fence; 716222865Sattilio choices[fence->ring == ring ? 0 : 1] = i; 71717658Sjulian } 718103647Sjhb } 719103647Sjhb 72017658Sjulian for (i = 0; i < 2; ++i) { 721116398Siedowse if (choices[i]) { 722116398Siedowse vm->id = choices[i]; 72338874Sache return rdev->vm_manager.active[choices[i]]; 724228632Savg } 725130164Sphk } 726116398Siedowse 727116398Siedowse /* should never happen */ 728116398Siedowse panic("%s: failed to allocate next VMID", __func__); 729130164Sphk return NULL; 730116398Siedowse} 73126100Sfsmp 732134089Sjhb/** 73326100Sfsmp * radeon_vm_fence - remember fence for vm 73417658Sjulian * 735131927Smarcel * @rdev: radeon_device pointer 736103647Sjhb * @vm: vm we want to fence 737131927Smarcel * @fence: fence to remember 73817658Sjulian * 739174898Srwatson * Fence the vm (cayman+). 74017658Sjulian * Set the fence used to protect page table and id. 741170307Sjeff * 742100209Sgallatin * Global and local mutex must be locked! 743170307Sjeff */ 74485202Spetervoid radeon_vm_fence(struct radeon_device *rdev, 74585202Speter struct radeon_vm *vm, 746214004Smarcel struct radeon_fence *fence) 74717658Sjulian{ 74817658Sjulian radeon_fence_unref(&rdev->vm_manager.active[vm->id]); 74917768Sjulian rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); 75043436Smsmith 751197071Sn_hibma radeon_fence_unref(&vm->fence); 752197071Sn_hibma vm->fence = radeon_fence_ref(fence); 753197071Sn_hibma} 754197071Sn_hibma 75543436Smsmith/** 75654248Smsmith * radeon_vm_bo_find - find the bo_va for a specific vm & bo 75754248Smsmith * 75854248Smsmith * @vm: requested vm 75954248Smsmith * @bo: requested buffer object 76054248Smsmith * 76143436Smsmith * Find @bo inside the requested vm (cayman+). 762228449Seadler * Search inside the @bos vm list for the requested vm 76343436Smsmith * Returns the found bo_va or NULL if none is found 764110859Salfred * 76550107Smsmith * Object has to be reserved! 76643436Smsmith */ 767110859Salfredstruct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, 768110859Salfred struct radeon_bo *bo) 76943436Smsmith{ 77043436Smsmith struct radeon_bo_va *bo_va; 77143436Smsmith 77255539Sluoqi list_for_each_entry(bo_va, &bo->va, bo_list) { 77355539Sluoqi if (bo_va->vm == vm) { 77455539Sluoqi return bo_va; 77555539Sluoqi } 77655539Sluoqi } 77755539Sluoqi return NULL; 77855539Sluoqi} 77955539Sluoqi 78055539Sluoqi/** 78155539Sluoqi * radeon_vm_bo_add - add a bo to a specific vm 782228449Seadler * 78355539Sluoqi * @rdev: radeon_device pointer 78455539Sluoqi * @vm: requested vm 78570063Sjhb * @bo: radeon buffer object 78655539Sluoqi * 78755539Sluoqi * Add @bo into the requested vm (cayman+). 78855539Sluoqi * Add @bo to the list of bos associated with the vm 78955539Sluoqi * Returns newly added bo_va or NULL for failure 79055539Sluoqi * 79155539Sluoqi * Object has to be reserved! 79255539Sluoqi */ 79355539Sluoqistruct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev, 794132866Snjl struct radeon_vm *vm, 795198408Sjhb struct radeon_bo *bo) 796172836Sjulian{ 79755539Sluoqi struct radeon_bo_va *bo_va; 79855539Sluoqi 799132866Snjl bo_va = malloc(sizeof(struct radeon_bo_va), 80055539Sluoqi DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 801132866Snjl if (bo_va == NULL) { 80255539Sluoqi return NULL; 80393496Sphk } 804173004Sjulian bo_va->vm = vm; 805173004Sjulian bo_va->bo = bo; 806173004Sjulian bo_va->soffset = 0; 807173004Sjulian bo_va->eoffset = 0; 808173004Sjulian bo_va->flags = 0; 809173004Sjulian bo_va->valid = false; 810173004Sjulian bo_va->ref_count = 1; 811173004Sjulian INIT_LIST_HEAD(&bo_va->bo_list); 812173004Sjulian INIT_LIST_HEAD(&bo_va->vm_list); 813173004Sjulian 814173004Sjulian sx_xlock(&vm->mutex); 815198408Sjhb list_add(&bo_va->vm_list, &vm->va); 816173004Sjulian list_add_tail(&bo_va->bo_list, &bo->va); 817173004Sjulian sx_xunlock(&vm->mutex); 818173004Sjulian 819173004Sjulian return bo_va; 820173004Sjulian} 821173004Sjulian 822173004Sjulian/** 823173004Sjulian * radeon_vm_bo_set_addr - set bos virtual address inside a vm 824242439Salfred * 825242439Salfred * @rdev: radeon_device pointer 826242439Salfred * @bo_va: bo_va to store the address 827242439Salfred * @soffset: requested offset of the buffer in the VM address space 82893496Sphk * @flags: attributes of pages (read/write/valid/etc.) 82993496Sphk * 830274366Spjd * Set offset of @bo_va (cayman+). 83193496Sphk * Validate and set the offset requested within the vm address space. 832242489Salfred * Returns 0 for success, error for failure. 833274366Spjd * 834110859Salfred * Object has to be reserved! 835274366Spjd */ 836274366Spjdint radeon_vm_bo_set_addr(struct radeon_device *rdev, 837274366Spjd struct radeon_bo_va *bo_va, 838274366Spjd uint64_t soffset, 83993496Sphk uint32_t flags) 84093496Sphk{ 841242439Salfred uint64_t size = radeon_bo_size(bo_va->bo); 84293496Sphk uint64_t eoffset, last_offset = 0; 84393496Sphk struct radeon_vm *vm = bo_va->vm; 84493496Sphk struct radeon_bo_va *tmp; 84593496Sphk struct list_head *head; 84693496Sphk unsigned last_pfn; 847242489Salfred 848242489Salfred if (soffset) { 849242439Salfred /* make sure object fit at this offset */ 850242439Salfred eoffset = soffset + size; 851242439Salfred if (soffset >= eoffset) { 85293496Sphk return -EINVAL; 85393496Sphk } 85493496Sphk 855175768Sru last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; 856175768Sru if (last_pfn > rdev->vm_manager.max_pfn) { 857175768Sru dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", 858175768Sru last_pfn, rdev->vm_manager.max_pfn); 859175768Sru return -EINVAL; 860175768Sru } 861175768Sru 862175768Sru } else { 863225516Sattilio eoffset = last_pfn = 0; 864225516Sattilio } 865225516Sattilio 866225516Sattilio sx_xlock(&vm->mutex); 867225516Sattilio head = &vm->va; 868175768Sru last_offset = 0; 869175768Sru list_for_each_entry(tmp, &vm->va, vm_list) { 870175768Sru if (bo_va == tmp) { 871175768Sru /* skip over currently modified bo */ 87293496Sphk continue; 873183527Speter } 874183527Speter 875183527Speter if (soffset >= last_offset && eoffset <= tmp->soffset) { 876183527Speter /* bo can be added before this one */ 877183527Speter break; 878183527Speter } 879183527Speter if (eoffset > tmp->soffset && soffset < tmp->eoffset) { 880183527Speter /* bo and tmp overlap, invalid offset */ 881183527Speter dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", 882183527Speter bo_va->bo, (unsigned)bo_va->soffset, tmp->bo, 883183527Speter (unsigned)tmp->soffset, (unsigned)tmp->eoffset); 884183527Speter sx_xunlock(&vm->mutex); 885194118Sjamie return -EINVAL; 886183527Speter } 887183527Speter last_offset = tmp->eoffset; 888183527Speter head = &tmp->vm_list; 889183527Speter } 890183527Speter 891 bo_va->soffset = soffset; 892 bo_va->eoffset = eoffset; 893 bo_va->flags = flags; 894 bo_va->valid = false; 895 list_move(&bo_va->vm_list, head); 896 897 sx_xunlock(&vm->mutex); 898 return 0; 899} 900 901/** 902 * radeon_vm_map_gart - get the physical address of a gart page 903 * 904 * @rdev: radeon_device pointer 905 * @addr: the unmapped addr 906 * 907 * Look up the physical address of the page that the pte resolves 908 * to (cayman+). 909 * Returns the physical address of the page. 910 */ 911uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) 912{ 913 uint64_t result; 914 915 /* page table offset */ 916 result = rdev->gart.pages_addr[addr >> PAGE_SHIFT]; 917 918 /* in case cpu page size != gpu page size*/ 919 /* 920 * FreeBSD port note: FreeBSD's PAGE_MASK is the inverse of 921 * Linux's one. That's why the test below doesn't inverse the 922 * constant. 923 */ 924 result |= addr & (PAGE_MASK); 925 926 return result; 927} 928 929/** 930 * radeon_vm_update_pdes - make sure that page directory is valid 931 * 932 * @rdev: radeon_device pointer 933 * @vm: requested vm 934 * @start: start of GPU address range 935 * @end: end of GPU address range 936 * 937 * Allocates new page tables if necessary 938 * and updates the page directory (cayman+). 939 * Returns 0 for success, error for failure. 940 * 941 * Global and local mutex must be locked! 942 */ 943static int radeon_vm_update_pdes(struct radeon_device *rdev, 944 struct radeon_vm *vm, 945 uint64_t start, uint64_t end) 946{ 947 static const uint32_t incr = RADEON_VM_PTE_COUNT * 8; 948 949 uint64_t last_pde = ~0, last_pt = ~0; 950 unsigned count = 0; 951 uint64_t pt_idx; 952 int r; 953 954 start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 955 end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE; 956 957 /* walk over the address space and update the page directory */ 958 for (pt_idx = start; pt_idx <= end; ++pt_idx) { 959 uint64_t pde, pt; 960 961 if (vm->page_tables[pt_idx]) 962 continue; 963 964retry: 965 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, 966 &vm->page_tables[pt_idx], 967 RADEON_VM_PTE_COUNT * 8, 968 RADEON_GPU_PAGE_SIZE, false); 969 970 if (r == -ENOMEM) { 971 r = radeon_vm_evict(rdev, vm); 972 if (r) 973 return r; 974 goto retry; 975 } else if (r) { 976 return r; 977 } 978 979 pde = vm->pd_gpu_addr + pt_idx * 8; 980 981 pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 982 983 if (((last_pde + 8 * count) != pde) || 984 ((last_pt + incr * count) != pt)) { 985 986 if (count) { 987 radeon_asic_vm_set_page(rdev, last_pde, 988 last_pt, count, incr, 989 RADEON_VM_PAGE_VALID); 990 } 991 992 count = 1; 993 last_pde = pde; 994 last_pt = pt; 995 } else { 996 ++count; 997 } 998 } 999 1000 if (count) { 1001 radeon_asic_vm_set_page(rdev, last_pde, last_pt, count, 1002 incr, RADEON_VM_PAGE_VALID); 1003 1004 } 1005 1006 return 0; 1007} 1008 1009/** 1010 * radeon_vm_update_ptes - make sure that page tables are valid 1011 * 1012 * @rdev: radeon_device pointer 1013 * @vm: requested vm 1014 * @start: start of GPU address range 1015 * @end: end of GPU address range 1016 * @dst: destination address to map to 1017 * @flags: mapping flags 1018 * 1019 * Update the page tables in the range @start - @end (cayman+). 1020 * 1021 * Global and local mutex must be locked! 1022 */ 1023static void radeon_vm_update_ptes(struct radeon_device *rdev, 1024 struct radeon_vm *vm, 1025 uint64_t start, uint64_t end, 1026 uint64_t dst, uint32_t flags) 1027{ 1028 static const uint64_t mask = RADEON_VM_PTE_COUNT - 1; 1029 1030 uint64_t last_pte = ~0, last_dst = ~0; 1031 unsigned count = 0; 1032 uint64_t addr; 1033 1034 start = start / RADEON_GPU_PAGE_SIZE; 1035 end = end / RADEON_GPU_PAGE_SIZE; 1036 1037 /* walk over the address space and update the page tables */ 1038 for (addr = start; addr < end; ) { 1039 uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE; 1040 unsigned nptes; 1041 uint64_t pte; 1042 1043 if ((addr & ~mask) == (end & ~mask)) 1044 nptes = end - addr; 1045 else 1046 nptes = RADEON_VM_PTE_COUNT - (addr & mask); 1047 1048 pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]); 1049 pte += (addr & mask) * 8; 1050 1051 if ((last_pte + 8 * count) != pte) { 1052 1053 if (count) { 1054 radeon_asic_vm_set_page(rdev, last_pte, 1055 last_dst, count, 1056 RADEON_GPU_PAGE_SIZE, 1057 flags); 1058 } 1059 1060 count = nptes; 1061 last_pte = pte; 1062 last_dst = dst; 1063 } else { 1064 count += nptes; 1065 } 1066 1067 addr += nptes; 1068 dst += nptes * RADEON_GPU_PAGE_SIZE; 1069 } 1070 1071 if (count) { 1072 radeon_asic_vm_set_page(rdev, last_pte, last_dst, count, 1073 RADEON_GPU_PAGE_SIZE, flags); 1074 } 1075} 1076 1077/** 1078 * radeon_vm_bo_update_pte - map a bo into the vm page table 1079 * 1080 * @rdev: radeon_device pointer 1081 * @vm: requested vm 1082 * @bo: radeon buffer object 1083 * @mem: ttm mem 1084 * 1085 * Fill in the page table entries for @bo (cayman+). 1086 * Returns 0 for success, -EINVAL for failure. 1087 * 1088 * Object have to be reserved & global and local mutex must be locked! 1089 */ 1090int radeon_vm_bo_update_pte(struct radeon_device *rdev, 1091 struct radeon_vm *vm, 1092 struct radeon_bo *bo, 1093 struct ttm_mem_reg *mem) 1094{ 1095 unsigned ridx = rdev->asic->vm.pt_ring_index; 1096 struct radeon_ring *ring = &rdev->ring[ridx]; 1097 struct radeon_semaphore *sem = NULL; 1098 struct radeon_bo_va *bo_va; 1099 unsigned nptes, npdes, ndw; 1100 uint64_t addr; 1101 int r; 1102 1103 /* nothing to do if vm isn't bound */ 1104 if (vm->page_directory == NULL) 1105 return 0; 1106 1107 bo_va = radeon_vm_bo_find(vm, bo); 1108 if (bo_va == NULL) { 1109 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); 1110 return -EINVAL; 1111 } 1112 1113 if (!bo_va->soffset) { 1114 dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n", 1115 bo, vm); 1116 return -EINVAL; 1117 } 1118 1119 if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL)) 1120 return 0; 1121 1122 bo_va->flags &= ~RADEON_VM_PAGE_VALID; 1123 bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; 1124 if (mem) { 1125 addr = mem->start << PAGE_SHIFT; 1126 if (mem->mem_type != TTM_PL_SYSTEM) { 1127 bo_va->flags |= RADEON_VM_PAGE_VALID; 1128 bo_va->valid = true; 1129 } 1130 if (mem->mem_type == TTM_PL_TT) { 1131 bo_va->flags |= RADEON_VM_PAGE_SYSTEM; 1132 } else { 1133 addr += rdev->vm_manager.vram_base_offset; 1134 } 1135 } else { 1136 addr = 0; 1137 bo_va->valid = false; 1138 } 1139 1140 if (vm->fence && radeon_fence_signaled(vm->fence)) { 1141 radeon_fence_unref(&vm->fence); 1142 } 1143 1144 if (vm->fence && vm->fence->ring != ridx) { 1145 r = radeon_semaphore_create(rdev, &sem); 1146 if (r) { 1147 return r; 1148 } 1149 } 1150 1151 nptes = radeon_bo_ngpu_pages(bo); 1152 1153 /* assume two extra pdes in case the mapping overlaps the borders */ 1154 npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2; 1155 1156 /* estimate number of dw needed */ 1157 /* semaphore, fence and padding */ 1158 ndw = 32; 1159 1160 if (RADEON_VM_BLOCK_SIZE > 11) 1161 /* reserve space for one header for every 2k dwords */ 1162 ndw += (nptes >> 11) * 4; 1163 else 1164 /* reserve space for one header for 1165 every (1 << BLOCK_SIZE) entries */ 1166 ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4; 1167 1168 /* reserve space for pte addresses */ 1169 ndw += nptes * 2; 1170 1171 /* reserve space for one header for every 2k dwords */ 1172 ndw += (npdes >> 11) * 4; 1173 1174 /* reserve space for pde addresses */ 1175 ndw += npdes * 2; 1176 1177 r = radeon_ring_lock(rdev, ring, ndw); 1178 if (r) { 1179 return r; 1180 } 1181 1182 if (sem && radeon_fence_need_sync(vm->fence, ridx)) { 1183 radeon_semaphore_sync_rings(rdev, sem, vm->fence->ring, ridx); 1184 radeon_fence_note_sync(vm->fence, ridx); 1185 } 1186 1187 r = radeon_vm_update_pdes(rdev, vm, bo_va->soffset, bo_va->eoffset); 1188 if (r) { 1189 radeon_ring_unlock_undo(rdev, ring); 1190 return r; 1191 } 1192 1193 radeon_vm_update_ptes(rdev, vm, bo_va->soffset, bo_va->eoffset, 1194 addr, bo_va->flags); 1195 1196 radeon_fence_unref(&vm->fence); 1197 r = radeon_fence_emit(rdev, &vm->fence, ridx); 1198 if (r) { 1199 radeon_ring_unlock_undo(rdev, ring); 1200 return r; 1201 } 1202 radeon_ring_unlock_commit(rdev, ring); 1203 radeon_semaphore_free(rdev, &sem, vm->fence); 1204 radeon_fence_unref(&vm->last_flush); 1205 1206 return 0; 1207} 1208 1209/** 1210 * radeon_vm_bo_rmv - remove a bo to a specific vm 1211 * 1212 * @rdev: radeon_device pointer 1213 * @bo_va: requested bo_va 1214 * 1215 * Remove @bo_va->bo from the requested vm (cayman+). 1216 * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and 1217 * remove the ptes for @bo_va in the page table. 1218 * Returns 0 for success. 1219 * 1220 * Object have to be reserved! 1221 */ 1222int radeon_vm_bo_rmv(struct radeon_device *rdev, 1223 struct radeon_bo_va *bo_va) 1224{ 1225 int r; 1226 1227 sx_xlock(&rdev->vm_manager.lock); 1228 sx_xlock(&bo_va->vm->mutex); 1229 r = radeon_vm_bo_update_pte(rdev, bo_va->vm, bo_va->bo, NULL); 1230 sx_xunlock(&rdev->vm_manager.lock); 1231 list_del(&bo_va->vm_list); 1232 sx_xunlock(&bo_va->vm->mutex); 1233 list_del(&bo_va->bo_list); 1234 1235 free(bo_va, DRM_MEM_DRIVER); 1236 return r; 1237} 1238 1239/** 1240 * radeon_vm_bo_invalidate - mark the bo as invalid 1241 * 1242 * @rdev: radeon_device pointer 1243 * @vm: requested vm 1244 * @bo: radeon buffer object 1245 * 1246 * Mark @bo as invalid (cayman+). 1247 */ 1248void radeon_vm_bo_invalidate(struct radeon_device *rdev, 1249 struct radeon_bo *bo) 1250{ 1251 struct radeon_bo_va *bo_va; 1252 1253 list_for_each_entry(bo_va, &bo->va, bo_list) { 1254 bo_va->valid = false; 1255 } 1256} 1257 1258/** 1259 * radeon_vm_init - initialize a vm instance 1260 * 1261 * @rdev: radeon_device pointer 1262 * @vm: requested vm 1263 * 1264 * Init @vm fields (cayman+). 1265 */ 1266void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) 1267{ 1268 vm->id = 0; 1269 vm->fence = NULL; 1270 sx_init(&vm->mutex, "drm__radeon_vm__mutex"); 1271 INIT_LIST_HEAD(&vm->list); 1272 INIT_LIST_HEAD(&vm->va); 1273} 1274 1275/** 1276 * radeon_vm_fini - tear down a vm instance 1277 * 1278 * @rdev: radeon_device pointer 1279 * @vm: requested vm 1280 * 1281 * Tear down @vm (cayman+). 1282 * Unbind the VM and remove all bos from the vm bo list 1283 */ 1284void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) 1285{ 1286 struct radeon_bo_va *bo_va, *tmp; 1287 int r; 1288 1289 sx_xlock(&rdev->vm_manager.lock); 1290 sx_xlock(&vm->mutex); 1291 radeon_vm_free_pt(rdev, vm); 1292 sx_xunlock(&rdev->vm_manager.lock); 1293 1294 if (!list_empty(&vm->va)) { 1295 dev_err(rdev->dev, "still active bo inside vm\n"); 1296 } 1297 list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) { 1298 list_del_init(&bo_va->vm_list); 1299 r = radeon_bo_reserve(bo_va->bo, false); 1300 if (!r) { 1301 list_del_init(&bo_va->bo_list); 1302 radeon_bo_unreserve(bo_va->bo); 1303 free(bo_va, DRM_MEM_DRIVER); 1304 } 1305 } 1306 radeon_fence_unref(&vm->fence); 1307 radeon_fence_unref(&vm->last_flush); 1308 sx_xunlock(&vm->mutex); 1309} 1310