1105197Ssam// SPDX-License-Identifier: MIT 2105197Ssam/* 3105197Ssam * Copyright �� 2021 Intel Corporation 4105197Ssam */ 5105197Ssam 6105197Ssam#include <linux/shmem_fs.h> 7105197Ssam 8105197Ssam#include <drm/ttm/ttm_placement.h> 9105197Ssam#include <drm/ttm/ttm_tt.h> 10105197Ssam#include <drm/drm_buddy.h> 11105197Ssam 12105197Ssam#include "i915_drv.h" 13105197Ssam#include "i915_ttm_buddy_manager.h" 14105197Ssam#include "intel_memory_region.h" 15105197Ssam#include "intel_region_ttm.h" 16105197Ssam 17105197Ssam#include "gem/i915_gem_mman.h" 18105197Ssam#include "gem/i915_gem_object.h" 19105197Ssam#include "gem/i915_gem_region.h" 20105197Ssam#include "gem/i915_gem_ttm.h" 21105197Ssam#include "gem/i915_gem_ttm_move.h" 22105197Ssam#include "gem/i915_gem_ttm_pm.h" 23105197Ssam#include "gt/intel_gpu_commands.h" 24105197Ssam 25105197Ssam#define I915_TTM_PRIO_PURGE 0 26105197Ssam#define I915_TTM_PRIO_NO_PAGES 1 27105197Ssam#define I915_TTM_PRIO_HAS_PAGES 2 28105197Ssam#define I915_TTM_PRIO_NEEDS_CPU_ACCESS 3 29105197Ssam 30105197Ssam/* 31105197Ssam * Size of struct ttm_place vector in on-stack struct ttm_placement allocs 32105197Ssam */ 33105197Ssam#define I915_TTM_MAX_PLACEMENTS INTEL_REGION_UNKNOWN 34105197Ssam 35105197Ssam/** 36105197Ssam * struct i915_ttm_tt - TTM page vector with additional private information 37105197Ssam * @ttm: The base TTM page vector. 38105197Ssam * @dev: The struct device used for dma mapping and unmapping. 39105197Ssam * @cached_rsgt: The cached scatter-gather table. 40105197Ssam * @is_shmem: Set if using shmem. 41105197Ssam * @filp: The shmem file, if using shmem backend. 42105197Ssam * 43105197Ssam * Note that DMA may be going on right up to the point where the page- 44105197Ssam * vector is unpopulated in delayed destroy. Hence keep the 45105197Ssam * scatter-gather table mapped and cached up to that point. This is 46105197Ssam * different from the cached gem object io scatter-gather table which 47105197Ssam * doesn't have an associated dma mapping. 48105197Ssam */ 49105197Ssamstruct i915_ttm_tt { 50105197Ssam struct ttm_tt ttm; 51105197Ssam struct device *dev; 52105197Ssam struct i915_refct_sgt cached_rsgt; 53105197Ssam 54105197Ssam bool is_shmem; 55105197Ssam struct file *filp; 56105197Ssam}; 57105197Ssam 58105197Ssamstatic const struct ttm_place sys_placement_flags = { 59105197Ssam .fpfn = 0, 60105197Ssam .lpfn = 0, 61105197Ssam .mem_type = I915_PL_SYSTEM, 62105197Ssam .flags = 0, 63105197Ssam}; 64105197Ssam 65105197Ssamstatic struct ttm_placement i915_sys_placement = { 66105197Ssam .num_placement = 1, 67105197Ssam .placement = &sys_placement_flags, 68105197Ssam}; 69105197Ssam 70105197Ssam/** 71105197Ssam * i915_ttm_sys_placement - Return the struct ttm_placement to be 72105197Ssam * used for an object in system memory. 73105197Ssam * 74105197Ssam * Rather than making the struct extern, use this 75105197Ssam * function. 76105197Ssam * 77105197Ssam * Return: A pointer to a static variable for sys placement. 78105197Ssam */ 79105197Ssamstruct ttm_placement *i915_ttm_sys_placement(void) 80105197Ssam{ 81105197Ssam return &i915_sys_placement; 82105197Ssam} 83105197Ssam 84105197Ssamstatic int i915_ttm_err_to_gem(int err) 85105197Ssam{ 86105197Ssam /* Fastpath */ 87105197Ssam if (likely(!err)) 88105197Ssam return 0; 89105197Ssam 90105197Ssam switch (err) { 91105197Ssam case -EBUSY: 92105197Ssam /* 93105197Ssam * TTM likes to convert -EDEADLK to -EBUSY, and wants us to 94105197Ssam * restart the operation, since we don't record the contending 95105197Ssam * lock. We use -EAGAIN to restart. 96105197Ssam */ 97105197Ssam return -EAGAIN; 98105197Ssam case -ENOSPC: 99105197Ssam /* 100105197Ssam * Memory type / region is full, and we can't evict. 101105197Ssam * Except possibly system, that returns -ENOMEM; 102105197Ssam */ 103105197Ssam return -ENXIO; 104105197Ssam default: 105105197Ssam break; 106105197Ssam } 107105197Ssam 108105197Ssam return err; 109105197Ssam} 110105197Ssam 111105197Ssamstatic enum ttm_caching 112105197Ssami915_ttm_select_tt_caching(const struct drm_i915_gem_object *obj) 113105197Ssam{ 114105197Ssam /* 115105197Ssam * Objects only allowed in system get cached cpu-mappings, or when 116105197Ssam * evicting lmem-only buffers to system for swapping. Other objects get 117105197Ssam * WC mapping for now. Even if in system. 118105197Ssam */ 119105197Ssam if (obj->mm.n_placements <= 1) 120105197Ssam return ttm_cached; 121105197Ssam 122105197Ssam return ttm_write_combined; 123105197Ssam} 124105197Ssam 125105197Ssamstatic void 126105197Ssami915_ttm_place_from_region(const struct intel_memory_region *mr, 127105197Ssam struct ttm_place *place, 128105197Ssam resource_size_t offset, 129105197Ssam resource_size_t size, 130105197Ssam unsigned int flags) 131105197Ssam{ 132105197Ssam memset(place, 0, sizeof(*place)); 133105197Ssam place->mem_type = intel_region_to_ttm_type(mr); 134105197Ssam 135105197Ssam if (mr->type == INTEL_MEMORY_SYSTEM) 136105197Ssam return; 137105197Ssam 138105197Ssam if (flags & I915_BO_ALLOC_CONTIGUOUS) 139105197Ssam place->flags |= TTM_PL_FLAG_CONTIGUOUS; 140105197Ssam if (offset != I915_BO_INVALID_OFFSET) { 141105197Ssam WARN_ON(overflows_type(offset >> PAGE_SHIFT, place->fpfn)); 142105197Ssam place->fpfn = offset >> PAGE_SHIFT; 143105197Ssam WARN_ON(overflows_type(place->fpfn + (size >> PAGE_SHIFT), place->lpfn)); 144105197Ssam place->lpfn = place->fpfn + (size >> PAGE_SHIFT); 145105197Ssam } else if (resource_size(&mr->io) && resource_size(&mr->io) < mr->total) { 146105197Ssam if (flags & I915_BO_ALLOC_GPU_ONLY) { 147105197Ssam place->flags |= TTM_PL_FLAG_TOPDOWN; 148105197Ssam } else { 149105197Ssam place->fpfn = 0; 150105197Ssam WARN_ON(overflows_type(resource_size(&mr->io) >> PAGE_SHIFT, place->lpfn)); 151105197Ssam place->lpfn = resource_size(&mr->io) >> PAGE_SHIFT; 152105197Ssam } 153105197Ssam } 154105197Ssam} 155105197Ssam 156105197Ssamstatic void 157105197Ssami915_ttm_placement_from_obj(const struct drm_i915_gem_object *obj, 158105197Ssam struct ttm_place *places, 159105197Ssam struct ttm_placement *placement) 160105197Ssam{ 161105197Ssam unsigned int num_allowed = obj->mm.n_placements; 162105197Ssam unsigned int flags = obj->flags; 163105197Ssam unsigned int i; 164105197Ssam 165105197Ssam i915_ttm_place_from_region(num_allowed ? obj->mm.placements[0] : 166105197Ssam obj->mm.region, &places[0], obj->bo_offset, 167105197Ssam obj->base.size, flags); 168105197Ssam places[0].flags |= TTM_PL_FLAG_DESIRED; 169105197Ssam 170105197Ssam /* Cache this on object? */ 171105197Ssam for (i = 0; i < num_allowed; ++i) { 172105197Ssam i915_ttm_place_from_region(obj->mm.placements[i], 173105197Ssam &places[i + 1], obj->bo_offset, 174105197Ssam obj->base.size, flags); 175105197Ssam places[i + 1].flags |= TTM_PL_FLAG_FALLBACK; 176105197Ssam } 177105197Ssam 178105197Ssam placement->num_placement = num_allowed + 1; 179105197Ssam placement->placement = places; 180105197Ssam} 181105197Ssam 182105197Ssamstatic int i915_ttm_tt_shmem_populate(struct ttm_device *bdev, 183105197Ssam struct ttm_tt *ttm, 184105197Ssam struct ttm_operation_ctx *ctx) 185105197Ssam{ 186105197Ssam struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev); 187105197Ssam struct intel_memory_region *mr = i915->mm.regions[INTEL_MEMORY_SYSTEM]; 188105197Ssam struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); 189105197Ssam const unsigned int max_segment = i915_sg_segment_size(i915->drm.dev); 190105197Ssam const size_t size = (size_t)ttm->num_pages << PAGE_SHIFT; 191105197Ssam struct file *filp = i915_tt->filp; 192105197Ssam struct sgt_iter sgt_iter; 193105197Ssam struct sg_table *st; 194105197Ssam struct page *page; 195105197Ssam unsigned long i; 196105197Ssam int err; 197105197Ssam 198105197Ssam if (!filp) { 199105197Ssam struct address_space *mapping; 200105197Ssam gfp_t mask; 201105197Ssam 202105197Ssam filp = shmem_file_setup("i915-shmem-tt", size, VM_NORESERVE); 203105197Ssam if (IS_ERR(filp)) 204105197Ssam return PTR_ERR(filp); 205105197Ssam 206105197Ssam mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 207105197Ssam 208105197Ssam mapping = filp->f_mapping; 209105197Ssam mapping_set_gfp_mask(mapping, mask); 210105197Ssam GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); 211105197Ssam 212105197Ssam i915_tt->filp = filp; 213105197Ssam } 214105197Ssam 215105197Ssam st = &i915_tt->cached_rsgt.table; 216105197Ssam err = shmem_sg_alloc_table(i915, st, size, mr, filp->f_mapping, 217105197Ssam max_segment); 218105197Ssam if (err) 219105197Ssam return err; 220105197Ssam 221105197Ssam err = dma_map_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 222105197Ssam DMA_ATTR_SKIP_CPU_SYNC); 223105197Ssam if (err) 224105197Ssam goto err_free_st; 225105197Ssam 226105197Ssam i = 0; 227105197Ssam for_each_sgt_page(page, sgt_iter, st) 228105197Ssam ttm->pages[i++] = page; 229105197Ssam 230105197Ssam if (ttm->page_flags & TTM_TT_FLAG_SWAPPED) 231105197Ssam ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; 232105197Ssam 233105197Ssam return 0; 234105197Ssam 235105197Ssamerr_free_st: 236105197Ssam shmem_sg_free_table(st, filp->f_mapping, false, false); 237105197Ssam 238105197Ssam return err; 239105197Ssam} 240105197Ssam 241105197Ssamstatic void i915_ttm_tt_shmem_unpopulate(struct ttm_tt *ttm) 242105197Ssam{ 243105197Ssam struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); 244105197Ssam bool backup = ttm->page_flags & TTM_TT_FLAG_SWAPPED; 245105197Ssam struct sg_table *st = &i915_tt->cached_rsgt.table; 246105197Ssam 247105197Ssam shmem_sg_free_table(st, file_inode(i915_tt->filp)->i_mapping, 248105197Ssam backup, backup); 249105197Ssam} 250105197Ssam 251105197Ssamstatic void i915_ttm_tt_release(struct kref *ref) 252105197Ssam{ 253105197Ssam struct i915_ttm_tt *i915_tt = 254105197Ssam container_of(ref, typeof(*i915_tt), cached_rsgt.kref); 255105197Ssam struct sg_table *st = &i915_tt->cached_rsgt.table; 256105197Ssam 257105197Ssam GEM_WARN_ON(st->sgl); 258105197Ssam 259105197Ssam kfree(i915_tt); 260105197Ssam} 261105197Ssam 262105197Ssamstatic const struct i915_refct_sgt_ops tt_rsgt_ops = { 263105197Ssam .release = i915_ttm_tt_release 264105197Ssam}; 265105197Ssam 266105197Ssamstatic struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo, 267105197Ssam uint32_t page_flags) 268105197Ssam{ 269105197Ssam struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), 270105197Ssam bdev); 271105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 272105197Ssam unsigned long ccs_pages = 0; 273105197Ssam enum ttm_caching caching; 274105197Ssam struct i915_ttm_tt *i915_tt; 275105197Ssam int ret; 276105197Ssam 277105197Ssam if (i915_ttm_is_ghost_object(bo)) 278105197Ssam return NULL; 279105197Ssam 280105197Ssam i915_tt = kzalloc(sizeof(*i915_tt), GFP_KERNEL); 281105197Ssam if (!i915_tt) 282105197Ssam return NULL; 283105197Ssam 284105197Ssam if (obj->flags & I915_BO_ALLOC_CPU_CLEAR && (!bo->resource || 285105197Ssam ttm_manager_type(bo->bdev, bo->resource->mem_type)->use_tt)) 286105197Ssam page_flags |= TTM_TT_FLAG_ZERO_ALLOC; 287105197Ssam 288105197Ssam caching = i915_ttm_select_tt_caching(obj); 289105197Ssam if (i915_gem_object_is_shrinkable(obj) && caching == ttm_cached) { 290105197Ssam page_flags |= TTM_TT_FLAG_EXTERNAL | 291105197Ssam TTM_TT_FLAG_EXTERNAL_MAPPABLE; 292105197Ssam i915_tt->is_shmem = true; 293105197Ssam } 294105197Ssam 295105197Ssam if (i915_gem_object_needs_ccs_pages(obj)) 296105197Ssam ccs_pages = DIV_ROUND_UP(DIV_ROUND_UP(bo->base.size, 297105197Ssam NUM_BYTES_PER_CCS_BYTE), 298105197Ssam PAGE_SIZE); 299105197Ssam 300105197Ssam ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, caching, ccs_pages); 301105197Ssam if (ret) 302105197Ssam goto err_free; 303105197Ssam 304105197Ssam __i915_refct_sgt_init(&i915_tt->cached_rsgt, bo->base.size, 305105197Ssam &tt_rsgt_ops); 306105197Ssam 307105197Ssam i915_tt->dev = obj->base.dev->dev; 308105197Ssam 309105197Ssam return &i915_tt->ttm; 310105197Ssam 311105197Ssamerr_free: 312105197Ssam kfree(i915_tt); 313105197Ssam return NULL; 314105197Ssam} 315105197Ssam 316105197Ssamstatic int i915_ttm_tt_populate(struct ttm_device *bdev, 317105197Ssam struct ttm_tt *ttm, 318105197Ssam struct ttm_operation_ctx *ctx) 319105197Ssam{ 320105197Ssam struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); 321105197Ssam 322105197Ssam if (i915_tt->is_shmem) 323105197Ssam return i915_ttm_tt_shmem_populate(bdev, ttm, ctx); 324105197Ssam 325105197Ssam return ttm_pool_alloc(&bdev->pool, ttm, ctx); 326105197Ssam} 327105197Ssam 328105197Ssamstatic void i915_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) 329105197Ssam{ 330105197Ssam struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); 331105197Ssam struct sg_table *st = &i915_tt->cached_rsgt.table; 332105197Ssam 333105197Ssam if (st->sgl) 334105197Ssam dma_unmap_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 0); 335105197Ssam 336105197Ssam if (i915_tt->is_shmem) { 337105197Ssam i915_ttm_tt_shmem_unpopulate(ttm); 338105197Ssam } else { 339105197Ssam sg_free_table(st); 340105197Ssam ttm_pool_free(&bdev->pool, ttm); 341105197Ssam } 342105197Ssam} 343105197Ssam 344105197Ssamstatic void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) 345105197Ssam{ 346105197Ssam struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); 347105197Ssam 348105197Ssam if (i915_tt->filp) 349105197Ssam fput(i915_tt->filp); 350105197Ssam 351105197Ssam ttm_tt_fini(ttm); 352105197Ssam i915_refct_sgt_put(&i915_tt->cached_rsgt); 353105197Ssam} 354105197Ssam 355105197Ssamstatic bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo, 356105197Ssam const struct ttm_place *place) 357105197Ssam{ 358105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 359105197Ssam 360105197Ssam if (i915_ttm_is_ghost_object(bo)) 361105197Ssam return false; 362105197Ssam 363105197Ssam /* 364105197Ssam * EXTERNAL objects should never be swapped out by TTM, instead we need 365105197Ssam * to handle that ourselves. TTM will already skip such objects for us, 366105197Ssam * but we would like to avoid grabbing locks for no good reason. 367105197Ssam */ 368105197Ssam if (bo->ttm && bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL) 369105197Ssam return false; 370105197Ssam 371105197Ssam /* Will do for now. Our pinned objects are still on TTM's LRU lists */ 372105197Ssam if (!i915_gem_object_evictable(obj)) 373105197Ssam return false; 374105197Ssam 375105197Ssam return ttm_bo_eviction_valuable(bo, place); 376105197Ssam} 377105197Ssam 378105197Ssamstatic void i915_ttm_evict_flags(struct ttm_buffer_object *bo, 379105197Ssam struct ttm_placement *placement) 380105197Ssam{ 381105197Ssam *placement = i915_sys_placement; 382105197Ssam} 383105197Ssam 384105197Ssam/** 385105197Ssam * i915_ttm_free_cached_io_rsgt - Free object cached LMEM information 386105197Ssam * @obj: The GEM object 387105197Ssam * This function frees any LMEM-related information that is cached on 388105197Ssam * the object. For example the radix tree for fast page lookup and the 389105197Ssam * cached refcounted sg-table 390105197Ssam */ 391105197Ssamvoid i915_ttm_free_cached_io_rsgt(struct drm_i915_gem_object *obj) 392105197Ssam{ 393105197Ssam struct radix_tree_iter iter; 394105197Ssam void __rcu **slot; 395105197Ssam 396105197Ssam if (!obj->ttm.cached_io_rsgt) 397105197Ssam return; 398105197Ssam 399105197Ssam rcu_read_lock(); 400105197Ssam radix_tree_for_each_slot(slot, &obj->ttm.get_io_page.radix, &iter, 0) 401105197Ssam radix_tree_delete(&obj->ttm.get_io_page.radix, iter.index); 402105197Ssam rcu_read_unlock(); 403105197Ssam 404105197Ssam i915_refct_sgt_put(obj->ttm.cached_io_rsgt); 405105197Ssam obj->ttm.cached_io_rsgt = NULL; 406105197Ssam} 407105197Ssam 408105197Ssam/** 409105197Ssam * i915_ttm_purge - Clear an object of its memory 410105197Ssam * @obj: The object 411105197Ssam * 412105197Ssam * This function is called to clear an object of it's memory when it is 413105197Ssam * marked as not needed anymore. 414105197Ssam * 415105197Ssam * Return: 0 on success, negative error code on failure. 416105197Ssam */ 417105197Ssamint i915_ttm_purge(struct drm_i915_gem_object *obj) 418105197Ssam{ 419105197Ssam struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 420105197Ssam struct i915_ttm_tt *i915_tt = 421105197Ssam container_of(bo->ttm, typeof(*i915_tt), ttm); 422105197Ssam struct ttm_operation_ctx ctx = { 423105197Ssam .interruptible = true, 424105197Ssam .no_wait_gpu = false, 425105197Ssam }; 426105197Ssam struct ttm_placement place = {}; 427105197Ssam int ret; 428105197Ssam 429105197Ssam if (obj->mm.madv == __I915_MADV_PURGED) 430105197Ssam return 0; 431105197Ssam 432105197Ssam ret = ttm_bo_validate(bo, &place, &ctx); 433105197Ssam if (ret) 434105197Ssam return ret; 435105197Ssam 436105197Ssam if (bo->ttm && i915_tt->filp) { 437105197Ssam /* 438105197Ssam * The below fput(which eventually calls shmem_truncate) might 439105197Ssam * be delayed by worker, so when directly called to purge the 440105197Ssam * pages(like by the shrinker) we should try to be more 441105197Ssam * aggressive and release the pages immediately. 442105197Ssam */ 443105197Ssam shmem_truncate_range(file_inode(i915_tt->filp), 444105197Ssam 0, (loff_t)-1); 445105197Ssam fput(fetch_and_zero(&i915_tt->filp)); 446105197Ssam } 447105197Ssam 448105197Ssam obj->write_domain = 0; 449105197Ssam obj->read_domains = 0; 450105197Ssam i915_ttm_adjust_gem_after_move(obj); 451105197Ssam i915_ttm_free_cached_io_rsgt(obj); 452105197Ssam obj->mm.madv = __I915_MADV_PURGED; 453105197Ssam 454105197Ssam return 0; 455105197Ssam} 456105197Ssam 457105197Ssamstatic int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags) 458105197Ssam{ 459105197Ssam struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 460105197Ssam struct i915_ttm_tt *i915_tt = 461105197Ssam container_of(bo->ttm, typeof(*i915_tt), ttm); 462105197Ssam struct ttm_operation_ctx ctx = { 463105197Ssam .interruptible = true, 464105197Ssam .no_wait_gpu = flags & I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT, 465105197Ssam }; 466105197Ssam struct ttm_placement place = {}; 467105197Ssam int ret; 468105197Ssam 469105197Ssam if (!bo->ttm || i915_ttm_cpu_maps_iomem(bo->resource)) 470105197Ssam return 0; 471105197Ssam 472105197Ssam GEM_BUG_ON(!i915_tt->is_shmem); 473105197Ssam 474105197Ssam if (!i915_tt->filp) 475105197Ssam return 0; 476105197Ssam 477105197Ssam ret = ttm_bo_wait_ctx(bo, &ctx); 478105197Ssam if (ret) 479105197Ssam return ret; 480105197Ssam 481105197Ssam switch (obj->mm.madv) { 482105197Ssam case I915_MADV_DONTNEED: 483105197Ssam return i915_ttm_purge(obj); 484105197Ssam case __I915_MADV_PURGED: 485105197Ssam return 0; 486105197Ssam } 487105197Ssam 488105197Ssam if (bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) 489105197Ssam return 0; 490105197Ssam 491105197Ssam bo->ttm->page_flags |= TTM_TT_FLAG_SWAPPED; 492105197Ssam ret = ttm_bo_validate(bo, &place, &ctx); 493105197Ssam if (ret) { 494105197Ssam bo->ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; 495105197Ssam return ret; 496105197Ssam } 497105197Ssam 498105197Ssam if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) 499105197Ssam __shmem_writeback(obj->base.size, i915_tt->filp->f_mapping); 500105197Ssam 501105197Ssam return 0; 502105197Ssam} 503105197Ssam 504105197Ssamstatic void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo) 505105197Ssam{ 506105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 507105197Ssam 508105197Ssam /* 509105197Ssam * This gets called twice by ttm, so long as we have a ttm resource or 510105197Ssam * ttm_tt then we can still safely call this. Due to pipeline-gutting, 511105197Ssam * we maybe have NULL bo->resource, but in that case we should always 512105197Ssam * have a ttm alive (like if the pages are swapped out). 513105197Ssam */ 514105197Ssam if ((bo->resource || bo->ttm) && !i915_ttm_is_ghost_object(bo)) { 515105197Ssam __i915_gem_object_pages_fini(obj); 516105197Ssam i915_ttm_free_cached_io_rsgt(obj); 517105197Ssam } 518105197Ssam} 519105197Ssam 520105197Ssamstatic struct i915_refct_sgt *i915_ttm_tt_get_st(struct ttm_tt *ttm) 521105197Ssam{ 522105197Ssam struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); 523105197Ssam struct sg_table *st; 524105197Ssam int ret; 525105197Ssam 526105197Ssam if (i915_tt->cached_rsgt.table.sgl) 527105197Ssam return i915_refct_sgt_get(&i915_tt->cached_rsgt); 528105197Ssam 529105197Ssam st = &i915_tt->cached_rsgt.table; 530105197Ssam ret = sg_alloc_table_from_pages_segment(st, 531105197Ssam ttm->pages, ttm->num_pages, 532105197Ssam 0, (unsigned long)ttm->num_pages << PAGE_SHIFT, 533105197Ssam i915_sg_segment_size(i915_tt->dev), GFP_KERNEL); 534105197Ssam if (ret) { 535105197Ssam st->sgl = NULL; 536105197Ssam return ERR_PTR(ret); 537105197Ssam } 538105197Ssam 539105197Ssam ret = dma_map_sgtable(i915_tt->dev, st, DMA_BIDIRECTIONAL, 0); 540105197Ssam if (ret) { 541105197Ssam sg_free_table(st); 542105197Ssam return ERR_PTR(ret); 543105197Ssam } 544105197Ssam 545105197Ssam return i915_refct_sgt_get(&i915_tt->cached_rsgt); 546105197Ssam} 547105197Ssam 548105197Ssam/** 549105197Ssam * i915_ttm_resource_get_st - Get a refcounted sg-table pointing to the 550105197Ssam * resource memory 551105197Ssam * @obj: The GEM object used for sg-table caching 552105197Ssam * @res: The struct ttm_resource for which an sg-table is requested. 553105197Ssam * 554105197Ssam * This function returns a refcounted sg-table representing the memory 555105197Ssam * pointed to by @res. If @res is the object's current resource it may also 556105197Ssam * cache the sg_table on the object or attempt to access an already cached 557105197Ssam * sg-table. The refcounted sg-table needs to be put when no-longer in use. 558105197Ssam * 559105197Ssam * Return: A valid pointer to a struct i915_refct_sgt or error pointer on 560105197Ssam * failure. 561105197Ssam */ 562105197Ssamstruct i915_refct_sgt * 563105197Ssami915_ttm_resource_get_st(struct drm_i915_gem_object *obj, 564105197Ssam struct ttm_resource *res) 565105197Ssam{ 566105197Ssam struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 567105197Ssam u32 page_alignment; 568105197Ssam 569105197Ssam if (!i915_ttm_gtt_binds_lmem(res)) 570105197Ssam return i915_ttm_tt_get_st(bo->ttm); 571105197Ssam 572105197Ssam page_alignment = bo->page_alignment << PAGE_SHIFT; 573105197Ssam if (!page_alignment) 574105197Ssam page_alignment = obj->mm.region->min_page_size; 575105197Ssam 576105197Ssam /* 577105197Ssam * If CPU mapping differs, we need to add the ttm_tt pages to 578105197Ssam * the resulting st. Might make sense for GGTT. 579105197Ssam */ 580105197Ssam GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(res)); 581105197Ssam if (bo->resource == res) { 582105197Ssam if (!obj->ttm.cached_io_rsgt) { 583105197Ssam struct i915_refct_sgt *rsgt; 584105197Ssam 585105197Ssam rsgt = intel_region_ttm_resource_to_rsgt(obj->mm.region, 586105197Ssam res, 587105197Ssam page_alignment); 588105197Ssam if (IS_ERR(rsgt)) 589105197Ssam return rsgt; 590105197Ssam 591105197Ssam obj->ttm.cached_io_rsgt = rsgt; 592105197Ssam } 593105197Ssam return i915_refct_sgt_get(obj->ttm.cached_io_rsgt); 594105197Ssam } 595105197Ssam 596105197Ssam return intel_region_ttm_resource_to_rsgt(obj->mm.region, res, 597105197Ssam page_alignment); 598105197Ssam} 599105197Ssam 600105197Ssamstatic int i915_ttm_truncate(struct drm_i915_gem_object *obj) 601105197Ssam{ 602105197Ssam struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 603105197Ssam long err; 604105197Ssam 605105197Ssam WARN_ON_ONCE(obj->mm.madv == I915_MADV_WILLNEED); 606105197Ssam 607105197Ssam err = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, 608105197Ssam true, 15 * HZ); 609105197Ssam if (err < 0) 610105197Ssam return err; 611105197Ssam if (err == 0) 612105197Ssam return -EBUSY; 613105197Ssam 614105197Ssam err = i915_ttm_move_notify(bo); 615105197Ssam if (err) 616105197Ssam return err; 617105197Ssam 618105197Ssam return i915_ttm_purge(obj); 619105197Ssam} 620105197Ssam 621105197Ssamstatic void i915_ttm_swap_notify(struct ttm_buffer_object *bo) 622105197Ssam{ 623105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 624105197Ssam int ret; 625105197Ssam 626105197Ssam if (i915_ttm_is_ghost_object(bo)) 627105197Ssam return; 628105197Ssam 629105197Ssam ret = i915_ttm_move_notify(bo); 630105197Ssam GEM_WARN_ON(ret); 631105197Ssam GEM_WARN_ON(obj->ttm.cached_io_rsgt); 632105197Ssam if (!ret && obj->mm.madv != I915_MADV_WILLNEED) 633105197Ssam i915_ttm_purge(obj); 634105197Ssam} 635105197Ssam 636105197Ssam/** 637105197Ssam * i915_ttm_resource_mappable - Return true if the ttm resource is CPU 638105197Ssam * accessible. 639105197Ssam * @res: The TTM resource to check. 640105197Ssam * 641105197Ssam * This is interesting on small-BAR systems where we may encounter lmem objects 642105197Ssam * that can't be accessed via the CPU. 643105197Ssam */ 644105197Ssambool i915_ttm_resource_mappable(struct ttm_resource *res) 645105197Ssam{ 646105197Ssam struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 647105197Ssam 648105197Ssam if (!i915_ttm_cpu_maps_iomem(res)) 649105197Ssam return true; 650105197Ssam 651105197Ssam return bman_res->used_visible_size == PFN_UP(bman_res->base.size); 652105197Ssam} 653105197Ssam 654105197Ssamstatic int i915_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) 655105197Ssam{ 656105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(mem->bo); 657105197Ssam bool unknown_state; 658105197Ssam 659105197Ssam if (i915_ttm_is_ghost_object(mem->bo)) 660105197Ssam return -EINVAL; 661105197Ssam 662105197Ssam if (!kref_get_unless_zero(&obj->base.refcount)) 663105197Ssam return -EINVAL; 664105197Ssam 665105197Ssam assert_object_held(obj); 666105197Ssam 667105197Ssam unknown_state = i915_gem_object_has_unknown_state(obj); 668105197Ssam i915_gem_object_put(obj); 669105197Ssam if (unknown_state) 670105197Ssam return -EINVAL; 671105197Ssam 672105197Ssam if (!i915_ttm_cpu_maps_iomem(mem)) 673105197Ssam return 0; 674105197Ssam 675105197Ssam if (!i915_ttm_resource_mappable(mem)) 676105197Ssam return -EINVAL; 677105197Ssam 678105197Ssam mem->bus.caching = ttm_write_combined; 679105197Ssam mem->bus.is_iomem = true; 680105197Ssam 681105197Ssam return 0; 682105197Ssam} 683105197Ssam 684105197Ssamstatic unsigned long i915_ttm_io_mem_pfn(struct ttm_buffer_object *bo, 685105197Ssam unsigned long page_offset) 686105197Ssam{ 687105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 688105197Ssam struct scatterlist *sg; 689105197Ssam unsigned long base; 690105197Ssam unsigned int ofs; 691105197Ssam 692105197Ssam GEM_BUG_ON(i915_ttm_is_ghost_object(bo)); 693105197Ssam GEM_WARN_ON(bo->ttm); 694105197Ssam 695105197Ssam base = obj->mm.region->iomap.base - obj->mm.region->region.start; 696105197Ssam sg = i915_gem_object_page_iter_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs); 697105197Ssam 698105197Ssam return ((base + sg_dma_address(sg)) >> PAGE_SHIFT) + ofs; 699105197Ssam} 700105197Ssam 701105197Ssamstatic int i915_ttm_access_memory(struct ttm_buffer_object *bo, 702105197Ssam unsigned long offset, void *buf, 703105197Ssam int len, int write) 704105197Ssam{ 705105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 706105197Ssam resource_size_t iomap = obj->mm.region->iomap.base - 707105197Ssam obj->mm.region->region.start; 708105197Ssam unsigned long page = offset >> PAGE_SHIFT; 709105197Ssam unsigned long bytes_left = len; 710105197Ssam 711105197Ssam /* 712105197Ssam * TODO: For now just let it fail if the resource is non-mappable, 713105197Ssam * otherwise we need to perform the memcpy from the gpu here, without 714105197Ssam * interfering with the object (like moving the entire thing). 715105197Ssam */ 716105197Ssam if (!i915_ttm_resource_mappable(bo->resource)) 717105197Ssam return -EIO; 718105197Ssam 719105197Ssam offset -= page << PAGE_SHIFT; 720105197Ssam do { 721105197Ssam unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); 722105197Ssam void __iomem *ptr; 723105197Ssam dma_addr_t daddr; 724105197Ssam 725105197Ssam daddr = i915_gem_object_get_dma_address(obj, page); 726105197Ssam ptr = ioremap_wc(iomap + daddr + offset, bytes); 727105197Ssam if (!ptr) 728105197Ssam return -EIO; 729105197Ssam 730105197Ssam if (write) 731105197Ssam memcpy_toio(ptr, buf, bytes); 732105197Ssam else 733105197Ssam memcpy_fromio(buf, ptr, bytes); 734105197Ssam iounmap(ptr); 735105197Ssam 736105197Ssam page++; 737105197Ssam buf += bytes; 738105197Ssam bytes_left -= bytes; 739105197Ssam offset = 0; 740105197Ssam } while (bytes_left); 741105197Ssam 742105197Ssam return len; 743105197Ssam} 744105197Ssam 745105197Ssam/* 746105197Ssam * All callbacks need to take care not to downcast a struct ttm_buffer_object 747105197Ssam * without checking its subclass, since it might be a TTM ghost object. 748105197Ssam */ 749105197Ssamstatic struct ttm_device_funcs i915_ttm_bo_driver = { 750105197Ssam .ttm_tt_create = i915_ttm_tt_create, 751105197Ssam .ttm_tt_populate = i915_ttm_tt_populate, 752105197Ssam .ttm_tt_unpopulate = i915_ttm_tt_unpopulate, 753105197Ssam .ttm_tt_destroy = i915_ttm_tt_destroy, 754105197Ssam .eviction_valuable = i915_ttm_eviction_valuable, 755105197Ssam .evict_flags = i915_ttm_evict_flags, 756105197Ssam .move = i915_ttm_move, 757105197Ssam .swap_notify = i915_ttm_swap_notify, 758105197Ssam .delete_mem_notify = i915_ttm_delete_mem_notify, 759105197Ssam .io_mem_reserve = i915_ttm_io_mem_reserve, 760105197Ssam .io_mem_pfn = i915_ttm_io_mem_pfn, 761105197Ssam .access_memory = i915_ttm_access_memory, 762105197Ssam}; 763105197Ssam 764105197Ssam/** 765105197Ssam * i915_ttm_driver - Return a pointer to the TTM device funcs 766105197Ssam * 767105197Ssam * Return: Pointer to statically allocated TTM device funcs. 768105197Ssam */ 769105197Ssamstruct ttm_device_funcs *i915_ttm_driver(void) 770105197Ssam{ 771105197Ssam return &i915_ttm_bo_driver; 772105197Ssam} 773105197Ssam 774105197Ssamstatic int __i915_ttm_get_pages(struct drm_i915_gem_object *obj, 775105197Ssam struct ttm_placement *placement) 776105197Ssam{ 777105197Ssam struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 778105197Ssam struct ttm_operation_ctx ctx = { 779105197Ssam .interruptible = true, 780105197Ssam .no_wait_gpu = false, 781105197Ssam }; 782105197Ssam int real_num_busy; 783105197Ssam int ret; 784105197Ssam 785105197Ssam /* First try only the requested placement. No eviction. */ 786105197Ssam real_num_busy = placement->num_placement; 787105197Ssam placement->num_placement = 1; 788105197Ssam ret = ttm_bo_validate(bo, placement, &ctx); 789105197Ssam if (ret) { 790105197Ssam ret = i915_ttm_err_to_gem(ret); 791105197Ssam /* 792105197Ssam * Anything that wants to restart the operation gets to 793105197Ssam * do that. 794105197Ssam */ 795105197Ssam if (ret == -EDEADLK || ret == -EINTR || ret == -ERESTARTSYS || 796105197Ssam ret == -EAGAIN) 797105197Ssam return ret; 798105197Ssam 799105197Ssam /* 800105197Ssam * If the initial attempt fails, allow all accepted placements, 801105197Ssam * evicting if necessary. 802105197Ssam */ 803105197Ssam placement->num_placement = real_num_busy; 804105197Ssam ret = ttm_bo_validate(bo, placement, &ctx); 805105197Ssam if (ret) 806105197Ssam return i915_ttm_err_to_gem(ret); 807105197Ssam } 808105197Ssam 809105197Ssam if (bo->ttm && !ttm_tt_is_populated(bo->ttm)) { 810105197Ssam ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx); 811105197Ssam if (ret) 812105197Ssam return ret; 813105197Ssam 814105197Ssam i915_ttm_adjust_domains_after_move(obj); 815105197Ssam i915_ttm_adjust_gem_after_move(obj); 816105197Ssam } 817105197Ssam 818105197Ssam if (!i915_gem_object_has_pages(obj)) { 819105197Ssam struct i915_refct_sgt *rsgt = 820105197Ssam i915_ttm_resource_get_st(obj, bo->resource); 821105197Ssam 822105197Ssam if (IS_ERR(rsgt)) 823105197Ssam return PTR_ERR(rsgt); 824105197Ssam 825105197Ssam GEM_BUG_ON(obj->mm.rsgt); 826105197Ssam obj->mm.rsgt = rsgt; 827105197Ssam __i915_gem_object_set_pages(obj, &rsgt->table); 828105197Ssam } 829105197Ssam 830105197Ssam GEM_BUG_ON(bo->ttm && ((obj->base.size >> PAGE_SHIFT) < bo->ttm->num_pages)); 831105197Ssam i915_ttm_adjust_lru(obj); 832105197Ssam return ret; 833105197Ssam} 834105197Ssam 835105197Ssamstatic int i915_ttm_get_pages(struct drm_i915_gem_object *obj) 836105197Ssam{ 837105197Ssam struct ttm_place places[I915_TTM_MAX_PLACEMENTS + 1]; 838105197Ssam struct ttm_placement placement; 839105197Ssam 840105197Ssam /* restricted by sg_alloc_table */ 841105197Ssam if (overflows_type(obj->base.size >> PAGE_SHIFT, unsigned int)) 842105197Ssam return -E2BIG; 843105197Ssam 844105197Ssam GEM_BUG_ON(obj->mm.n_placements > I915_TTM_MAX_PLACEMENTS); 845105197Ssam 846105197Ssam /* Move to the requested placement. */ 847105197Ssam i915_ttm_placement_from_obj(obj, places, &placement); 848105197Ssam 849105197Ssam return __i915_ttm_get_pages(obj, &placement); 850105197Ssam} 851105197Ssam 852105197Ssam/** 853105197Ssam * DOC: Migration vs eviction 854105197Ssam * 855105197Ssam * GEM migration may not be the same as TTM migration / eviction. If 856105197Ssam * the TTM core decides to evict an object it may be evicted to a 857105197Ssam * TTM memory type that is not in the object's allowable GEM regions, or 858105197Ssam * in fact theoretically to a TTM memory type that doesn't correspond to 859105197Ssam * a GEM memory region. In that case the object's GEM region is not 860105197Ssam * updated, and the data is migrated back to the GEM region at 861105197Ssam * get_pages time. TTM may however set up CPU ptes to the object even 862105197Ssam * when it is evicted. 863105197Ssam * Gem forced migration using the i915_ttm_migrate() op, is allowed even 864105197Ssam * to regions that are not in the object's list of allowable placements. 865105197Ssam */ 866105197Ssamstatic int __i915_ttm_migrate(struct drm_i915_gem_object *obj, 867105197Ssam struct intel_memory_region *mr, 868105197Ssam unsigned int flags) 869105197Ssam{ 870105197Ssam struct ttm_place requested; 871105197Ssam struct ttm_placement placement; 872105197Ssam int ret; 873105197Ssam 874105197Ssam i915_ttm_place_from_region(mr, &requested, obj->bo_offset, 875105197Ssam obj->base.size, flags); 876105197Ssam placement.num_placement = 1; 877105197Ssam placement.placement = &requested; 878105197Ssam 879105197Ssam ret = __i915_ttm_get_pages(obj, &placement); 880105197Ssam if (ret) 881105197Ssam return ret; 882105197Ssam 883105197Ssam /* 884105197Ssam * Reinitialize the region bindings. This is primarily 885105197Ssam * required for objects where the new region is not in 886105197Ssam * its allowable placements. 887105197Ssam */ 888105197Ssam if (obj->mm.region != mr) { 889105197Ssam i915_gem_object_release_memory_region(obj); 890105197Ssam i915_gem_object_init_memory_region(obj, mr); 891105197Ssam } 892105197Ssam 893105197Ssam return 0; 894105197Ssam} 895105197Ssam 896105197Ssamstatic int i915_ttm_migrate(struct drm_i915_gem_object *obj, 897105197Ssam struct intel_memory_region *mr, 898105197Ssam unsigned int flags) 899105197Ssam{ 900105197Ssam return __i915_ttm_migrate(obj, mr, flags); 901105197Ssam} 902105197Ssam 903105197Ssamstatic void i915_ttm_put_pages(struct drm_i915_gem_object *obj, 904105197Ssam struct sg_table *st) 905105197Ssam{ 906105197Ssam /* 907105197Ssam * We're currently not called from a shrinker, so put_pages() 908105197Ssam * typically means the object is about to destroyed, or called 909105197Ssam * from move_notify(). So just avoid doing much for now. 910105197Ssam * If the object is not destroyed next, The TTM eviction logic 911105197Ssam * and shrinkers will move it out if needed. 912105197Ssam */ 913105197Ssam 914105197Ssam if (obj->mm.rsgt) 915105197Ssam i915_refct_sgt_put(fetch_and_zero(&obj->mm.rsgt)); 916105197Ssam} 917105197Ssam 918105197Ssam/** 919105197Ssam * i915_ttm_adjust_lru - Adjust an object's position on relevant LRU lists. 920105197Ssam * @obj: The object 921105197Ssam */ 922105197Ssamvoid i915_ttm_adjust_lru(struct drm_i915_gem_object *obj) 923105197Ssam{ 924105197Ssam struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 925105197Ssam struct i915_ttm_tt *i915_tt = 926105197Ssam container_of(bo->ttm, typeof(*i915_tt), ttm); 927105197Ssam bool shrinkable = 928105197Ssam bo->ttm && i915_tt->filp && ttm_tt_is_populated(bo->ttm); 929105197Ssam 930105197Ssam /* 931105197Ssam * Don't manipulate the TTM LRUs while in TTM bo destruction. 932105197Ssam * We're called through i915_ttm_delete_mem_notify(). 933105197Ssam */ 934105197Ssam if (!kref_read(&bo->kref)) 935105197Ssam return; 936105197Ssam 937105197Ssam /* 938105197Ssam * We skip managing the shrinker LRU in set_pages() and just manage 939105197Ssam * everything here. This does at least solve the issue with having 940105197Ssam * temporary shmem mappings(like with evicted lmem) not being visible to 941105197Ssam * the shrinker. Only our shmem objects are shrinkable, everything else 942105197Ssam * we keep as unshrinkable. 943105197Ssam * 944105197Ssam * To make sure everything plays nice we keep an extra shrink pin in TTM 945105197Ssam * if the underlying pages are not currently shrinkable. Once we release 946105197Ssam * our pin, like when the pages are moved to shmem, the pages will then 947105197Ssam * be added to the shrinker LRU, assuming the caller isn't also holding 948105197Ssam * a pin. 949105197Ssam * 950105197Ssam * TODO: consider maybe also bumping the shrinker list here when we have 951105197Ssam * already unpinned it, which should give us something more like an LRU. 952105197Ssam * 953105197Ssam * TODO: There is a small window of opportunity for this function to 954105197Ssam * get called from eviction after we've dropped the last GEM refcount, 955105197Ssam * but before the TTM deleted flag is set on the object. Avoid 956105197Ssam * adjusting the shrinker list in such cases, since the object is 957105197Ssam * not available to the shrinker anyway due to its zero refcount. 958105197Ssam * To fix this properly we should move to a TTM shrinker LRU list for 959105197Ssam * these objects. 960105197Ssam */ 961105197Ssam if (kref_get_unless_zero(&obj->base.refcount)) { 962105197Ssam if (shrinkable != obj->mm.ttm_shrinkable) { 963105197Ssam if (shrinkable) { 964105197Ssam if (obj->mm.madv == I915_MADV_WILLNEED) 965105197Ssam __i915_gem_object_make_shrinkable(obj); 966105197Ssam else 967105197Ssam __i915_gem_object_make_purgeable(obj); 968105197Ssam } else { 969105197Ssam i915_gem_object_make_unshrinkable(obj); 970105197Ssam } 971105197Ssam 972105197Ssam obj->mm.ttm_shrinkable = shrinkable; 973105197Ssam } 974105197Ssam i915_gem_object_put(obj); 975105197Ssam } 976105197Ssam 977105197Ssam /* 978105197Ssam * Put on the correct LRU list depending on the MADV status 979105197Ssam */ 980105197Ssam spin_lock(&bo->bdev->lru_lock); 981105197Ssam if (shrinkable) { 982105197Ssam /* Try to keep shmem_tt from being considered for shrinking. */ 983105197Ssam bo->priority = TTM_MAX_BO_PRIORITY - 1; 984105197Ssam } else if (obj->mm.madv != I915_MADV_WILLNEED) { 985105197Ssam bo->priority = I915_TTM_PRIO_PURGE; 986105197Ssam } else if (!i915_gem_object_has_pages(obj)) { 987105197Ssam bo->priority = I915_TTM_PRIO_NO_PAGES; 988105197Ssam } else { 989105197Ssam struct ttm_resource_manager *man = 990105197Ssam ttm_manager_type(bo->bdev, bo->resource->mem_type); 991105197Ssam 992105197Ssam /* 993105197Ssam * If we need to place an LMEM resource which doesn't need CPU 994105197Ssam * access then we should try not to victimize mappable objects 995105197Ssam * first, since we likely end up stealing more of the mappable 996105197Ssam * portion. And likewise when we try to find space for a mappble 997105197Ssam * object, we know not to ever victimize objects that don't 998105197Ssam * occupy any mappable pages. 999105197Ssam */ 1000105197Ssam if (i915_ttm_cpu_maps_iomem(bo->resource) && 1001105197Ssam i915_ttm_buddy_man_visible_size(man) < man->size && 1002105197Ssam !(obj->flags & I915_BO_ALLOC_GPU_ONLY)) 1003105197Ssam bo->priority = I915_TTM_PRIO_NEEDS_CPU_ACCESS; 1004105197Ssam else 1005105197Ssam bo->priority = I915_TTM_PRIO_HAS_PAGES; 1006105197Ssam } 1007105197Ssam 1008105197Ssam ttm_bo_move_to_lru_tail(bo); 1009105197Ssam spin_unlock(&bo->bdev->lru_lock); 1010105197Ssam} 1011105197Ssam 1012105197Ssam/* 1013105197Ssam * TTM-backed gem object destruction requires some clarification. 1014105197Ssam * Basically we have two possibilities here. We can either rely on the 1015105197Ssam * i915 delayed destruction and put the TTM object when the object 1016105197Ssam * is idle. This would be detected by TTM which would bypass the 1017105197Ssam * TTM delayed destroy handling. The other approach is to put the TTM 1018105197Ssam * object early and rely on the TTM destroyed handling, and then free 1019105197Ssam * the leftover parts of the GEM object once TTM's destroyed list handling is 1020105197Ssam * complete. For now, we rely on the latter for two reasons: 1021105197Ssam * a) TTM can evict an object even when it's on the delayed destroy list, 1022105197Ssam * which in theory allows for complete eviction. 1023105197Ssam * b) There is work going on in TTM to allow freeing an object even when 1024105197Ssam * it's not idle, and using the TTM destroyed list handling could help us 1025105197Ssam * benefit from that. 1026105197Ssam */ 1027105197Ssamstatic void i915_ttm_delayed_free(struct drm_i915_gem_object *obj) 1028105197Ssam{ 1029105197Ssam GEM_BUG_ON(!obj->ttm.created); 1030105197Ssam 1031105197Ssam ttm_bo_put(i915_gem_to_ttm(obj)); 1032105197Ssam} 1033105197Ssam 1034105197Ssamstatic vm_fault_t vm_fault_ttm(struct vm_fault *vmf) 1035105197Ssam{ 1036105197Ssam struct vm_area_struct *area = vmf->vma; 1037105197Ssam struct ttm_buffer_object *bo = area->vm_private_data; 1038105197Ssam struct drm_device *dev = bo->base.dev; 1039105197Ssam struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 1040105197Ssam intel_wakeref_t wakeref = 0; 1041105197Ssam vm_fault_t ret; 1042105197Ssam int idx; 1043105197Ssam 1044105197Ssam /* Sanity check that we allow writing into this object */ 1045105197Ssam if (unlikely(i915_gem_object_is_readonly(obj) && 1046105197Ssam area->vm_flags & VM_WRITE)) 1047105197Ssam return VM_FAULT_SIGBUS; 1048105197Ssam 1049105197Ssam ret = ttm_bo_vm_reserve(bo, vmf); 1050105197Ssam if (ret) 1051105197Ssam return ret; 1052105197Ssam 1053105197Ssam if (obj->mm.madv != I915_MADV_WILLNEED) { 1054105197Ssam dma_resv_unlock(bo->base.resv); 1055105197Ssam return VM_FAULT_SIGBUS; 1056105197Ssam } 1057105197Ssam 1058105197Ssam /* 1059105197Ssam * This must be swapped out with shmem ttm_tt (pipeline-gutting). 1060105197Ssam * Calling ttm_bo_validate() here with TTM_PL_SYSTEM should only go as 1061105197Ssam * far as far doing a ttm_bo_move_null(), which should skip all the 1062105197Ssam * other junk. 1063105197Ssam */ 1064105197Ssam if (!bo->resource) { 1065105197Ssam struct ttm_operation_ctx ctx = { 1066105197Ssam .interruptible = true, 1067105197Ssam .no_wait_gpu = true, /* should be idle already */ 1068105197Ssam }; 1069105197Ssam int err; 1070105197Ssam 1071105197Ssam GEM_BUG_ON(!bo->ttm || !(bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED)); 1072105197Ssam 1073105197Ssam err = ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx); 1074105197Ssam if (err) { 1075105197Ssam dma_resv_unlock(bo->base.resv); 1076105197Ssam return VM_FAULT_SIGBUS; 1077105197Ssam } 1078105197Ssam } else if (!i915_ttm_resource_mappable(bo->resource)) { 1079105197Ssam int err = -ENODEV; 1080105197Ssam int i; 1081105197Ssam 1082105197Ssam for (i = 0; i < obj->mm.n_placements; i++) { 1083105197Ssam struct intel_memory_region *mr = obj->mm.placements[i]; 1084105197Ssam unsigned int flags; 1085105197Ssam 1086105197Ssam if (!resource_size(&mr->io) && mr->type != INTEL_MEMORY_SYSTEM) 1087105197Ssam continue; 1088105197Ssam 1089105197Ssam flags = obj->flags; 1090105197Ssam flags &= ~I915_BO_ALLOC_GPU_ONLY; 1091105197Ssam err = __i915_ttm_migrate(obj, mr, flags); 1092105197Ssam if (!err) 1093105197Ssam break; 1094105197Ssam } 1095105197Ssam 1096105197Ssam if (err) { 1097105197Ssam drm_dbg_ratelimited(dev, 1098105197Ssam "Unable to make resource CPU accessible(err = %pe)\n", 1099105197Ssam ERR_PTR(err)); 1100105197Ssam dma_resv_unlock(bo->base.resv); 1101105197Ssam ret = VM_FAULT_SIGBUS; 1102105197Ssam goto out_rpm; 1103105197Ssam } 1104105197Ssam } 1105105197Ssam 1106105197Ssam if (i915_ttm_cpu_maps_iomem(bo->resource)) 1107105197Ssam wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); 1108105197Ssam 1109105197Ssam if (drm_dev_enter(dev, &idx)) { 1110105197Ssam ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, 1111105197Ssam TTM_BO_VM_NUM_PREFAULT); 1112105197Ssam drm_dev_exit(idx); 1113105197Ssam } else { 1114105197Ssam ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); 1115105197Ssam } 1116105197Ssam 1117105197Ssam if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 1118105197Ssam goto out_rpm; 1119105197Ssam 1120105197Ssam /* 1121105197Ssam * ttm_bo_vm_reserve() already has dma_resv_lock. 1122105197Ssam * userfault_count is protected by dma_resv lock and rpm wakeref. 1123105197Ssam */ 1124105197Ssam if (ret == VM_FAULT_NOPAGE && wakeref && !obj->userfault_count) { 1125105197Ssam obj->userfault_count = 1; 1126105197Ssam spin_lock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); 1127105197Ssam list_add(&obj->userfault_link, &to_i915(obj->base.dev)->runtime_pm.lmem_userfault_list); 1128105197Ssam spin_unlock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); 1129105197Ssam 1130105197Ssam GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(bo->resource)); 1131105197Ssam } 1132105197Ssam 1133105197Ssam if (wakeref & CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) 1134105197Ssam intel_wakeref_auto(&to_i915(obj->base.dev)->runtime_pm.userfault_wakeref, 1135105197Ssam msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); 1136105197Ssam 1137105197Ssam i915_ttm_adjust_lru(obj); 1138105197Ssam 1139105197Ssam dma_resv_unlock(bo->base.resv); 1140105197Ssam 1141105197Ssamout_rpm: 1142105197Ssam if (wakeref) 1143105197Ssam intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); 1144105197Ssam 1145105197Ssam return ret; 1146105197Ssam} 1147105197Ssam 1148105197Ssamstatic int 1149105197Ssamvm_access_ttm(struct vm_area_struct *area, unsigned long addr, 1150105197Ssam void *buf, int len, int write) 1151105197Ssam{ 1152105197Ssam struct drm_i915_gem_object *obj = 1153105197Ssam i915_ttm_to_gem(area->vm_private_data); 1154105197Ssam 1155105197Ssam if (i915_gem_object_is_readonly(obj) && write) 1156105197Ssam return -EACCES; 1157105197Ssam 1158105197Ssam return ttm_bo_vm_access(area, addr, buf, len, write); 1159105197Ssam} 1160105197Ssam 1161105197Ssamstatic void ttm_vm_open(struct vm_area_struct *vma) 1162105197Ssam{ 1163105197Ssam struct drm_i915_gem_object *obj = 1164105197Ssam i915_ttm_to_gem(vma->vm_private_data); 1165105197Ssam 1166105197Ssam GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); 1167105197Ssam i915_gem_object_get(obj); 1168105197Ssam} 1169105197Ssam 1170105197Ssamstatic void ttm_vm_close(struct vm_area_struct *vma) 1171105197Ssam{ 1172105197Ssam struct drm_i915_gem_object *obj = 1173105197Ssam i915_ttm_to_gem(vma->vm_private_data); 1174105197Ssam 1175105197Ssam GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); 1176105197Ssam i915_gem_object_put(obj); 1177105197Ssam} 1178105197Ssam 1179105197Ssamstatic const struct vm_operations_struct vm_ops_ttm = { 1180105197Ssam .fault = vm_fault_ttm, 1181105197Ssam .access = vm_access_ttm, 1182105197Ssam .open = ttm_vm_open, 1183105197Ssam .close = ttm_vm_close, 1184105197Ssam}; 1185105197Ssam 1186105197Ssamstatic u64 i915_ttm_mmap_offset(struct drm_i915_gem_object *obj) 1187105197Ssam{ 1188105197Ssam /* The ttm_bo must be allocated with I915_BO_ALLOC_USER */ 1189105197Ssam GEM_BUG_ON(!drm_mm_node_allocated(&obj->base.vma_node.vm_node)); 1190105197Ssam 1191105197Ssam return drm_vma_node_offset_addr(&obj->base.vma_node); 1192105197Ssam} 1193105197Ssam 1194105197Ssamstatic void i915_ttm_unmap_virtual(struct drm_i915_gem_object *obj) 1195105197Ssam{ 1196105197Ssam struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 1197105197Ssam intel_wakeref_t wakeref = 0; 1198105197Ssam 1199105197Ssam assert_object_held_shared(obj); 1200105197Ssam 1201105197Ssam if (i915_ttm_cpu_maps_iomem(bo->resource)) { 1202105197Ssam wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); 1203105197Ssam 1204105197Ssam /* userfault_count is protected by obj lock and rpm wakeref. */ 1205105197Ssam if (obj->userfault_count) { 1206105197Ssam spin_lock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); 1207105197Ssam list_del(&obj->userfault_link); 1208105197Ssam spin_unlock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); 1209105197Ssam obj->userfault_count = 0; 1210 } 1211 } 1212 1213 GEM_WARN_ON(obj->userfault_count); 1214 1215 ttm_bo_unmap_virtual(i915_gem_to_ttm(obj)); 1216 1217 if (wakeref) 1218 intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); 1219} 1220 1221static const struct drm_i915_gem_object_ops i915_gem_ttm_obj_ops = { 1222 .name = "i915_gem_object_ttm", 1223 .flags = I915_GEM_OBJECT_IS_SHRINKABLE | 1224 I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST, 1225 1226 .get_pages = i915_ttm_get_pages, 1227 .put_pages = i915_ttm_put_pages, 1228 .truncate = i915_ttm_truncate, 1229 .shrink = i915_ttm_shrink, 1230 1231 .adjust_lru = i915_ttm_adjust_lru, 1232 .delayed_free = i915_ttm_delayed_free, 1233 .migrate = i915_ttm_migrate, 1234 1235 .mmap_offset = i915_ttm_mmap_offset, 1236 .unmap_virtual = i915_ttm_unmap_virtual, 1237 .mmap_ops = &vm_ops_ttm, 1238}; 1239 1240void i915_ttm_bo_destroy(struct ttm_buffer_object *bo) 1241{ 1242 struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); 1243 1244 i915_gem_object_release_memory_region(obj); 1245 mutex_destroy(&obj->ttm.get_io_page.lock); 1246 1247 if (obj->ttm.created) { 1248 /* 1249 * We freely manage the shrinker LRU outide of the mm.pages life 1250 * cycle. As a result when destroying the object we should be 1251 * extra paranoid and ensure we remove it from the LRU, before 1252 * we free the object. 1253 * 1254 * Touching the ttm_shrinkable outside of the object lock here 1255 * should be safe now that the last GEM object ref was dropped. 1256 */ 1257 if (obj->mm.ttm_shrinkable) 1258 i915_gem_object_make_unshrinkable(obj); 1259 1260 i915_ttm_backup_free(obj); 1261 1262 /* This releases all gem object bindings to the backend. */ 1263 __i915_gem_free_object(obj); 1264 1265 call_rcu(&obj->rcu, __i915_gem_free_object_rcu); 1266 } else { 1267 __i915_gem_object_fini(obj); 1268 } 1269} 1270 1271/* 1272 * __i915_gem_ttm_object_init - Initialize a ttm-backed i915 gem object 1273 * @mem: The initial memory region for the object. 1274 * @obj: The gem object. 1275 * @size: Object size in bytes. 1276 * @flags: gem object flags. 1277 * 1278 * Return: 0 on success, negative error code on failure. 1279 */ 1280int __i915_gem_ttm_object_init(struct intel_memory_region *mem, 1281 struct drm_i915_gem_object *obj, 1282 resource_size_t offset, 1283 resource_size_t size, 1284 resource_size_t page_size, 1285 unsigned int flags) 1286{ 1287 static struct lock_class_key lock_class; 1288 struct drm_i915_private *i915 = mem->i915; 1289 struct ttm_operation_ctx ctx = { 1290 .interruptible = true, 1291 .no_wait_gpu = false, 1292 }; 1293 enum ttm_bo_type bo_type; 1294 int ret; 1295 1296 drm_gem_private_object_init(&i915->drm, &obj->base, size); 1297 i915_gem_object_init(obj, &i915_gem_ttm_obj_ops, &lock_class, flags); 1298 1299 obj->bo_offset = offset; 1300 1301 /* Don't put on a region list until we're either locked or fully initialized. */ 1302 obj->mm.region = mem; 1303 INIT_LIST_HEAD(&obj->mm.region_link); 1304 1305 INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN); 1306 mutex_init(&obj->ttm.get_io_page.lock); 1307 bo_type = (obj->flags & I915_BO_ALLOC_USER) ? ttm_bo_type_device : 1308 ttm_bo_type_kernel; 1309 1310 obj->base.vma_node.driver_private = i915_gem_to_ttm(obj); 1311 1312 /* Forcing the page size is kernel internal only */ 1313 GEM_BUG_ON(page_size && obj->mm.n_placements); 1314 1315 /* 1316 * Keep an extra shrink pin to prevent the object from being made 1317 * shrinkable too early. If the ttm_tt is ever allocated in shmem, we 1318 * drop the pin. The TTM backend manages the shrinker LRU itself, 1319 * outside of the normal mm.pages life cycle. 1320 */ 1321 i915_gem_object_make_unshrinkable(obj); 1322 1323 /* 1324 * If this function fails, it will call the destructor, but 1325 * our caller still owns the object. So no freeing in the 1326 * destructor until obj->ttm.created is true. 1327 * Similarly, in delayed_destroy, we can't call ttm_bo_put() 1328 * until successful initialization. 1329 */ 1330 ret = ttm_bo_init_reserved(&i915->bdev, i915_gem_to_ttm(obj), bo_type, 1331 &i915_sys_placement, page_size >> PAGE_SHIFT, 1332 &ctx, NULL, NULL, i915_ttm_bo_destroy); 1333 1334 /* 1335 * XXX: The ttm_bo_init_reserved() functions returns -ENOSPC if the size 1336 * is too big to add vma. The direct function that returns -ENOSPC is 1337 * drm_mm_insert_node_in_range(). To handle the same error as other code 1338 * that returns -E2BIG when the size is too large, it converts -ENOSPC to 1339 * -E2BIG. 1340 */ 1341 if (size >> PAGE_SHIFT > INT_MAX && ret == -ENOSPC) 1342 ret = -E2BIG; 1343 1344 if (ret) 1345 return i915_ttm_err_to_gem(ret); 1346 1347 obj->ttm.created = true; 1348 i915_gem_object_release_memory_region(obj); 1349 i915_gem_object_init_memory_region(obj, mem); 1350 i915_ttm_adjust_domains_after_move(obj); 1351 i915_ttm_adjust_gem_after_move(obj); 1352 i915_gem_object_unlock(obj); 1353 1354 return 0; 1355} 1356 1357static const struct intel_memory_region_ops ttm_system_region_ops = { 1358 .init_object = __i915_gem_ttm_object_init, 1359 .release = intel_region_ttm_fini, 1360}; 1361 1362struct intel_memory_region * 1363i915_gem_ttm_system_setup(struct drm_i915_private *i915, 1364 u16 type, u16 instance) 1365{ 1366 struct intel_memory_region *mr; 1367 1368 mr = intel_memory_region_create(i915, 0, 1369 totalram_pages() << PAGE_SHIFT, 1370 PAGE_SIZE, 0, 0, 1371 type, instance, 1372 &ttm_system_region_ops); 1373 if (IS_ERR(mr)) 1374 return mr; 1375 1376 intel_memory_region_set_name(mr, "system-ttm"); 1377 return mr; 1378} 1379