1/* 2 * Copyright �� 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25#include <linux/highmem.h> 26#include <linux/sched/mm.h> 27 28#include <drm/drm_cache.h> 29 30#include "display/intel_frontbuffer.h" 31#include "pxp/intel_pxp.h" 32 33#include "i915_drv.h" 34#include "i915_file_private.h" 35#include "i915_gem_clflush.h" 36#include "i915_gem_context.h" 37#include "i915_gem_dmabuf.h" 38#include "i915_gem_mman.h" 39#include "i915_gem_object.h" 40#include "i915_gem_ttm.h" 41#include "i915_memcpy.h" 42#include "i915_trace.h" 43 44static struct pool slab_objects; 45 46static const struct drm_gem_object_funcs i915_gem_object_funcs; 47 48unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915, 49 enum i915_cache_level level) 50{ 51 if (drm_WARN_ON(&i915->drm, level >= I915_MAX_CACHE_LEVEL)) 52 return 0; 53 54 return INTEL_INFO(i915)->cachelevel_to_pat[level]; 55} 56 57bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj, 58 enum i915_cache_level lvl) 59{ 60 /* 61 * In case the pat_index is set by user space, this kernel mode 62 * driver should leave the coherency to be managed by user space, 63 * simply return true here. 64 */ 65 if (obj->pat_set_by_user) 66 return true; 67 68 /* 69 * Otherwise the pat_index should have been converted from cache_level 70 * so that the following comparison is valid. 71 */ 72 return obj->pat_index == i915_gem_get_pat_index(obj_to_i915(obj), lvl); 73} 74 75struct drm_i915_gem_object *i915_gem_object_alloc(void) 76{ 77 struct drm_i915_gem_object *obj; 78 79#ifdef __linux__ 80 obj = kmem_cache_zalloc(slab_objects, GFP_KERNEL); 81#else 82 obj = pool_get(&slab_objects, PR_WAITOK | PR_ZERO); 83#endif 84 if (!obj) 85 return NULL; 86 obj->base.funcs = &i915_gem_object_funcs; 87 88 return obj; 89} 90 91void i915_gem_object_free(struct drm_i915_gem_object *obj) 92{ 93#ifdef __linux__ 94 return kmem_cache_free(slab_objects, obj); 95#else 96 pool_put(&slab_objects, obj); 97#endif 98} 99 100void i915_gem_object_init(struct drm_i915_gem_object *obj, 101 const struct drm_i915_gem_object_ops *ops, 102 struct lock_class_key *key, unsigned flags) 103{ 104 /* 105 * A gem object is embedded both in a struct ttm_buffer_object :/ and 106 * in a drm_i915_gem_object. Make sure they are aliased. 107 */ 108 BUILD_BUG_ON(offsetof(typeof(*obj), base) != 109 offsetof(typeof(*obj), __do_not_access.base)); 110 111 mtx_init(&obj->vma.lock, IPL_NONE); 112 INIT_LIST_HEAD(&obj->vma.list); 113 114 INIT_LIST_HEAD(&obj->mm.link); 115 116 INIT_LIST_HEAD(&obj->lut_list); 117 mtx_init(&obj->lut_lock, IPL_NONE); 118 119 mtx_init(&obj->mmo.lock, IPL_NONE); 120 obj->mmo.offsets = RB_ROOT; 121 122 init_rcu_head(&obj->rcu); 123 124 obj->ops = ops; 125 GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS); 126 obj->flags = flags; 127 128 obj->mm.madv = I915_MADV_WILLNEED; 129 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); 130 rw_init(&obj->mm.get_page.lock, "mmget"); 131 INIT_RADIX_TREE(&obj->mm.get_dma_page.radix, GFP_KERNEL | __GFP_NOWARN); 132 rw_init(&obj->mm.get_dma_page.lock, "mmgetd"); 133} 134 135/** 136 * __i915_gem_object_fini - Clean up a GEM object initialization 137 * @obj: The gem object to cleanup 138 * 139 * This function cleans up gem object fields that are set up by 140 * drm_gem_private_object_init() and i915_gem_object_init(). 141 * It's primarily intended as a helper for backends that need to 142 * clean up the gem object in separate steps. 143 */ 144void __i915_gem_object_fini(struct drm_i915_gem_object *obj) 145{ 146 mutex_destroy(&obj->mm.get_page.lock); 147 mutex_destroy(&obj->mm.get_dma_page.lock); 148 dma_resv_fini(&obj->base._resv); 149} 150 151/** 152 * i915_gem_object_set_cache_coherency - Mark up the object's coherency levels 153 * for a given cache_level 154 * @obj: #drm_i915_gem_object 155 * @cache_level: cache level 156 */ 157void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 158 unsigned int cache_level) 159{ 160 struct drm_i915_private *i915 = to_i915(obj->base.dev); 161 162 obj->pat_index = i915_gem_get_pat_index(i915, cache_level); 163 164 if (cache_level != I915_CACHE_NONE) 165 obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | 166 I915_BO_CACHE_COHERENT_FOR_WRITE); 167 else if (HAS_LLC(i915)) 168 obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ; 169 else 170 obj->cache_coherent = 0; 171 172 obj->cache_dirty = 173 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) && 174 !IS_DGFX(i915); 175} 176 177/** 178 * i915_gem_object_set_pat_index - set PAT index to be used in PTE encode 179 * @obj: #drm_i915_gem_object 180 * @pat_index: PAT index 181 * 182 * This is a clone of i915_gem_object_set_cache_coherency taking pat index 183 * instead of cache_level as its second argument. 184 */ 185void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj, 186 unsigned int pat_index) 187{ 188 struct drm_i915_private *i915 = to_i915(obj->base.dev); 189 190 if (obj->pat_index == pat_index) 191 return; 192 193 obj->pat_index = pat_index; 194 195 if (pat_index != i915_gem_get_pat_index(i915, I915_CACHE_NONE)) 196 obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | 197 I915_BO_CACHE_COHERENT_FOR_WRITE); 198 else if (HAS_LLC(i915)) 199 obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ; 200 else 201 obj->cache_coherent = 0; 202 203 obj->cache_dirty = 204 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) && 205 !IS_DGFX(i915); 206} 207 208bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj) 209{ 210 struct drm_i915_private *i915 = to_i915(obj->base.dev); 211 212 /* 213 * This is purely from a security perspective, so we simply don't care 214 * about non-userspace objects being able to bypass the LLC. 215 */ 216 if (!(obj->flags & I915_BO_ALLOC_USER)) 217 return false; 218 219 /* 220 * Always flush cache for UMD objects at creation time. 221 */ 222 if (obj->pat_set_by_user) 223 return true; 224 225 /* 226 * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it 227 * possible for userspace to bypass the GTT caching bits set by the 228 * kernel, as per the given object cache_level. This is troublesome 229 * since the heavy flush we apply when first gathering the pages is 230 * skipped if the kernel thinks the object is coherent with the GPU. As 231 * a result it might be possible to bypass the cache and read the 232 * contents of the page directly, which could be stale data. If it's 233 * just a case of userspace shooting themselves in the foot then so be 234 * it, but since i915 takes the stance of always zeroing memory before 235 * handing it to userspace, we need to prevent this. 236 */ 237 return (IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915)); 238} 239 240static void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) 241{ 242 struct drm_i915_gem_object *obj = to_intel_bo(gem); 243 struct drm_i915_file_private *fpriv = file->driver_priv; 244 struct i915_lut_handle bookmark = {}; 245 struct i915_mmap_offset *mmo, *mn; 246 struct i915_lut_handle *lut, *ln; 247 DRM_LIST_HEAD(close); 248 249 spin_lock(&obj->lut_lock); 250 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) { 251 struct i915_gem_context *ctx = lut->ctx; 252 253 if (ctx && ctx->file_priv == fpriv) { 254 i915_gem_context_get(ctx); 255 list_move(&lut->obj_link, &close); 256 } 257 258 /* Break long locks, and carefully continue on from this spot */ 259 if (&ln->obj_link != &obj->lut_list) { 260 list_add_tail(&bookmark.obj_link, &ln->obj_link); 261 if (cond_resched_lock(&obj->lut_lock)) 262 list_safe_reset_next(&bookmark, ln, obj_link); 263 __list_del_entry(&bookmark.obj_link); 264 } 265 } 266 spin_unlock(&obj->lut_lock); 267 268 spin_lock(&obj->mmo.lock); 269 rbtree_postorder_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset) 270 drm_vma_node_revoke(&mmo->vma_node, file); 271 spin_unlock(&obj->mmo.lock); 272 273 list_for_each_entry_safe(lut, ln, &close, obj_link) { 274 struct i915_gem_context *ctx = lut->ctx; 275 struct i915_vma *vma; 276 277 /* 278 * We allow the process to have multiple handles to the same 279 * vma, in the same fd namespace, by virtue of flink/open. 280 */ 281 282 mutex_lock(&ctx->lut_mutex); 283 vma = radix_tree_delete(&ctx->handles_vma, lut->handle); 284 if (vma) { 285 GEM_BUG_ON(vma->obj != obj); 286 GEM_BUG_ON(!atomic_read(&vma->open_count)); 287 i915_vma_close(vma); 288 } 289 mutex_unlock(&ctx->lut_mutex); 290 291 i915_gem_context_put(lut->ctx); 292 i915_lut_handle_free(lut); 293 i915_gem_object_put(obj); 294 } 295} 296 297void __i915_gem_free_object_rcu(struct rcu_head *head) 298{ 299 struct drm_i915_gem_object *obj = 300 container_of(head, typeof(*obj), rcu); 301 struct drm_i915_private *i915 = to_i915(obj->base.dev); 302 303#ifdef __OpenBSD__ 304 if (obj->base.uao) 305 uao_detach(obj->base.uao); 306#endif 307 308 i915_gem_object_free(obj); 309 310 GEM_BUG_ON(!atomic_read(&i915->mm.free_count)); 311 atomic_dec(&i915->mm.free_count); 312} 313 314static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) 315{ 316 /* Skip serialisation and waking the device if known to be not used. */ 317 318 if (obj->userfault_count && !IS_DGFX(to_i915(obj->base.dev))) 319 i915_gem_object_release_mmap_gtt(obj); 320 321 if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) { 322 struct i915_mmap_offset *mmo, *mn; 323 324 i915_gem_object_release_mmap_offset(obj); 325 326 rbtree_postorder_for_each_entry_safe(mmo, mn, 327 &obj->mmo.offsets, 328 offset) { 329 drm_vma_offset_remove(obj->base.dev->vma_offset_manager, 330 &mmo->vma_node); 331 kfree(mmo); 332 } 333 obj->mmo.offsets = RB_ROOT; 334 } 335} 336 337/** 338 * __i915_gem_object_pages_fini - Clean up pages use of a gem object 339 * @obj: The gem object to clean up 340 * 341 * This function cleans up usage of the object mm.pages member. It 342 * is intended for backends that need to clean up a gem object in 343 * separate steps and needs to be called when the object is idle before 344 * the object's backing memory is freed. 345 */ 346void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) 347{ 348 assert_object_held_shared(obj); 349 350 if (!list_empty(&obj->vma.list)) { 351 struct i915_vma *vma; 352 353 spin_lock(&obj->vma.lock); 354 while ((vma = list_first_entry_or_null(&obj->vma.list, 355 struct i915_vma, 356 obj_link))) { 357 GEM_BUG_ON(vma->obj != obj); 358 spin_unlock(&obj->vma.lock); 359 360 i915_vma_destroy(vma); 361 362 spin_lock(&obj->vma.lock); 363 } 364 spin_unlock(&obj->vma.lock); 365 } 366 367 __i915_gem_object_free_mmaps(obj); 368 369 atomic_set(&obj->mm.pages_pin_count, 0); 370 371 /* 372 * dma_buf_unmap_attachment() requires reservation to be 373 * locked. The imported GEM shouldn't share reservation lock 374 * and ttm_bo_cleanup_memtype_use() shouldn't be invoked for 375 * dma-buf, so it's safe to take the lock. 376 */ 377 if (obj->base.import_attach) 378 i915_gem_object_lock(obj, NULL); 379 380 __i915_gem_object_put_pages(obj); 381 382 if (obj->base.import_attach) 383 i915_gem_object_unlock(obj); 384 385 GEM_BUG_ON(i915_gem_object_has_pages(obj)); 386} 387 388void __i915_gem_free_object(struct drm_i915_gem_object *obj) 389{ 390 trace_i915_gem_object_destroy(obj); 391 392 GEM_BUG_ON(!list_empty(&obj->lut_list)); 393 394 bitmap_free(obj->bit_17); 395 396 if (obj->base.import_attach) 397 drm_prime_gem_destroy(&obj->base, NULL); 398 399 drm_gem_free_mmap_offset(&obj->base); 400 401 if (obj->ops->release) 402 obj->ops->release(obj); 403 404 if (obj->mm.n_placements > 1) 405 kfree(obj->mm.placements); 406 407 if (obj->shares_resv_from) 408 i915_vm_resv_put(obj->shares_resv_from); 409 410 __i915_gem_object_fini(obj); 411} 412 413static void __i915_gem_free_objects(struct drm_i915_private *i915, 414 struct llist_node *freed) 415{ 416 struct drm_i915_gem_object *obj, *on; 417 418 llist_for_each_entry_safe(obj, on, freed, freed) { 419 might_sleep(); 420 if (obj->ops->delayed_free) { 421 obj->ops->delayed_free(obj); 422 continue; 423 } 424 425 __i915_gem_object_pages_fini(obj); 426 __i915_gem_free_object(obj); 427 428 /* But keep the pointer alive for RCU-protected lookups */ 429 call_rcu(&obj->rcu, __i915_gem_free_object_rcu); 430 cond_resched(); 431 } 432} 433 434void i915_gem_flush_free_objects(struct drm_i915_private *i915) 435{ 436 struct llist_node *freed = llist_del_all(&i915->mm.free_list); 437 438 if (unlikely(freed)) 439 __i915_gem_free_objects(i915, freed); 440} 441 442static void __i915_gem_free_work(struct work_struct *work) 443{ 444 struct drm_i915_private *i915 = 445 container_of(work, struct drm_i915_private, mm.free_work); 446 447 i915_gem_flush_free_objects(i915); 448} 449 450static void i915_gem_free_object(struct drm_gem_object *gem_obj) 451{ 452 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 453 struct drm_i915_private *i915 = to_i915(obj->base.dev); 454 455 GEM_BUG_ON(i915_gem_object_is_framebuffer(obj)); 456 457 /* 458 * Before we free the object, make sure any pure RCU-only 459 * read-side critical sections are complete, e.g. 460 * i915_gem_busy_ioctl(). For the corresponding synchronized 461 * lookup see i915_gem_object_lookup_rcu(). 462 */ 463 atomic_inc(&i915->mm.free_count); 464 465 /* 466 * Since we require blocking on struct_mutex to unbind the freed 467 * object from the GPU before releasing resources back to the 468 * system, we can not do that directly from the RCU callback (which may 469 * be a softirq context), but must instead then defer that work onto a 470 * kthread. We use the RCU callback rather than move the freed object 471 * directly onto the work queue so that we can mix between using the 472 * worker and performing frees directly from subsequent allocations for 473 * crude but effective memory throttling. 474 */ 475 476 if (llist_add(&obj->freed, &i915->mm.free_list)) 477 queue_work(i915->wq, &i915->mm.free_work); 478} 479 480void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, 481 enum fb_op_origin origin) 482{ 483 struct intel_frontbuffer *front; 484 485 front = i915_gem_object_get_frontbuffer(obj); 486 if (front) { 487 intel_frontbuffer_flush(front, origin); 488 intel_frontbuffer_put(front); 489 } 490} 491 492void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, 493 enum fb_op_origin origin) 494{ 495 struct intel_frontbuffer *front; 496 497 front = i915_gem_object_get_frontbuffer(obj); 498 if (front) { 499 intel_frontbuffer_invalidate(front, origin); 500 intel_frontbuffer_put(front); 501 } 502} 503 504static void 505i915_gem_object_read_from_page_kmap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) 506{ 507 pgoff_t idx = offset >> PAGE_SHIFT; 508 void *src_map; 509 void *src_ptr; 510 511 src_map = kmap_atomic(i915_gem_object_get_page(obj, idx)); 512 513 src_ptr = src_map + offset_in_page(offset); 514 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 515 drm_clflush_virt_range(src_ptr, size); 516 memcpy(dst, src_ptr, size); 517 518 kunmap_atomic(src_map); 519} 520 521static void 522i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) 523{ 524 pgoff_t idx = offset >> PAGE_SHIFT; 525 dma_addr_t dma = i915_gem_object_get_dma_address(obj, idx); 526 void __iomem *src_map; 527 void __iomem *src_ptr; 528 529 src_map = io_mapping_map_wc(&obj->mm.region->iomap, 530 dma - obj->mm.region->region.start, 531 PAGE_SIZE); 532 533 src_ptr = src_map + offset_in_page(offset); 534 if (!i915_memcpy_from_wc(dst, (void __force *)src_ptr, size)) 535 memcpy_fromio(dst, src_ptr, size); 536 537 io_mapping_unmap(src_map); 538} 539 540static bool object_has_mappable_iomem(struct drm_i915_gem_object *obj) 541{ 542 GEM_BUG_ON(!i915_gem_object_has_iomem(obj)); 543 544 if (IS_DGFX(to_i915(obj->base.dev))) 545 return i915_ttm_resource_mappable(i915_gem_to_ttm(obj)->resource); 546 547 return true; 548} 549 550/** 551 * i915_gem_object_read_from_page - read data from the page of a GEM object 552 * @obj: GEM object to read from 553 * @offset: offset within the object 554 * @dst: buffer to store the read data 555 * @size: size to read 556 * 557 * Reads data from @obj at the specified offset. The requested region to read 558 * from can't cross a page boundary. The caller must ensure that @obj pages 559 * are pinned and that @obj is synced wrt. any related writes. 560 * 561 * Return: %0 on success or -ENODEV if the type of @obj's backing store is 562 * unsupported. 563 */ 564int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) 565{ 566 GEM_BUG_ON(overflows_type(offset >> PAGE_SHIFT, pgoff_t)); 567 GEM_BUG_ON(offset >= obj->base.size); 568 GEM_BUG_ON(offset_in_page(offset) > PAGE_SIZE - size); 569 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 570 571 if (i915_gem_object_has_struct_page(obj)) 572 i915_gem_object_read_from_page_kmap(obj, offset, dst, size); 573 else if (i915_gem_object_has_iomem(obj) && object_has_mappable_iomem(obj)) 574 i915_gem_object_read_from_page_iomap(obj, offset, dst, size); 575 else 576 return -ENODEV; 577 578 return 0; 579} 580 581/** 582 * i915_gem_object_evictable - Whether object is likely evictable after unbind. 583 * @obj: The object to check 584 * 585 * This function checks whether the object is likely unvictable after unbind. 586 * If the object is not locked when checking, the result is only advisory. 587 * If the object is locked when checking, and the function returns true, 588 * then an eviction should indeed be possible. But since unlocked vma 589 * unpinning and unbinding is currently possible, the object can actually 590 * become evictable even if this function returns false. 591 * 592 * Return: true if the object may be evictable. False otherwise. 593 */ 594bool i915_gem_object_evictable(struct drm_i915_gem_object *obj) 595{ 596 struct i915_vma *vma; 597 int pin_count = atomic_read(&obj->mm.pages_pin_count); 598 599 if (!pin_count) 600 return true; 601 602 spin_lock(&obj->vma.lock); 603 list_for_each_entry(vma, &obj->vma.list, obj_link) { 604 if (i915_vma_is_pinned(vma)) { 605 spin_unlock(&obj->vma.lock); 606 return false; 607 } 608 if (atomic_read(&vma->pages_count)) 609 pin_count--; 610 } 611 spin_unlock(&obj->vma.lock); 612 GEM_WARN_ON(pin_count < 0); 613 614 return pin_count == 0; 615} 616 617/** 618 * i915_gem_object_migratable - Whether the object is migratable out of the 619 * current region. 620 * @obj: Pointer to the object. 621 * 622 * Return: Whether the object is allowed to be resident in other 623 * regions than the current while pages are present. 624 */ 625bool i915_gem_object_migratable(struct drm_i915_gem_object *obj) 626{ 627 struct intel_memory_region *mr = READ_ONCE(obj->mm.region); 628 629 if (!mr) 630 return false; 631 632 return obj->mm.n_placements > 1; 633} 634 635/** 636 * i915_gem_object_has_struct_page - Whether the object is page-backed 637 * @obj: The object to query. 638 * 639 * This function should only be called while the object is locked or pinned, 640 * otherwise the page backing may change under the caller. 641 * 642 * Return: True if page-backed, false otherwise. 643 */ 644bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) 645{ 646#ifdef CONFIG_LOCKDEP 647 if (IS_DGFX(to_i915(obj->base.dev)) && 648 i915_gem_object_evictable((void __force *)obj)) 649 assert_object_held_shared(obj); 650#endif 651 return obj->mem_flags & I915_BO_FLAG_STRUCT_PAGE; 652} 653 654/** 655 * i915_gem_object_has_iomem - Whether the object is iomem-backed 656 * @obj: The object to query. 657 * 658 * This function should only be called while the object is locked or pinned, 659 * otherwise the iomem backing may change under the caller. 660 * 661 * Return: True if iomem-backed, false otherwise. 662 */ 663bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj) 664{ 665#ifdef CONFIG_LOCKDEP 666 if (IS_DGFX(to_i915(obj->base.dev)) && 667 i915_gem_object_evictable((void __force *)obj)) 668 assert_object_held_shared(obj); 669#endif 670 return obj->mem_flags & I915_BO_FLAG_IOMEM; 671} 672 673/** 674 * i915_gem_object_can_migrate - Whether an object likely can be migrated 675 * 676 * @obj: The object to migrate 677 * @id: The region intended to migrate to 678 * 679 * Check whether the object backend supports migration to the 680 * given region. Note that pinning may affect the ability to migrate as 681 * returned by this function. 682 * 683 * This function is primarily intended as a helper for checking the 684 * possibility to migrate objects and might be slightly less permissive 685 * than i915_gem_object_migrate() when it comes to objects with the 686 * I915_BO_ALLOC_USER flag set. 687 * 688 * Return: true if migration is possible, false otherwise. 689 */ 690bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, 691 enum intel_region_id id) 692{ 693 struct drm_i915_private *i915 = to_i915(obj->base.dev); 694 unsigned int num_allowed = obj->mm.n_placements; 695 struct intel_memory_region *mr; 696 unsigned int i; 697 698 GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN); 699 GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); 700 701 mr = i915->mm.regions[id]; 702 if (!mr) 703 return false; 704 705 if (!IS_ALIGNED(obj->base.size, mr->min_page_size)) 706 return false; 707 708 if (obj->mm.region == mr) 709 return true; 710 711 if (!i915_gem_object_evictable(obj)) 712 return false; 713 714 if (!obj->ops->migrate) 715 return false; 716 717 if (!(obj->flags & I915_BO_ALLOC_USER)) 718 return true; 719 720 if (num_allowed == 0) 721 return false; 722 723 for (i = 0; i < num_allowed; ++i) { 724 if (mr == obj->mm.placements[i]) 725 return true; 726 } 727 728 return false; 729} 730 731/** 732 * i915_gem_object_migrate - Migrate an object to the desired region id 733 * @obj: The object to migrate. 734 * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may 735 * not be successful in evicting other objects to make room for this object. 736 * @id: The region id to migrate to. 737 * 738 * Attempt to migrate the object to the desired memory region. The 739 * object backend must support migration and the object may not be 740 * pinned, (explicitly pinned pages or pinned vmas). The object must 741 * be locked. 742 * On successful completion, the object will have pages pointing to 743 * memory in the new region, but an async migration task may not have 744 * completed yet, and to accomplish that, i915_gem_object_wait_migration() 745 * must be called. 746 * 747 * Note: the @ww parameter is not used yet, but included to make sure 748 * callers put some effort into obtaining a valid ww ctx if one is 749 * available. 750 * 751 * Return: 0 on success. Negative error code on failure. In particular may 752 * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance 753 * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and 754 * -EBUSY if the object is pinned. 755 */ 756int i915_gem_object_migrate(struct drm_i915_gem_object *obj, 757 struct i915_gem_ww_ctx *ww, 758 enum intel_region_id id) 759{ 760 return __i915_gem_object_migrate(obj, ww, id, obj->flags); 761} 762 763/** 764 * __i915_gem_object_migrate - Migrate an object to the desired region id, with 765 * control of the extra flags 766 * @obj: The object to migrate. 767 * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may 768 * not be successful in evicting other objects to make room for this object. 769 * @id: The region id to migrate to. 770 * @flags: The object flags. Normally just obj->flags. 771 * 772 * Attempt to migrate the object to the desired memory region. The 773 * object backend must support migration and the object may not be 774 * pinned, (explicitly pinned pages or pinned vmas). The object must 775 * be locked. 776 * On successful completion, the object will have pages pointing to 777 * memory in the new region, but an async migration task may not have 778 * completed yet, and to accomplish that, i915_gem_object_wait_migration() 779 * must be called. 780 * 781 * Note: the @ww parameter is not used yet, but included to make sure 782 * callers put some effort into obtaining a valid ww ctx if one is 783 * available. 784 * 785 * Return: 0 on success. Negative error code on failure. In particular may 786 * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance 787 * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and 788 * -EBUSY if the object is pinned. 789 */ 790int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, 791 struct i915_gem_ww_ctx *ww, 792 enum intel_region_id id, 793 unsigned int flags) 794{ 795 struct drm_i915_private *i915 = to_i915(obj->base.dev); 796 struct intel_memory_region *mr; 797 798 GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN); 799 GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); 800 assert_object_held(obj); 801 802 mr = i915->mm.regions[id]; 803 GEM_BUG_ON(!mr); 804 805 if (!i915_gem_object_can_migrate(obj, id)) 806 return -EINVAL; 807 808 if (!obj->ops->migrate) { 809 if (GEM_WARN_ON(obj->mm.region != mr)) 810 return -EINVAL; 811 return 0; 812 } 813 814 return obj->ops->migrate(obj, mr, flags); 815} 816 817/** 818 * i915_gem_object_placement_possible - Check whether the object can be 819 * placed at certain memory type 820 * @obj: Pointer to the object 821 * @type: The memory type to check 822 * 823 * Return: True if the object can be placed in @type. False otherwise. 824 */ 825bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, 826 enum intel_memory_type type) 827{ 828 unsigned int i; 829 830 if (!obj->mm.n_placements) { 831 switch (type) { 832 case INTEL_MEMORY_LOCAL: 833 return i915_gem_object_has_iomem(obj); 834 case INTEL_MEMORY_SYSTEM: 835 return i915_gem_object_has_pages(obj); 836 default: 837 /* Ignore stolen for now */ 838 GEM_BUG_ON(1); 839 return false; 840 } 841 } 842 843 for (i = 0; i < obj->mm.n_placements; i++) { 844 if (obj->mm.placements[i]->type == type) 845 return true; 846 } 847 848 return false; 849} 850 851/** 852 * i915_gem_object_needs_ccs_pages - Check whether the object requires extra 853 * pages when placed in system-memory, in order to save and later restore the 854 * flat-CCS aux state when the object is moved between local-memory and 855 * system-memory 856 * @obj: Pointer to the object 857 * 858 * Return: True if the object needs extra ccs pages. False otherwise. 859 */ 860bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj) 861{ 862 bool lmem_placement = false; 863 int i; 864 865 if (!HAS_FLAT_CCS(to_i915(obj->base.dev))) 866 return false; 867 868 if (obj->flags & I915_BO_ALLOC_CCS_AUX) 869 return true; 870 871 for (i = 0; i < obj->mm.n_placements; i++) { 872 /* Compression is not allowed for the objects with smem placement */ 873 if (obj->mm.placements[i]->type == INTEL_MEMORY_SYSTEM) 874 return false; 875 if (!lmem_placement && 876 obj->mm.placements[i]->type == INTEL_MEMORY_LOCAL) 877 lmem_placement = true; 878 } 879 880 return lmem_placement; 881} 882 883void i915_gem_init__objects(struct drm_i915_private *i915) 884{ 885 INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); 886} 887 888void i915_objects_module_exit(void) 889{ 890#ifdef __linux__ 891 kmem_cache_destroy(slab_objects); 892#else 893 pool_destroy(&slab_objects); 894#endif 895} 896 897int __init i915_objects_module_init(void) 898{ 899#ifdef __linux__ 900 slab_objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); 901 if (!slab_objects) 902 return -ENOMEM; 903#else 904 pool_init(&slab_objects, sizeof(struct drm_i915_gem_object), 905 CACHELINESIZE, IPL_NONE, 0, "drmobj", NULL); 906#endif 907 908 return 0; 909} 910 911static const struct drm_gem_object_funcs i915_gem_object_funcs = { 912 .free = i915_gem_free_object, 913 .close = i915_gem_close_object, 914 .export = i915_gem_prime_export, 915}; 916 917/** 918 * i915_gem_object_get_moving_fence - Get the object's moving fence if any 919 * @obj: The object whose moving fence to get. 920 * @fence: The resulting fence 921 * 922 * A non-signaled moving fence means that there is an async operation 923 * pending on the object that needs to be waited on before setting up 924 * any GPU- or CPU PTEs to the object's pages. 925 * 926 * Return: Negative error code or 0 for success. 927 */ 928int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, 929 struct dma_fence **fence) 930{ 931 return dma_resv_get_singleton(obj->base.resv, DMA_RESV_USAGE_KERNEL, 932 fence); 933} 934 935/** 936 * i915_gem_object_wait_moving_fence - Wait for the object's moving fence if any 937 * @obj: The object whose moving fence to wait for. 938 * @intr: Whether to wait interruptible. 939 * 940 * If the moving fence signaled without an error, it is detached from the 941 * object and put. 942 * 943 * Return: 0 if successful, -ERESTARTSYS if the wait was interrupted, 944 * negative error code if the async operation represented by the 945 * moving fence failed. 946 */ 947int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, 948 bool intr) 949{ 950 long ret; 951 952 assert_object_held(obj); 953 954 ret = dma_resv_wait_timeout(obj->base. resv, DMA_RESV_USAGE_KERNEL, 955 intr, MAX_SCHEDULE_TIMEOUT); 956 if (!ret) 957 ret = -ETIME; 958 else if (ret > 0 && i915_gem_object_has_unknown_state(obj)) 959 ret = -EIO; 960 961 return ret < 0 ? ret : 0; 962} 963 964/* 965 * i915_gem_object_has_unknown_state - Return true if the object backing pages are 966 * in an unknown_state. This means that userspace must NEVER be allowed to touch 967 * the pages, with either the GPU or CPU. 968 * 969 * ONLY valid to be called after ensuring that all kernel fences have signalled 970 * (in particular the fence for moving/clearing the object). 971 */ 972bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj) 973{ 974 /* 975 * The below barrier pairs with the dma_fence_signal() in 976 * __memcpy_work(). We should only sample the unknown_state after all 977 * the kernel fences have signalled. 978 */ 979 smp_rmb(); 980 return obj->mm.unknown_state; 981} 982 983#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 984#include "selftests/huge_gem_object.c" 985#include "selftests/huge_pages.c" 986#include "selftests/i915_gem_migrate.c" 987#include "selftests/i915_gem_object.c" 988#include "selftests/i915_gem_coherency.c" 989#endif 990