1/* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright �� 2016 Intel Corporation 5 */ 6 7#ifndef __I915_GEM_OBJECT_H__ 8#define __I915_GEM_OBJECT_H__ 9 10#include <drm/drm_gem.h> 11#include <drm/drm_file.h> 12#include <drm/drm_device.h> 13 14#include "intel_memory_region.h" 15#include "i915_gem_object_types.h" 16#include "i915_gem_gtt.h" 17#include "i915_gem_ww.h" 18#include "i915_vma_types.h" 19 20enum intel_region_id; 21 22#define obj_to_i915(obj__) to_i915((obj__)->base.dev) 23 24static inline bool i915_gem_object_size_2big(u64 size) 25{ 26 struct drm_i915_gem_object *obj; 27 28 if (overflows_type(size, obj->base.size)) 29 return true; 30 31 return false; 32} 33 34unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915, 35 enum i915_cache_level level); 36bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj, 37 enum i915_cache_level lvl); 38void i915_gem_init__objects(struct drm_i915_private *i915); 39 40void i915_objects_module_exit(void); 41int i915_objects_module_init(void); 42 43struct drm_i915_gem_object *i915_gem_object_alloc(void); 44void i915_gem_object_free(struct drm_i915_gem_object *obj); 45 46void i915_gem_object_init(struct drm_i915_gem_object *obj, 47 const struct drm_i915_gem_object_ops *ops, 48 struct lock_class_key *key, 49 unsigned alloc_flags); 50 51void __i915_gem_object_fini(struct drm_i915_gem_object *obj); 52 53struct drm_i915_gem_object * 54i915_gem_object_create_shmem(struct drm_i915_private *i915, 55 resource_size_t size); 56struct drm_i915_gem_object * 57i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, 58 const void *data, resource_size_t size); 59struct drm_i915_gem_object * 60__i915_gem_object_create_user(struct drm_i915_private *i915, u64 size, 61 struct intel_memory_region **placements, 62 unsigned int n_placements); 63 64extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops; 65 66void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, 67 struct sg_table *pages, 68 bool needs_clflush); 69 70int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 71 const struct drm_i915_gem_pwrite *args); 72int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 73 const struct drm_i915_gem_pread *args); 74 75int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); 76void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, 77 struct sg_table *pages); 78void i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 79 struct sg_table *pages); 80 81void i915_gem_flush_free_objects(struct drm_i915_private *i915); 82 83struct sg_table * 84__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj); 85 86/** 87 * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle 88 * @file: DRM file private date 89 * @handle: userspace handle 90 * 91 * Returns: 92 * 93 * A pointer to the object named by the handle if such exists on @filp, NULL 94 * otherwise. This object is only valid whilst under the RCU read lock, and 95 * note carefully the object may be in the process of being destroyed. 96 */ 97static inline struct drm_i915_gem_object * 98i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) 99{ 100#ifdef CONFIG_LOCKDEP 101 WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); 102#endif 103 return idr_find(&file->object_idr, handle); 104} 105 106static inline struct drm_i915_gem_object * 107i915_gem_object_get_rcu(struct drm_i915_gem_object *obj) 108{ 109 if (obj && !kref_get_unless_zero(&obj->base.refcount)) 110 obj = NULL; 111 112 return obj; 113} 114 115static inline struct drm_i915_gem_object * 116i915_gem_object_lookup(struct drm_file *file, u32 handle) 117{ 118 struct drm_i915_gem_object *obj; 119 120 rcu_read_lock(); 121 obj = i915_gem_object_lookup_rcu(file, handle); 122 obj = i915_gem_object_get_rcu(obj); 123 rcu_read_unlock(); 124 125 return obj; 126} 127 128__deprecated 129struct drm_gem_object * 130drm_gem_object_lookup(struct drm_file *file, u32 handle); 131 132__attribute__((nonnull)) 133static inline struct drm_i915_gem_object * 134i915_gem_object_get(struct drm_i915_gem_object *obj) 135{ 136 drm_gem_object_get(&obj->base); 137 return obj; 138} 139 140__attribute__((nonnull)) 141static inline void 142i915_gem_object_put(struct drm_i915_gem_object *obj) 143{ 144 __drm_gem_object_put(&obj->base); 145} 146 147#define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv) 148 149/* 150 * If more than one potential simultaneous locker, assert held. 151 */ 152static inline void assert_object_held_shared(const struct drm_i915_gem_object *obj) 153{ 154 /* 155 * Note mm list lookup is protected by 156 * kref_get_unless_zero(). 157 */ 158 if (IS_ENABLED(CONFIG_LOCKDEP) && 159 kref_read(&obj->base.refcount) > 0) 160 assert_object_held(obj); 161} 162 163static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj, 164 struct i915_gem_ww_ctx *ww, 165 bool intr) 166{ 167 int ret; 168 169 if (intr) 170 ret = dma_resv_lock_interruptible(obj->base.resv, ww ? &ww->ctx : NULL); 171 else 172 ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL); 173 174 if (!ret && ww) { 175 i915_gem_object_get(obj); 176 list_add_tail(&obj->obj_link, &ww->obj_list); 177 } 178 if (ret == -EALREADY) 179 ret = 0; 180 181 if (ret == -EDEADLK) { 182 i915_gem_object_get(obj); 183 ww->contended = obj; 184 } 185 186 return ret; 187} 188 189static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj, 190 struct i915_gem_ww_ctx *ww) 191{ 192 return __i915_gem_object_lock(obj, ww, ww && ww->intr); 193} 194 195static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj, 196 struct i915_gem_ww_ctx *ww) 197{ 198 WARN_ON(ww && !ww->intr); 199 return __i915_gem_object_lock(obj, ww, true); 200} 201 202static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj, 203 struct i915_gem_ww_ctx *ww) 204{ 205 if (!ww) 206 return dma_resv_trylock(obj->base.resv); 207 else 208 return ww_mutex_trylock(&obj->base.resv->lock, &ww->ctx); 209} 210 211static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) 212{ 213 if (obj->ops->adjust_lru) 214 obj->ops->adjust_lru(obj); 215 216 dma_resv_unlock(obj->base.resv); 217} 218 219static inline void 220i915_gem_object_set_readonly(struct drm_i915_gem_object *obj) 221{ 222 obj->flags |= I915_BO_READONLY; 223} 224 225static inline bool 226i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj) 227{ 228 return obj->flags & I915_BO_READONLY; 229} 230 231static inline bool 232i915_gem_object_is_contiguous(const struct drm_i915_gem_object *obj) 233{ 234 return obj->flags & I915_BO_ALLOC_CONTIGUOUS; 235} 236 237static inline bool 238i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj) 239{ 240 return obj->flags & I915_BO_ALLOC_VOLATILE; 241} 242 243static inline void 244i915_gem_object_set_volatile(struct drm_i915_gem_object *obj) 245{ 246 obj->flags |= I915_BO_ALLOC_VOLATILE; 247} 248 249static inline bool 250i915_gem_object_has_tiling_quirk(struct drm_i915_gem_object *obj) 251{ 252 return test_bit(I915_TILING_QUIRK_BIT, &obj->flags); 253} 254 255static inline void 256i915_gem_object_set_tiling_quirk(struct drm_i915_gem_object *obj) 257{ 258 set_bit(I915_TILING_QUIRK_BIT, &obj->flags); 259} 260 261static inline void 262i915_gem_object_clear_tiling_quirk(struct drm_i915_gem_object *obj) 263{ 264 clear_bit(I915_TILING_QUIRK_BIT, &obj->flags); 265} 266 267static inline bool 268i915_gem_object_is_protected(const struct drm_i915_gem_object *obj) 269{ 270 return obj->flags & I915_BO_PROTECTED; 271} 272 273static inline bool 274i915_gem_object_type_has(const struct drm_i915_gem_object *obj, 275 unsigned long flags) 276{ 277 return obj->ops->flags & flags; 278} 279 280bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj); 281 282bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj); 283 284static inline bool 285i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) 286{ 287 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE); 288} 289 290static inline bool 291i915_gem_object_has_self_managed_shrink_list(const struct drm_i915_gem_object *obj) 292{ 293 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST); 294} 295 296static inline bool 297i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj) 298{ 299 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_PROXY); 300} 301 302static inline bool 303i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj) 304{ 305 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP); 306} 307 308static inline bool 309i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj) 310{ 311 return READ_ONCE(obj->frontbuffer) || obj->is_dpt; 312} 313 314static inline unsigned int 315i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj) 316{ 317 return obj->tiling_and_stride & TILING_MASK; 318} 319 320static inline bool 321i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj) 322{ 323 return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; 324} 325 326static inline unsigned int 327i915_gem_object_get_stride(const struct drm_i915_gem_object *obj) 328{ 329 return obj->tiling_and_stride & STRIDE_MASK; 330} 331 332static inline unsigned int 333i915_gem_tile_height(unsigned int tiling) 334{ 335 GEM_BUG_ON(!tiling); 336 return tiling == I915_TILING_Y ? 32 : 8; 337} 338 339static inline unsigned int 340i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj) 341{ 342 return i915_gem_tile_height(i915_gem_object_get_tiling(obj)); 343} 344 345static inline unsigned int 346i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj) 347{ 348 return (i915_gem_object_get_stride(obj) * 349 i915_gem_object_get_tile_height(obj)); 350} 351 352int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, 353 unsigned int tiling, unsigned int stride); 354 355/** 356 * __i915_gem_object_page_iter_get_sg - helper to find the target scatterlist 357 * pointer and the target page position using pgoff_t n input argument and 358 * i915_gem_object_page_iter 359 * @obj: i915 GEM buffer object 360 * @iter: i915 GEM buffer object page iterator 361 * @n: page offset 362 * @offset: searched physical offset, 363 * it will be used for returning physical page offset value 364 * 365 * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 366 * Takes and releases the RCU lock to search the radix_tree of 367 * i915_gem_object_page_iter. 368 * 369 * Returns: 370 * The target scatterlist pointer and the target page position. 371 * 372 * Recommended to use wrapper macro: i915_gem_object_page_iter_get_sg() 373 */ 374struct scatterlist * 375__i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj, 376 struct i915_gem_object_page_iter *iter, 377 pgoff_t n, 378 unsigned int *offset); 379 380/** 381 * i915_gem_object_page_iter_get_sg - wrapper macro for 382 * __i915_gem_object_page_iter_get_sg() 383 * @obj: i915 GEM buffer object 384 * @it: i915 GEM buffer object page iterator 385 * @n: page offset 386 * @offset: searched physical offset, 387 * it will be used for returning physical page offset value 388 * 389 * Context: Takes and releases the mutex lock of the i915_gem_object_page_iter. 390 * Takes and releases the RCU lock to search the radix_tree of 391 * i915_gem_object_page_iter. 392 * 393 * Returns: 394 * The target scatterlist pointer and the target page position. 395 * 396 * In order to avoid the truncation of the input parameter, it checks the page 397 * offset n's type from the input parameter before calling 398 * __i915_gem_object_page_iter_get_sg(). 399 */ 400#define i915_gem_object_page_iter_get_sg(obj, it, n, offset) ({ \ 401 static_assert(castable_to_type(n, pgoff_t)); \ 402 __i915_gem_object_page_iter_get_sg(obj, it, n, offset); \ 403}) 404 405/** 406 * __i915_gem_object_get_sg - helper to find the target scatterlist 407 * pointer and the target page position using pgoff_t n input argument and 408 * drm_i915_gem_object. It uses an internal shmem scatterlist lookup function. 409 * @obj: i915 GEM buffer object 410 * @n: page offset 411 * @offset: searched physical offset, 412 * it will be used for returning physical page offset value 413 * 414 * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 415 * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 416 * 417 * Returns: 418 * The target scatterlist pointer and the target page position. 419 * 420 * Recommended to use wrapper macro: i915_gem_object_get_sg() 421 * See also __i915_gem_object_page_iter_get_sg() 422 */ 423static inline struct scatterlist * 424__i915_gem_object_get_sg(struct drm_i915_gem_object *obj, pgoff_t n, 425 unsigned int *offset) 426{ 427 return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_page, n, offset); 428} 429 430/** 431 * i915_gem_object_get_sg - wrapper macro for __i915_gem_object_get_sg() 432 * @obj: i915 GEM buffer object 433 * @n: page offset 434 * @offset: searched physical offset, 435 * it will be used for returning physical page offset value 436 * 437 * Returns: 438 * The target scatterlist pointer and the target page position. 439 * 440 * In order to avoid the truncation of the input parameter, it checks the page 441 * offset n's type from the input parameter before calling 442 * __i915_gem_object_get_sg(). 443 * See also __i915_gem_object_page_iter_get_sg() 444 */ 445#define i915_gem_object_get_sg(obj, n, offset) ({ \ 446 static_assert(castable_to_type(n, pgoff_t)); \ 447 __i915_gem_object_get_sg(obj, n, offset); \ 448}) 449 450/** 451 * __i915_gem_object_get_sg_dma - helper to find the target scatterlist 452 * pointer and the target page position using pgoff_t n input argument and 453 * drm_i915_gem_object. It uses an internal DMA mapped scatterlist lookup function 454 * @obj: i915 GEM buffer object 455 * @n: page offset 456 * @offset: searched physical offset, 457 * it will be used for returning physical page offset value 458 * 459 * It uses drm_i915_gem_object's internal DMA mapped scatterlist lookup function 460 * as i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg(). 461 * 462 * Returns: 463 * The target scatterlist pointer and the target page position. 464 * 465 * Recommended to use wrapper macro: i915_gem_object_get_sg_dma() 466 * See also __i915_gem_object_page_iter_get_sg() 467 */ 468static inline struct scatterlist * 469__i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj, pgoff_t n, 470 unsigned int *offset) 471{ 472 return __i915_gem_object_page_iter_get_sg(obj, &obj->mm.get_dma_page, n, offset); 473} 474 475/** 476 * i915_gem_object_get_sg_dma - wrapper macro for __i915_gem_object_get_sg_dma() 477 * @obj: i915 GEM buffer object 478 * @n: page offset 479 * @offset: searched physical offset, 480 * it will be used for returning physical page offset value 481 * 482 * Returns: 483 * The target scatterlist pointer and the target page position. 484 * 485 * In order to avoid the truncation of the input parameter, it checks the page 486 * offset n's type from the input parameter before calling 487 * __i915_gem_object_get_sg_dma(). 488 * See also __i915_gem_object_page_iter_get_sg() 489 */ 490#define i915_gem_object_get_sg_dma(obj, n, offset) ({ \ 491 static_assert(castable_to_type(n, pgoff_t)); \ 492 __i915_gem_object_get_sg_dma(obj, n, offset); \ 493}) 494 495/** 496 * __i915_gem_object_get_page - helper to find the target page with a page offset 497 * @obj: i915 GEM buffer object 498 * @n: page offset 499 * 500 * It uses drm_i915_gem_object's internal shmem scatterlist lookup function as 501 * i915_gem_object_page_iter and calls __i915_gem_object_page_iter_get_sg() 502 * internally. 503 * 504 * Returns: 505 * The target page pointer. 506 * 507 * Recommended to use wrapper macro: i915_gem_object_get_page() 508 * See also __i915_gem_object_page_iter_get_sg() 509 */ 510struct page * 511__i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n); 512 513/** 514 * i915_gem_object_get_page - wrapper macro for __i915_gem_object_get_page 515 * @obj: i915 GEM buffer object 516 * @n: page offset 517 * 518 * Returns: 519 * The target page pointer. 520 * 521 * In order to avoid the truncation of the input parameter, it checks the page 522 * offset n's type from the input parameter before calling 523 * __i915_gem_object_get_page(). 524 * See also __i915_gem_object_page_iter_get_sg() 525 */ 526#define i915_gem_object_get_page(obj, n) ({ \ 527 static_assert(castable_to_type(n, pgoff_t)); \ 528 __i915_gem_object_get_page(obj, n); \ 529}) 530 531/** 532 * __i915_gem_object_get_dirty_page - helper to find the target page with a page 533 * offset 534 * @obj: i915 GEM buffer object 535 * @n: page offset 536 * 537 * It works like i915_gem_object_get_page(), but it marks the returned page dirty. 538 * 539 * Returns: 540 * The target page pointer. 541 * 542 * Recommended to use wrapper macro: i915_gem_object_get_dirty_page() 543 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 544 */ 545struct page * 546__i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n); 547 548/** 549 * i915_gem_object_get_dirty_page - wrapper macro for __i915_gem_object_get_dirty_page 550 * @obj: i915 GEM buffer object 551 * @n: page offset 552 * 553 * Returns: 554 * The target page pointer. 555 * 556 * In order to avoid the truncation of the input parameter, it checks the page 557 * offset n's type from the input parameter before calling 558 * __i915_gem_object_get_dirty_page(). 559 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_page() 560 */ 561#define i915_gem_object_get_dirty_page(obj, n) ({ \ 562 static_assert(castable_to_type(n, pgoff_t)); \ 563 __i915_gem_object_get_dirty_page(obj, n); \ 564}) 565 566/** 567 * __i915_gem_object_get_dma_address_len - helper to get bus addresses of 568 * targeted DMA mapped scatterlist from i915 GEM buffer object and it's length 569 * @obj: i915 GEM buffer object 570 * @n: page offset 571 * @len: DMA mapped scatterlist's DMA bus addresses length to return 572 * 573 * Returns: 574 * Bus addresses of targeted DMA mapped scatterlist 575 * 576 * Recommended to use wrapper macro: i915_gem_object_get_dma_address_len() 577 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 578 */ 579dma_addr_t 580__i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, pgoff_t n, 581 unsigned int *len); 582 583/** 584 * i915_gem_object_get_dma_address_len - wrapper macro for 585 * __i915_gem_object_get_dma_address_len 586 * @obj: i915 GEM buffer object 587 * @n: page offset 588 * @len: DMA mapped scatterlist's DMA bus addresses length to return 589 * 590 * Returns: 591 * Bus addresses of targeted DMA mapped scatterlist 592 * 593 * In order to avoid the truncation of the input parameter, it checks the page 594 * offset n's type from the input parameter before calling 595 * __i915_gem_object_get_dma_address_len(). 596 * See also __i915_gem_object_page_iter_get_sg() and 597 * __i915_gem_object_get_dma_address_len() 598 */ 599#define i915_gem_object_get_dma_address_len(obj, n, len) ({ \ 600 static_assert(castable_to_type(n, pgoff_t)); \ 601 __i915_gem_object_get_dma_address_len(obj, n, len); \ 602}) 603 604/** 605 * __i915_gem_object_get_dma_address - helper to get bus addresses of 606 * targeted DMA mapped scatterlist from i915 GEM buffer object 607 * @obj: i915 GEM buffer object 608 * @n: page offset 609 * 610 * Returns: 611 * Bus addresses of targeted DMA mapped scatterlis 612 * 613 * Recommended to use wrapper macro: i915_gem_object_get_dma_address() 614 * See also __i915_gem_object_page_iter_get_sg() and __i915_gem_object_get_sg_dma() 615 */ 616dma_addr_t 617__i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n); 618 619/** 620 * i915_gem_object_get_dma_address - wrapper macro for 621 * __i915_gem_object_get_dma_address 622 * @obj: i915 GEM buffer object 623 * @n: page offset 624 * 625 * Returns: 626 * Bus addresses of targeted DMA mapped scatterlist 627 * 628 * In order to avoid the truncation of the input parameter, it checks the page 629 * offset n's type from the input parameter before calling 630 * __i915_gem_object_get_dma_address(). 631 * See also __i915_gem_object_page_iter_get_sg() and 632 * __i915_gem_object_get_dma_address() 633 */ 634#define i915_gem_object_get_dma_address(obj, n) ({ \ 635 static_assert(castable_to_type(n, pgoff_t)); \ 636 __i915_gem_object_get_dma_address(obj, n); \ 637}) 638 639void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, 640 struct sg_table *pages); 641 642int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 643int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); 644 645static inline int __must_check 646i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 647{ 648 assert_object_held(obj); 649 650 if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) 651 return 0; 652 653 return __i915_gem_object_get_pages(obj); 654} 655 656int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj); 657 658static inline bool 659i915_gem_object_has_pages(struct drm_i915_gem_object *obj) 660{ 661 return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages)); 662} 663 664static inline void 665__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) 666{ 667 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 668 669 atomic_inc(&obj->mm.pages_pin_count); 670} 671 672static inline bool 673i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) 674{ 675 return atomic_read(&obj->mm.pages_pin_count); 676} 677 678static inline void 679__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 680{ 681 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 682 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 683 684 atomic_dec(&obj->mm.pages_pin_count); 685} 686 687static inline void 688i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) 689{ 690 __i915_gem_object_unpin_pages(obj); 691} 692 693int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj); 694int i915_gem_object_truncate(struct drm_i915_gem_object *obj); 695 696/** 697 * i915_gem_object_pin_map - return a contiguous mapping of the entire object 698 * @obj: the object to map into kernel address space 699 * @type: the type of mapping, used to select pgprot_t 700 * 701 * Calls i915_gem_object_pin_pages() to prevent reaping of the object's 702 * pages and then returns a contiguous mapping of the backing storage into 703 * the kernel address space. Based on the @type of mapping, the PTE will be 704 * set to either WriteBack or WriteCombine (via pgprot_t). 705 * 706 * The caller is responsible for calling i915_gem_object_unpin_map() when the 707 * mapping is no longer required. 708 * 709 * Returns the pointer through which to access the mapped object, or an 710 * ERR_PTR() on error. 711 */ 712void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, 713 enum i915_map_type type); 714 715void *__must_check i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, 716 enum i915_map_type type); 717 718void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, 719 unsigned long offset, 720 unsigned long size); 721static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj) 722{ 723 __i915_gem_object_flush_map(obj, 0, obj->base.size); 724} 725 726/** 727 * i915_gem_object_unpin_map - releases an earlier mapping 728 * @obj: the object to unmap 729 * 730 * After pinning the object and mapping its pages, once you are finished 731 * with your access, call i915_gem_object_unpin_map() to release the pin 732 * upon the mapping. Once the pin count reaches zero, that mapping may be 733 * removed. 734 */ 735static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) 736{ 737 i915_gem_object_unpin_pages(obj); 738} 739 740void __i915_gem_object_release_map(struct drm_i915_gem_object *obj); 741 742int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj, 743 unsigned int *needs_clflush); 744int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj, 745 unsigned int *needs_clflush); 746#define CLFLUSH_BEFORE BIT(0) 747#define CLFLUSH_AFTER BIT(1) 748#define CLFLUSH_FLAGS (CLFLUSH_BEFORE | CLFLUSH_AFTER) 749 750static inline void 751i915_gem_object_finish_access(struct drm_i915_gem_object *obj) 752{ 753 i915_gem_object_unpin_pages(obj); 754} 755 756int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, 757 struct dma_fence **fence); 758int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, 759 bool intr); 760bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj); 761 762void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, 763 unsigned int cache_level); 764void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj, 765 unsigned int pat_index); 766bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj); 767void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj); 768void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj); 769bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj); 770 771int __must_check 772i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write); 773int __must_check 774i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); 775int __must_check 776i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); 777struct i915_vma * __must_check 778i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 779 struct i915_gem_ww_ctx *ww, 780 u32 alignment, 781 const struct i915_gtt_view *view, 782 unsigned int flags); 783 784void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj); 785void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 786void __i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj); 787void __i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 788void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj); 789 790static inline void __start_cpu_write(struct drm_i915_gem_object *obj) 791{ 792 obj->read_domains = I915_GEM_DOMAIN_CPU; 793 obj->write_domain = I915_GEM_DOMAIN_CPU; 794 if (i915_gem_cpu_write_needs_clflush(obj)) 795 obj->cache_dirty = true; 796} 797 798void i915_gem_fence_wait_priority(struct dma_fence *fence, 799 const struct i915_sched_attr *attr); 800 801int i915_gem_object_wait(struct drm_i915_gem_object *obj, 802 unsigned int flags, 803 long timeout); 804int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 805 unsigned int flags, 806 const struct i915_sched_attr *attr); 807 808int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size); 809 810bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj); 811 812void __i915_gem_free_object_rcu(struct rcu_head *head); 813 814void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj); 815 816void __i915_gem_free_object(struct drm_i915_gem_object *obj); 817 818bool i915_gem_object_evictable(struct drm_i915_gem_object *obj); 819 820bool i915_gem_object_migratable(struct drm_i915_gem_object *obj); 821 822int i915_gem_object_migrate(struct drm_i915_gem_object *obj, 823 struct i915_gem_ww_ctx *ww, 824 enum intel_region_id id); 825int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, 826 struct i915_gem_ww_ctx *ww, 827 enum intel_region_id id, 828 unsigned int flags); 829 830bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, 831 enum intel_region_id id); 832 833int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, 834 unsigned int flags); 835 836bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, 837 enum intel_memory_type type); 838 839bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj); 840 841int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st, 842 size_t size, struct intel_memory_region *mr, 843 struct address_space *mapping, 844 unsigned int max_segment); 845void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping, 846 bool dirty, bool backup); 847void __shmem_writeback(size_t size, struct address_space *mapping); 848 849#ifdef CONFIG_MMU_NOTIFIER 850static inline bool 851i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) 852{ 853 return obj->userptr.notifier.mm; 854} 855 856int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj); 857int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj); 858int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj); 859#else 860static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; } 861 862static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 863static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 864static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; } 865 866#endif 867 868#endif 869