1/* $NetBSD: i915_vma.h,v 1.9 2021/12/19 12:40:43 riastradh Exp $ */ 2 3/* 4 * Copyright �� 2016 Intel Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 */ 26 27#ifndef __I915_VMA_H__ 28#define __I915_VMA_H__ 29 30#include <linux/io-mapping.h> 31#include <linux/rbtree.h> 32 33#include <drm/drm_mm.h> 34 35#include "gem/i915_gem_object.h" 36 37#include "i915_gem_gtt.h" 38#include "i915_gem_fence_reg.h" 39 40#include "i915_active.h" 41#include "i915_request.h" 42#include "i915_vma_types.h" 43 44void i915_vma_tree_init(struct drm_i915_gem_object *); 45 46struct i915_vma * 47i915_vma_instance(struct drm_i915_gem_object *obj, 48 struct i915_address_space *vm, 49 const struct i915_ggtt_view *view); 50 51void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags); 52#define I915_VMA_RELEASE_MAP BIT(0) 53 54static inline bool i915_vma_is_active(struct i915_vma *vma) 55{ 56 return !i915_active_is_idle(&vma->active); 57} 58 59int __must_check __i915_vma_move_to_active(struct i915_vma *vma, 60 struct i915_request *rq); 61int __must_check i915_vma_move_to_active(struct i915_vma *vma, 62 struct i915_request *rq, 63 unsigned int flags); 64 65#ifdef __NetBSD__ 66#define __i915_vma_flags(v) ((unsigned long *)&(v)->flags) 67#define __i915_vma_flags_const(v) ((const unsigned long *)&(v)->flags) 68#else 69#define __i915_vma_flags(v) ((unsigned long *)&(v)->flags.counter) 70#define __i915_vma_flags_const(v) ((const unsigned long *)&(v)->flags.counter) 71#endif 72 73 74static inline bool i915_vma_is_ggtt(const struct i915_vma *vma) 75{ 76 return test_bit(I915_VMA_GGTT_BIT, __i915_vma_flags_const(vma)); 77} 78 79static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma) 80{ 81 return test_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags_const(vma)); 82} 83 84static inline void i915_vma_set_ggtt_write(struct i915_vma *vma) 85{ 86 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 87 set_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma)); 88} 89 90static inline bool i915_vma_unset_ggtt_write(struct i915_vma *vma) 91{ 92 return test_and_clear_bit(I915_VMA_GGTT_WRITE_BIT, 93 __i915_vma_flags(vma)); 94} 95 96void i915_vma_flush_writes(struct i915_vma *vma); 97 98static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma) 99{ 100 return test_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags_const(vma)); 101} 102 103static inline bool i915_vma_set_userfault(struct i915_vma *vma) 104{ 105 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); 106 return test_and_set_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma)); 107} 108 109static inline void i915_vma_unset_userfault(struct i915_vma *vma) 110{ 111 return clear_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma)); 112} 113 114static inline bool i915_vma_has_userfault(const struct i915_vma *vma) 115{ 116 return test_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags_const(vma)); 117} 118 119static inline bool i915_vma_is_closed(const struct i915_vma *vma) 120{ 121 return !list_empty(&vma->closed_link); 122} 123 124static inline u32 i915_ggtt_offset(const struct i915_vma *vma) 125{ 126 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 127 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 128 GEM_BUG_ON(upper_32_bits(vma->node.start)); 129 GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1)); 130 return lower_32_bits(vma->node.start); 131} 132 133static inline u32 i915_ggtt_pin_bias(const struct i915_vma *vma) 134{ 135 return i915_vm_to_ggtt(vma->vm)->pin_bias; 136} 137 138static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) 139{ 140 i915_gem_object_get(vma->obj); 141 return vma; 142} 143 144static inline struct i915_vma *i915_vma_tryget(struct i915_vma *vma) 145{ 146 if (likely(kref_get_unless_zero(&vma->obj->base.refcount))) 147 return vma; 148 149 return NULL; 150} 151 152static inline void i915_vma_put(struct i915_vma *vma) 153{ 154 i915_gem_object_put(vma->obj); 155} 156 157static __always_inline ptrdiff_t ptrdiff(const void *a, const void *b) 158{ 159 return a - b; 160} 161 162static inline long 163i915_vma_compare(struct i915_vma *vma, 164 struct i915_address_space *vm, 165 const struct i915_ggtt_view *view) 166{ 167 ptrdiff_t cmp; 168 169 GEM_BUG_ON(view && !i915_is_ggtt(vm)); 170 171 cmp = ptrdiff(vma->vm, vm); 172 if (cmp) 173 return cmp; 174 175 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0); 176 cmp = vma->ggtt_view.type; 177 if (!view) 178 return cmp; 179 180 cmp -= view->type; 181 if (cmp) 182 return cmp; 183 184 assert_i915_gem_gtt_types(); 185 186 /* ggtt_view.type also encodes its size so that we both distinguish 187 * different views using it as a "type" and also use a compact (no 188 * accessing of uninitialised padding bytes) memcmp without storing 189 * an extra parameter or adding more code. 190 * 191 * To ensure that the memcmp is valid for all branches of the union, 192 * even though the code looks like it is just comparing one branch, 193 * we assert above that all branches have the same address, and that 194 * each branch has a unique type/size. 195 */ 196 BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL); 197 BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED); 198 BUILD_BUG_ON(I915_GGTT_VIEW_ROTATED >= I915_GGTT_VIEW_REMAPPED); 199 BUILD_BUG_ON(offsetof(typeof(*view), rotated) != 200 offsetof(typeof(*view), partial)); 201 BUILD_BUG_ON(offsetof(typeof(*view), rotated) != 202 offsetof(typeof(*view), remapped)); 203 return memcmp(&vma->ggtt_view.partial, &view->partial, view->type); 204} 205 206struct i915_vma_work *i915_vma_work(void); 207int i915_vma_bind(struct i915_vma *vma, 208 enum i915_cache_level cache_level, 209 u32 flags, 210 struct i915_vma_work *work); 211 212bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color); 213bool i915_vma_misplaced(const struct i915_vma *vma, 214 u64 size, u64 alignment, u64 flags); 215void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); 216void i915_vma_revoke_mmap(struct i915_vma *vma); 217int __i915_vma_unbind(struct i915_vma *vma); 218int __must_check i915_vma_unbind(struct i915_vma *vma); 219void i915_vma_unlink_ctx(struct i915_vma *vma); 220void i915_vma_close(struct i915_vma *vma); 221void i915_vma_reopen(struct i915_vma *vma); 222 223static inline struct i915_vma *__i915_vma_get(struct i915_vma *vma) 224{ 225 if (kref_get_unless_zero(&vma->ref)) 226 return vma; 227 228 return NULL; 229} 230 231void i915_vma_release(struct kref *ref); 232static inline void __i915_vma_put(struct i915_vma *vma) 233{ 234 kref_put(&vma->ref, i915_vma_release); 235} 236 237#define assert_vma_held(vma) dma_resv_assert_held((vma)->resv) 238 239static inline void i915_vma_lock(struct i915_vma *vma) 240{ 241 dma_resv_lock(vma->resv, NULL); 242} 243 244static inline void i915_vma_unlock(struct i915_vma *vma) 245{ 246 dma_resv_unlock(vma->resv); 247} 248 249int __must_check 250i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags); 251int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags); 252 253static inline int i915_vma_pin_count(struct i915_vma *vma) 254{ 255 return atomic_read(&vma->flags) & I915_VMA_PIN_MASK; 256} 257 258static inline bool i915_vma_is_pinned(struct i915_vma *vma) 259{ 260 return i915_vma_pin_count(vma); 261} 262 263static inline void __i915_vma_pin(struct i915_vma *vma) 264{ 265 atomic_inc(&vma->flags); 266 GEM_BUG_ON(!i915_vma_is_pinned(vma)); 267} 268 269static inline void __i915_vma_unpin(struct i915_vma *vma) 270{ 271 GEM_BUG_ON(!i915_vma_is_pinned(vma)); 272 atomic_dec(&vma->flags); 273} 274 275static inline void i915_vma_unpin(struct i915_vma *vma) 276{ 277 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 278 __i915_vma_unpin(vma); 279} 280 281static inline bool i915_vma_is_bound(struct i915_vma *vma, 282 unsigned int where) 283{ 284 return atomic_read(&vma->flags) & where; 285} 286 287static inline bool i915_node_color_differs(const struct drm_mm_node *node, 288 unsigned long color) 289{ 290 return drm_mm_node_allocated(node) && node->color != color; 291} 292 293/** 294 * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture 295 * @vma: VMA to iomap 296 * 297 * The passed in VMA has to be pinned in the global GTT mappable region. 298 * An extra pinning of the VMA is acquired for the return iomapping, 299 * the caller must call i915_vma_unpin_iomap to relinquish the pinning 300 * after the iomapping is no longer required. 301 * 302 * Returns a valid iomapped pointer or ERR_PTR. 303 */ 304#ifdef __NetBSD__ 305# define __iomem __i915_vma_iomem 306#endif 307void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); 308#ifdef __NetBSD__ 309# undef __iomem 310#endif 311 312#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x)) 313 314/** 315 * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap 316 * @vma: VMA to unpin 317 * 318 * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). 319 * 320 * This function is only valid to be called on a VMA previously 321 * iomapped by the caller with i915_vma_pin_iomap(). 322 */ 323void i915_vma_unpin_iomap(struct i915_vma *vma); 324 325static inline struct page *i915_vma_first_page(struct i915_vma *vma) 326{ 327 GEM_BUG_ON(!vma->pages); 328#ifdef __NetBSD__ 329 return vma->pages->sgl->sg_pgs[0]; 330#else 331 return sg_page(vma->pages->sgl); 332#endif 333} 334 335/** 336 * i915_vma_pin_fence - pin fencing state 337 * @vma: vma to pin fencing for 338 * 339 * This pins the fencing state (whether tiled or untiled) to make sure the 340 * vma (and its object) is ready to be used as a scanout target. Fencing 341 * status must be synchronize first by calling i915_vma_get_fence(): 342 * 343 * The resulting fence pin reference must be released again with 344 * i915_vma_unpin_fence(). 345 * 346 * Returns: 347 * 348 * True if the vma has a fence, false otherwise. 349 */ 350int __must_check i915_vma_pin_fence(struct i915_vma *vma); 351int __must_check i915_vma_revoke_fence(struct i915_vma *vma); 352 353int __i915_vma_pin_fence(struct i915_vma *vma); 354 355static inline void __i915_vma_unpin_fence(struct i915_vma *vma) 356{ 357 GEM_BUG_ON(atomic_read(&vma->fence->pin_count) <= 0); 358 atomic_dec(&vma->fence->pin_count); 359} 360 361/** 362 * i915_vma_unpin_fence - unpin fencing state 363 * @vma: vma to unpin fencing for 364 * 365 * This releases the fence pin reference acquired through 366 * i915_vma_pin_fence. It will handle both objects with and without an 367 * attached fence correctly, callers do not need to distinguish this. 368 */ 369static inline void 370i915_vma_unpin_fence(struct i915_vma *vma) 371{ 372 if (vma->fence) 373 __i915_vma_unpin_fence(vma); 374} 375 376void i915_vma_parked(struct intel_gt *gt); 377 378#define for_each_until(cond) if (cond) break; else 379 380/** 381 * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object. 382 * @V: the #i915_vma iterator 383 * @OBJ: the #drm_i915_gem_object 384 * 385 * GGTT VMA are placed at the being of the object's vma_list, see 386 * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA, 387 * or the list is empty ofc. 388 */ 389#define for_each_ggtt_vma(V, OBJ) \ 390 list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \ 391 for_each_until(!i915_vma_is_ggtt(V)) 392 393struct i915_vma *i915_vma_alloc(void); 394void i915_vma_free(struct i915_vma *vma); 395 396struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma); 397void i915_vma_make_shrinkable(struct i915_vma *vma); 398void i915_vma_make_purgeable(struct i915_vma *vma); 399 400static inline int i915_vma_sync(struct i915_vma *vma) 401{ 402 /* Wait for the asynchronous bindings and pending GPU reads */ 403 return i915_active_wait(&vma->active); 404} 405 406#endif 407