1/* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright �� 2014-2016 Intel Corporation 5 */ 6 7#include <linux/anon_inodes.h> 8#include <linux/mman.h> 9#include <linux/pfn_t.h> 10#include <linux/sizes.h> 11 12#include <drm/drm_cache.h> 13 14#include "gt/intel_gt.h" 15#include "gt/intel_gt_requests.h" 16 17#include "i915_drv.h" 18#include "i915_gem_evict.h" 19#include "i915_gem_gtt.h" 20#include "i915_gem_ioctls.h" 21#include "i915_gem_object.h" 22#include "i915_gem_mman.h" 23#include "i915_mm.h" 24#include "i915_trace.h" 25#include "i915_user_extensions.h" 26#include "i915_gem_ttm.h" 27#include "i915_vma.h" 28 29static inline bool 30__vma_matches(struct vm_area_struct *vma, struct file *filp, 31 unsigned long addr, unsigned long size) 32{ 33 if (vma->vm_file != filp) 34 return false; 35 36 return vma->vm_start == addr && 37 (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); 38} 39 40/** 41 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address 42 * it is mapped to. 43 * @dev: drm device 44 * @data: ioctl data blob 45 * @file: drm file 46 * 47 * While the mapping holds a reference on the contents of the object, it doesn't 48 * imply a ref on the object itself. 49 * 50 * IMPORTANT: 51 * 52 * DRM driver writers who look a this function as an example for how to do GEM 53 * mmap support, please don't implement mmap support like here. The modern way 54 * to implement DRM mmap support is with an mmap offset ioctl (like 55 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. 56 * That way debug tooling like valgrind will understand what's going on, hiding 57 * the mmap call in a driver private ioctl will break that. The i915 driver only 58 * does cpu mmaps this way because we didn't know better. 59 */ 60int 61i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 62 struct drm_file *file) 63{ 64 struct drm_i915_private *i915 = to_i915(dev); 65 struct drm_i915_gem_mmap *args = data; 66 struct drm_i915_gem_object *obj; 67 unsigned long addr; 68 69 /* 70 * mmap ioctl is disallowed for all discrete platforms, 71 * and for all platforms with GRAPHICS_VER > 12. 72 */ 73 if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0)) 74 return -EOPNOTSUPP; 75 76 if (args->flags & ~(I915_MMAP_WC)) 77 return -EINVAL; 78 79 if (args->flags & I915_MMAP_WC && !pat_enabled()) 80 return -ENODEV; 81 82 obj = i915_gem_object_lookup(file, args->handle); 83 if (!obj) 84 return -ENOENT; 85 86 /* prime objects have no backing filp to GEM mmap 87 * pages from. 88 */ 89 if (!obj->base.filp) { 90 addr = -ENXIO; 91 goto err; 92 } 93 94 if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { 95 addr = -EINVAL; 96 goto err; 97 } 98 99 addr = vm_mmap(obj->base.filp, 0, args->size, 100 PROT_READ | PROT_WRITE, MAP_SHARED, 101 args->offset); 102 if (IS_ERR_VALUE(addr)) 103 goto err; 104 105 if (args->flags & I915_MMAP_WC) { 106 struct mm_struct *mm = current->mm; 107 struct vm_area_struct *vma; 108 109 if (mmap_write_lock_killable(mm)) { 110 addr = -EINTR; 111 goto err; 112 } 113 vma = find_vma(mm, addr); 114 if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) 115 vma->vm_page_prot = 116 pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 117 else 118 addr = -ENOMEM; 119 mmap_write_unlock(mm); 120 if (IS_ERR_VALUE(addr)) 121 goto err; 122 } 123 i915_gem_object_put(obj); 124 125 args->addr_ptr = (u64)addr; 126 return 0; 127 128err: 129 i915_gem_object_put(obj); 130 return addr; 131} 132 133static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) 134{ 135 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; 136} 137 138/** 139 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps 140 * 141 * A history of the GTT mmap interface: 142 * 143 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to 144 * aligned and suitable for fencing, and still fit into the available 145 * mappable space left by the pinned display objects. A classic problem 146 * we called the page-fault-of-doom where we would ping-pong between 147 * two objects that could not fit inside the GTT and so the memcpy 148 * would page one object in at the expense of the other between every 149 * single byte. 150 * 151 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none 152 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the 153 * object is too large for the available space (or simply too large 154 * for the mappable aperture!), a view is created instead and faulted 155 * into userspace. (This view is aligned and sized appropriately for 156 * fenced access.) 157 * 158 * 2 - Recognise WC as a separate cache domain so that we can flush the 159 * delayed writes via GTT before performing direct access via WC. 160 * 161 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial 162 * pagefault; swapin remains transparent. 163 * 164 * 4 - Support multiple fault handlers per object depending on object's 165 * backing storage (a.k.a. MMAP_OFFSET). 166 * 167 * Restrictions: 168 * 169 * * snoopable objects cannot be accessed via the GTT. It can cause machine 170 * hangs on some architectures, corruption on others. An attempt to service 171 * a GTT page fault from a snoopable object will generate a SIGBUS. 172 * 173 * * the object must be able to fit into RAM (physical memory, though no 174 * limited to the mappable aperture). 175 * 176 * 177 * Caveats: 178 * 179 * * a new GTT page fault will synchronize rendering from the GPU and flush 180 * all data to system memory. Subsequent access will not be synchronized. 181 * 182 * * all mappings are revoked on runtime device suspend. 183 * 184 * * there are only 8, 16 or 32 fence registers to share between all users 185 * (older machines require fence register for display and blitter access 186 * as well). Contention of the fence registers will cause the previous users 187 * to be unmapped and any new access will generate new page faults. 188 * 189 * * running out of memory while servicing a fault may generate a SIGBUS, 190 * rather than the expected SIGSEGV. 191 */ 192int i915_gem_mmap_gtt_version(void) 193{ 194 return 4; 195} 196 197static inline struct i915_gtt_view 198compute_partial_view(const struct drm_i915_gem_object *obj, 199 pgoff_t page_offset, 200 unsigned int chunk) 201{ 202 struct i915_gtt_view view; 203 204 if (i915_gem_object_is_tiled(obj)) 205 chunk = roundup(chunk, tile_row_pages(obj) ?: 1); 206 207 view.type = I915_GTT_VIEW_PARTIAL; 208 view.partial.offset = rounddown(page_offset, chunk); 209 view.partial.size = 210 min_t(unsigned int, chunk, 211 (obj->base.size >> PAGE_SHIFT) - view.partial.offset); 212 213 /* If the partial covers the entire object, just create a normal VMA. */ 214 if (chunk >= obj->base.size >> PAGE_SHIFT) 215 view.type = I915_GTT_VIEW_NORMAL; 216 217 return view; 218} 219 220static vm_fault_t i915_error_to_vmf_fault(int err) 221{ 222 switch (err) { 223 default: 224 WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err); 225 fallthrough; 226 case -EIO: /* shmemfs failure from swap device */ 227 case -EFAULT: /* purged object */ 228 case -ENODEV: /* bad object, how did you get here! */ 229 case -ENXIO: /* unable to access backing store (on device) */ 230 return VM_FAULT_SIGBUS; 231 232 case -ENOMEM: /* our allocation failure */ 233 return VM_FAULT_OOM; 234 235 case 0: 236 case -EAGAIN: 237 case -ENOSPC: /* transient failure to evict? */ 238 case -ENOBUFS: /* temporarily out of fences? */ 239 case -ERESTARTSYS: 240 case -EINTR: 241 case -EBUSY: 242 /* 243 * EBUSY is ok: this just means that another thread 244 * already did the job. 245 */ 246 return VM_FAULT_NOPAGE; 247 } 248} 249 250static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) 251{ 252 struct vm_area_struct *area = vmf->vma; 253 struct i915_mmap_offset *mmo = area->vm_private_data; 254 struct drm_i915_gem_object *obj = mmo->obj; 255 resource_size_t iomap; 256 int err; 257 258 /* Sanity check that we allow writing into this object */ 259 if (unlikely(i915_gem_object_is_readonly(obj) && 260 area->vm_flags & VM_WRITE)) 261 return VM_FAULT_SIGBUS; 262 263 if (i915_gem_object_lock_interruptible(obj, NULL)) 264 return VM_FAULT_NOPAGE; 265 266 err = i915_gem_object_pin_pages(obj); 267 if (err) 268 goto out; 269 270 iomap = -1; 271 if (!i915_gem_object_has_struct_page(obj)) { 272 iomap = obj->mm.region->iomap.base; 273 iomap -= obj->mm.region->region.start; 274 } 275 276 /* PTEs are revoked in obj->ops->put_pages() */ 277 err = remap_io_sg(area, 278 area->vm_start, area->vm_end - area->vm_start, 279 obj->mm.pages->sgl, iomap); 280 281 if (area->vm_flags & VM_WRITE) { 282 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 283 obj->mm.dirty = true; 284 } 285 286 i915_gem_object_unpin_pages(obj); 287 288out: 289 i915_gem_object_unlock(obj); 290 return i915_error_to_vmf_fault(err); 291} 292 293static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) 294{ 295#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) 296 struct vm_area_struct *area = vmf->vma; 297 struct i915_mmap_offset *mmo = area->vm_private_data; 298 struct drm_i915_gem_object *obj = mmo->obj; 299 struct drm_device *dev = obj->base.dev; 300 struct drm_i915_private *i915 = to_i915(dev); 301 struct intel_runtime_pm *rpm = &i915->runtime_pm; 302 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 303 bool write = area->vm_flags & VM_WRITE; 304 struct i915_gem_ww_ctx ww; 305 intel_wakeref_t wakeref; 306 struct i915_vma *vma; 307 pgoff_t page_offset; 308 int srcu; 309 int ret; 310 311 /* We don't use vmf->pgoff since that has the fake offset */ 312 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; 313 314 trace_i915_gem_object_fault(obj, page_offset, true, write); 315 316 wakeref = intel_runtime_pm_get(rpm); 317 318 i915_gem_ww_ctx_init(&ww, true); 319retry: 320 ret = i915_gem_object_lock(obj, &ww); 321 if (ret) 322 goto err_rpm; 323 324 /* Sanity check that we allow writing into this object */ 325 if (i915_gem_object_is_readonly(obj) && write) { 326 ret = -EFAULT; 327 goto err_rpm; 328 } 329 330 ret = i915_gem_object_pin_pages(obj); 331 if (ret) 332 goto err_rpm; 333 334 ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu); 335 if (ret) 336 goto err_pages; 337 338 /* Now pin it into the GTT as needed */ 339 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0, 340 PIN_MAPPABLE | 341 PIN_NONBLOCK /* NOWARN */ | 342 PIN_NOEVICT); 343 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { 344 /* Use a partial view if it is bigger than available space */ 345 struct i915_gtt_view view = 346 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); 347 unsigned int flags; 348 349 flags = PIN_MAPPABLE | PIN_NOSEARCH; 350 if (view.type == I915_GTT_VIEW_NORMAL) 351 flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ 352 353 /* 354 * Userspace is now writing through an untracked VMA, abandon 355 * all hope that the hardware is able to track future writes. 356 */ 357 358 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); 359 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { 360 flags = PIN_MAPPABLE; 361 view.type = I915_GTT_VIEW_PARTIAL; 362 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); 363 } 364 365 /* 366 * The entire mappable GGTT is pinned? Unexpected! 367 * Try to evict the object we locked too, as normally we skip it 368 * due to lack of short term pinning inside execbuf. 369 */ 370 if (vma == ERR_PTR(-ENOSPC)) { 371 ret = mutex_lock_interruptible(&ggtt->vm.mutex); 372 if (!ret) { 373 ret = i915_gem_evict_vm(&ggtt->vm, &ww, NULL); 374 mutex_unlock(&ggtt->vm.mutex); 375 } 376 if (ret) 377 goto err_reset; 378 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); 379 } 380 } 381 if (IS_ERR(vma)) { 382 ret = PTR_ERR(vma); 383 goto err_reset; 384 } 385 386 /* Access to snoopable pages through the GTT is incoherent. */ 387 /* 388 * For objects created by userspace through GEM_CREATE with pat_index 389 * set by set_pat extension, coherency is managed by userspace, make 390 * sure we don't fail handling the vm fault by calling 391 * i915_gem_object_has_cache_level() which always return true for such 392 * objects. Otherwise this helper function would fall back to checking 393 * whether the object is un-cached. 394 */ 395 if (!(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) || 396 HAS_LLC(i915))) { 397 ret = -EFAULT; 398 goto err_unpin; 399 } 400 401 ret = i915_vma_pin_fence(vma); 402 if (ret) 403 goto err_unpin; 404 405 /* Finally, remap it using the new GTT offset */ 406 ret = remap_io_mapping(area, 407 area->vm_start + (vma->gtt_view.partial.offset << PAGE_SHIFT), 408 (ggtt->gmadr.start + i915_ggtt_offset(vma)) >> PAGE_SHIFT, 409 min_t(u64, vma->size, area->vm_end - area->vm_start), 410 &ggtt->iomap); 411 if (ret) 412 goto err_fence; 413 414 assert_rpm_wakelock_held(rpm); 415 416 /* Mark as being mmapped into userspace for later revocation */ 417 mutex_lock(&to_gt(i915)->ggtt->vm.mutex); 418 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) 419 list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list); 420 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); 421 422 /* Track the mmo associated with the fenced vma */ 423 vma->mmo = mmo; 424 425 if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) 426 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 427 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); 428 429 if (write) { 430 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 431 i915_vma_set_ggtt_write(vma); 432 obj->mm.dirty = true; 433 } 434 435err_fence: 436 i915_vma_unpin_fence(vma); 437err_unpin: 438 __i915_vma_unpin(vma); 439err_reset: 440 intel_gt_reset_unlock(ggtt->vm.gt, srcu); 441err_pages: 442 i915_gem_object_unpin_pages(obj); 443err_rpm: 444 if (ret == -EDEADLK) { 445 ret = i915_gem_ww_ctx_backoff(&ww); 446 if (!ret) 447 goto retry; 448 } 449 i915_gem_ww_ctx_fini(&ww); 450 intel_runtime_pm_put(rpm, wakeref); 451 return i915_error_to_vmf_fault(ret); 452} 453 454static int 455vm_access(struct vm_area_struct *area, unsigned long addr, 456 void *buf, int len, int write) 457{ 458 struct i915_mmap_offset *mmo = area->vm_private_data; 459 struct drm_i915_gem_object *obj = mmo->obj; 460 struct i915_gem_ww_ctx ww; 461 void *vaddr; 462 int err = 0; 463 464 if (i915_gem_object_is_readonly(obj) && write) 465 return -EACCES; 466 467 addr -= area->vm_start; 468 if (range_overflows_t(u64, addr, len, obj->base.size)) 469 return -EINVAL; 470 471 i915_gem_ww_ctx_init(&ww, true); 472retry: 473 err = i915_gem_object_lock(obj, &ww); 474 if (err) 475 goto out; 476 477 /* As this is primarily for debugging, let's focus on simplicity */ 478 vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); 479 if (IS_ERR(vaddr)) { 480 err = PTR_ERR(vaddr); 481 goto out; 482 } 483 484 if (write) { 485 memcpy(vaddr + addr, buf, len); 486 __i915_gem_object_flush_map(obj, addr, len); 487 } else { 488 memcpy(buf, vaddr + addr, len); 489 } 490 491 i915_gem_object_unpin_map(obj); 492out: 493 if (err == -EDEADLK) { 494 err = i915_gem_ww_ctx_backoff(&ww); 495 if (!err) 496 goto retry; 497 } 498 i915_gem_ww_ctx_fini(&ww); 499 500 if (err) 501 return err; 502 503 return len; 504} 505 506void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) 507{ 508 struct i915_vma *vma; 509 510 GEM_BUG_ON(!obj->userfault_count); 511 512 for_each_ggtt_vma(vma, obj) 513 i915_vma_revoke_mmap(vma); 514 515 GEM_BUG_ON(obj->userfault_count); 516} 517 518/* 519 * It is vital that we remove the page mapping if we have mapped a tiled 520 * object through the GTT and then lose the fence register due to 521 * resource pressure. Similarly if the object has been moved out of the 522 * aperture, than pages mapped into userspace must be revoked. Removing the 523 * mapping will then trigger a page fault on the next user access, allowing 524 * fixup by vm_fault_gtt(). 525 */ 526void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) 527{ 528 struct drm_i915_private *i915 = to_i915(obj->base.dev); 529 intel_wakeref_t wakeref; 530 531 /* 532 * Serialisation between user GTT access and our code depends upon 533 * revoking the CPU's PTE whilst the mutex is held. The next user 534 * pagefault then has to wait until we release the mutex. 535 * 536 * Note that RPM complicates somewhat by adding an additional 537 * requirement that operations to the GGTT be made holding the RPM 538 * wakeref. 539 */ 540 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 541 mutex_lock(&to_gt(i915)->ggtt->vm.mutex); 542 543 if (!obj->userfault_count) 544 goto out; 545 546 __i915_gem_object_release_mmap_gtt(obj); 547 548 /* 549 * Ensure that the CPU's PTE are revoked and there are not outstanding 550 * memory transactions from userspace before we return. The TLB 551 * flushing implied above by changing the PTE above *should* be 552 * sufficient, an extra barrier here just provides us with a bit 553 * of paranoid documentation about our requirement to serialise 554 * memory writes before touching registers / GSM. 555 */ 556 wmb(); 557 558out: 559 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); 560 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 561} 562 563void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj) 564{ 565 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); 566 struct ttm_device *bdev = bo->bdev; 567 568 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); 569 570 /* 571 * We have exclusive access here via runtime suspend. All other callers 572 * must first grab the rpm wakeref. 573 */ 574 GEM_BUG_ON(!obj->userfault_count); 575 list_del(&obj->userfault_link); 576 obj->userfault_count = 0; 577} 578 579void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) 580{ 581 struct i915_mmap_offset *mmo, *mn; 582 583 if (obj->ops->unmap_virtual) 584 obj->ops->unmap_virtual(obj); 585 586 spin_lock(&obj->mmo.lock); 587 rbtree_postorder_for_each_entry_safe(mmo, mn, 588 &obj->mmo.offsets, offset) { 589 /* 590 * vma_node_unmap for GTT mmaps handled already in 591 * __i915_gem_object_release_mmap_gtt 592 */ 593 if (mmo->mmap_type == I915_MMAP_TYPE_GTT) 594 continue; 595 596 spin_unlock(&obj->mmo.lock); 597 drm_vma_node_unmap(&mmo->vma_node, 598 obj->base.dev->anon_inode->i_mapping); 599 spin_lock(&obj->mmo.lock); 600 } 601 spin_unlock(&obj->mmo.lock); 602} 603 604static struct i915_mmap_offset * 605lookup_mmo(struct drm_i915_gem_object *obj, 606 enum i915_mmap_type mmap_type) 607{ 608 struct rb_node *rb; 609 610 spin_lock(&obj->mmo.lock); 611 rb = obj->mmo.offsets.rb_node; 612 while (rb) { 613 struct i915_mmap_offset *mmo = 614 rb_entry(rb, typeof(*mmo), offset); 615 616 if (mmo->mmap_type == mmap_type) { 617 spin_unlock(&obj->mmo.lock); 618 return mmo; 619 } 620 621 if (mmo->mmap_type < mmap_type) 622 rb = rb->rb_right; 623 else 624 rb = rb->rb_left; 625 } 626 spin_unlock(&obj->mmo.lock); 627 628 return NULL; 629} 630 631static struct i915_mmap_offset * 632insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo) 633{ 634 struct rb_node *rb, **p; 635 636 spin_lock(&obj->mmo.lock); 637 rb = NULL; 638 p = &obj->mmo.offsets.rb_node; 639 while (*p) { 640 struct i915_mmap_offset *pos; 641 642 rb = *p; 643 pos = rb_entry(rb, typeof(*pos), offset); 644 645 if (pos->mmap_type == mmo->mmap_type) { 646 spin_unlock(&obj->mmo.lock); 647 drm_vma_offset_remove(obj->base.dev->vma_offset_manager, 648 &mmo->vma_node); 649 kfree(mmo); 650 return pos; 651 } 652 653 if (pos->mmap_type < mmo->mmap_type) 654 p = &rb->rb_right; 655 else 656 p = &rb->rb_left; 657 } 658 rb_link_node(&mmo->offset, rb, p); 659 rb_insert_color(&mmo->offset, &obj->mmo.offsets); 660 spin_unlock(&obj->mmo.lock); 661 662 return mmo; 663} 664 665static struct i915_mmap_offset * 666mmap_offset_attach(struct drm_i915_gem_object *obj, 667 enum i915_mmap_type mmap_type, 668 struct drm_file *file) 669{ 670 struct drm_i915_private *i915 = to_i915(obj->base.dev); 671 struct i915_mmap_offset *mmo; 672 int err; 673 674 GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops); 675 676 mmo = lookup_mmo(obj, mmap_type); 677 if (mmo) 678 goto out; 679 680 mmo = kmalloc(sizeof(*mmo), GFP_KERNEL); 681 if (!mmo) 682 return ERR_PTR(-ENOMEM); 683 684 mmo->obj = obj; 685 mmo->mmap_type = mmap_type; 686 drm_vma_node_reset(&mmo->vma_node); 687 688 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, 689 &mmo->vma_node, obj->base.size / PAGE_SIZE); 690 if (likely(!err)) 691 goto insert; 692 693 /* Attempt to reap some mmap space from dead objects */ 694 err = intel_gt_retire_requests_timeout(to_gt(i915), MAX_SCHEDULE_TIMEOUT, 695 NULL); 696 if (err) 697 goto err; 698 699 i915_gem_drain_freed_objects(i915); 700 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, 701 &mmo->vma_node, obj->base.size / PAGE_SIZE); 702 if (err) 703 goto err; 704 705insert: 706 mmo = insert_mmo(obj, mmo); 707 GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo); 708out: 709 if (file) 710 drm_vma_node_allow_once(&mmo->vma_node, file); 711 return mmo; 712 713err: 714 kfree(mmo); 715 return ERR_PTR(err); 716} 717 718static int 719__assign_mmap_offset(struct drm_i915_gem_object *obj, 720 enum i915_mmap_type mmap_type, 721 u64 *offset, struct drm_file *file) 722{ 723 struct i915_mmap_offset *mmo; 724 725 if (i915_gem_object_never_mmap(obj)) 726 return -ENODEV; 727 728 if (obj->ops->mmap_offset) { 729 if (mmap_type != I915_MMAP_TYPE_FIXED) 730 return -ENODEV; 731 732 *offset = obj->ops->mmap_offset(obj); 733 return 0; 734 } 735 736 if (mmap_type == I915_MMAP_TYPE_FIXED) 737 return -ENODEV; 738 739 if (mmap_type != I915_MMAP_TYPE_GTT && 740 !i915_gem_object_has_struct_page(obj) && 741 !i915_gem_object_has_iomem(obj)) 742 return -ENODEV; 743 744 mmo = mmap_offset_attach(obj, mmap_type, file); 745 if (IS_ERR(mmo)) 746 return PTR_ERR(mmo); 747 748 *offset = drm_vma_node_offset_addr(&mmo->vma_node); 749 return 0; 750} 751 752static int 753__assign_mmap_offset_handle(struct drm_file *file, 754 u32 handle, 755 enum i915_mmap_type mmap_type, 756 u64 *offset) 757{ 758 struct drm_i915_gem_object *obj; 759 int err; 760 761 obj = i915_gem_object_lookup(file, handle); 762 if (!obj) 763 return -ENOENT; 764 765 err = i915_gem_object_lock_interruptible(obj, NULL); 766 if (err) 767 goto out_put; 768 err = __assign_mmap_offset(obj, mmap_type, offset, file); 769 i915_gem_object_unlock(obj); 770out_put: 771 i915_gem_object_put(obj); 772 return err; 773} 774 775int 776i915_gem_dumb_mmap_offset(struct drm_file *file, 777 struct drm_device *dev, 778 u32 handle, 779 u64 *offset) 780{ 781 struct drm_i915_private *i915 = to_i915(dev); 782 enum i915_mmap_type mmap_type; 783 784 if (HAS_LMEM(to_i915(dev))) 785 mmap_type = I915_MMAP_TYPE_FIXED; 786 else if (pat_enabled()) 787 mmap_type = I915_MMAP_TYPE_WC; 788 else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 789 return -ENODEV; 790 else 791 mmap_type = I915_MMAP_TYPE_GTT; 792 793 return __assign_mmap_offset_handle(file, handle, mmap_type, offset); 794} 795 796/** 797 * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing 798 * @dev: DRM device 799 * @data: GTT mapping ioctl data 800 * @file: GEM object info 801 * 802 * Simply returns the fake offset to userspace so it can mmap it. 803 * The mmap call will end up in drm_gem_mmap(), which will set things 804 * up so we can get faults in the handler above. 805 * 806 * The fault handler will take care of binding the object into the GTT 807 * (since it may have been evicted to make room for something), allocating 808 * a fence register, and mapping the appropriate aperture address into 809 * userspace. 810 */ 811int 812i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, 813 struct drm_file *file) 814{ 815 struct drm_i915_private *i915 = to_i915(dev); 816 struct drm_i915_gem_mmap_offset *args = data; 817 enum i915_mmap_type type; 818 int err; 819 820 /* 821 * Historically we failed to check args.pad and args.offset 822 * and so we cannot use those fields for user input and we cannot 823 * add -EINVAL for them as the ABI is fixed, i.e. old userspace 824 * may be feeding in garbage in those fields. 825 * 826 * if (args->pad) return -EINVAL; is verbotten! 827 */ 828 829 err = i915_user_extensions(u64_to_user_ptr(args->extensions), 830 NULL, 0, NULL); 831 if (err) 832 return err; 833 834 switch (args->flags) { 835 case I915_MMAP_OFFSET_GTT: 836 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 837 return -ENODEV; 838 type = I915_MMAP_TYPE_GTT; 839 break; 840 841 case I915_MMAP_OFFSET_WC: 842 if (!pat_enabled()) 843 return -ENODEV; 844 type = I915_MMAP_TYPE_WC; 845 break; 846 847 case I915_MMAP_OFFSET_WB: 848 type = I915_MMAP_TYPE_WB; 849 break; 850 851 case I915_MMAP_OFFSET_UC: 852 if (!pat_enabled()) 853 return -ENODEV; 854 type = I915_MMAP_TYPE_UC; 855 break; 856 857 case I915_MMAP_OFFSET_FIXED: 858 type = I915_MMAP_TYPE_FIXED; 859 break; 860 861 default: 862 return -EINVAL; 863 } 864 865 return __assign_mmap_offset_handle(file, args->handle, type, &args->offset); 866} 867 868static void vm_open(struct vm_area_struct *vma) 869{ 870 struct i915_mmap_offset *mmo = vma->vm_private_data; 871 struct drm_i915_gem_object *obj = mmo->obj; 872 873 GEM_BUG_ON(!obj); 874 i915_gem_object_get(obj); 875} 876 877static void vm_close(struct vm_area_struct *vma) 878{ 879 struct i915_mmap_offset *mmo = vma->vm_private_data; 880 struct drm_i915_gem_object *obj = mmo->obj; 881 882 GEM_BUG_ON(!obj); 883 i915_gem_object_put(obj); 884} 885 886static const struct vm_operations_struct vm_ops_gtt = { 887 .fault = vm_fault_gtt, 888 .access = vm_access, 889 .open = vm_open, 890 .close = vm_close, 891}; 892 893static const struct vm_operations_struct vm_ops_cpu = { 894 .fault = vm_fault_cpu, 895 .access = vm_access, 896 .open = vm_open, 897 .close = vm_close, 898}; 899 900static int singleton_release(struct inode *inode, struct file *file) 901{ 902 struct drm_i915_private *i915 = file->private_data; 903 904 cmpxchg(&i915->gem.mmap_singleton, file, NULL); 905 drm_dev_put(&i915->drm); 906 907 return 0; 908} 909 910static const struct file_operations singleton_fops = { 911 .owner = THIS_MODULE, 912 .release = singleton_release, 913}; 914 915static struct file *mmap_singleton(struct drm_i915_private *i915) 916{ 917 struct file *file; 918 919 file = get_file_active(&i915->gem.mmap_singleton); 920 if (file) 921 return file; 922 923 file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR); 924 if (IS_ERR(file)) 925 return file; 926 927 /* Everyone shares a single global address space */ 928 file->f_mapping = i915->drm.anon_inode->i_mapping; 929 930 smp_store_mb(i915->gem.mmap_singleton, file); 931 drm_dev_get(&i915->drm); 932 933 return file; 934} 935 936static int 937i915_gem_object_mmap(struct drm_i915_gem_object *obj, 938 struct i915_mmap_offset *mmo, 939 struct vm_area_struct *vma) 940{ 941 struct drm_i915_private *i915 = to_i915(obj->base.dev); 942 struct drm_device *dev = &i915->drm; 943 struct file *anon; 944 945 if (i915_gem_object_is_readonly(obj)) { 946 if (vma->vm_flags & VM_WRITE) { 947 i915_gem_object_put(obj); 948 return -EINVAL; 949 } 950 vm_flags_clear(vma, VM_MAYWRITE); 951 } 952 953 anon = mmap_singleton(to_i915(dev)); 954 if (IS_ERR(anon)) { 955 i915_gem_object_put(obj); 956 return PTR_ERR(anon); 957 } 958 959 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); 960 961 /* 962 * We keep the ref on mmo->obj, not vm_file, but we require 963 * vma->vm_file->f_mapping, see vma_link(), for later revocation. 964 * Our userspace is accustomed to having per-file resource cleanup 965 * (i.e. contexts, objects and requests) on their close(fd), which 966 * requires avoiding extraneous references to their filp, hence why 967 * we prefer to use an anonymous file for their mmaps. 968 */ 969 vma_set_file(vma, anon); 970 /* Drop the initial creation reference, the vma is now holding one. */ 971 fput(anon); 972 973 if (obj->ops->mmap_ops) { 974 vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); 975 vma->vm_ops = obj->ops->mmap_ops; 976 vma->vm_private_data = obj->base.vma_node.driver_private; 977 return 0; 978 } 979 980 vma->vm_private_data = mmo; 981 982 switch (mmo->mmap_type) { 983 case I915_MMAP_TYPE_WC: 984 vma->vm_page_prot = 985 pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 986 vma->vm_ops = &vm_ops_cpu; 987 break; 988 989 case I915_MMAP_TYPE_FIXED: 990 GEM_WARN_ON(1); 991 fallthrough; 992 case I915_MMAP_TYPE_WB: 993 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); 994 vma->vm_ops = &vm_ops_cpu; 995 break; 996 997 case I915_MMAP_TYPE_UC: 998 vma->vm_page_prot = 999 pgprot_noncached(vm_get_page_prot(vma->vm_flags)); 1000 vma->vm_ops = &vm_ops_cpu; 1001 break; 1002 1003 case I915_MMAP_TYPE_GTT: 1004 vma->vm_page_prot = 1005 pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 1006 vma->vm_ops = &vm_ops_gtt; 1007 break; 1008 } 1009 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); 1010 1011 return 0; 1012} 1013 1014/* 1015 * This overcomes the limitation in drm_gem_mmap's assignment of a 1016 * drm_gem_object as the vma->vm_private_data. Since we need to 1017 * be able to resolve multiple mmap offsets which could be tied 1018 * to a single gem object. 1019 */ 1020int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) 1021{ 1022 struct drm_vma_offset_node *node; 1023 struct drm_file *priv = filp->private_data; 1024 struct drm_device *dev = priv->minor->dev; 1025 struct drm_i915_gem_object *obj = NULL; 1026 struct i915_mmap_offset *mmo = NULL; 1027 1028 if (drm_dev_is_unplugged(dev)) 1029 return -ENODEV; 1030 1031 rcu_read_lock(); 1032 drm_vma_offset_lock_lookup(dev->vma_offset_manager); 1033 node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, 1034 vma->vm_pgoff, 1035 vma_pages(vma)); 1036 if (node && drm_vma_node_is_allowed(node, priv)) { 1037 /* 1038 * Skip 0-refcnted objects as it is in the process of being 1039 * destroyed and will be invalid when the vma manager lock 1040 * is released. 1041 */ 1042 if (!node->driver_private) { 1043 mmo = container_of(node, struct i915_mmap_offset, vma_node); 1044 obj = i915_gem_object_get_rcu(mmo->obj); 1045 1046 GEM_BUG_ON(obj && obj->ops->mmap_ops); 1047 } else { 1048 obj = i915_gem_object_get_rcu 1049 (container_of(node, struct drm_i915_gem_object, 1050 base.vma_node)); 1051 1052 GEM_BUG_ON(obj && !obj->ops->mmap_ops); 1053 } 1054 } 1055 drm_vma_offset_unlock_lookup(dev->vma_offset_manager); 1056 rcu_read_unlock(); 1057 if (!obj) 1058 return node ? -EACCES : -EINVAL; 1059 1060 return i915_gem_object_mmap(obj, mmo, vma); 1061} 1062 1063int i915_gem_fb_mmap(struct drm_i915_gem_object *obj, struct vm_area_struct *vma) 1064{ 1065 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1066 struct drm_device *dev = &i915->drm; 1067 struct i915_mmap_offset *mmo = NULL; 1068 enum i915_mmap_type mmap_type; 1069 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1070 1071 if (drm_dev_is_unplugged(dev)) 1072 return -ENODEV; 1073 1074 /* handle ttm object */ 1075 if (obj->ops->mmap_ops) { 1076 /* 1077 * ttm fault handler, ttm_bo_vm_fault_reserved() uses fake offset 1078 * to calculate page offset so set that up. 1079 */ 1080 vma->vm_pgoff += drm_vma_node_start(&obj->base.vma_node); 1081 } else { 1082 /* handle stolen and smem objects */ 1083 mmap_type = i915_ggtt_has_aperture(ggtt) ? I915_MMAP_TYPE_GTT : I915_MMAP_TYPE_WC; 1084 mmo = mmap_offset_attach(obj, mmap_type, NULL); 1085 if (IS_ERR(mmo)) 1086 return PTR_ERR(mmo); 1087 } 1088 1089 /* 1090 * When we install vm_ops for mmap we are too late for 1091 * the vm_ops->open() which increases the ref_count of 1092 * this obj and then it gets decreased by the vm_ops->close(). 1093 * To balance this increase the obj ref_count here. 1094 */ 1095 obj = i915_gem_object_get(obj); 1096 return i915_gem_object_mmap(obj, mmo, vma); 1097} 1098 1099#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1100#include "selftests/i915_gem_mman.c" 1101#endif 1102