i915_gem.c revision 293837
1/* 2 * Copyright �� 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 */ 53 54#include <sys/cdefs.h> 55__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 293837 2016-01-13 19:52:25Z dumbbell $"); 56 57#include <dev/drm2/drmP.h> 58#include <dev/drm2/drm.h> 59#include <dev/drm2/i915/i915_drm.h> 60#include <dev/drm2/i915/i915_drv.h> 61#include <dev/drm2/i915/intel_drv.h> 62#include <dev/drm2/i915/intel_ringbuffer.h> 63 64#include <sys/resourcevar.h> 65#include <sys/sched.h> 66#include <sys/sf_buf.h> 67 68#include <vm/vm.h> 69#include <vm/vm_pageout.h> 70 71#include <machine/md_var.h> 72 73#define __user 74#define __force 75#define __iomem 76#define __must_check 77#define to_user_ptr(x) ((void *)(uintptr_t)(x)) 78#define offset_in_page(x) ((x) & PAGE_MASK) 79#define page_to_phys(x) VM_PAGE_TO_PHYS(x) 80 81static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); 82static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); 83static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 84 unsigned alignment, 85 bool map_and_fenceable); 86static int i915_gem_phys_pwrite(struct drm_device *dev, 87 struct drm_i915_gem_object *obj, 88 struct drm_i915_gem_pwrite *args, 89 struct drm_file *file); 90 91static void i915_gem_write_fence(struct drm_device *dev, int reg, 92 struct drm_i915_gem_object *obj); 93static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 94 struct drm_i915_fence_reg *fence, 95 bool enable); 96 97static void i915_gem_lowmem(void *arg); 98static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 99 100static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 101 off_t start, off_t end); 102static void i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, 103 off_t start, off_t end); 104 105static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, 106 bool *fresh); 107 108MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 109long i915_gem_wired_pages_cnt; 110 111static bool cpu_cache_is_coherent(struct drm_device *dev, 112 enum i915_cache_level level) 113{ 114 return HAS_LLC(dev) || level != I915_CACHE_NONE; 115} 116 117static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 118{ 119 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) 120 return true; 121 122 return obj->pin_display; 123} 124 125static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) 126{ 127 if (obj->tiling_mode) 128 i915_gem_release_mmap(obj); 129 130 /* As we do not have an associated fence register, we will force 131 * a tiling change if we ever need to acquire one. 132 */ 133 obj->fence_dirty = false; 134 obj->fence_reg = I915_FENCE_REG_NONE; 135} 136 137/* some bookkeeping */ 138static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 139 size_t size) 140{ 141 dev_priv->mm.object_count++; 142 dev_priv->mm.object_memory += size; 143} 144 145static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 146 size_t size) 147{ 148 dev_priv->mm.object_count--; 149 dev_priv->mm.object_memory -= size; 150} 151 152static int 153i915_gem_wait_for_error(struct drm_device *dev) 154{ 155 struct drm_i915_private *dev_priv = dev->dev_private; 156 int ret; 157 158 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 159 return 0; 160 161 mtx_lock(&dev_priv->error_completion_lock); 162 while (dev_priv->error_completion == 0) { 163 ret = -msleep(&dev_priv->error_completion, 164 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 165 if (ret == -ERESTART) 166 ret = -ERESTARTSYS; 167 if (ret != 0) { 168 mtx_unlock(&dev_priv->error_completion_lock); 169 return ret; 170 } 171 } 172 mtx_unlock(&dev_priv->error_completion_lock); 173 174 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 175 /* GPU is hung, bump the completion count to account for 176 * the token we just consumed so that we never hit zero and 177 * end up waiting upon a subsequent completion event that 178 * will never happen. 179 */ 180 mtx_lock(&dev_priv->error_completion_lock); 181 dev_priv->error_completion++; 182 mtx_unlock(&dev_priv->error_completion_lock); 183 } 184 return 0; 185} 186 187int i915_mutex_lock_interruptible(struct drm_device *dev) 188{ 189 int ret; 190 191 ret = i915_gem_wait_for_error(dev); 192 if (ret) 193 return ret; 194 195 /* 196 * interruptible shall it be. might indeed be if dev_lock is 197 * changed to sx 198 */ 199 ret = -sx_xlock_sig(&dev->dev_struct_lock); 200 if (ret) 201 return ret; 202 203 return 0; 204} 205 206static inline bool 207i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 208{ 209 return !obj->active; 210} 211 212int 213i915_gem_init_ioctl(struct drm_device *dev, void *data, 214 struct drm_file *file) 215{ 216 struct drm_i915_gem_init *args = data; 217 drm_i915_private_t *dev_priv = dev->dev_private; 218 int ret; 219 220 if (drm_core_check_feature(dev, DRIVER_MODESET)) 221 return -ENODEV; 222 223 if (args->gtt_start >= args->gtt_end || 224 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 225 return -EINVAL; 226 227 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) 228 return -EBUSY; 229 230 /* GEM with user mode setting was never supported on ilk and later. */ 231 if (INTEL_INFO(dev)->gen >= 5) 232 return -ENODEV; 233 234 /* 235 * XXXKIB. The second-time initialization should be guarded 236 * against. 237 */ 238 DRM_LOCK(dev); 239 ret = i915_gem_init_global_gtt(dev, args->gtt_start, 240 args->gtt_end, args->gtt_end); 241 DRM_UNLOCK(dev); 242 243 return ret; 244} 245 246int 247i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 248 struct drm_file *file) 249{ 250 struct drm_i915_private *dev_priv = dev->dev_private; 251 struct drm_i915_gem_get_aperture *args = data; 252 struct drm_i915_gem_object *obj; 253 size_t pinned; 254 255 pinned = 0; 256 DRM_LOCK(dev); 257 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) 258 if (obj->pin_count) 259 pinned += obj->gtt_space->size; 260 DRM_UNLOCK(dev); 261 262 args->aper_size = dev_priv->mm.gtt_total; 263 args->aper_available_size = args->aper_size - pinned; 264 265 return 0; 266} 267 268static int 269i915_gem_create(struct drm_file *file, 270 struct drm_device *dev, 271 uint64_t size, 272 uint32_t *handle_p) 273{ 274 struct drm_i915_gem_object *obj; 275 int ret; 276 u32 handle; 277 278 size = roundup(size, PAGE_SIZE); 279 if (size == 0) 280 return -EINVAL; 281 282 /* Allocate the new object */ 283 obj = i915_gem_alloc_object(dev, size); 284 if (obj == NULL) 285 return -ENOMEM; 286 287 ret = drm_gem_handle_create(file, &obj->base, &handle); 288 if (ret) { 289 drm_gem_object_release(&obj->base); 290 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 291 free(obj, DRM_I915_GEM); 292 return ret; 293 } 294 295 /* drop reference from allocate - handle holds it now */ 296 drm_gem_object_unreference(&obj->base); 297 CTR2(KTR_DRM, "object_create %p %x", obj, size); 298 299 *handle_p = handle; 300 return 0; 301} 302 303int 304i915_gem_dumb_create(struct drm_file *file, 305 struct drm_device *dev, 306 struct drm_mode_create_dumb *args) 307{ 308 /* have to work out size/pitch and return them */ 309 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 310 args->size = args->pitch * args->height; 311 return i915_gem_create(file, dev, 312 args->size, &args->handle); 313} 314 315int i915_gem_dumb_destroy(struct drm_file *file, 316 struct drm_device *dev, 317 uint32_t handle) 318{ 319 return drm_gem_handle_delete(file, handle); 320} 321 322/** 323 * Creates a new mm object and returns a handle to it. 324 */ 325int 326i915_gem_create_ioctl(struct drm_device *dev, void *data, 327 struct drm_file *file) 328{ 329 struct drm_i915_gem_create *args = data; 330 331 return i915_gem_create(file, dev, 332 args->size, &args->handle); 333} 334 335static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 336{ 337 drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 338 339 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 340 obj->tiling_mode != I915_TILING_NONE; 341} 342 343static inline int 344__copy_to_user_inatomic(void __user *to, const void *from, unsigned n) 345{ 346 return (copyout_nofault(from, to, n) != 0 ? n : 0); 347} 348static inline unsigned long 349__copy_from_user_inatomic_nocache(void *to, const void __user *from, 350 unsigned long n) 351{ 352 353 /* 354 * XXXKIB. Equivalent Linux function is implemented using 355 * MOVNTI for aligned moves. For unaligned head and tail, 356 * normal move is performed. As such, it is not incorrect, if 357 * only somewhat slower, to use normal copyin. All uses 358 * except shmem_pwrite_fast() have the destination mapped WC. 359 */ 360 return ((copyin_nofault(__DECONST(void *, from), to, n) != 0 ? n : 0)); 361} 362static inline int 363fault_in_multipages_readable(const char __user *uaddr, int size) 364{ 365 char c; 366 int ret = 0; 367 const char __user *end = uaddr + size - 1; 368 369 if (unlikely(size == 0)) 370 return ret; 371 372 while (uaddr <= end) { 373 ret = -copyin(uaddr, &c, 1); 374 if (ret != 0) 375 return -EFAULT; 376 uaddr += PAGE_SIZE; 377 } 378 379 /* Check whether the range spilled into the next page. */ 380 if (((unsigned long)uaddr & ~PAGE_MASK) == 381 ((unsigned long)end & ~PAGE_MASK)) { 382 ret = -copyin(end, &c, 1); 383 } 384 385 return ret; 386} 387 388static inline int 389fault_in_multipages_writeable(char __user *uaddr, int size) 390{ 391 int ret = 0; 392 char __user *end = uaddr + size - 1; 393 394 if (unlikely(size == 0)) 395 return ret; 396 397 /* 398 * Writing zeroes into userspace here is OK, because we know that if 399 * the zero gets there, we'll be overwriting it. 400 */ 401 while (uaddr <= end) { 402 ret = subyte(uaddr, 0); 403 if (ret != 0) 404 return -EFAULT; 405 uaddr += PAGE_SIZE; 406 } 407 408 /* Check whether the range spilled into the next page. */ 409 if (((unsigned long)uaddr & ~PAGE_MASK) == 410 ((unsigned long)end & ~PAGE_MASK)) 411 ret = subyte(end, 0); 412 413 return ret; 414} 415 416static inline int 417__copy_to_user_swizzled(char __user *cpu_vaddr, 418 const char *gpu_vaddr, int gpu_offset, 419 int length) 420{ 421 int ret, cpu_offset = 0; 422 423 while (length > 0) { 424 int cacheline_end = roundup2(gpu_offset + 1, 64); 425 int this_length = min(cacheline_end - gpu_offset, length); 426 int swizzled_gpu_offset = gpu_offset ^ 64; 427 428 ret = __copy_to_user(cpu_vaddr + cpu_offset, 429 gpu_vaddr + swizzled_gpu_offset, 430 this_length); 431 if (ret) 432 return ret + length; 433 434 cpu_offset += this_length; 435 gpu_offset += this_length; 436 length -= this_length; 437 } 438 439 return 0; 440} 441 442static inline int 443__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 444 const char __user *cpu_vaddr, 445 int length) 446{ 447 int ret, cpu_offset = 0; 448 449 while (length > 0) { 450 int cacheline_end = roundup2(gpu_offset + 1, 64); 451 int this_length = min(cacheline_end - gpu_offset, length); 452 int swizzled_gpu_offset = gpu_offset ^ 64; 453 454 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 455 cpu_vaddr + cpu_offset, 456 this_length); 457 if (ret) 458 return ret + length; 459 460 cpu_offset += this_length; 461 gpu_offset += this_length; 462 length -= this_length; 463 } 464 465 return 0; 466} 467 468/* Per-page copy function for the shmem pread fastpath. 469 * Flushes invalid cachelines before reading the target if 470 * needs_clflush is set. */ 471static int 472shmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length, 473 char __user *user_data, 474 bool page_do_bit17_swizzling, bool needs_clflush) 475{ 476 char *vaddr; 477 struct sf_buf *sf; 478 int ret; 479 480 if (unlikely(page_do_bit17_swizzling)) 481 return -EINVAL; 482 483 sched_pin(); 484 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 485 if (sf == NULL) { 486 sched_unpin(); 487 return (-EFAULT); 488 } 489 vaddr = (char *)sf_buf_kva(sf); 490 if (needs_clflush) 491 drm_clflush_virt_range(vaddr + shmem_page_offset, 492 page_length); 493 ret = __copy_to_user_inatomic(user_data, 494 vaddr + shmem_page_offset, 495 page_length); 496 sf_buf_free(sf); 497 sched_unpin(); 498 499 return ret ? -EFAULT : 0; 500} 501 502static void 503shmem_clflush_swizzled_range(char *addr, unsigned long length, 504 bool swizzled) 505{ 506 if (unlikely(swizzled)) { 507 unsigned long start = (unsigned long) addr; 508 unsigned long end = (unsigned long) addr + length; 509 510 /* For swizzling simply ensure that we always flush both 511 * channels. Lame, but simple and it works. Swizzled 512 * pwrite/pread is far from a hotpath - current userspace 513 * doesn't use it at all. */ 514 start = rounddown2(start, 128); 515 end = roundup2(end, 128); 516 517 drm_clflush_virt_range((void *)start, end - start); 518 } else { 519 drm_clflush_virt_range(addr, length); 520 } 521 522} 523 524/* Only difference to the fast-path function is that this can handle bit17 525 * and uses non-atomic copy and kmap functions. */ 526static int 527shmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length, 528 char __user *user_data, 529 bool page_do_bit17_swizzling, bool needs_clflush) 530{ 531 char *vaddr; 532 struct sf_buf *sf; 533 int ret; 534 535 sf = sf_buf_alloc(page, 0); 536 vaddr = (char *)sf_buf_kva(sf); 537 if (needs_clflush) 538 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 539 page_length, 540 page_do_bit17_swizzling); 541 542 if (page_do_bit17_swizzling) 543 ret = __copy_to_user_swizzled(user_data, 544 vaddr, shmem_page_offset, 545 page_length); 546 else 547 ret = __copy_to_user(user_data, 548 vaddr + shmem_page_offset, 549 page_length); 550 sf_buf_free(sf); 551 552 return ret ? - EFAULT : 0; 553} 554 555static int 556i915_gem_shmem_pread(struct drm_device *dev, 557 struct drm_i915_gem_object *obj, 558 struct drm_i915_gem_pread *args, 559 struct drm_file *file) 560{ 561 char __user *user_data; 562 ssize_t remain, sremain; 563 off_t offset, soffset; 564 int shmem_page_offset, page_length, ret = 0; 565 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 566 int prefaulted = 0; 567 int needs_clflush = 0; 568 569 user_data = to_user_ptr(args->data_ptr); 570 sremain = remain = args->size; 571 572 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 573 574 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 575 /* If we're not in the cpu read domain, set ourself into the gtt 576 * read domain and manually flush cachelines (if required). This 577 * optimizes for the case when the gpu will dirty the data 578 * anyway again before the next pread happens. */ 579 needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level); 580 ret = i915_gem_object_set_to_gtt_domain(obj, false); 581 if (ret) 582 return ret; 583 } 584 585 soffset = offset = args->offset; 586 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain); 587 if (ret) 588 return ret; 589 590 i915_gem_object_pin_pages(obj); 591 592 VM_OBJECT_WLOCK(obj->base.vm_obj); 593 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 594 OFF_TO_IDX(offset));; page = vm_page_next(page)) { 595 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 596 597 if (remain <= 0) 598 break; 599 600 /* Operation in this page 601 * 602 * shmem_page_offset = offset within page in shmem file 603 * page_length = bytes to copy for this page 604 */ 605 shmem_page_offset = offset_in_page(offset); 606 page_length = remain; 607 if ((shmem_page_offset + page_length) > PAGE_SIZE) 608 page_length = PAGE_SIZE - shmem_page_offset; 609 610 page_do_bit17_swizzling = obj_do_bit17_swizzling && 611 (page_to_phys(page) & (1 << 17)) != 0; 612 613 ret = shmem_pread_fast(page, shmem_page_offset, page_length, 614 user_data, page_do_bit17_swizzling, 615 needs_clflush); 616 if (ret == 0) 617 goto next_page; 618 619 DRM_UNLOCK(dev); 620 621 if (likely(!i915_prefault_disable) && !prefaulted) { 622 ret = fault_in_multipages_writeable(user_data, remain); 623 /* Userspace is tricking us, but we've already clobbered 624 * its pages with the prefault and promised to write the 625 * data up to the first fault. Hence ignore any errors 626 * and just continue. */ 627 (void)ret; 628 prefaulted = 1; 629 } 630 631 ret = shmem_pread_slow(page, shmem_page_offset, page_length, 632 user_data, page_do_bit17_swizzling, 633 needs_clflush); 634 635 DRM_LOCK(dev); 636 637next_page: 638 vm_page_reference(page); 639 640 if (ret) 641 goto out; 642 643 remain -= page_length; 644 user_data += page_length; 645 offset += page_length; 646 VM_OBJECT_WLOCK(obj->base.vm_obj); 647 } 648 649out: 650 i915_gem_object_unpin_pages(obj); 651 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain); 652 653 return ret; 654} 655 656/** 657 * Reads data from the object referenced by handle. 658 * 659 * On error, the contents of *data are undefined. 660 */ 661int 662i915_gem_pread_ioctl(struct drm_device *dev, void *data, 663 struct drm_file *file) 664{ 665 struct drm_i915_gem_pread *args = data; 666 struct drm_i915_gem_object *obj; 667 int ret = 0; 668 669 if (args->size == 0) 670 return 0; 671 672 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE)) 673 return -EFAULT; 674 675 ret = i915_mutex_lock_interruptible(dev); 676 if (ret) 677 return ret; 678 679 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 680 if (&obj->base == NULL) { 681 ret = -ENOENT; 682 goto unlock; 683 } 684 685 /* Bounds check source. */ 686 if (args->offset > obj->base.size || 687 args->size > obj->base.size - args->offset) { 688 ret = -EINVAL; 689 goto out; 690 } 691 692#if 1 693 KIB_NOTYET(); 694#else 695 /* prime objects have no backing filp to GEM pread/pwrite 696 * pages from. 697 */ 698 if (!obj->base.filp) { 699 ret = -EINVAL; 700 goto out; 701 } 702#endif 703 704 CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size); 705 706 ret = i915_gem_shmem_pread(dev, obj, args, file); 707 708out: 709 drm_gem_object_unreference(&obj->base); 710unlock: 711 DRM_UNLOCK(dev); 712 return ret; 713} 714 715/* This is the fast write path which cannot handle 716 * page faults in the source data 717 */ 718 719static inline int 720fast_user_write(struct drm_device *dev, 721 off_t page_base, int page_offset, 722 char __user *user_data, 723 int length) 724{ 725 void __iomem *vaddr_atomic; 726 void *vaddr; 727 unsigned long unwritten; 728 729 vaddr_atomic = pmap_mapdev_attr(dev->agp->base + page_base, 730 length, PAT_WRITE_COMBINING); 731 /* We can use the cpu mem copy function because this is X86. */ 732 vaddr = (char __force*)vaddr_atomic + page_offset; 733 unwritten = __copy_from_user_inatomic_nocache(vaddr, 734 user_data, length); 735 pmap_unmapdev((vm_offset_t)vaddr_atomic, length); 736 return unwritten; 737} 738 739/** 740 * This is the fast pwrite path, where we copy the data directly from the 741 * user into the GTT, uncached. 742 */ 743static int 744i915_gem_gtt_pwrite_fast(struct drm_device *dev, 745 struct drm_i915_gem_object *obj, 746 struct drm_i915_gem_pwrite *args, 747 struct drm_file *file) 748{ 749 ssize_t remain; 750 off_t offset, page_base; 751 char __user *user_data; 752 int page_offset, page_length, ret; 753 754 ret = i915_gem_object_pin(obj, 0, true); 755 /* XXXKIB ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); */ 756 if (ret) 757 goto out; 758 759 ret = i915_gem_object_set_to_gtt_domain(obj, true); 760 if (ret) 761 goto out_unpin; 762 763 ret = i915_gem_object_put_fence(obj); 764 if (ret) 765 goto out_unpin; 766 767 user_data = to_user_ptr(args->data_ptr); 768 remain = args->size; 769 770 offset = obj->gtt_offset + args->offset; 771 772 while (remain > 0) { 773 /* Operation in this page 774 * 775 * page_base = page offset within aperture 776 * page_offset = offset within page 777 * page_length = bytes to copy for this page 778 */ 779 page_base = offset & ~PAGE_MASK; 780 page_offset = offset_in_page(offset); 781 page_length = remain; 782 if ((page_offset + remain) > PAGE_SIZE) 783 page_length = PAGE_SIZE - page_offset; 784 785 /* If we get a fault while copying data, then (presumably) our 786 * source page isn't available. Return the error and we'll 787 * retry in the slow path. 788 */ 789 if (fast_user_write(dev, page_base, 790 page_offset, user_data, page_length)) { 791 ret = -EFAULT; 792 goto out_unpin; 793 } 794 795 remain -= page_length; 796 user_data += page_length; 797 offset += page_length; 798 } 799 800out_unpin: 801 i915_gem_object_unpin(obj); 802out: 803 return ret; 804} 805 806/* Per-page copy function for the shmem pwrite fastpath. 807 * Flushes invalid cachelines before writing to the target if 808 * needs_clflush_before is set and flushes out any written cachelines after 809 * writing if needs_clflush is set. */ 810static int 811shmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length, 812 char __user *user_data, 813 bool page_do_bit17_swizzling, 814 bool needs_clflush_before, 815 bool needs_clflush_after) 816{ 817 char *vaddr; 818 struct sf_buf *sf; 819 int ret; 820 821 if (unlikely(page_do_bit17_swizzling)) 822 return -EINVAL; 823 824 sched_pin(); 825 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 826 if (sf == NULL) { 827 sched_unpin(); 828 return (-EFAULT); 829 } 830 vaddr = (char *)sf_buf_kva(sf); 831 if (needs_clflush_before) 832 drm_clflush_virt_range(vaddr + shmem_page_offset, 833 page_length); 834 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, 835 user_data, 836 page_length); 837 if (needs_clflush_after) 838 drm_clflush_virt_range(vaddr + shmem_page_offset, 839 page_length); 840 sf_buf_free(sf); 841 sched_unpin(); 842 843 return ret ? -EFAULT : 0; 844} 845 846/* Only difference to the fast-path function is that this can handle bit17 847 * and uses non-atomic copy and kmap functions. */ 848static int 849shmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length, 850 char __user *user_data, 851 bool page_do_bit17_swizzling, 852 bool needs_clflush_before, 853 bool needs_clflush_after) 854{ 855 char *vaddr; 856 struct sf_buf *sf; 857 int ret; 858 859 sf = sf_buf_alloc(page, 0); 860 vaddr = (char *)sf_buf_kva(sf); 861 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 862 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 863 page_length, 864 page_do_bit17_swizzling); 865 if (page_do_bit17_swizzling) 866 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 867 user_data, 868 page_length); 869 else 870 ret = __copy_from_user(vaddr + shmem_page_offset, 871 user_data, 872 page_length); 873 if (needs_clflush_after) 874 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 875 page_length, 876 page_do_bit17_swizzling); 877 sf_buf_free(sf); 878 879 return ret ? -EFAULT : 0; 880} 881 882static int 883i915_gem_shmem_pwrite(struct drm_device *dev, 884 struct drm_i915_gem_object *obj, 885 struct drm_i915_gem_pwrite *args, 886 struct drm_file *file) 887{ 888 ssize_t remain, sremain; 889 off_t offset, soffset; 890 char __user *user_data; 891 int shmem_page_offset, page_length, ret = 0; 892 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 893 int hit_slowpath = 0; 894 int needs_clflush_after = 0; 895 int needs_clflush_before = 0; 896 897 user_data = to_user_ptr(args->data_ptr); 898 sremain = remain = args->size; 899 900 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 901 902 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 903 /* If we're not in the cpu write domain, set ourself into the gtt 904 * write domain and manually flush cachelines (if required). This 905 * optimizes for the case when the gpu will use the data 906 * right away and we therefore have to clflush anyway. */ 907 needs_clflush_after = cpu_write_needs_clflush(obj); 908 ret = i915_gem_object_set_to_gtt_domain(obj, true); 909 if (ret) 910 return ret; 911 } 912 /* Same trick applies to invalidate partially written cachelines read 913 * before writing. */ 914 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 915 needs_clflush_before = 916 !cpu_cache_is_coherent(dev, obj->cache_level); 917 918 soffset = offset = args->offset; 919 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain); 920 if (ret) 921 return ret; 922 923 i915_gem_object_pin_pages(obj); 924 925 obj->dirty = 1; 926 927 VM_OBJECT_WLOCK(obj->base.vm_obj); 928 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 929 OFF_TO_IDX(offset));; page = vm_page_next(page)) { 930 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 931 int partial_cacheline_write; 932 933 if (remain <= 0) 934 break; 935 936 /* Operation in this page 937 * 938 * shmem_page_offset = offset within page in shmem file 939 * page_length = bytes to copy for this page 940 */ 941 shmem_page_offset = offset_in_page(offset); 942 943 page_length = remain; 944 if ((shmem_page_offset + page_length) > PAGE_SIZE) 945 page_length = PAGE_SIZE - shmem_page_offset; 946 947 /* If we don't overwrite a cacheline completely we need to be 948 * careful to have up-to-date data by first clflushing. Don't 949 * overcomplicate things and flush the entire patch. */ 950 partial_cacheline_write = needs_clflush_before && 951 ((shmem_page_offset | page_length) 952 & (cpu_clflush_line_size - 1)); 953 954 page_do_bit17_swizzling = obj_do_bit17_swizzling && 955 (page_to_phys(page) & (1 << 17)) != 0; 956 957 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 958 user_data, page_do_bit17_swizzling, 959 partial_cacheline_write, 960 needs_clflush_after); 961 if (ret == 0) 962 goto next_page; 963 964 hit_slowpath = 1; 965 DRM_UNLOCK(dev); 966 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 967 user_data, page_do_bit17_swizzling, 968 partial_cacheline_write, 969 needs_clflush_after); 970 971 DRM_LOCK(dev); 972 973next_page: 974 vm_page_dirty(page); 975 vm_page_reference(page); 976 977 if (ret) 978 goto out; 979 980 remain -= page_length; 981 user_data += page_length; 982 offset += page_length; 983 VM_OBJECT_WLOCK(obj->base.vm_obj); 984 } 985 986out: 987 i915_gem_object_unpin_pages(obj); 988 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain); 989 990 if (hit_slowpath) { 991 /* 992 * Fixup: Flush cpu caches in case we didn't flush the dirty 993 * cachelines in-line while writing and the object moved 994 * out of the cpu write domain while we've dropped the lock. 995 */ 996 if (!needs_clflush_after && 997 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 998 i915_gem_clflush_object(obj); 999 i915_gem_chipset_flush(dev); 1000 } 1001 } 1002 1003 if (needs_clflush_after) 1004 i915_gem_chipset_flush(dev); 1005 1006 return ret; 1007} 1008 1009/** 1010 * Writes data to the object referenced by handle. 1011 * 1012 * On error, the contents of the buffer that were to be modified are undefined. 1013 */ 1014int 1015i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 1016 struct drm_file *file) 1017{ 1018 struct drm_i915_gem_pwrite *args = data; 1019 struct drm_i915_gem_object *obj; 1020 int ret; 1021 1022 if (args->size == 0) 1023 return 0; 1024 1025 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ)) 1026 return -EFAULT; 1027 1028 if (likely(!i915_prefault_disable)) { 1029 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), 1030 args->size); 1031 if (ret) 1032 return -EFAULT; 1033 } 1034 1035 ret = i915_mutex_lock_interruptible(dev); 1036 if (ret) 1037 return ret; 1038 1039 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1040 if (&obj->base == NULL) { 1041 ret = -ENOENT; 1042 goto unlock; 1043 } 1044 1045 /* Bounds check destination. */ 1046 if (args->offset > obj->base.size || 1047 args->size > obj->base.size - args->offset) { 1048 ret = -EINVAL; 1049 goto out; 1050 } 1051 1052#if 1 1053 KIB_NOTYET(); 1054#else 1055 /* prime objects have no backing filp to GEM pread/pwrite 1056 * pages from. 1057 */ 1058 if (!obj->base.filp) { 1059 ret = -EINVAL; 1060 goto out; 1061 } 1062#endif 1063 1064 CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size); 1065 1066 ret = -EFAULT; 1067 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1068 * it would end up going through the fenced access, and we'll get 1069 * different detiling behavior between reading and writing. 1070 * pread/pwrite currently are reading and writing from the CPU 1071 * perspective, requiring manual detiling by the client. 1072 */ 1073 if (obj->phys_obj) { 1074 ret = i915_gem_phys_pwrite(dev, obj, args, file); 1075 goto out; 1076 } 1077 1078 if (obj->tiling_mode == I915_TILING_NONE && 1079 obj->base.write_domain != I915_GEM_DOMAIN_CPU && 1080 cpu_write_needs_clflush(obj)) { 1081 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); 1082 /* Note that the gtt paths might fail with non-page-backed user 1083 * pointers (e.g. gtt mappings when moving data between 1084 * textures). Fallback to the shmem path in that case. */ 1085 } 1086 1087 if (ret == -EFAULT || ret == -ENOSPC) 1088 ret = i915_gem_shmem_pwrite(dev, obj, args, file); 1089 1090out: 1091 drm_gem_object_unreference(&obj->base); 1092unlock: 1093 DRM_UNLOCK(dev); 1094 return ret; 1095} 1096 1097static int 1098i915_gem_check_wedge(struct drm_i915_private *dev_priv) 1099{ 1100 DRM_LOCK_ASSERT(dev_priv->dev); 1101 1102 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 1103 bool recovery_complete; 1104 1105 /* Give the error handler a chance to run. */ 1106 mtx_lock(&dev_priv->error_completion_lock); 1107 recovery_complete = (&dev_priv->error_completion) > 0; 1108 mtx_unlock(&dev_priv->error_completion_lock); 1109 1110 return (recovery_complete ? -EIO : -EAGAIN); 1111 } 1112 1113 return 0; 1114} 1115 1116/* 1117 * Compare seqno against outstanding lazy request. Emit a request if they are 1118 * equal. 1119 */ 1120static int 1121i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) 1122{ 1123 int ret; 1124 1125 DRM_LOCK_ASSERT(ring->dev); 1126 1127 ret = 0; 1128 if (seqno == ring->outstanding_lazy_request) { 1129 struct drm_i915_gem_request *request; 1130 1131 request = malloc(sizeof(*request), DRM_I915_GEM, 1132 M_WAITOK | M_ZERO); 1133 1134 ret = i915_add_request(ring, NULL, request); 1135 if (ret != 0) { 1136 free(request, DRM_I915_GEM); 1137 return ret; 1138 } 1139 1140 MPASS(seqno == request->seqno); 1141 } 1142 return ret; 1143} 1144 1145static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 1146 bool interruptible) 1147{ 1148 drm_i915_private_t *dev_priv = ring->dev->dev_private; 1149 int ret = 0, flags; 1150 1151 if (i915_seqno_passed(ring->get_seqno(ring), seqno)) 1152 return 0; 1153 1154 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 1155 1156 mtx_lock(&dev_priv->irq_lock); 1157 if (!ring->irq_get(ring)) { 1158 mtx_unlock(&dev_priv->irq_lock); 1159 return -ENODEV; 1160 } 1161 1162 flags = interruptible ? PCATCH : 0; 1163 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 1164 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 1165 ret == 0) { 1166 ret = -msleep(ring, &dev_priv->irq_lock, flags, "915gwr", 0); 1167 if (ret == -ERESTART) 1168 ret = -ERESTARTSYS; 1169 } 1170 ring->irq_put(ring); 1171 mtx_unlock(&dev_priv->irq_lock); 1172 1173 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, ret); 1174 1175 return ret; 1176} 1177 1178/** 1179 * Waits for a sequence number to be signaled, and cleans up the 1180 * request and object lists appropriately for that event. 1181 */ 1182int 1183i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno) 1184{ 1185 struct drm_device *dev = ring->dev; 1186 struct drm_i915_private *dev_priv = dev->dev_private; 1187 int ret; 1188 1189 KASSERT(seqno != 0, ("Zero seqno")); 1190 1191 ret = i915_gem_check_wedge(dev_priv); 1192 if (ret) 1193 return ret; 1194 1195 ret = i915_gem_check_olr(ring, seqno); 1196 if (ret) 1197 return ret; 1198 1199 ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible); 1200 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 1201 ret = -EAGAIN; 1202 1203 return ret; 1204} 1205 1206/** 1207 * Ensures that all rendering to the object has completed and the object is 1208 * safe to unbind from the GTT or access from the CPU. 1209 */ 1210static __must_check int 1211i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 1212{ 1213 int ret; 1214 1215 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 1216 ("In GPU write domain")); 1217 1218 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, 1219 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, 1220 obj->active, obj->last_rendering_seqno); 1221 if (obj->active) { 1222 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno); 1223 if (ret != 0) 1224 return (ret); 1225 i915_gem_retire_requests_ring(obj->ring); 1226 } 1227 1228 return 0; 1229} 1230 1231int 1232i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1233 struct drm_file *file) 1234{ 1235 struct drm_i915_gem_set_domain *args = data; 1236 struct drm_i915_gem_object *obj; 1237 uint32_t read_domains = args->read_domains; 1238 uint32_t write_domain = args->write_domain; 1239 int ret; 1240 1241 /* Only handle setting domains to types used by the CPU. */ 1242 if (write_domain & I915_GEM_GPU_DOMAINS) 1243 return -EINVAL; 1244 1245 if (read_domains & I915_GEM_GPU_DOMAINS) 1246 return -EINVAL; 1247 1248 /* Having something in the write domain implies it's in the read 1249 * domain, and only that read domain. Enforce that in the request. 1250 */ 1251 if (write_domain != 0 && read_domains != write_domain) 1252 return -EINVAL; 1253 1254 ret = i915_mutex_lock_interruptible(dev); 1255 if (ret) 1256 return ret; 1257 1258 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1259 if (&obj->base == NULL) { 1260 ret = -ENOENT; 1261 goto unlock; 1262 } 1263 1264 if (read_domains & I915_GEM_DOMAIN_GTT) { 1265 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1266 1267 /* Silently promote "you're not bound, there was nothing to do" 1268 * to success, since the client was just asking us to 1269 * make sure everything was done. 1270 */ 1271 if (ret == -EINVAL) 1272 ret = 0; 1273 } else { 1274 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1275 } 1276 1277 drm_gem_object_unreference(&obj->base); 1278unlock: 1279 DRM_UNLOCK(dev); 1280 return ret; 1281} 1282 1283/** 1284 * Called when user space has done writes to this buffer 1285 */ 1286int 1287i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1288 struct drm_file *file) 1289{ 1290 struct drm_i915_gem_sw_finish *args = data; 1291 struct drm_i915_gem_object *obj; 1292 int ret = 0; 1293 1294 ret = i915_mutex_lock_interruptible(dev); 1295 if (ret) 1296 return ret; 1297 1298 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1299 if (&obj->base == NULL) { 1300 ret = -ENOENT; 1301 goto unlock; 1302 } 1303 1304 /* Pinned buffers may be scanout, so flush the cache */ 1305 if (obj->pin_count) 1306 i915_gem_object_flush_cpu_write_domain(obj); 1307 1308 drm_gem_object_unreference(&obj->base); 1309unlock: 1310 DRM_UNLOCK(dev); 1311 return ret; 1312} 1313 1314/** 1315 * Maps the contents of an object, returning the address it is mapped 1316 * into. 1317 * 1318 * While the mapping holds a reference on the contents of the object, it doesn't 1319 * imply a ref on the object itself. 1320 */ 1321int 1322i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1323 struct drm_file *file) 1324{ 1325 struct drm_i915_gem_mmap *args = data; 1326 struct drm_gem_object *obj; 1327 struct proc *p; 1328 vm_map_t map; 1329 vm_offset_t addr; 1330 vm_size_t size; 1331 int error, rv; 1332 1333 obj = drm_gem_object_lookup(dev, file, args->handle); 1334 if (obj == NULL) 1335 return -ENOENT; 1336 1337 error = 0; 1338 if (args->size == 0) 1339 goto out; 1340 p = curproc; 1341 map = &p->p_vmspace->vm_map; 1342 size = round_page(args->size); 1343 PROC_LOCK(p); 1344 if (map->size + size > lim_cur_proc(p, RLIMIT_VMEM)) { 1345 PROC_UNLOCK(p); 1346 error = -ENOMEM; 1347 goto out; 1348 } 1349 PROC_UNLOCK(p); 1350 1351 addr = 0; 1352 vm_object_reference(obj->vm_obj); 1353 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1354 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1355 VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1356 if (rv != KERN_SUCCESS) { 1357 vm_object_deallocate(obj->vm_obj); 1358 error = -vm_mmap_to_errno(rv); 1359 } else { 1360 args->addr_ptr = (uint64_t)addr; 1361 } 1362out: 1363 drm_gem_object_unreference(obj); 1364 return (error); 1365} 1366 1367static int 1368i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1369 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1370{ 1371 1372 *color = 0; /* XXXKIB */ 1373 return (0); 1374} 1375 1376/** 1377 * i915_gem_fault - fault a page into the GTT 1378 * vma: VMA in question 1379 * vmf: fault info 1380 * 1381 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 1382 * from userspace. The fault handler takes care of binding the object to 1383 * the GTT (if needed), allocating and programming a fence register (again, 1384 * only if needed based on whether the old reg is still valid or the object 1385 * is tiled) and inserting a new PTE into the faulting process. 1386 * 1387 * Note that the faulting process may involve evicting existing objects 1388 * from the GTT and/or fence registers to make room. So performance may 1389 * suffer if the GTT working set is large or there are few fence registers 1390 * left. 1391 */ 1392 1393int i915_intr_pf; 1394 1395static int 1396i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1397 vm_page_t *mres) 1398{ 1399 struct drm_gem_object *gem_obj; 1400 struct drm_i915_gem_object *obj; 1401 struct drm_device *dev; 1402 drm_i915_private_t *dev_priv; 1403 vm_page_t page, oldpage; 1404 int cause, ret; 1405 bool write; 1406 1407 gem_obj = vm_obj->handle; 1408 obj = to_intel_bo(gem_obj); 1409 dev = obj->base.dev; 1410 dev_priv = dev->dev_private; 1411#if 0 1412 write = (prot & VM_PROT_WRITE) != 0; 1413#else 1414 write = true; 1415#endif 1416 vm_object_pip_add(vm_obj, 1); 1417 1418 /* 1419 * Remove the placeholder page inserted by vm_fault() from the 1420 * object before dropping the object lock. If 1421 * i915_gem_release_mmap() is active in parallel on this gem 1422 * object, then it owns the drm device sx and might find the 1423 * placeholder already. Then, since the page is busy, 1424 * i915_gem_release_mmap() sleeps waiting for the busy state 1425 * of the page cleared. We will be not able to acquire drm 1426 * device lock until i915_gem_release_mmap() is able to make a 1427 * progress. 1428 */ 1429 if (*mres != NULL) { 1430 oldpage = *mres; 1431 vm_page_lock(oldpage); 1432 vm_page_remove(oldpage); 1433 vm_page_unlock(oldpage); 1434 *mres = NULL; 1435 } else 1436 oldpage = NULL; 1437 VM_OBJECT_WUNLOCK(vm_obj); 1438retry: 1439 cause = ret = 0; 1440 page = NULL; 1441 1442 if (i915_intr_pf) { 1443 ret = i915_mutex_lock_interruptible(dev); 1444 if (ret != 0) { 1445 cause = 10; 1446 goto out; 1447 } 1448 } else 1449 DRM_LOCK(dev); 1450 1451 /* 1452 * Since the object lock was dropped, other thread might have 1453 * faulted on the same GTT address and instantiated the 1454 * mapping for the page. Recheck. 1455 */ 1456 VM_OBJECT_WLOCK(vm_obj); 1457 page = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1458 if (page != NULL) { 1459 if (vm_page_busied(page)) { 1460 DRM_UNLOCK(dev); 1461 vm_page_lock(page); 1462 VM_OBJECT_WUNLOCK(vm_obj); 1463 vm_page_busy_sleep(page, "915pee"); 1464 goto retry; 1465 } 1466 goto have_page; 1467 } else 1468 VM_OBJECT_WUNLOCK(vm_obj); 1469 1470 /* Now bind it into the GTT if needed */ 1471 if (!obj->map_and_fenceable) { 1472 ret = i915_gem_object_unbind(obj); 1473 if (ret != 0) { 1474 cause = 20; 1475 goto unlock; 1476 } 1477 } 1478 if (!obj->gtt_space) { 1479 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1480 if (ret != 0) { 1481 cause = 30; 1482 goto unlock; 1483 } 1484 1485 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1486 if (ret != 0) { 1487 cause = 40; 1488 goto unlock; 1489 } 1490 } 1491 1492 if (!obj->has_global_gtt_mapping) 1493 i915_gem_gtt_bind_object(obj, obj->cache_level); 1494 1495 ret = i915_gem_object_get_fence(obj); 1496 if (ret != 0) { 1497 cause = 50; 1498 goto unlock; 1499 } 1500 1501 if (i915_gem_object_is_inactive(obj)) 1502 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1503 1504 obj->fault_mappable = true; 1505 VM_OBJECT_WLOCK(vm_obj); 1506 page = PHYS_TO_VM_PAGE(dev->agp->base + obj->gtt_offset + offset); 1507 KASSERT((page->flags & PG_FICTITIOUS) != 0, 1508 ("physical address %#jx not fictitious", 1509 (uintmax_t)(dev->agp->base + obj->gtt_offset + offset))); 1510 if (page == NULL) { 1511 VM_OBJECT_WUNLOCK(vm_obj); 1512 cause = 60; 1513 ret = -EFAULT; 1514 goto unlock; 1515 } 1516 KASSERT((page->flags & PG_FICTITIOUS) != 0, 1517 ("not fictitious %p", page)); 1518 KASSERT(page->wire_count == 1, ("wire_count not 1 %p", page)); 1519 1520 if (vm_page_busied(page)) { 1521 DRM_UNLOCK(dev); 1522 vm_page_lock(page); 1523 VM_OBJECT_WUNLOCK(vm_obj); 1524 vm_page_busy_sleep(page, "915pbs"); 1525 goto retry; 1526 } 1527 if (vm_page_insert(page, vm_obj, OFF_TO_IDX(offset))) { 1528 DRM_UNLOCK(dev); 1529 VM_OBJECT_WUNLOCK(vm_obj); 1530 VM_WAIT; 1531 goto retry; 1532 } 1533 page->valid = VM_PAGE_BITS_ALL; 1534have_page: 1535 *mres = page; 1536 vm_page_xbusy(page); 1537 1538 CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 1539 page->phys_addr); 1540 DRM_UNLOCK(dev); 1541 if (oldpage != NULL) { 1542 vm_page_lock(oldpage); 1543 vm_page_free(oldpage); 1544 vm_page_unlock(oldpage); 1545 } 1546 vm_object_pip_wakeup(vm_obj); 1547 return (VM_PAGER_OK); 1548 1549unlock: 1550 DRM_UNLOCK(dev); 1551out: 1552 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1553 CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, 1554 -ret, cause); 1555 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1556 kern_yield(PRI_USER); 1557 goto retry; 1558 } 1559 VM_OBJECT_WLOCK(vm_obj); 1560 vm_object_pip_wakeup(vm_obj); 1561 return (VM_PAGER_ERROR); 1562} 1563 1564static void 1565i915_gem_pager_dtor(void *handle) 1566{ 1567 struct drm_gem_object *obj; 1568 struct drm_device *dev; 1569 1570 obj = handle; 1571 dev = obj->dev; 1572 1573 DRM_LOCK(dev); 1574 drm_gem_free_mmap_offset(obj); 1575 i915_gem_release_mmap(to_intel_bo(obj)); 1576 drm_gem_object_unreference(obj); 1577 DRM_UNLOCK(dev); 1578} 1579 1580struct cdev_pager_ops i915_gem_pager_ops = { 1581 .cdev_pg_fault = i915_gem_pager_fault, 1582 .cdev_pg_ctor = i915_gem_pager_ctor, 1583 .cdev_pg_dtor = i915_gem_pager_dtor 1584}; 1585 1586/** 1587 * i915_gem_release_mmap - remove physical page mappings 1588 * @obj: obj in question 1589 * 1590 * Preserve the reservation of the mmapping with the DRM core code, but 1591 * relinquish ownership of the pages back to the system. 1592 * 1593 * It is vital that we remove the page mapping if we have mapped a tiled 1594 * object through the GTT and then lose the fence register due to 1595 * resource pressure. Similarly if the object has been moved out of the 1596 * aperture, than pages mapped into userspace must be revoked. Removing the 1597 * mapping will then trigger a page fault on the next user access, allowing 1598 * fixup by i915_gem_fault(). 1599 */ 1600void 1601i915_gem_release_mmap(struct drm_i915_gem_object *obj) 1602{ 1603 vm_object_t devobj; 1604 vm_page_t page; 1605 int i, page_count; 1606 1607 if (!obj->fault_mappable) 1608 return; 1609 1610 CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 1611 OFF_TO_IDX(obj->base.size)); 1612 devobj = cdev_pager_lookup(obj); 1613 if (devobj != NULL) { 1614 page_count = OFF_TO_IDX(obj->base.size); 1615 1616 VM_OBJECT_WLOCK(devobj); 1617retry: 1618 for (i = 0; i < page_count; i++) { 1619 page = vm_page_lookup(devobj, i); 1620 if (page == NULL) 1621 continue; 1622 if (vm_page_sleep_if_busy(page, "915unm")) 1623 goto retry; 1624 cdev_pager_free_page(devobj, page); 1625 } 1626 VM_OBJECT_WUNLOCK(devobj); 1627 vm_object_deallocate(devobj); 1628 } 1629 1630 obj->fault_mappable = false; 1631} 1632 1633static uint32_t 1634i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1635{ 1636 uint32_t gtt_size; 1637 1638 if (INTEL_INFO(dev)->gen >= 4 || 1639 tiling_mode == I915_TILING_NONE) 1640 return size; 1641 1642 /* Previous chips need a power-of-two fence region when tiling */ 1643 if (INTEL_INFO(dev)->gen == 3) 1644 gtt_size = 1024*1024; 1645 else 1646 gtt_size = 512*1024; 1647 1648 while (gtt_size < size) 1649 gtt_size <<= 1; 1650 1651 return gtt_size; 1652} 1653 1654/** 1655 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1656 * @obj: object to check 1657 * 1658 * Return the required GTT alignment for an object, taking into account 1659 * potential fence register mapping. 1660 */ 1661static uint32_t 1662i915_gem_get_gtt_alignment(struct drm_device *dev, 1663 uint32_t size, 1664 int tiling_mode) 1665{ 1666 /* 1667 * Minimum alignment is 4k (GTT page size), but might be greater 1668 * if a fence register is needed for the object. 1669 */ 1670 if (INTEL_INFO(dev)->gen >= 4 || 1671 tiling_mode == I915_TILING_NONE) 1672 return 4096; 1673 1674 /* 1675 * Previous chips need to be aligned to the size of the smallest 1676 * fence register that can contain the object. 1677 */ 1678 return i915_gem_get_gtt_size(dev, size, tiling_mode); 1679} 1680 1681/** 1682 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an 1683 * unfenced object 1684 * @dev: the device 1685 * @size: size of the object 1686 * @tiling_mode: tiling mode of the object 1687 * 1688 * Return the required GTT alignment for an object, only taking into account 1689 * unfenced tiled surface requirements. 1690 */ 1691uint32_t 1692i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, 1693 uint32_t size, 1694 int tiling_mode) 1695{ 1696 /* 1697 * Minimum alignment is 4k (GTT page size) for sane hw. 1698 */ 1699 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || 1700 tiling_mode == I915_TILING_NONE) 1701 return 4096; 1702 1703 /* Previous hardware however needs to be aligned to a power-of-two 1704 * tile height. The simplest method for determining this is to reuse 1705 * the power-of-tile object size. 1706 */ 1707 return i915_gem_get_gtt_size(dev, size, tiling_mode); 1708} 1709 1710int 1711i915_gem_mmap_gtt(struct drm_file *file, 1712 struct drm_device *dev, 1713 uint32_t handle, 1714 uint64_t *offset) 1715{ 1716 struct drm_i915_private *dev_priv = dev->dev_private; 1717 struct drm_i915_gem_object *obj; 1718 int ret; 1719 1720 ret = i915_mutex_lock_interruptible(dev); 1721 if (ret) 1722 return ret; 1723 1724 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1725 if (&obj->base == NULL) { 1726 ret = -ENOENT; 1727 goto unlock; 1728 } 1729 1730 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1731 ret = -E2BIG; 1732 goto out; 1733 } 1734 1735 if (obj->madv != I915_MADV_WILLNEED) { 1736 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1737 ret = -EINVAL; 1738 goto out; 1739 } 1740 1741 ret = drm_gem_create_mmap_offset(&obj->base); 1742 if (ret) 1743 goto out; 1744 1745 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1746 DRM_GEM_MAPPING_KEY; 1747 1748out: 1749 drm_gem_object_unreference(&obj->base); 1750unlock: 1751 DRM_UNLOCK(dev); 1752 return ret; 1753} 1754 1755/** 1756 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1757 * @dev: DRM device 1758 * @data: GTT mapping ioctl data 1759 * @file: GEM object info 1760 * 1761 * Simply returns the fake offset to userspace so it can mmap it. 1762 * The mmap call will end up in drm_gem_mmap(), which will set things 1763 * up so we can get faults in the handler above. 1764 * 1765 * The fault handler will take care of binding the object into the GTT 1766 * (since it may have been evicted to make room for something), allocating 1767 * a fence register, and mapping the appropriate aperture address into 1768 * userspace. 1769 */ 1770int 1771i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1772 struct drm_file *file) 1773{ 1774 struct drm_i915_gem_mmap_gtt *args = data; 1775 1776 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); 1777} 1778 1779/* Immediately discard the backing storage */ 1780static void 1781i915_gem_object_truncate(struct drm_i915_gem_object *obj) 1782{ 1783 vm_object_t vm_obj; 1784 1785 vm_obj = obj->base.vm_obj; 1786 VM_OBJECT_WLOCK(vm_obj); 1787 vm_object_page_remove(vm_obj, 0, 0, false); 1788 VM_OBJECT_WUNLOCK(vm_obj); 1789 drm_gem_free_mmap_offset(&obj->base); 1790 obj->madv = I915_MADV_PURGED_INTERNAL; 1791} 1792 1793static inline int 1794i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 1795{ 1796 return obj->madv == I915_MADV_DONTNEED; 1797} 1798 1799static void 1800i915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj, 1801 vm_pindex_t si, vm_pindex_t ei) 1802{ 1803 vm_object_t vm_obj; 1804 vm_page_t page; 1805 vm_pindex_t i; 1806 1807 vm_obj = obj->base.vm_obj; 1808 VM_OBJECT_ASSERT_LOCKED(vm_obj); 1809 for (i = si, page = vm_page_lookup(vm_obj, i); i < ei; 1810 page = vm_page_next(page), i++) { 1811 KASSERT(page->pindex == i, ("pindex %jx %jx", 1812 (uintmax_t)page->pindex, (uintmax_t)i)); 1813 vm_page_lock(page); 1814 vm_page_unwire(page, PQ_INACTIVE); 1815 if (page->wire_count == 0) 1816 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1817 vm_page_unlock(page); 1818 } 1819} 1820 1821#define GEM_PARANOID_CHECK_GTT 0 1822#if GEM_PARANOID_CHECK_GTT 1823static void 1824i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 1825 int page_count) 1826{ 1827 struct drm_i915_private *dev_priv; 1828 vm_paddr_t pa; 1829 unsigned long start, end; 1830 u_int i; 1831 int j; 1832 1833 dev_priv = dev->dev_private; 1834 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 1835 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 1836 for (i = start; i < end; i++) { 1837 pa = intel_gtt_read_pte_paddr(i); 1838 for (j = 0; j < page_count; j++) { 1839 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 1840 panic("Page %p in GTT pte index %d pte %x", 1841 ma[i], i, intel_gtt_read_pte(i)); 1842 } 1843 } 1844 } 1845} 1846#endif 1847 1848static void 1849i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, 1850 off_t start, off_t end) 1851{ 1852 vm_object_t vm_obj; 1853 1854 vm_obj = obj->base.vm_obj; 1855 VM_OBJECT_WLOCK(vm_obj); 1856 i915_gem_object_put_pages_range_locked(obj, 1857 OFF_TO_IDX(trunc_page(start)), OFF_TO_IDX(round_page(end))); 1858 VM_OBJECT_WUNLOCK(vm_obj); 1859} 1860 1861static void 1862i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 1863{ 1864 int page_count = obj->base.size / PAGE_SIZE; 1865 int i; 1866 1867 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 1868 1869 if (obj->tiling_mode != I915_TILING_NONE) 1870 i915_gem_object_save_bit_17_swizzle(obj); 1871 1872 if (obj->madv == I915_MADV_DONTNEED) 1873 obj->dirty = 0; 1874 1875 VM_OBJECT_WLOCK(obj->base.vm_obj); 1876#if GEM_PARANOID_CHECK_GTT 1877 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 1878#endif 1879 for (i = 0; i < page_count; i++) { 1880 vm_page_t page = obj->pages[i]; 1881 1882 if (obj->dirty) 1883 vm_page_dirty(page); 1884 1885 if (obj->madv == I915_MADV_WILLNEED) 1886 vm_page_reference(page); 1887 1888 vm_page_lock(page); 1889 vm_page_unwire(obj->pages[i], PQ_ACTIVE); 1890 vm_page_unlock(page); 1891 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1892 } 1893 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 1894 obj->dirty = 0; 1895 1896 free(obj->pages, DRM_I915_GEM); 1897 obj->pages = NULL; 1898} 1899 1900static int 1901i915_gpu_is_active(struct drm_device *dev) 1902{ 1903 drm_i915_private_t *dev_priv = dev->dev_private; 1904 1905 return (!list_empty(&dev_priv->mm.flushing_list) || 1906 !list_empty(&dev_priv->mm.active_list)); 1907} 1908 1909static void 1910i915_gem_lowmem(void *arg) 1911{ 1912 struct drm_device *dev; 1913 struct drm_i915_private *dev_priv; 1914 struct drm_i915_gem_object *obj, *next; 1915 int cnt, cnt_fail, cnt_total; 1916 1917 dev = arg; 1918 dev_priv = dev->dev_private; 1919 1920 if (!sx_try_xlock(&dev->dev_struct_lock)) 1921 return; 1922 1923 CTR0(KTR_DRM, "gem_lowmem"); 1924 1925rescan: 1926 /* first scan for clean buffers */ 1927 i915_gem_retire_requests(dev); 1928 1929 cnt_total = cnt_fail = cnt = 0; 1930 1931 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 1932 mm_list) { 1933 if (i915_gem_object_is_purgeable(obj)) { 1934 if (i915_gem_object_unbind(obj) != 0) 1935 cnt_total++; 1936 } else 1937 cnt_total++; 1938 } 1939 1940 /* second pass, evict/count anything still on the inactive list */ 1941 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 1942 mm_list) { 1943 if (i915_gem_object_unbind(obj) == 0) 1944 cnt++; 1945 else 1946 cnt_fail++; 1947 } 1948 1949 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 1950 /* 1951 * We are desperate for pages, so as a last resort, wait 1952 * for the GPU to finish and discard whatever we can. 1953 * This has a dramatic impact to reduce the number of 1954 * OOM-killer events whilst running the GPU aggressively. 1955 */ 1956 if (i915_gpu_idle(dev) == 0) 1957 goto rescan; 1958 } 1959 DRM_UNLOCK(dev); 1960} 1961 1962static int 1963i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 1964 off_t start, off_t end) 1965{ 1966 vm_object_t vm_obj; 1967 vm_page_t page; 1968 vm_pindex_t si, ei, i; 1969 bool need_swizzle, fresh; 1970 1971 need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0; 1972 vm_obj = obj->base.vm_obj; 1973 si = OFF_TO_IDX(trunc_page(start)); 1974 ei = OFF_TO_IDX(round_page(end)); 1975 VM_OBJECT_WLOCK(vm_obj); 1976 for (i = si; i < ei; i++) { 1977 page = i915_gem_wire_page(vm_obj, i, &fresh); 1978 if (page == NULL) 1979 goto failed; 1980 if (need_swizzle && fresh) 1981 i915_gem_object_do_bit_17_swizzle_page(obj, page); 1982 } 1983 VM_OBJECT_WUNLOCK(vm_obj); 1984 return (0); 1985failed: 1986 i915_gem_object_put_pages_range_locked(obj, si, i); 1987 VM_OBJECT_WUNLOCK(vm_obj); 1988 return (-EIO); 1989} 1990 1991static int 1992i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 1993 int flags) 1994{ 1995 vm_object_t vm_obj; 1996 vm_page_t page; 1997 vm_pindex_t i, page_count; 1998 int res; 1999 2000 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2001 2002 page_count = OFF_TO_IDX(obj->base.size); 2003 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2004 M_WAITOK); 2005 res = i915_gem_object_get_pages_range(obj, 0, obj->base.size); 2006 if (res != 0) { 2007 free(obj->pages, DRM_I915_GEM); 2008 obj->pages = NULL; 2009 return (res); 2010 } 2011 vm_obj = obj->base.vm_obj; 2012 VM_OBJECT_WLOCK(vm_obj); 2013 for (i = 0, page = vm_page_lookup(vm_obj, 0); i < page_count; 2014 i++, page = vm_page_next(page)) { 2015 KASSERT(page->pindex == i, ("pindex %jx %jx", 2016 (uintmax_t)page->pindex, (uintmax_t)i)); 2017 obj->pages[i] = page; 2018 } 2019 VM_OBJECT_WUNLOCK(vm_obj); 2020 return (0); 2021} 2022 2023void 2024i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2025 struct intel_ring_buffer *ring, uint32_t seqno) 2026{ 2027 struct drm_device *dev = obj->base.dev; 2028 struct drm_i915_private *dev_priv = dev->dev_private; 2029 struct drm_i915_fence_reg *reg; 2030 2031 KASSERT(ring != NULL, ("NULL ring")); 2032 obj->ring = ring; 2033 2034 /* Add a reference if we're newly entering the active list. */ 2035 if (!obj->active) { 2036 drm_gem_object_reference(&obj->base); 2037 obj->active = 1; 2038 } 2039 2040 /* Move from whatever list we were on to the tail of execution. */ 2041 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2042 list_move_tail(&obj->ring_list, &ring->active_list); 2043 2044 obj->last_rendering_seqno = seqno; 2045 if (obj->fenced_gpu_access) { 2046 obj->last_fenced_seqno = seqno; 2047 2048 /* Bump MRU to take account of the delayed flush */ 2049 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2050 reg = &dev_priv->fence_regs[obj->fence_reg]; 2051 list_move_tail(®->lru_list, 2052 &dev_priv->mm.fence_list); 2053 } 2054 } 2055} 2056 2057static void 2058i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2059{ 2060 list_del_init(&obj->ring_list); 2061 obj->last_rendering_seqno = 0; 2062 obj->last_fenced_seqno = 0; 2063} 2064 2065static void 2066i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2067{ 2068 struct drm_device *dev = obj->base.dev; 2069 drm_i915_private_t *dev_priv = dev->dev_private; 2070 2071 KASSERT(obj->active, ("Object not active")); 2072 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2073 2074 i915_gem_object_move_off_active(obj); 2075} 2076 2077static void 2078i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2079{ 2080 struct drm_device *dev = obj->base.dev; 2081 struct drm_i915_private *dev_priv = dev->dev_private; 2082 2083 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2084 2085 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2086 KASSERT(obj->active, ("Object not active")); 2087 obj->ring = NULL; 2088 2089 i915_gem_object_move_off_active(obj); 2090 obj->fenced_gpu_access = false; 2091 2092 obj->active = 0; 2093 obj->pending_gpu_write = false; 2094 drm_gem_object_unreference(&obj->base); 2095 2096#if 1 2097 KIB_NOTYET(); 2098#else 2099 WARN_ON(i915_verify_lists(dev)); 2100#endif 2101} 2102 2103static u32 2104i915_gem_get_seqno(struct drm_device *dev) 2105{ 2106 drm_i915_private_t *dev_priv = dev->dev_private; 2107 u32 seqno = dev_priv->next_seqno; 2108 2109 /* reserve 0 for non-seqno */ 2110 if (++dev_priv->next_seqno == 0) 2111 dev_priv->next_seqno = 1; 2112 2113 return seqno; 2114} 2115 2116u32 2117i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2118{ 2119 if (ring->outstanding_lazy_request == 0) 2120 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2121 2122 return ring->outstanding_lazy_request; 2123} 2124 2125int 2126i915_add_request(struct intel_ring_buffer *ring, 2127 struct drm_file *file, 2128 struct drm_i915_gem_request *request) 2129{ 2130 drm_i915_private_t *dev_priv = ring->dev->dev_private; 2131 struct drm_i915_file_private *file_priv; 2132 uint32_t seqno; 2133 u32 request_ring_position; 2134 int was_empty; 2135 int ret; 2136 2137 KASSERT(request != NULL, ("NULL request in add")); 2138 DRM_LOCK_ASSERT(ring->dev); 2139 2140 seqno = i915_gem_next_request_seqno(ring); 2141 request_ring_position = intel_ring_get_tail(ring); 2142 2143 ret = ring->add_request(ring, &seqno); 2144 if (ret != 0) 2145 return ret; 2146 2147 CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); 2148 2149 request->seqno = seqno; 2150 request->ring = ring; 2151 request->tail = request_ring_position; 2152 request->emitted_jiffies = ticks; 2153 was_empty = list_empty(&ring->request_list); 2154 list_add_tail(&request->list, &ring->request_list); 2155 2156 if (file) { 2157 file_priv = file->driver_priv; 2158 2159 mtx_lock(&file_priv->mm.lck); 2160 request->file_priv = file_priv; 2161 list_add_tail(&request->client_list, 2162 &file_priv->mm.request_list); 2163 mtx_unlock(&file_priv->mm.lck); 2164 } 2165 2166 ring->outstanding_lazy_request = 0; 2167 2168 if (!dev_priv->mm.suspended) { 2169 if (i915_enable_hangcheck) { 2170 callout_schedule(&dev_priv->hangcheck_timer, 2171 DRM_I915_HANGCHECK_PERIOD); 2172 } 2173 if (was_empty) 2174 taskqueue_enqueue_timeout(dev_priv->tq, 2175 &dev_priv->mm.retire_task, hz); 2176 } 2177 2178 return 0; 2179} 2180 2181static inline void 2182i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2183{ 2184 struct drm_i915_file_private *file_priv = request->file_priv; 2185 2186 if (!file_priv) 2187 return; 2188 2189 DRM_LOCK_ASSERT(request->ring->dev); 2190 2191 mtx_lock(&file_priv->mm.lck); 2192 if (request->file_priv) { 2193 list_del(&request->client_list); 2194 request->file_priv = NULL; 2195 } 2196 mtx_unlock(&file_priv->mm.lck); 2197} 2198 2199static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2200 struct intel_ring_buffer *ring) 2201{ 2202 if (ring->dev != NULL) 2203 DRM_LOCK_ASSERT(ring->dev); 2204 2205 while (!list_empty(&ring->request_list)) { 2206 struct drm_i915_gem_request *request; 2207 2208 request = list_first_entry(&ring->request_list, 2209 struct drm_i915_gem_request, 2210 list); 2211 2212 list_del(&request->list); 2213 i915_gem_request_remove_from_client(request); 2214 free(request, DRM_I915_GEM); 2215 } 2216 2217 while (!list_empty(&ring->active_list)) { 2218 struct drm_i915_gem_object *obj; 2219 2220 obj = list_first_entry(&ring->active_list, 2221 struct drm_i915_gem_object, 2222 ring_list); 2223 2224 obj->base.write_domain = 0; 2225 list_del_init(&obj->gpu_write_list); 2226 i915_gem_object_move_to_inactive(obj); 2227 } 2228} 2229 2230static void i915_gem_reset_fences(struct drm_device *dev) 2231{ 2232 struct drm_i915_private *dev_priv = dev->dev_private; 2233 int i; 2234 2235 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2236 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2237 2238 i915_gem_write_fence(dev, i, NULL); 2239 2240 if (reg->obj) 2241 i915_gem_object_fence_lost(reg->obj); 2242 2243 reg->pin_count = 0; 2244 reg->obj = NULL; 2245 INIT_LIST_HEAD(®->lru_list); 2246 } 2247 2248 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 2249} 2250 2251void i915_gem_reset(struct drm_device *dev) 2252{ 2253 struct drm_i915_private *dev_priv = dev->dev_private; 2254 struct drm_i915_gem_object *obj; 2255 struct intel_ring_buffer *ring; 2256 int i; 2257 2258 for_each_ring(ring, dev_priv, i) 2259 i915_gem_reset_ring_lists(dev_priv, ring); 2260 2261 /* Remove anything from the flushing lists. The GPU cache is likely 2262 * to be lost on reset along with the data, so simply move the 2263 * lost bo to the inactive list. 2264 */ 2265 while (!list_empty(&dev_priv->mm.flushing_list)) { 2266 obj = list_first_entry(&dev_priv->mm.flushing_list, 2267 struct drm_i915_gem_object, 2268 mm_list); 2269 2270 obj->base.write_domain = 0; 2271 list_del_init(&obj->gpu_write_list); 2272 i915_gem_object_move_to_inactive(obj); 2273 } 2274 2275 /* Move everything out of the GPU domains to ensure we do any 2276 * necessary invalidation upon reuse. 2277 */ 2278 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2279 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2280 } 2281 2282 /* The fence registers are invalidated so clear them out */ 2283 i915_gem_reset_fences(dev); 2284} 2285 2286/** 2287 * This function clears the request list as sequence numbers are passed. 2288 */ 2289void 2290i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2291{ 2292 uint32_t seqno; 2293 int i; 2294 2295 if (list_empty(&ring->request_list)) 2296 return; 2297 2298 seqno = ring->get_seqno(ring); 2299 CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 2300 2301 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) 2302 if (seqno >= ring->sync_seqno[i]) 2303 ring->sync_seqno[i] = 0; 2304 2305 while (!list_empty(&ring->request_list)) { 2306 struct drm_i915_gem_request *request; 2307 2308 request = list_first_entry(&ring->request_list, 2309 struct drm_i915_gem_request, 2310 list); 2311 2312 if (!i915_seqno_passed(seqno, request->seqno)) 2313 break; 2314 2315 CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 2316 ring->name, seqno); 2317 ring->last_retired_head = request->tail; 2318 2319 list_del(&request->list); 2320 i915_gem_request_remove_from_client(request); 2321 free(request, DRM_I915_GEM); 2322 } 2323 2324 /* Move any buffers on the active list that are no longer referenced 2325 * by the ringbuffer to the flushing/inactive lists as appropriate. 2326 */ 2327 while (!list_empty(&ring->active_list)) { 2328 struct drm_i915_gem_object *obj; 2329 2330 obj = list_first_entry(&ring->active_list, 2331 struct drm_i915_gem_object, 2332 ring_list); 2333 2334 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2335 break; 2336 2337 if (obj->base.write_domain != 0) 2338 i915_gem_object_move_to_flushing(obj); 2339 else 2340 i915_gem_object_move_to_inactive(obj); 2341 } 2342 2343 if (ring->trace_irq_seqno && 2344 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2345 struct drm_i915_private *dev_priv = ring->dev->dev_private; 2346 mtx_lock(&dev_priv->irq_lock); 2347 ring->irq_put(ring); 2348 mtx_unlock(&dev_priv->irq_lock); 2349 ring->trace_irq_seqno = 0; 2350 } 2351} 2352 2353void 2354i915_gem_retire_requests(struct drm_device *dev) 2355{ 2356 drm_i915_private_t *dev_priv = dev->dev_private; 2357 struct intel_ring_buffer *ring; 2358 int i; 2359 2360 for_each_ring(ring, dev_priv, i) 2361 i915_gem_retire_requests_ring(ring); 2362} 2363 2364static void 2365i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2366 uint32_t flush_domains) 2367{ 2368 struct drm_i915_gem_object *obj, *next; 2369 uint32_t old_write_domain; 2370 2371 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2372 gpu_write_list) { 2373 if (obj->base.write_domain & flush_domains) { 2374 old_write_domain = obj->base.write_domain; 2375 obj->base.write_domain = 0; 2376 list_del_init(&obj->gpu_write_list); 2377 i915_gem_object_move_to_active(obj, ring, 2378 i915_gem_next_request_seqno(ring)); 2379 2380 CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", 2381 obj, obj->base.read_domains, old_write_domain); 2382 } 2383 } 2384} 2385 2386int 2387i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2388 uint32_t flush_domains) 2389{ 2390 int ret; 2391 2392 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2393 return 0; 2394 2395 CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, 2396 flush_domains); 2397 ret = ring->flush(ring, invalidate_domains, flush_domains); 2398 if (ret) 2399 return ret; 2400 2401 if (flush_domains & I915_GEM_GPU_DOMAINS) 2402 i915_gem_process_flushing_list(ring, flush_domains); 2403 return 0; 2404} 2405 2406static void 2407i915_gem_retire_task_handler(void *arg, int pending) 2408{ 2409 drm_i915_private_t *dev_priv; 2410 struct drm_device *dev; 2411 struct intel_ring_buffer *ring; 2412 bool idle; 2413 int i; 2414 2415 dev_priv = arg; 2416 dev = dev_priv->dev; 2417 2418 /* Come back later if the device is busy... */ 2419 if (!sx_try_xlock(&dev->dev_struct_lock)) { 2420 taskqueue_enqueue_timeout(dev_priv->tq, 2421 &dev_priv->mm.retire_task, hz); 2422 return; 2423 } 2424 2425 CTR0(KTR_DRM, "retire_task"); 2426 2427 i915_gem_retire_requests(dev); 2428 2429 /* Send a periodic flush down the ring so we don't hold onto GEM 2430 * objects indefinitely. 2431 */ 2432 idle = true; 2433 for_each_ring(ring, dev_priv, i) { 2434 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 2435 2436 if (!list_empty(&ring->gpu_write_list)) { 2437 struct drm_i915_gem_request *request; 2438 int ret; 2439 2440 ret = i915_gem_flush_ring(ring, 2441 0, I915_GEM_GPU_DOMAINS); 2442 request = malloc(sizeof(*request), DRM_I915_GEM, 2443 M_WAITOK | M_ZERO); 2444 if (ret || request == NULL || 2445 i915_add_request(ring, NULL, request)) 2446 free(request, DRM_I915_GEM); 2447 } 2448 2449 idle &= list_empty(&ring->request_list); 2450 } 2451 2452 if (!dev_priv->mm.suspended && !idle) 2453 taskqueue_enqueue_timeout(dev_priv->tq, 2454 &dev_priv->mm.retire_task, hz); 2455 2456 DRM_UNLOCK(dev); 2457} 2458 2459int 2460i915_gem_object_sync(struct drm_i915_gem_object *obj, 2461 struct intel_ring_buffer *to) 2462{ 2463 struct intel_ring_buffer *from = obj->ring; 2464 u32 seqno; 2465 int ret, idx; 2466 2467 if (from == NULL || to == from) 2468 return 0; 2469 2470 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) 2471 return i915_gem_object_wait_rendering(obj); 2472 2473 idx = intel_ring_sync_index(from, to); 2474 2475 seqno = obj->last_rendering_seqno; 2476 if (seqno <= from->sync_seqno[idx]) 2477 return 0; 2478 2479 if (seqno == from->outstanding_lazy_request) { 2480 struct drm_i915_gem_request *request; 2481 2482 request = malloc(sizeof(*request), DRM_I915_GEM, 2483 M_WAITOK | M_ZERO); 2484 ret = i915_add_request(from, NULL, request); 2485 if (ret) { 2486 free(request, DRM_I915_GEM); 2487 return ret; 2488 } 2489 seqno = request->seqno; 2490 } 2491 2492 2493 ret = to->sync_to(to, from, seqno); 2494 if (!ret) 2495 from->sync_seqno[idx] = seqno; 2496 2497 return ret; 2498} 2499 2500static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2501{ 2502 u32 old_write_domain, old_read_domains; 2503 2504 /* Act a barrier for all accesses through the GTT */ 2505 mb(); 2506 2507 /* Force a pagefault for domain tracking on next user access */ 2508 i915_gem_release_mmap(obj); 2509 2510 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2511 return; 2512 2513 old_read_domains = obj->base.read_domains; 2514 old_write_domain = obj->base.write_domain; 2515 2516 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2517 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2518 2519 CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2520 obj, old_read_domains, old_write_domain); 2521} 2522 2523/** 2524 * Unbinds an object from the GTT aperture. 2525 */ 2526int 2527i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2528{ 2529 drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 2530 int ret = 0; 2531 2532 if (obj->gtt_space == NULL) 2533 return 0; 2534 2535 if (obj->pin_count) 2536 return -EINVAL; 2537 2538 ret = i915_gem_object_finish_gpu(obj); 2539 if (ret == -ERESTARTSYS || ret == -EINTR) 2540 return ret; 2541 2542 i915_gem_object_finish_gtt(obj); 2543 2544 if (ret == 0) 2545 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2546 if (ret == -ERESTARTSYS || ret == -EINTR) 2547 return ret; 2548 if (ret != 0) { 2549 i915_gem_clflush_object(obj); 2550 obj->base.read_domains = obj->base.write_domain = 2551 I915_GEM_DOMAIN_CPU; 2552 } 2553 2554 /* release the fence reg _after_ flushing */ 2555 ret = i915_gem_object_put_fence(obj); 2556 if (ret) 2557 return ret; 2558 2559 if (obj->has_global_gtt_mapping) 2560 i915_gem_gtt_unbind_object(obj); 2561 if (obj->has_aliasing_ppgtt_mapping) { 2562 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2563 obj->has_aliasing_ppgtt_mapping = 0; 2564 } 2565 i915_gem_gtt_finish_object(obj); 2566 2567 i915_gem_object_put_pages_gtt(obj); 2568 2569 list_del_init(&obj->gtt_list); 2570 list_del_init(&obj->mm_list); 2571 obj->map_and_fenceable = true; 2572 2573 drm_mm_put_block(obj->gtt_space); 2574 obj->gtt_space = NULL; 2575 obj->gtt_offset = 0; 2576 2577 if (i915_gem_object_is_purgeable(obj)) 2578 i915_gem_object_truncate(obj); 2579 CTR1(KTR_DRM, "object_unbind %p", obj); 2580 2581 return ret; 2582} 2583 2584static int 2585i915_ring_idle(struct intel_ring_buffer *ring) 2586{ 2587 int ret; 2588 2589 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2590 return 0; 2591 2592 if (!list_empty(&ring->gpu_write_list)) { 2593 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2594 I915_GEM_GPU_DOMAINS); 2595 if (ret != 0) 2596 return ret; 2597 } 2598 2599 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring))); 2600} 2601 2602int i915_gpu_idle(struct drm_device *dev) 2603{ 2604 drm_i915_private_t *dev_priv = dev->dev_private; 2605 struct intel_ring_buffer *ring; 2606 int ret, i; 2607 2608 /* Flush everything onto the inactive list. */ 2609 for_each_ring(ring, dev_priv, i) { 2610 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); 2611 if (ret) 2612 return ret; 2613 2614 ret = i915_ring_idle(ring); 2615 if (ret) 2616 return ret; 2617 2618 /* Is the device fubar? */ 2619 if (!list_empty(&ring->gpu_write_list)) 2620 return -EBUSY; 2621 } 2622 2623 return 0; 2624} 2625 2626static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, 2627 struct drm_i915_gem_object *obj) 2628{ 2629 drm_i915_private_t *dev_priv = dev->dev_private; 2630 uint64_t val; 2631 2632 if (obj) { 2633 u32 size = obj->gtt_space->size; 2634 2635 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2636 0xfffff000) << 32; 2637 val |= obj->gtt_offset & 0xfffff000; 2638 val |= (uint64_t)((obj->stride / 128) - 1) << 2639 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2640 2641 if (obj->tiling_mode == I915_TILING_Y) 2642 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2643 val |= I965_FENCE_REG_VALID; 2644 } else 2645 val = 0; 2646 2647 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); 2648 POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); 2649} 2650 2651static void i965_write_fence_reg(struct drm_device *dev, int reg, 2652 struct drm_i915_gem_object *obj) 2653{ 2654 drm_i915_private_t *dev_priv = dev->dev_private; 2655 uint64_t val; 2656 2657 if (obj) { 2658 u32 size = obj->gtt_space->size; 2659 2660 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2661 0xfffff000) << 32; 2662 val |= obj->gtt_offset & 0xfffff000; 2663 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2664 if (obj->tiling_mode == I915_TILING_Y) 2665 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2666 val |= I965_FENCE_REG_VALID; 2667 } else 2668 val = 0; 2669 2670 I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); 2671 POSTING_READ(FENCE_REG_965_0 + reg * 8); 2672} 2673 2674static void i915_write_fence_reg(struct drm_device *dev, int reg, 2675 struct drm_i915_gem_object *obj) 2676{ 2677 drm_i915_private_t *dev_priv = dev->dev_private; 2678 u32 val; 2679 2680 if (obj) { 2681 u32 size = obj->gtt_space->size; 2682 int pitch_val; 2683 int tile_width; 2684 2685 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 2686 (size & -size) != size || 2687 (obj->gtt_offset & (size - 1))) 2688 printf( 2689 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 2690 obj->gtt_offset, obj->map_and_fenceable, size); 2691 2692 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 2693 tile_width = 128; 2694 else 2695 tile_width = 512; 2696 2697 /* Note: pitch better be a power of two tile widths */ 2698 pitch_val = obj->stride / tile_width; 2699 pitch_val = ffs(pitch_val) - 1; 2700 2701 val = obj->gtt_offset; 2702 if (obj->tiling_mode == I915_TILING_Y) 2703 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2704 val |= I915_FENCE_SIZE_BITS(size); 2705 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2706 val |= I830_FENCE_REG_VALID; 2707 } else 2708 val = 0; 2709 2710 if (reg < 8) 2711 reg = FENCE_REG_830_0 + reg * 4; 2712 else 2713 reg = FENCE_REG_945_8 + (reg - 8) * 4; 2714 2715 I915_WRITE(reg, val); 2716 POSTING_READ(reg); 2717} 2718 2719static void i830_write_fence_reg(struct drm_device *dev, int reg, 2720 struct drm_i915_gem_object *obj) 2721{ 2722 drm_i915_private_t *dev_priv = dev->dev_private; 2723 uint32_t val; 2724 2725 if (obj) { 2726 u32 size = obj->gtt_space->size; 2727 uint32_t pitch_val; 2728 2729 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 2730 (size & -size) != size || 2731 (obj->gtt_offset & (size - 1))) 2732 printf( 2733 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 2734 obj->gtt_offset, size); 2735 2736 pitch_val = obj->stride / 128; 2737 pitch_val = ffs(pitch_val) - 1; 2738 2739 val = obj->gtt_offset; 2740 if (obj->tiling_mode == I915_TILING_Y) 2741 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2742 val |= I830_FENCE_SIZE_BITS(size); 2743 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2744 val |= I830_FENCE_REG_VALID; 2745 } else 2746 val = 0; 2747 2748 I915_WRITE(FENCE_REG_830_0 + reg * 4, val); 2749 POSTING_READ(FENCE_REG_830_0 + reg * 4); 2750} 2751 2752static void i915_gem_write_fence(struct drm_device *dev, int reg, 2753 struct drm_i915_gem_object *obj) 2754{ 2755 switch (INTEL_INFO(dev)->gen) { 2756 case 7: 2757 case 6: sandybridge_write_fence_reg(dev, reg, obj); break; 2758 case 5: 2759 case 4: i965_write_fence_reg(dev, reg, obj); break; 2760 case 3: i915_write_fence_reg(dev, reg, obj); break; 2761 case 2: i830_write_fence_reg(dev, reg, obj); break; 2762 default: break; 2763 } 2764} 2765 2766static inline int fence_number(struct drm_i915_private *dev_priv, 2767 struct drm_i915_fence_reg *fence) 2768{ 2769 return fence - dev_priv->fence_regs; 2770} 2771 2772static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 2773 struct drm_i915_fence_reg *fence, 2774 bool enable) 2775{ 2776 struct drm_device *dev = obj->base.dev; 2777 struct drm_i915_private *dev_priv = dev->dev_private; 2778 int fence_reg = fence_number(dev_priv, fence); 2779 2780 i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL); 2781 2782 if (enable) { 2783 obj->fence_reg = fence_reg; 2784 fence->obj = obj; 2785 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); 2786 } else { 2787 obj->fence_reg = I915_FENCE_REG_NONE; 2788 fence->obj = NULL; 2789 list_del_init(&fence->lru_list); 2790 } 2791} 2792 2793static int 2794i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) 2795{ 2796 int ret; 2797 2798 if (obj->fenced_gpu_access) { 2799 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 2800 ret = i915_gem_flush_ring(obj->ring, 2801 0, obj->base.write_domain); 2802 if (ret) 2803 return ret; 2804 } 2805 2806 obj->fenced_gpu_access = false; 2807 } 2808 2809 if (obj->last_fenced_seqno) { 2810 ret = i915_wait_request(obj->ring, 2811 obj->last_fenced_seqno); 2812 if (ret) 2813 return ret; 2814 2815 obj->last_fenced_seqno = 0; 2816 } 2817 2818 /* Ensure that all CPU reads are completed before installing a fence 2819 * and all writes before removing the fence. 2820 */ 2821 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 2822 mb(); 2823 2824 return 0; 2825} 2826 2827int 2828i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 2829{ 2830 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2831 int ret; 2832 2833 ret = i915_gem_object_flush_fence(obj); 2834 if (ret) 2835 return ret; 2836 2837 if (obj->fence_reg == I915_FENCE_REG_NONE) 2838 return 0; 2839 2840 i915_gem_object_update_fence(obj, 2841 &dev_priv->fence_regs[obj->fence_reg], 2842 false); 2843 i915_gem_object_fence_lost(obj); 2844 2845 return 0; 2846} 2847 2848static struct drm_i915_fence_reg * 2849i915_find_fence_reg(struct drm_device *dev) 2850{ 2851 struct drm_i915_private *dev_priv = dev->dev_private; 2852 struct drm_i915_fence_reg *reg, *avail; 2853 int i; 2854 2855 /* First try to find a free reg */ 2856 avail = NULL; 2857 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 2858 reg = &dev_priv->fence_regs[i]; 2859 if (!reg->obj) 2860 return reg; 2861 2862 if (!reg->pin_count) 2863 avail = reg; 2864 } 2865 2866 if (avail == NULL) 2867 return NULL; 2868 2869 /* None available, try to steal one or wait for a user to finish */ 2870 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 2871 if (reg->pin_count) 2872 continue; 2873 2874 return reg; 2875 } 2876 2877 return NULL; 2878} 2879 2880/** 2881 * i915_gem_object_get_fence - set up fencing for an object 2882 * @obj: object to map through a fence reg 2883 * 2884 * When mapping objects through the GTT, userspace wants to be able to write 2885 * to them without having to worry about swizzling if the object is tiled. 2886 * This function walks the fence regs looking for a free one for @obj, 2887 * stealing one if it can't find any. 2888 * 2889 * It then sets up the reg based on the object's properties: address, pitch 2890 * and tiling format. 2891 * 2892 * For an untiled surface, this removes any existing fence. 2893 */ 2894int 2895i915_gem_object_get_fence(struct drm_i915_gem_object *obj) 2896{ 2897 struct drm_device *dev = obj->base.dev; 2898 struct drm_i915_private *dev_priv = dev->dev_private; 2899 bool enable = obj->tiling_mode != I915_TILING_NONE; 2900 struct drm_i915_fence_reg *reg; 2901 int ret; 2902 2903 /* Have we updated the tiling parameters upon the object and so 2904 * will need to serialise the write to the associated fence register? 2905 */ 2906 if (obj->fence_dirty) { 2907 ret = i915_gem_object_flush_fence(obj); 2908 if (ret) 2909 return ret; 2910 } 2911 2912 /* Just update our place in the LRU if our fence is getting reused. */ 2913 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2914 reg = &dev_priv->fence_regs[obj->fence_reg]; 2915 if (!obj->fence_dirty) { 2916 list_move_tail(®->lru_list, 2917 &dev_priv->mm.fence_list); 2918 return 0; 2919 } 2920 } else if (enable) { 2921 reg = i915_find_fence_reg(dev); 2922 if (reg == NULL) 2923 return -EDEADLK; 2924 2925 if (reg->obj) { 2926 struct drm_i915_gem_object *old = reg->obj; 2927 2928 ret = i915_gem_object_flush_fence(old); 2929 if (ret) 2930 return ret; 2931 2932 i915_gem_object_fence_lost(old); 2933 } 2934 } else 2935 return 0; 2936 2937 i915_gem_object_update_fence(obj, reg, enable); 2938 obj->fence_dirty = false; 2939 2940 return 0; 2941} 2942 2943/** 2944 * Finds free space in the GTT aperture and binds the object there. 2945 */ 2946static int 2947i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2948 unsigned alignment, 2949 bool map_and_fenceable) 2950{ 2951 struct drm_device *dev = obj->base.dev; 2952 drm_i915_private_t *dev_priv = dev->dev_private; 2953 struct drm_mm_node *free_space; 2954 u32 size, fence_size, fence_alignment, unfenced_alignment; 2955 bool mappable, fenceable; 2956 int ret; 2957 2958 if (obj->madv != I915_MADV_WILLNEED) { 2959 DRM_ERROR("Attempting to bind a purgeable object\n"); 2960 return -EINVAL; 2961 } 2962 2963 fence_size = i915_gem_get_gtt_size(dev, 2964 obj->base.size, 2965 obj->tiling_mode); 2966 fence_alignment = i915_gem_get_gtt_alignment(dev, 2967 obj->base.size, 2968 obj->tiling_mode); 2969 unfenced_alignment = 2970 i915_gem_get_unfenced_gtt_alignment(dev, 2971 obj->base.size, 2972 obj->tiling_mode); 2973 2974 if (alignment == 0) 2975 alignment = map_and_fenceable ? fence_alignment : 2976 unfenced_alignment; 2977 if (map_and_fenceable && alignment & (fence_alignment - 1)) { 2978 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2979 return -EINVAL; 2980 } 2981 2982 size = map_and_fenceable ? fence_size : obj->base.size; 2983 2984 /* If the object is bigger than the entire aperture, reject it early 2985 * before evicting everything in a vain attempt to find space. 2986 */ 2987 if (obj->base.size > 2988 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2989 DRM_ERROR("Attempting to bind an object larger than the aperture\n"); 2990 return -E2BIG; 2991 } 2992 2993 search_free: 2994 if (map_and_fenceable) 2995 free_space = drm_mm_search_free_in_range( 2996 &dev_priv->mm.gtt_space, size, alignment, 0, 2997 dev_priv->mm.gtt_mappable_end, 0); 2998 else 2999 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 3000 size, alignment, 0); 3001 if (free_space != NULL) { 3002 if (map_and_fenceable) 3003 obj->gtt_space = drm_mm_get_block_range_generic( 3004 free_space, size, alignment, 0, 0, 3005 dev_priv->mm.gtt_mappable_end, 1); 3006 else 3007 obj->gtt_space = drm_mm_get_block_generic(free_space, 3008 size, alignment, 0, 1); 3009 } 3010 if (obj->gtt_space == NULL) { 3011 ret = i915_gem_evict_something(dev, size, alignment, 3012 map_and_fenceable); 3013 if (ret != 0) 3014 return ret; 3015 goto search_free; 3016 } 3017 ret = i915_gem_object_get_pages_gtt(obj, 0); 3018 if (ret) { 3019 drm_mm_put_block(obj->gtt_space); 3020 obj->gtt_space = NULL; 3021 /* 3022 * i915_gem_object_get_pages_gtt() cannot return 3023 * ENOMEM, since we use vm_page_grab(). 3024 */ 3025 return ret; 3026 } 3027 3028 ret = i915_gem_gtt_prepare_object(obj); 3029 if (ret) { 3030 i915_gem_object_put_pages_gtt(obj); 3031 drm_mm_put_block(obj->gtt_space); 3032 obj->gtt_space = NULL; 3033 if (i915_gem_evict_everything(dev, false)) 3034 return ret; 3035 goto search_free; 3036 } 3037 3038 if (!dev_priv->mm.aliasing_ppgtt) 3039 i915_gem_gtt_bind_object(obj, obj->cache_level); 3040 3041 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 3042 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3043 3044 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 3045 ("Object in gpu read domain")); 3046 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 3047 ("Object in gpu write domain")); 3048 3049 obj->gtt_offset = obj->gtt_space->start; 3050 3051 fenceable = 3052 obj->gtt_space->size == fence_size && 3053 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 3054 3055 mappable = 3056 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 3057 3058 obj->map_and_fenceable = mappable && fenceable; 3059 3060 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 3061 obj->base.size, map_and_fenceable); 3062 return 0; 3063} 3064 3065void 3066i915_gem_clflush_object(struct drm_i915_gem_object *obj) 3067{ 3068 /* If we don't have a page list set up, then we're not pinned 3069 * to GPU, and we can ignore the cache flush because it'll happen 3070 * again at bind time. 3071 */ 3072 if (obj->pages == NULL) 3073 return; 3074 3075 /* If the GPU is snooping the contents of the CPU cache, 3076 * we do not need to manually clear the CPU cache lines. However, 3077 * the caches are only snooped when the render cache is 3078 * flushed/invalidated. As we always have to emit invalidations 3079 * and flushes when moving into and out of the RENDER domain, correct 3080 * snooping behaviour occurs naturally as the result of our domain 3081 * tracking. 3082 */ 3083 if (obj->cache_level != I915_CACHE_NONE) 3084 return; 3085 3086 CTR1(KTR_DRM, "object_clflush %p", obj); 3087 3088 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 3089} 3090 3091/** Flushes the GTT write domain for the object if it's dirty. */ 3092static void 3093i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 3094{ 3095 uint32_t old_write_domain; 3096 3097 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 3098 return; 3099 3100 /* No actual flushing is required for the GTT write domain. Writes 3101 * to it immediately go to main memory as far as we know, so there's 3102 * no chipset flush. It also doesn't land in render cache. 3103 * 3104 * However, we do have to enforce the order so that all writes through 3105 * the GTT land before any writes to the device, such as updates to 3106 * the GATT itself. 3107 */ 3108 wmb(); 3109 3110 old_write_domain = obj->base.write_domain; 3111 obj->base.write_domain = 0; 3112 3113 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 3114 obj->base.read_domains, old_write_domain); 3115} 3116 3117/** Flushes the CPU write domain for the object if it's dirty. */ 3118static void 3119i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 3120{ 3121 uint32_t old_write_domain; 3122 3123 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 3124 return; 3125 3126 i915_gem_clflush_object(obj); 3127 intel_gtt_chipset_flush(); 3128 old_write_domain = obj->base.write_domain; 3129 obj->base.write_domain = 0; 3130 3131 CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 3132 obj->base.read_domains, old_write_domain); 3133} 3134 3135static int 3136i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 3137{ 3138 3139 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 3140 return (0); 3141 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 3142} 3143 3144/** 3145 * Moves a single object to the GTT read, and possibly write domain. 3146 * 3147 * This function returns when the move is complete, including waiting on 3148 * flushes to occur. 3149 */ 3150int 3151i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 3152{ 3153 drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 3154 uint32_t old_write_domain, old_read_domains; 3155 int ret; 3156 3157 /* Not valid to be called on unbound objects. */ 3158 if (obj->gtt_space == NULL) 3159 return -EINVAL; 3160 3161 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 3162 return 0; 3163 3164 ret = i915_gem_object_flush_gpu_write_domain(obj); 3165 if (ret) 3166 return ret; 3167 3168 if (obj->pending_gpu_write || write) { 3169 ret = i915_gem_object_wait_rendering(obj); 3170 if (ret) 3171 return (ret); 3172 } 3173 3174 i915_gem_object_flush_cpu_write_domain(obj); 3175 3176 old_write_domain = obj->base.write_domain; 3177 old_read_domains = obj->base.read_domains; 3178 3179 /* It should now be out of any other write domains, and we can update 3180 * the domain values for our changes. 3181 */ 3182 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 3183 ("In GTT write domain")); 3184 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3185 if (write) { 3186 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 3187 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 3188 obj->dirty = 1; 3189 } 3190 3191 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 3192 old_read_domains, old_write_domain); 3193 3194 /* And bump the LRU for this access */ 3195 if (i915_gem_object_is_inactive(obj)) 3196 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3197 3198 return 0; 3199} 3200 3201int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3202 enum i915_cache_level cache_level) 3203{ 3204 struct drm_device *dev = obj->base.dev; 3205 drm_i915_private_t *dev_priv = dev->dev_private; 3206 int ret; 3207 3208 if (obj->cache_level == cache_level) 3209 return 0; 3210 3211 if (obj->pin_count) { 3212 DRM_DEBUG("can not change the cache level of pinned objects\n"); 3213 return -EBUSY; 3214 } 3215 3216 if (obj->gtt_space) { 3217 ret = i915_gem_object_finish_gpu(obj); 3218 if (ret) 3219 return ret; 3220 3221 i915_gem_object_finish_gtt(obj); 3222 3223 /* Before SandyBridge, you could not use tiling or fence 3224 * registers with snooped memory, so relinquish any fences 3225 * currently pointing to our region in the aperture. 3226 */ 3227 if (INTEL_INFO(obj->base.dev)->gen < 6) { 3228 ret = i915_gem_object_put_fence(obj); 3229 if (ret) 3230 return ret; 3231 } 3232 3233 if (obj->has_global_gtt_mapping) 3234 i915_gem_gtt_bind_object(obj, cache_level); 3235 if (obj->has_aliasing_ppgtt_mapping) 3236 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 3237 obj, cache_level); 3238 } 3239 3240 if (cache_level == I915_CACHE_NONE) { 3241 u32 old_read_domains, old_write_domain; 3242 3243 /* If we're coming from LLC cached, then we haven't 3244 * actually been tracking whether the data is in the 3245 * CPU cache or not, since we only allow one bit set 3246 * in obj->write_domain and have been skipping the clflushes. 3247 * Just set it to the CPU cache for now. 3248 */ 3249 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 3250 ("obj %p in CPU write domain", obj)); 3251 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 3252 ("obj %p in CPU read domain", obj)); 3253 3254 old_read_domains = obj->base.read_domains; 3255 old_write_domain = obj->base.write_domain; 3256 3257 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3258 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3259 3260 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 3261 obj, old_read_domains, old_write_domain); 3262 } 3263 3264 obj->cache_level = cache_level; 3265 return 0; 3266} 3267 3268static bool is_pin_display(struct drm_i915_gem_object *obj) 3269{ 3270 /* There are 3 sources that pin objects: 3271 * 1. The display engine (scanouts, sprites, cursors); 3272 * 2. Reservations for execbuffer; 3273 * 3. The user. 3274 * 3275 * We can ignore reservations as we hold the struct_mutex and 3276 * are only called outside of the reservation path. The user 3277 * can only increment pin_count once, and so if after 3278 * subtracting the potential reference by the user, any pin_count 3279 * remains, it must be due to another use by the display engine. 3280 */ 3281 return obj->pin_count - !!obj->user_pin_count; 3282} 3283 3284int 3285i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 3286 u32 alignment, 3287 struct intel_ring_buffer *pipelined) 3288{ 3289 u32 old_read_domains, old_write_domain; 3290 int ret; 3291 3292 ret = i915_gem_object_flush_gpu_write_domain(obj); 3293 if (ret) 3294 return ret; 3295 3296 if (pipelined != obj->ring) { 3297 ret = i915_gem_object_sync(obj, pipelined); 3298 if (ret) 3299 return ret; 3300 } 3301 3302 /* Mark the pin_display early so that we account for the 3303 * display coherency whilst setting up the cache domains. 3304 */ 3305 obj->pin_display = true; 3306 3307 /* The display engine is not coherent with the LLC cache on gen6. As 3308 * a result, we make sure that the pinning that is about to occur is 3309 * done with uncached PTEs. This is lowest common denominator for all 3310 * chipsets. 3311 * 3312 * However for gen6+, we could do better by using the GFDT bit instead 3313 * of uncaching, which would allow us to flush all the LLC-cached data 3314 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 3315 */ 3316 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 3317 if (ret) 3318 goto err_unpin_display; 3319 3320 /* As the user may map the buffer once pinned in the display plane 3321 * (e.g. libkms for the bootup splash), we have to ensure that we 3322 * always use map_and_fenceable for all scanout buffers. 3323 */ 3324 ret = i915_gem_object_pin(obj, alignment, true); 3325 if (ret) 3326 goto err_unpin_display; 3327 3328 i915_gem_object_flush_cpu_write_domain(obj); 3329 3330 old_write_domain = obj->base.write_domain; 3331 old_read_domains = obj->base.read_domains; 3332 3333 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 3334 ("obj %p in GTT write domain", obj)); 3335 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3336 3337 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 3338 obj, old_read_domains, obj->base.write_domain); 3339 3340 return 0; 3341 3342err_unpin_display: 3343 obj->pin_display = is_pin_display(obj); 3344 return ret; 3345} 3346 3347void 3348i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) 3349{ 3350 i915_gem_object_unpin(obj); 3351 obj->pin_display = is_pin_display(obj); 3352} 3353 3354int 3355i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 3356{ 3357 int ret; 3358 3359 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 3360 return 0; 3361 3362 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3363 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 3364 if (ret) 3365 return ret; 3366 } 3367 3368 ret = i915_gem_object_wait_rendering(obj); 3369 if (ret) 3370 return ret; 3371 3372 /* Ensure that we invalidate the GPU's caches and TLBs. */ 3373 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 3374 return 0; 3375} 3376 3377/** 3378 * Moves a single object to the CPU read, and possibly write domain. 3379 * 3380 * This function returns when the move is complete, including waiting on 3381 * flushes to occur. 3382 */ 3383int 3384i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 3385{ 3386 uint32_t old_write_domain, old_read_domains; 3387 int ret; 3388 3389 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 3390 return 0; 3391 3392 ret = i915_gem_object_flush_gpu_write_domain(obj); 3393 if (ret) 3394 return ret; 3395 3396 if (write || obj->pending_gpu_write) { 3397 ret = i915_gem_object_wait_rendering(obj); 3398 if (ret) 3399 return ret; 3400 } 3401 3402 i915_gem_object_flush_gtt_write_domain(obj); 3403 3404 old_write_domain = obj->base.write_domain; 3405 old_read_domains = obj->base.read_domains; 3406 3407 /* Flush the CPU cache if it's still invalid. */ 3408 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 3409 i915_gem_clflush_object(obj); 3410 3411 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 3412 } 3413 3414 /* It should now be out of any other write domains, and we can update 3415 * the domain values for our changes. 3416 */ 3417 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 3418 ("In cpu write domain")); 3419 3420 /* If we're writing through the CPU, then the GPU read domains will 3421 * need to be invalidated at next use. 3422 */ 3423 if (write) { 3424 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3425 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3426 } 3427 3428 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 3429 old_read_domains, old_write_domain); 3430 3431 return 0; 3432} 3433 3434/* Throttle our rendering by waiting until the ring has completed our requests 3435 * emitted over 20 msec ago. 3436 * 3437 * Note that if we were to use the current jiffies each time around the loop, 3438 * we wouldn't escape the function with any frames outstanding if the time to 3439 * render a frame was over 20ms. 3440 * 3441 * This should get us reasonable parallelism between CPU and GPU but also 3442 * relatively low latency when blocking on a particular request to finish. 3443 */ 3444static int 3445i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 3446{ 3447 struct drm_i915_private *dev_priv = dev->dev_private; 3448 struct drm_i915_file_private *file_priv = file->driver_priv; 3449 unsigned long recent_enough = ticks - (20 * hz / 1000); 3450 struct drm_i915_gem_request *request; 3451 struct intel_ring_buffer *ring = NULL; 3452 u32 seqno = 0; 3453 int ret; 3454 3455 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 3456 return -EIO; 3457 3458 mtx_lock(&file_priv->mm.lck); 3459 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 3460 if (time_after_eq(request->emitted_jiffies, recent_enough)) 3461 break; 3462 ring = request->ring; 3463 seqno = request->seqno; 3464 } 3465 mtx_unlock(&file_priv->mm.lck); 3466 if (seqno == 0) 3467 return 0; 3468 3469 ret = __wait_seqno(ring, seqno, true); 3470 if (ret == 0) 3471 taskqueue_enqueue_timeout(dev_priv->tq, 3472 &dev_priv->mm.retire_task, 0); 3473 3474 return ret; 3475} 3476 3477int 3478i915_gem_object_pin(struct drm_i915_gem_object *obj, 3479 uint32_t alignment, 3480 bool map_and_fenceable) 3481{ 3482 int ret; 3483 3484 if (obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT) 3485 return -EBUSY; 3486 3487 if (obj->gtt_space != NULL) { 3488 if ((alignment && obj->gtt_offset & (alignment - 1)) || 3489 (map_and_fenceable && !obj->map_and_fenceable)) { 3490 DRM_DEBUG("bo is already pinned with incorrect alignment:" 3491 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 3492 " obj->map_and_fenceable=%d\n", 3493 obj->gtt_offset, alignment, 3494 map_and_fenceable, 3495 obj->map_and_fenceable); 3496 ret = i915_gem_object_unbind(obj); 3497 if (ret) 3498 return ret; 3499 } 3500 } 3501 3502 if (obj->gtt_space == NULL) { 3503 ret = i915_gem_object_bind_to_gtt(obj, alignment, 3504 map_and_fenceable); 3505 if (ret) 3506 return ret; 3507 } 3508 3509 if (!obj->has_global_gtt_mapping && map_and_fenceable) 3510 i915_gem_gtt_bind_object(obj, obj->cache_level); 3511 3512 obj->pin_count++; 3513 obj->pin_mappable |= map_and_fenceable; 3514 3515 return 0; 3516} 3517 3518void 3519i915_gem_object_unpin(struct drm_i915_gem_object *obj) 3520{ 3521 3522 KASSERT(obj->pin_count != 0, ("zero pin count")); 3523 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 3524 3525 if (--obj->pin_count == 0) 3526 obj->pin_mappable = false; 3527} 3528 3529int 3530i915_gem_pin_ioctl(struct drm_device *dev, void *data, 3531 struct drm_file *file) 3532{ 3533 struct drm_i915_gem_pin *args = data; 3534 struct drm_i915_gem_object *obj; 3535 struct drm_gem_object *gobj; 3536 int ret; 3537 3538 ret = i915_mutex_lock_interruptible(dev); 3539 if (ret) 3540 return ret; 3541 3542 gobj = drm_gem_object_lookup(dev, file, args->handle); 3543 if (gobj == NULL) { 3544 ret = -ENOENT; 3545 goto unlock; 3546 } 3547 obj = to_intel_bo(gobj); 3548 3549 if (obj->madv != I915_MADV_WILLNEED) { 3550 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 3551 ret = -EINVAL; 3552 goto out; 3553 } 3554 3555 if (obj->pin_filp != NULL && obj->pin_filp != file) { 3556 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3557 args->handle); 3558 ret = -EINVAL; 3559 goto out; 3560 } 3561 3562 obj->user_pin_count++; 3563 obj->pin_filp = file; 3564 if (obj->user_pin_count == 1) { 3565 ret = i915_gem_object_pin(obj, args->alignment, true); 3566 if (ret) 3567 goto out; 3568 } 3569 3570 /* XXX - flush the CPU caches for pinned objects 3571 * as the X server doesn't manage domains yet 3572 */ 3573 i915_gem_object_flush_cpu_write_domain(obj); 3574 args->offset = obj->gtt_offset; 3575out: 3576 drm_gem_object_unreference(&obj->base); 3577unlock: 3578 DRM_UNLOCK(dev); 3579 return ret; 3580} 3581 3582int 3583i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3584 struct drm_file *file) 3585{ 3586 struct drm_i915_gem_pin *args = data; 3587 struct drm_i915_gem_object *obj; 3588 int ret; 3589 3590 ret = i915_mutex_lock_interruptible(dev); 3591 if (ret) 3592 return ret; 3593 3594 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3595 if (&obj->base == NULL) { 3596 ret = -ENOENT; 3597 goto unlock; 3598 } 3599 3600 if (obj->pin_filp != file) { 3601 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3602 args->handle); 3603 ret = -EINVAL; 3604 goto out; 3605 } 3606 obj->user_pin_count--; 3607 if (obj->user_pin_count == 0) { 3608 obj->pin_filp = NULL; 3609 i915_gem_object_unpin(obj); 3610 } 3611 3612out: 3613 drm_gem_object_unreference(&obj->base); 3614unlock: 3615 DRM_UNLOCK(dev); 3616 return ret; 3617} 3618 3619int 3620i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3621 struct drm_file *file) 3622{ 3623 struct drm_i915_gem_busy *args = data; 3624 struct drm_i915_gem_object *obj; 3625 int ret; 3626 3627 ret = i915_mutex_lock_interruptible(dev); 3628 if (ret) 3629 return ret; 3630 3631 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3632 if (&obj->base == NULL) { 3633 ret = -ENOENT; 3634 goto unlock; 3635 } 3636 3637 args->busy = obj->active; 3638 if (args->busy) { 3639 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3640 ret = i915_gem_flush_ring(obj->ring, 3641 0, obj->base.write_domain); 3642 } else { 3643 ret = i915_gem_check_olr(obj->ring, 3644 obj->last_rendering_seqno); 3645 } 3646 3647 i915_gem_retire_requests_ring(obj->ring); 3648 args->busy = obj->active; 3649 } 3650 3651 drm_gem_object_unreference(&obj->base); 3652unlock: 3653 DRM_UNLOCK(dev); 3654 return ret; 3655} 3656 3657int 3658i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 3659 struct drm_file *file_priv) 3660{ 3661 return i915_gem_ring_throttle(dev, file_priv); 3662} 3663 3664int 3665i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 3666 struct drm_file *file_priv) 3667{ 3668 struct drm_i915_gem_madvise *args = data; 3669 struct drm_i915_gem_object *obj; 3670 int ret; 3671 3672 switch (args->madv) { 3673 case I915_MADV_DONTNEED: 3674 case I915_MADV_WILLNEED: 3675 break; 3676 default: 3677 return -EINVAL; 3678 } 3679 3680 ret = i915_mutex_lock_interruptible(dev); 3681 if (ret) 3682 return ret; 3683 3684 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 3685 if (&obj->base == NULL) { 3686 ret = -ENOENT; 3687 goto unlock; 3688 } 3689 3690 if (obj->pin_count) { 3691 ret = -EINVAL; 3692 goto out; 3693 } 3694 3695 if (obj->madv != I915_MADV_PURGED_INTERNAL) 3696 obj->madv = args->madv; 3697 3698 /* if the object is no longer attached, discard its backing storage */ 3699 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 3700 i915_gem_object_truncate(obj); 3701 3702 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 3703 3704out: 3705 drm_gem_object_unreference(&obj->base); 3706unlock: 3707 DRM_UNLOCK(dev); 3708 return ret; 3709} 3710 3711struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, 3712 size_t size) 3713{ 3714 struct drm_i915_private *dev_priv; 3715 struct drm_i915_gem_object *obj; 3716 3717 dev_priv = dev->dev_private; 3718 3719 obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 3720 3721 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 3722 free(obj, DRM_I915_GEM); 3723 return NULL; 3724 } 3725 3726 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3727 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3728 3729 if (HAS_LLC(dev)) { 3730 /* On some devices, we can have the GPU use the LLC (the CPU 3731 * cache) for about a 10% performance improvement 3732 * compared to uncached. Graphics requests other than 3733 * display scanout are coherent with the CPU in 3734 * accessing this cache. This means in this mode we 3735 * don't need to clflush on the CPU side, and on the 3736 * GPU side we only need to flush internal caches to 3737 * get data visible to the CPU. 3738 * 3739 * However, we maintain the display planes as UC, and so 3740 * need to rebind when first used as such. 3741 */ 3742 obj->cache_level = I915_CACHE_LLC; 3743 } else 3744 obj->cache_level = I915_CACHE_NONE; 3745 obj->base.driver_private = NULL; 3746 obj->fence_reg = I915_FENCE_REG_NONE; 3747 INIT_LIST_HEAD(&obj->mm_list); 3748 INIT_LIST_HEAD(&obj->gtt_list); 3749 INIT_LIST_HEAD(&obj->ring_list); 3750 INIT_LIST_HEAD(&obj->exec_list); 3751 INIT_LIST_HEAD(&obj->gpu_write_list); 3752 obj->madv = I915_MADV_WILLNEED; 3753 /* Avoid an unnecessary call to unbind on the first bind. */ 3754 obj->map_and_fenceable = true; 3755 3756 i915_gem_info_add_obj(dev_priv, size); 3757 3758 return obj; 3759} 3760 3761int i915_gem_init_object(struct drm_gem_object *obj) 3762{ 3763 printf("i915_gem_init_object called\n"); 3764 3765 return 0; 3766} 3767 3768void i915_gem_free_object(struct drm_gem_object *gem_obj) 3769{ 3770 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 3771 struct drm_device *dev = obj->base.dev; 3772 drm_i915_private_t *dev_priv = dev->dev_private; 3773 3774 CTR1(KTR_DRM, "object_destroy_tail %p", obj); 3775 3776 if (obj->phys_obj) 3777 i915_gem_detach_phys_object(dev, obj); 3778 3779 obj->pin_count = 0; 3780 if (i915_gem_object_unbind(obj) == -ERESTARTSYS) { 3781 bool was_interruptible; 3782 3783 was_interruptible = dev_priv->mm.interruptible; 3784 dev_priv->mm.interruptible = false; 3785 3786 if (i915_gem_object_unbind(obj)) 3787 printf("i915_gem_free_object: unbind\n"); 3788 3789 dev_priv->mm.interruptible = was_interruptible; 3790 } 3791 3792 drm_gem_free_mmap_offset(&obj->base); 3793 drm_gem_object_release(&obj->base); 3794 i915_gem_info_remove_obj(dev_priv, obj->base.size); 3795 3796 free(obj->bit_17, DRM_I915_GEM); 3797 free(obj, DRM_I915_GEM); 3798} 3799 3800int 3801i915_gem_idle(struct drm_device *dev) 3802{ 3803 drm_i915_private_t *dev_priv = dev->dev_private; 3804 int ret; 3805 3806 DRM_LOCK(dev); 3807 3808 if (dev_priv->mm.suspended) { 3809 DRM_UNLOCK(dev); 3810 return 0; 3811 } 3812 3813 ret = i915_gpu_idle(dev); 3814 if (ret) { 3815 DRM_UNLOCK(dev); 3816 return ret; 3817 } 3818 i915_gem_retire_requests(dev); 3819 3820 /* Under UMS, be paranoid and evict. */ 3821 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 3822 ret = i915_gem_evict_everything(dev, false); 3823 if (ret) { 3824 DRM_UNLOCK(dev); 3825 return ret; 3826 } 3827 } 3828 3829 i915_gem_reset_fences(dev); 3830 3831 /* Hack! Don't let anybody do execbuf while we don't control the chip. 3832 * We need to replace this with a semaphore, or something. 3833 * And not confound mm.suspended! 3834 */ 3835 dev_priv->mm.suspended = 1; 3836 callout_stop(&dev_priv->hangcheck_timer); 3837 3838 i915_kernel_lost_context(dev); 3839 i915_gem_cleanup_ringbuffer(dev); 3840 3841 DRM_UNLOCK(dev); 3842 3843 /* Cancel the retire work handler, which should be idle now. */ 3844 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 3845 3846 return ret; 3847} 3848 3849void i915_gem_init_swizzling(struct drm_device *dev) 3850{ 3851 drm_i915_private_t *dev_priv = dev->dev_private; 3852 3853 if (INTEL_INFO(dev)->gen < 5 || 3854 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 3855 return; 3856 3857 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 3858 DISP_TILE_SURFACE_SWIZZLING); 3859 3860 if (IS_GEN5(dev)) 3861 return; 3862 3863 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 3864 if (IS_GEN6(dev)) 3865 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 3866 else 3867 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 3868} 3869 3870int 3871i915_gem_init_hw(struct drm_device *dev) 3872{ 3873 drm_i915_private_t *dev_priv = dev->dev_private; 3874 int ret; 3875 3876 i915_gem_init_swizzling(dev); 3877 3878 ret = intel_init_render_ring_buffer(dev); 3879 if (ret) 3880 return ret; 3881 3882 if (HAS_BSD(dev)) { 3883 ret = intel_init_bsd_ring_buffer(dev); 3884 if (ret) 3885 goto cleanup_render_ring; 3886 } 3887 3888 if (HAS_BLT(dev)) { 3889 ret = intel_init_blt_ring_buffer(dev); 3890 if (ret) 3891 goto cleanup_bsd_ring; 3892 } 3893 3894 dev_priv->next_seqno = 1; 3895 3896 /* 3897 * XXX: There was some w/a described somewhere suggesting loading 3898 * contexts before PPGTT. 3899 */ 3900 i915_gem_context_init(dev); 3901 i915_gem_init_ppgtt(dev); 3902 3903 return 0; 3904 3905cleanup_bsd_ring: 3906 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 3907cleanup_render_ring: 3908 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 3909 return ret; 3910} 3911 3912static bool 3913intel_enable_ppgtt(struct drm_device *dev) 3914{ 3915 if (i915_enable_ppgtt >= 0) 3916 return i915_enable_ppgtt; 3917 3918 /* Disable ppgtt on SNB if VT-d is on. */ 3919 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled) 3920 return false; 3921 3922 return true; 3923} 3924 3925int i915_gem_init(struct drm_device *dev) 3926{ 3927 struct drm_i915_private *dev_priv = dev->dev_private; 3928 unsigned long gtt_size, mappable_size; 3929 int ret; 3930 3931 gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT; 3932 mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT; 3933 3934 DRM_LOCK(dev); 3935 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { 3936 /* PPGTT pdes are stolen from global gtt ptes, so shrink the 3937 * aperture accordingly when using aliasing ppgtt. */ 3938 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; 3939 3940 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); 3941 3942 ret = i915_gem_init_aliasing_ppgtt(dev); 3943 if (ret) { 3944 DRM_UNLOCK(dev); 3945 return ret; 3946 } 3947 } else { 3948 /* Let GEM Manage all of the aperture. 3949 * 3950 * However, leave one page at the end still bound to the scratch 3951 * page. There are a number of places where the hardware 3952 * apparently prefetches past the end of the object, and we've 3953 * seen multiple hangs with the GPU head pointer stuck in a 3954 * batchbuffer bound at the last page of the aperture. One page 3955 * should be enough to keep any prefetching inside of the 3956 * aperture. 3957 */ 3958 i915_gem_init_global_gtt(dev, 0, mappable_size, 3959 gtt_size); 3960 } 3961 3962 ret = i915_gem_init_hw(dev); 3963 DRM_UNLOCK(dev); 3964 if (ret) { 3965 i915_gem_cleanup_aliasing_ppgtt(dev); 3966 return ret; 3967 } 3968 3969 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ 3970 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 3971 dev_priv->dri1.allow_batchbuffer = 1; 3972 return 0; 3973} 3974 3975void 3976i915_gem_cleanup_ringbuffer(struct drm_device *dev) 3977{ 3978 drm_i915_private_t *dev_priv = dev->dev_private; 3979 struct intel_ring_buffer *ring; 3980 int i; 3981 3982 for_each_ring(ring, dev_priv, i) 3983 intel_cleanup_ring_buffer(ring); 3984} 3985 3986int 3987i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 3988 struct drm_file *file_priv) 3989{ 3990 drm_i915_private_t *dev_priv = dev->dev_private; 3991 int ret; 3992 3993 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3994 return 0; 3995 3996 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 3997 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 3998 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 3999 } 4000 4001 DRM_LOCK(dev); 4002 dev_priv->mm.suspended = 0; 4003 4004 ret = i915_gem_init_hw(dev); 4005 if (ret != 0) { 4006 DRM_UNLOCK(dev); 4007 return ret; 4008 } 4009 4010 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 4011 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 4012 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 4013 DRM_UNLOCK(dev); 4014 4015 ret = drm_irq_install(dev); 4016 if (ret) 4017 goto cleanup_ringbuffer; 4018 4019 return 0; 4020 4021cleanup_ringbuffer: 4022 DRM_LOCK(dev); 4023 i915_gem_cleanup_ringbuffer(dev); 4024 dev_priv->mm.suspended = 1; 4025 DRM_UNLOCK(dev); 4026 4027 return ret; 4028} 4029 4030int 4031i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 4032 struct drm_file *file_priv) 4033{ 4034 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4035 return 0; 4036 4037 drm_irq_uninstall(dev); 4038 return i915_gem_idle(dev); 4039} 4040 4041void 4042i915_gem_lastclose(struct drm_device *dev) 4043{ 4044 int ret; 4045 4046 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4047 return; 4048 4049 ret = i915_gem_idle(dev); 4050 if (ret) 4051 DRM_ERROR("failed to idle hardware: %d\n", ret); 4052} 4053 4054static void 4055init_ring_lists(struct intel_ring_buffer *ring) 4056{ 4057 INIT_LIST_HEAD(&ring->active_list); 4058 INIT_LIST_HEAD(&ring->request_list); 4059 INIT_LIST_HEAD(&ring->gpu_write_list); 4060} 4061 4062void 4063i915_gem_load(struct drm_device *dev) 4064{ 4065 int i; 4066 drm_i915_private_t *dev_priv = dev->dev_private; 4067 4068 INIT_LIST_HEAD(&dev_priv->mm.active_list); 4069 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 4070 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 4071 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 4072 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 4073 for (i = 0; i < I915_NUM_RINGS; i++) 4074 init_ring_lists(&dev_priv->rings[i]); 4075 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 4076 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 4077 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 4078 i915_gem_retire_task_handler, dev_priv); 4079 dev_priv->error_completion = 0; 4080 4081 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 4082 if (IS_GEN3(dev)) { 4083 I915_WRITE(MI_ARB_STATE, 4084 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 4085 } 4086 4087 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 4088 4089 /* Old X drivers will take 0-2 for front, back, depth buffers */ 4090 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4091 dev_priv->fence_reg_start = 3; 4092 4093 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 4094 dev_priv->num_fence_regs = 16; 4095 else 4096 dev_priv->num_fence_regs = 8; 4097 4098 /* Initialize fence registers to zero */ 4099 i915_gem_reset_fences(dev); 4100 4101 i915_gem_detect_bit_6_swizzle(dev); 4102 dev_priv->mm.interruptible = true; 4103 4104 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 4105 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 4106} 4107 4108void 4109i915_gem_unload(struct drm_device *dev) 4110{ 4111 struct drm_i915_private *dev_priv; 4112 4113 dev_priv = dev->dev_private; 4114 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 4115} 4116 4117/* 4118 * Create a physically contiguous memory object for this object 4119 * e.g. for cursor + overlay regs 4120 */ 4121static int i915_gem_init_phys_object(struct drm_device *dev, 4122 int id, int size, int align) 4123{ 4124 drm_i915_private_t *dev_priv = dev->dev_private; 4125 struct drm_i915_gem_phys_object *phys_obj; 4126 int ret; 4127 4128 if (dev_priv->mm.phys_objs[id - 1] || !size) 4129 return 0; 4130 4131 phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), 4132 DRM_I915_GEM, M_WAITOK | M_ZERO); 4133 4134 phys_obj->id = id; 4135 4136 phys_obj->handle = drm_pci_alloc(dev, size, align, BUS_SPACE_MAXADDR); 4137 if (!phys_obj->handle) { 4138 ret = -ENOMEM; 4139 goto kfree_obj; 4140 } 4141 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 4142 size / PAGE_SIZE, PAT_WRITE_COMBINING); 4143 4144 dev_priv->mm.phys_objs[id - 1] = phys_obj; 4145 4146 return 0; 4147kfree_obj: 4148 free(phys_obj, DRM_I915_GEM); 4149 return ret; 4150} 4151 4152static void i915_gem_free_phys_object(struct drm_device *dev, int id) 4153{ 4154 drm_i915_private_t *dev_priv = dev->dev_private; 4155 struct drm_i915_gem_phys_object *phys_obj; 4156 4157 if (!dev_priv->mm.phys_objs[id - 1]) 4158 return; 4159 4160 phys_obj = dev_priv->mm.phys_objs[id - 1]; 4161 if (phys_obj->cur_obj) { 4162 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 4163 } 4164 4165 drm_pci_free(dev, phys_obj->handle); 4166 free(phys_obj, DRM_I915_GEM); 4167 dev_priv->mm.phys_objs[id - 1] = NULL; 4168} 4169 4170void i915_gem_free_all_phys_object(struct drm_device *dev) 4171{ 4172 int i; 4173 4174 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 4175 i915_gem_free_phys_object(dev, i); 4176} 4177 4178void i915_gem_detach_phys_object(struct drm_device *dev, 4179 struct drm_i915_gem_object *obj) 4180{ 4181 vm_page_t page; 4182 struct sf_buf *sf; 4183 char *vaddr, *dst; 4184 int i, page_count; 4185 4186 if (!obj->phys_obj) 4187 return; 4188 vaddr = obj->phys_obj->handle->vaddr; 4189 4190 page_count = obj->base.size / PAGE_SIZE; 4191 VM_OBJECT_WLOCK(obj->base.vm_obj); 4192 for (i = 0; i < page_count; i++) { 4193 page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4194 if (page == NULL) 4195 continue; /* XXX */ 4196 4197 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4198 sf = sf_buf_alloc(page, 0); 4199 if (sf != NULL) { 4200 dst = (char *)sf_buf_kva(sf); 4201 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 4202 sf_buf_free(sf); 4203 } 4204 drm_clflush_pages(&page, 1); 4205 4206 VM_OBJECT_WLOCK(obj->base.vm_obj); 4207 vm_page_reference(page); 4208 vm_page_lock(page); 4209 vm_page_dirty(page); 4210 vm_page_unwire(page, PQ_INACTIVE); 4211 vm_page_unlock(page); 4212 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4213 } 4214 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4215 intel_gtt_chipset_flush(); 4216 4217 obj->phys_obj->cur_obj = NULL; 4218 obj->phys_obj = NULL; 4219} 4220 4221int 4222i915_gem_attach_phys_object(struct drm_device *dev, 4223 struct drm_i915_gem_object *obj, 4224 int id, 4225 int align) 4226{ 4227 drm_i915_private_t *dev_priv = dev->dev_private; 4228 vm_page_t page; 4229 struct sf_buf *sf; 4230 char *dst, *src; 4231 int ret = 0; 4232 int page_count; 4233 int i; 4234 4235 if (id > I915_MAX_PHYS_OBJECT) 4236 return -EINVAL; 4237 4238 if (obj->phys_obj) { 4239 if (obj->phys_obj->id == id) 4240 return 0; 4241 i915_gem_detach_phys_object(dev, obj); 4242 } 4243 4244 /* create a new object */ 4245 if (!dev_priv->mm.phys_objs[id - 1]) { 4246 ret = i915_gem_init_phys_object(dev, id, 4247 obj->base.size, align); 4248 if (ret) { 4249 DRM_ERROR("failed to init phys object %d size: %zu\n", 4250 id, obj->base.size); 4251 return ret; 4252 } 4253 } 4254 4255 /* bind to the object */ 4256 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4257 obj->phys_obj->cur_obj = obj; 4258 4259 page_count = obj->base.size / PAGE_SIZE; 4260 4261 VM_OBJECT_WLOCK(obj->base.vm_obj); 4262 for (i = 0; i < page_count; i++) { 4263 page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4264 if (page == NULL) { 4265 ret = -EIO; 4266 break; 4267 } 4268 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4269 sf = sf_buf_alloc(page, 0); 4270 src = (char *)sf_buf_kva(sf); 4271 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 4272 memcpy(dst, src, PAGE_SIZE); 4273 sf_buf_free(sf); 4274 4275 VM_OBJECT_WLOCK(obj->base.vm_obj); 4276 4277 vm_page_reference(page); 4278 vm_page_lock(page); 4279 vm_page_unwire(page, PQ_INACTIVE); 4280 vm_page_unlock(page); 4281 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4282 } 4283 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4284 4285 return ret; 4286} 4287 4288static int 4289i915_gem_phys_pwrite(struct drm_device *dev, 4290 struct drm_i915_gem_object *obj, 4291 struct drm_i915_gem_pwrite *args, 4292 struct drm_file *file_priv) 4293{ 4294 void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset; 4295 char __user *user_data = to_user_ptr(args->data_ptr); 4296 4297 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 4298 unsigned long unwritten; 4299 4300 /* The physical object once assigned is fixed for the lifetime 4301 * of the obj, so we can safely drop the lock and continue 4302 * to access vaddr. 4303 */ 4304 DRM_UNLOCK(dev); 4305 unwritten = copy_from_user(vaddr, user_data, args->size); 4306 DRM_LOCK(dev); 4307 if (unwritten) 4308 return -EFAULT; 4309 } 4310 4311 i915_gem_chipset_flush(dev); 4312 return 0; 4313} 4314 4315void i915_gem_release(struct drm_device *dev, struct drm_file *file) 4316{ 4317 struct drm_i915_file_private *file_priv = file->driver_priv; 4318 4319 /* Clean up our request list when the client is going away, so that 4320 * later retire_requests won't dereference our soon-to-be-gone 4321 * file_priv. 4322 */ 4323 mtx_lock(&file_priv->mm.lck); 4324 while (!list_empty(&file_priv->mm.request_list)) { 4325 struct drm_i915_gem_request *request; 4326 4327 request = list_first_entry(&file_priv->mm.request_list, 4328 struct drm_i915_gem_request, 4329 client_list); 4330 list_del(&request->client_list); 4331 request->file_priv = NULL; 4332 } 4333 mtx_unlock(&file_priv->mm.lck); 4334} 4335 4336static vm_page_t 4337i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh) 4338{ 4339 vm_page_t page; 4340 int rv; 4341 4342 VM_OBJECT_ASSERT_WLOCKED(object); 4343 page = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 4344 if (page->valid != VM_PAGE_BITS_ALL) { 4345 if (vm_pager_has_page(object, pindex, NULL, NULL)) { 4346 rv = vm_pager_get_pages(object, &page, 1, NULL, NULL); 4347 if (rv != VM_PAGER_OK) { 4348 vm_page_lock(page); 4349 vm_page_free(page); 4350 vm_page_unlock(page); 4351 return (NULL); 4352 } 4353 if (fresh != NULL) 4354 *fresh = true; 4355 } else { 4356 pmap_zero_page(page); 4357 page->valid = VM_PAGE_BITS_ALL; 4358 page->dirty = 0; 4359 if (fresh != NULL) 4360 *fresh = false; 4361 } 4362 } else if (fresh != NULL) { 4363 *fresh = false; 4364 } 4365 vm_page_lock(page); 4366 vm_page_wire(page); 4367 vm_page_unlock(page); 4368 vm_page_xunbusy(page); 4369 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 4370 return (page); 4371} 4372 4373#undef __user 4374#undef __force 4375#undef __iomem 4376#undef __must_check 4377#undef to_user_ptr 4378#undef offset_in_page 4379#undef page_to_phys 4380