i915_gem.c revision 267548
1/*- 2 * Copyright �� 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 */ 53 54#include <sys/cdefs.h> 55__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 267548 2014-06-16 18:15:27Z attilio $"); 56 57#include <dev/drm2/drmP.h> 58#include <dev/drm2/drm.h> 59#include <dev/drm2/i915/i915_drm.h> 60#include <dev/drm2/i915/i915_drv.h> 61#include <dev/drm2/i915/intel_drv.h> 62#include <dev/drm2/i915/intel_ringbuffer.h> 63#include <sys/resourcevar.h> 64#include <sys/sched.h> 65#include <sys/sf_buf.h> 66 67#include <vm/vm.h> 68#include <vm/vm_pageout.h> 69 70static void i915_gem_object_flush_cpu_write_domain( 71 struct drm_i915_gem_object *obj); 72static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 73 int tiling_mode); 74static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 75 uint32_t size, int tiling_mode); 76static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 77 unsigned alignment, bool map_and_fenceable); 78static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 79 int flags); 80static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 81static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 82 bool write); 83static void i915_gem_object_set_to_full_cpu_read_domain( 84 struct drm_i915_gem_object *obj); 85static int i915_gem_object_set_cpu_read_domain_range( 86 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 87static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 88static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 89static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 90static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 91static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 92static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 93static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 94 uint32_t flush_domains); 95static void i915_gem_clear_fence_reg(struct drm_device *dev, 96 struct drm_i915_fence_reg *reg); 97static void i915_gem_reset_fences(struct drm_device *dev); 98static void i915_gem_retire_task_handler(void *arg, int pending); 99static int i915_gem_phys_pwrite(struct drm_device *dev, 100 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 101 uint64_t size, struct drm_file *file_priv); 102static void i915_gem_lowmem(void *arg); 103 104MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 105long i915_gem_wired_pages_cnt; 106 107static void 108i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 109{ 110 111 dev_priv->mm.object_count++; 112 dev_priv->mm.object_memory += size; 113} 114 115static void 116i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 117{ 118 119 dev_priv->mm.object_count--; 120 dev_priv->mm.object_memory -= size; 121} 122 123static int 124i915_gem_wait_for_error(struct drm_device *dev) 125{ 126 struct drm_i915_private *dev_priv; 127 int ret; 128 129 dev_priv = dev->dev_private; 130 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 131 return (0); 132 133 mtx_lock(&dev_priv->error_completion_lock); 134 while (dev_priv->error_completion == 0) { 135 ret = -msleep(&dev_priv->error_completion, 136 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 137 if (ret != 0) { 138 mtx_unlock(&dev_priv->error_completion_lock); 139 return (ret); 140 } 141 } 142 mtx_unlock(&dev_priv->error_completion_lock); 143 144 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 145 mtx_lock(&dev_priv->error_completion_lock); 146 dev_priv->error_completion++; 147 mtx_unlock(&dev_priv->error_completion_lock); 148 } 149 return (0); 150} 151 152int 153i915_mutex_lock_interruptible(struct drm_device *dev) 154{ 155 struct drm_i915_private *dev_priv; 156 int ret; 157 158 dev_priv = dev->dev_private; 159 ret = i915_gem_wait_for_error(dev); 160 if (ret != 0) 161 return (ret); 162 163 /* 164 * interruptible shall it be. might indeed be if dev_lock is 165 * changed to sx 166 */ 167 ret = sx_xlock_sig(&dev->dev_struct_lock); 168 if (ret != 0) 169 return (-ret); 170 171 return (0); 172} 173 174 175static void 176i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 177{ 178 struct drm_device *dev; 179 drm_i915_private_t *dev_priv; 180 int ret; 181 182 dev = obj->base.dev; 183 dev_priv = dev->dev_private; 184 185 ret = i915_gem_object_unbind(obj); 186 if (ret == -ERESTART) { 187 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 188 return; 189 } 190 191 CTR1(KTR_DRM, "object_destroy_tail %p", obj); 192 drm_gem_free_mmap_offset(&obj->base); 193 drm_gem_object_release(&obj->base); 194 i915_gem_info_remove_obj(dev_priv, obj->base.size); 195 196 free(obj->page_cpu_valid, DRM_I915_GEM); 197 free(obj->bit_17, DRM_I915_GEM); 198 free(obj, DRM_I915_GEM); 199} 200 201void 202i915_gem_free_object(struct drm_gem_object *gem_obj) 203{ 204 struct drm_i915_gem_object *obj; 205 struct drm_device *dev; 206 207 obj = to_intel_bo(gem_obj); 208 dev = obj->base.dev; 209 210 while (obj->pin_count > 0) 211 i915_gem_object_unpin(obj); 212 213 if (obj->phys_obj != NULL) 214 i915_gem_detach_phys_object(dev, obj); 215 216 i915_gem_free_object_tail(obj); 217} 218 219static void 220init_ring_lists(struct intel_ring_buffer *ring) 221{ 222 223 INIT_LIST_HEAD(&ring->active_list); 224 INIT_LIST_HEAD(&ring->request_list); 225 INIT_LIST_HEAD(&ring->gpu_write_list); 226} 227 228void 229i915_gem_load(struct drm_device *dev) 230{ 231 drm_i915_private_t *dev_priv; 232 int i; 233 234 dev_priv = dev->dev_private; 235 236 INIT_LIST_HEAD(&dev_priv->mm.active_list); 237 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 238 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 239 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 240 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 241 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 242 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 243 for (i = 0; i < I915_NUM_RINGS; i++) 244 init_ring_lists(&dev_priv->rings[i]); 245 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 246 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 247 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 248 i915_gem_retire_task_handler, dev_priv); 249 dev_priv->error_completion = 0; 250 251 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 252 if (IS_GEN3(dev)) { 253 u32 tmp = I915_READ(MI_ARB_STATE); 254 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { 255 /* 256 * arb state is a masked write, so set bit + 257 * bit in mask. 258 */ 259 tmp = MI_ARB_C3_LP_WRITE_ENABLE | 260 (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); 261 I915_WRITE(MI_ARB_STATE, tmp); 262 } 263 } 264 265 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 266 267 /* Old X drivers will take 0-2 for front, back, depth buffers */ 268 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 269 dev_priv->fence_reg_start = 3; 270 271 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 272 IS_G33(dev)) 273 dev_priv->num_fence_regs = 16; 274 else 275 dev_priv->num_fence_regs = 8; 276 277 /* Initialize fence registers to zero */ 278 for (i = 0; i < dev_priv->num_fence_regs; i++) { 279 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); 280 } 281 i915_gem_detect_bit_6_swizzle(dev); 282 dev_priv->mm.interruptible = true; 283 284 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 285 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 286} 287 288int 289i915_gem_do_init(struct drm_device *dev, unsigned long start, 290 unsigned long mappable_end, unsigned long end) 291{ 292 drm_i915_private_t *dev_priv; 293 unsigned long mappable; 294 int error; 295 296 dev_priv = dev->dev_private; 297 mappable = min(end, mappable_end) - start; 298 299 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 300 301 dev_priv->mm.gtt_start = start; 302 dev_priv->mm.gtt_mappable_end = mappable_end; 303 dev_priv->mm.gtt_end = end; 304 dev_priv->mm.gtt_total = end - start; 305 dev_priv->mm.mappable_gtt_total = mappable; 306 307 /* Take over this portion of the GTT */ 308 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 309 device_printf(dev->device, 310 "taking over the fictitious range 0x%lx-0x%lx\n", 311 dev->agp->base + start, dev->agp->base + start + mappable); 312 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 313 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 314 return (error); 315} 316 317int 318i915_gem_init_ioctl(struct drm_device *dev, void *data, 319 struct drm_file *file) 320{ 321 struct drm_i915_gem_init *args; 322 drm_i915_private_t *dev_priv; 323 324 dev_priv = dev->dev_private; 325 args = data; 326 327 if (args->gtt_start >= args->gtt_end || 328 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 329 return (-EINVAL); 330 331 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) 332 return (-EBUSY); 333 /* 334 * XXXKIB. The second-time initialization should be guarded 335 * against. 336 */ 337 return (i915_gem_do_init(dev, args->gtt_start, args->gtt_end, 338 args->gtt_end)); 339} 340 341int 342i915_gem_idle(struct drm_device *dev) 343{ 344 drm_i915_private_t *dev_priv; 345 int ret; 346 347 dev_priv = dev->dev_private; 348 if (dev_priv->mm.suspended) 349 return (0); 350 351 ret = i915_gpu_idle(dev, true); 352 if (ret != 0) 353 return (ret); 354 355 /* Under UMS, be paranoid and evict. */ 356 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 357 ret = i915_gem_evict_inactive(dev, false); 358 if (ret != 0) 359 return ret; 360 } 361 362 i915_gem_reset_fences(dev); 363 364 /* Hack! Don't let anybody do execbuf while we don't control the chip. 365 * We need to replace this with a semaphore, or something. 366 * And not confound mm.suspended! 367 */ 368 dev_priv->mm.suspended = 1; 369 callout_stop(&dev_priv->hangcheck_timer); 370 371 i915_kernel_lost_context(dev); 372 i915_gem_cleanup_ringbuffer(dev); 373 374 /* Cancel the retire work handler, which should be idle now. */ 375 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 376 return (ret); 377} 378 379void 380i915_gem_init_swizzling(struct drm_device *dev) 381{ 382 drm_i915_private_t *dev_priv; 383 384 dev_priv = dev->dev_private; 385 386 if (INTEL_INFO(dev)->gen < 5 || 387 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 388 return; 389 390 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 391 DISP_TILE_SURFACE_SWIZZLING); 392 393 if (IS_GEN5(dev)) 394 return; 395 396 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 397 if (IS_GEN6(dev)) 398 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); 399 else 400 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); 401} 402 403void 404i915_gem_init_ppgtt(struct drm_device *dev) 405{ 406 drm_i915_private_t *dev_priv; 407 struct i915_hw_ppgtt *ppgtt; 408 uint32_t pd_offset, pd_entry; 409 vm_paddr_t pt_addr; 410 struct intel_ring_buffer *ring; 411 u_int first_pd_entry_in_global_pt, i; 412 413 dev_priv = dev->dev_private; 414 ppgtt = dev_priv->mm.aliasing_ppgtt; 415 if (ppgtt == NULL) 416 return; 417 418 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 419 for (i = 0; i < ppgtt->num_pd_entries; i++) { 420 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 421 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 422 pd_entry |= GEN6_PDE_VALID; 423 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 424 } 425 intel_gtt_read_pte(first_pd_entry_in_global_pt); 426 427 pd_offset = ppgtt->pd_offset; 428 pd_offset /= 64; /* in cachelines, */ 429 pd_offset <<= 16; 430 431 if (INTEL_INFO(dev)->gen == 6) { 432 uint32_t ecochk = I915_READ(GAM_ECOCHK); 433 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 434 ECOCHK_PPGTT_CACHE64B); 435 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 436 } else if (INTEL_INFO(dev)->gen >= 7) { 437 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 438 /* GFX_MODE is per-ring on gen7+ */ 439 } 440 441 for (i = 0; i < I915_NUM_RINGS; i++) { 442 ring = &dev_priv->rings[i]; 443 444 if (INTEL_INFO(dev)->gen >= 7) 445 I915_WRITE(RING_MODE_GEN7(ring), 446 GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 447 448 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 449 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 450 } 451} 452 453int 454i915_gem_init_hw(struct drm_device *dev) 455{ 456 drm_i915_private_t *dev_priv; 457 int ret; 458 459 dev_priv = dev->dev_private; 460 461 i915_gem_init_swizzling(dev); 462 463 ret = intel_init_render_ring_buffer(dev); 464 if (ret != 0) 465 return (ret); 466 467 if (HAS_BSD(dev)) { 468 ret = intel_init_bsd_ring_buffer(dev); 469 if (ret != 0) 470 goto cleanup_render_ring; 471 } 472 473 if (HAS_BLT(dev)) { 474 ret = intel_init_blt_ring_buffer(dev); 475 if (ret != 0) 476 goto cleanup_bsd_ring; 477 } 478 479 dev_priv->next_seqno = 1; 480 i915_gem_init_ppgtt(dev); 481 return (0); 482 483cleanup_bsd_ring: 484 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 485cleanup_render_ring: 486 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 487 return (ret); 488} 489 490int 491i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 492 struct drm_file *file) 493{ 494 struct drm_i915_private *dev_priv; 495 struct drm_i915_gem_get_aperture *args; 496 struct drm_i915_gem_object *obj; 497 size_t pinned; 498 499 dev_priv = dev->dev_private; 500 args = data; 501 502 if (!(dev->driver->driver_features & DRIVER_GEM)) 503 return (-ENODEV); 504 505 pinned = 0; 506 DRM_LOCK(dev); 507 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 508 pinned += obj->gtt_space->size; 509 DRM_UNLOCK(dev); 510 511 args->aper_size = dev_priv->mm.gtt_total; 512 args->aper_available_size = args->aper_size - pinned; 513 514 return (0); 515} 516 517int 518i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 519 bool map_and_fenceable) 520{ 521 struct drm_device *dev; 522 struct drm_i915_private *dev_priv; 523 int ret; 524 525 dev = obj->base.dev; 526 dev_priv = dev->dev_private; 527 528 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 529 ("Max pin count")); 530 531 if (obj->gtt_space != NULL) { 532 if ((alignment && obj->gtt_offset & (alignment - 1)) || 533 (map_and_fenceable && !obj->map_and_fenceable)) { 534 DRM_DEBUG("bo is already pinned with incorrect alignment:" 535 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 536 " obj->map_and_fenceable=%d\n", 537 obj->gtt_offset, alignment, 538 map_and_fenceable, 539 obj->map_and_fenceable); 540 ret = i915_gem_object_unbind(obj); 541 if (ret != 0) 542 return (ret); 543 } 544 } 545 546 if (obj->gtt_space == NULL) { 547 ret = i915_gem_object_bind_to_gtt(obj, alignment, 548 map_and_fenceable); 549 if (ret) 550 return (ret); 551 } 552 553 if (obj->pin_count++ == 0 && !obj->active) 554 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 555 obj->pin_mappable |= map_and_fenceable; 556 557#if 1 558 KIB_NOTYET(); 559#else 560 WARN_ON(i915_verify_lists(dev)); 561#endif 562 return (0); 563} 564 565void 566i915_gem_object_unpin(struct drm_i915_gem_object *obj) 567{ 568 struct drm_device *dev; 569 drm_i915_private_t *dev_priv; 570 571 dev = obj->base.dev; 572 dev_priv = dev->dev_private; 573 574#if 1 575 KIB_NOTYET(); 576#else 577 WARN_ON(i915_verify_lists(dev)); 578#endif 579 580 KASSERT(obj->pin_count != 0, ("zero pin count")); 581 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 582 583 if (--obj->pin_count == 0) { 584 if (!obj->active) 585 list_move_tail(&obj->mm_list, 586 &dev_priv->mm.inactive_list); 587 obj->pin_mappable = false; 588 } 589#if 1 590 KIB_NOTYET(); 591#else 592 WARN_ON(i915_verify_lists(dev)); 593#endif 594} 595 596int 597i915_gem_pin_ioctl(struct drm_device *dev, void *data, 598 struct drm_file *file) 599{ 600 struct drm_i915_gem_pin *args; 601 struct drm_i915_gem_object *obj; 602 struct drm_gem_object *gobj; 603 int ret; 604 605 args = data; 606 607 ret = i915_mutex_lock_interruptible(dev); 608 if (ret != 0) 609 return ret; 610 611 gobj = drm_gem_object_lookup(dev, file, args->handle); 612 if (gobj == NULL) { 613 ret = -ENOENT; 614 goto unlock; 615 } 616 obj = to_intel_bo(gobj); 617 618 if (obj->madv != I915_MADV_WILLNEED) { 619 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 620 ret = -EINVAL; 621 goto out; 622 } 623 624 if (obj->pin_filp != NULL && obj->pin_filp != file) { 625 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 626 args->handle); 627 ret = -EINVAL; 628 goto out; 629 } 630 631 obj->user_pin_count++; 632 obj->pin_filp = file; 633 if (obj->user_pin_count == 1) { 634 ret = i915_gem_object_pin(obj, args->alignment, true); 635 if (ret != 0) 636 goto out; 637 } 638 639 /* XXX - flush the CPU caches for pinned objects 640 * as the X server doesn't manage domains yet 641 */ 642 i915_gem_object_flush_cpu_write_domain(obj); 643 args->offset = obj->gtt_offset; 644out: 645 drm_gem_object_unreference(&obj->base); 646unlock: 647 DRM_UNLOCK(dev); 648 return (ret); 649} 650 651int 652i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 653 struct drm_file *file) 654{ 655 struct drm_i915_gem_pin *args; 656 struct drm_i915_gem_object *obj; 657 int ret; 658 659 args = data; 660 ret = i915_mutex_lock_interruptible(dev); 661 if (ret != 0) 662 return (ret); 663 664 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 665 if (&obj->base == NULL) { 666 ret = -ENOENT; 667 goto unlock; 668 } 669 670 if (obj->pin_filp != file) { 671 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 672 args->handle); 673 ret = -EINVAL; 674 goto out; 675 } 676 obj->user_pin_count--; 677 if (obj->user_pin_count == 0) { 678 obj->pin_filp = NULL; 679 i915_gem_object_unpin(obj); 680 } 681 682out: 683 drm_gem_object_unreference(&obj->base); 684unlock: 685 DRM_UNLOCK(dev); 686 return (ret); 687} 688 689int 690i915_gem_busy_ioctl(struct drm_device *dev, void *data, 691 struct drm_file *file) 692{ 693 struct drm_i915_gem_busy *args; 694 struct drm_i915_gem_object *obj; 695 struct drm_i915_gem_request *request; 696 int ret; 697 698 args = data; 699 700 ret = i915_mutex_lock_interruptible(dev); 701 if (ret != 0) 702 return ret; 703 704 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 705 if (&obj->base == NULL) { 706 ret = -ENOENT; 707 goto unlock; 708 } 709 710 args->busy = obj->active; 711 if (args->busy) { 712 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 713 ret = i915_gem_flush_ring(obj->ring, 714 0, obj->base.write_domain); 715 } else if (obj->ring->outstanding_lazy_request == 716 obj->last_rendering_seqno) { 717 request = malloc(sizeof(*request), DRM_I915_GEM, 718 M_WAITOK | M_ZERO); 719 ret = i915_add_request(obj->ring, NULL, request); 720 if (ret != 0) 721 free(request, DRM_I915_GEM); 722 } 723 724 i915_gem_retire_requests_ring(obj->ring); 725 args->busy = obj->active; 726 } 727 728 drm_gem_object_unreference(&obj->base); 729unlock: 730 DRM_UNLOCK(dev); 731 return (ret); 732} 733 734static int 735i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 736{ 737 struct drm_i915_private *dev_priv; 738 struct drm_i915_file_private *file_priv; 739 unsigned long recent_enough; 740 struct drm_i915_gem_request *request; 741 struct intel_ring_buffer *ring; 742 u32 seqno; 743 int ret; 744 745 dev_priv = dev->dev_private; 746 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 747 return (-EIO); 748 749 file_priv = file->driver_priv; 750 recent_enough = ticks - (20 * hz / 1000); 751 ring = NULL; 752 seqno = 0; 753 754 mtx_lock(&file_priv->mm.lck); 755 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 756 if (time_after_eq(request->emitted_jiffies, recent_enough)) 757 break; 758 ring = request->ring; 759 seqno = request->seqno; 760 } 761 mtx_unlock(&file_priv->mm.lck); 762 if (seqno == 0) 763 return (0); 764 765 ret = 0; 766 mtx_lock(&ring->irq_lock); 767 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 768 if (ring->irq_get(ring)) { 769 while (ret == 0 && 770 !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 771 atomic_load_acq_int(&dev_priv->mm.wedged))) 772 ret = -msleep(ring, &ring->irq_lock, PCATCH, 773 "915thr", 0); 774 ring->irq_put(ring); 775 if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged)) 776 ret = -EIO; 777 } else if (_intel_wait_for(dev, 778 i915_seqno_passed(ring->get_seqno(ring), seqno) || 779 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 780 ret = -EBUSY; 781 } 782 } 783 mtx_unlock(&ring->irq_lock); 784 785 if (ret == 0) 786 taskqueue_enqueue_timeout(dev_priv->tq, 787 &dev_priv->mm.retire_task, 0); 788 789 return (ret); 790} 791 792int 793i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 794 struct drm_file *file_priv) 795{ 796 797 return (i915_gem_ring_throttle(dev, file_priv)); 798} 799 800int 801i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 802 struct drm_file *file_priv) 803{ 804 struct drm_i915_gem_madvise *args; 805 struct drm_i915_gem_object *obj; 806 int ret; 807 808 args = data; 809 switch (args->madv) { 810 case I915_MADV_DONTNEED: 811 case I915_MADV_WILLNEED: 812 break; 813 default: 814 return (-EINVAL); 815 } 816 817 ret = i915_mutex_lock_interruptible(dev); 818 if (ret != 0) 819 return (ret); 820 821 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 822 if (&obj->base == NULL) { 823 ret = -ENOENT; 824 goto unlock; 825 } 826 827 if (obj->pin_count != 0) { 828 ret = -EINVAL; 829 goto out; 830 } 831 832 if (obj->madv != I915_MADV_PURGED_INTERNAL) 833 obj->madv = args->madv; 834 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 835 i915_gem_object_truncate(obj); 836 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 837 838out: 839 drm_gem_object_unreference(&obj->base); 840unlock: 841 DRM_UNLOCK(dev); 842 return (ret); 843} 844 845void 846i915_gem_cleanup_ringbuffer(struct drm_device *dev) 847{ 848 drm_i915_private_t *dev_priv; 849 int i; 850 851 dev_priv = dev->dev_private; 852 for (i = 0; i < I915_NUM_RINGS; i++) 853 intel_cleanup_ring_buffer(&dev_priv->rings[i]); 854} 855 856int 857i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 858 struct drm_file *file_priv) 859{ 860 drm_i915_private_t *dev_priv; 861 int ret, i; 862 863 if (drm_core_check_feature(dev, DRIVER_MODESET)) 864 return (0); 865 dev_priv = dev->dev_private; 866 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 867 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 868 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 869 } 870 871 dev_priv->mm.suspended = 0; 872 873 ret = i915_gem_init_hw(dev); 874 if (ret != 0) { 875 return (ret); 876 } 877 878 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 879 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 880 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 881 for (i = 0; i < I915_NUM_RINGS; i++) { 882 KASSERT(list_empty(&dev_priv->rings[i].active_list), 883 ("ring %d active list", i)); 884 KASSERT(list_empty(&dev_priv->rings[i].request_list), 885 ("ring %d request list", i)); 886 } 887 888 DRM_UNLOCK(dev); 889 ret = drm_irq_install(dev); 890 DRM_LOCK(dev); 891 if (ret) 892 goto cleanup_ringbuffer; 893 894 return (0); 895 896cleanup_ringbuffer: 897 i915_gem_cleanup_ringbuffer(dev); 898 dev_priv->mm.suspended = 1; 899 900 return (ret); 901} 902 903int 904i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 905 struct drm_file *file_priv) 906{ 907 908 if (drm_core_check_feature(dev, DRIVER_MODESET)) 909 return 0; 910 911 drm_irq_uninstall(dev); 912 return (i915_gem_idle(dev)); 913} 914 915int 916i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 917 uint32_t *handle_p) 918{ 919 struct drm_i915_gem_object *obj; 920 uint32_t handle; 921 int ret; 922 923 size = roundup(size, PAGE_SIZE); 924 if (size == 0) 925 return (-EINVAL); 926 927 obj = i915_gem_alloc_object(dev, size); 928 if (obj == NULL) 929 return (-ENOMEM); 930 931 handle = 0; 932 ret = drm_gem_handle_create(file, &obj->base, &handle); 933 if (ret != 0) { 934 drm_gem_object_release(&obj->base); 935 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 936 free(obj, DRM_I915_GEM); 937 return (-ret); 938 } 939 940 /* drop reference from allocate - handle holds it now */ 941 drm_gem_object_unreference(&obj->base); 942 CTR2(KTR_DRM, "object_create %p %x", obj, size); 943 *handle_p = handle; 944 return (0); 945} 946 947int 948i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 949 struct drm_mode_create_dumb *args) 950{ 951 952 /* have to work out size/pitch and return them */ 953 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 954 args->size = args->pitch * args->height; 955 return (i915_gem_create(file, dev, args->size, &args->handle)); 956} 957 958int 959i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 960 uint32_t handle) 961{ 962 963 return (drm_gem_handle_delete(file, handle)); 964} 965 966int 967i915_gem_create_ioctl(struct drm_device *dev, void *data, 968 struct drm_file *file) 969{ 970 struct drm_i915_gem_create *args = data; 971 972 return (i915_gem_create(file, dev, args->size, &args->handle)); 973} 974 975static int 976i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 977 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 978 struct drm_file *file) 979{ 980 vm_object_t vm_obj; 981 vm_page_t m; 982 struct sf_buf *sf; 983 vm_offset_t mkva; 984 vm_pindex_t obj_pi; 985 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 986 987 if (obj->gtt_offset != 0 && rw == UIO_READ) 988 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 989 else 990 do_bit17_swizzling = 0; 991 992 obj->dirty = 1; 993 vm_obj = obj->base.vm_obj; 994 ret = 0; 995 996 VM_OBJECT_WLOCK(vm_obj); 997 vm_object_pip_add(vm_obj, 1); 998 while (size > 0) { 999 obj_pi = OFF_TO_IDX(offset); 1000 obj_po = offset & PAGE_MASK; 1001 1002 m = i915_gem_wire_page(vm_obj, obj_pi); 1003 VM_OBJECT_WUNLOCK(vm_obj); 1004 1005 sched_pin(); 1006 sf = sf_buf_alloc(m, SFB_CPUPRIVATE); 1007 mkva = sf_buf_kva(sf); 1008 length = min(size, PAGE_SIZE - obj_po); 1009 while (length > 0) { 1010 if (do_bit17_swizzling && 1011 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 1012 cnt = roundup2(obj_po + 1, 64); 1013 cnt = min(cnt - obj_po, length); 1014 swizzled_po = obj_po ^ 64; 1015 } else { 1016 cnt = length; 1017 swizzled_po = obj_po; 1018 } 1019 if (rw == UIO_READ) 1020 ret = -copyout_nofault( 1021 (char *)mkva + swizzled_po, 1022 (void *)(uintptr_t)data_ptr, cnt); 1023 else 1024 ret = -copyin_nofault( 1025 (void *)(uintptr_t)data_ptr, 1026 (char *)mkva + swizzled_po, cnt); 1027 if (ret != 0) 1028 break; 1029 data_ptr += cnt; 1030 size -= cnt; 1031 length -= cnt; 1032 offset += cnt; 1033 obj_po += cnt; 1034 } 1035 sf_buf_free(sf); 1036 sched_unpin(); 1037 VM_OBJECT_WLOCK(vm_obj); 1038 if (rw == UIO_WRITE) 1039 vm_page_dirty(m); 1040 vm_page_reference(m); 1041 vm_page_lock(m); 1042 vm_page_unwire(m, PQ_ACTIVE); 1043 vm_page_unlock(m); 1044 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1045 1046 if (ret != 0) 1047 break; 1048 } 1049 vm_object_pip_wakeup(vm_obj); 1050 VM_OBJECT_WUNLOCK(vm_obj); 1051 1052 return (ret); 1053} 1054 1055static int 1056i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 1057 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 1058{ 1059 vm_offset_t mkva; 1060 vm_pindex_t obj_pi; 1061 int obj_po, ret; 1062 1063 obj_pi = OFF_TO_IDX(offset); 1064 obj_po = offset & PAGE_MASK; 1065 1066 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 1067 IDX_TO_OFF(obj_pi), size, PAT_WRITE_COMBINING); 1068 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva + 1069 obj_po, size); 1070 pmap_unmapdev(mkva, size); 1071 return (ret); 1072} 1073 1074static int 1075i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 1076 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 1077{ 1078 struct drm_i915_gem_object *obj; 1079 vm_page_t *ma; 1080 vm_offset_t start, end; 1081 int npages, ret; 1082 1083 if (size == 0) 1084 return (0); 1085 start = trunc_page(data_ptr); 1086 end = round_page(data_ptr + size); 1087 npages = howmany(end - start, PAGE_SIZE); 1088 ma = malloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 1089 M_ZERO); 1090 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 1091 (vm_offset_t)data_ptr, size, 1092 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 1093 if (npages == -1) { 1094 ret = -EFAULT; 1095 goto free_ma; 1096 } 1097 1098 ret = i915_mutex_lock_interruptible(dev); 1099 if (ret != 0) 1100 goto unlocked; 1101 1102 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1103 if (&obj->base == NULL) { 1104 ret = -ENOENT; 1105 goto unlock; 1106 } 1107 if (offset > obj->base.size || size > obj->base.size - offset) { 1108 ret = -EINVAL; 1109 goto out; 1110 } 1111 1112 if (rw == UIO_READ) { 1113 CTR3(KTR_DRM, "object_pread %p %jx %jx", obj, offset, size); 1114 ret = i915_gem_object_set_cpu_read_domain_range(obj, 1115 offset, size); 1116 if (ret != 0) 1117 goto out; 1118 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1119 UIO_READ, file); 1120 } else { 1121 if (obj->phys_obj) { 1122 CTR3(KTR_DRM, "object_phys_write %p %jx %jx", obj, 1123 offset, size); 1124 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1125 size, file); 1126 } else if (obj->gtt_space && 1127 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1128 CTR3(KTR_DRM, "object_gtt_write %p %jx %jx", obj, 1129 offset, size); 1130 ret = i915_gem_object_pin(obj, 0, true); 1131 if (ret != 0) 1132 goto out; 1133 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1134 if (ret != 0) 1135 goto out_unpin; 1136 ret = i915_gem_object_put_fence(obj); 1137 if (ret != 0) 1138 goto out_unpin; 1139 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1140 offset, file); 1141out_unpin: 1142 i915_gem_object_unpin(obj); 1143 } else { 1144 CTR3(KTR_DRM, "object_pwrite %p %jx %jx", obj, 1145 offset, size); 1146 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1147 if (ret != 0) 1148 goto out; 1149 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1150 UIO_WRITE, file); 1151 } 1152 } 1153out: 1154 drm_gem_object_unreference(&obj->base); 1155unlock: 1156 DRM_UNLOCK(dev); 1157unlocked: 1158 vm_page_unhold_pages(ma, npages); 1159free_ma: 1160 free(ma, DRM_I915_GEM); 1161 return (ret); 1162} 1163 1164int 1165i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1166{ 1167 struct drm_i915_gem_pread *args; 1168 1169 args = data; 1170 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1171 args->offset, UIO_READ, file)); 1172} 1173 1174int 1175i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1176{ 1177 struct drm_i915_gem_pwrite *args; 1178 1179 args = data; 1180 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1181 args->offset, UIO_WRITE, file)); 1182} 1183 1184int 1185i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1186 struct drm_file *file) 1187{ 1188 struct drm_i915_gem_set_domain *args; 1189 struct drm_i915_gem_object *obj; 1190 uint32_t read_domains; 1191 uint32_t write_domain; 1192 int ret; 1193 1194 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1195 return (-ENODEV); 1196 1197 args = data; 1198 read_domains = args->read_domains; 1199 write_domain = args->write_domain; 1200 1201 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1202 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1203 (write_domain != 0 && read_domains != write_domain)) 1204 return (-EINVAL); 1205 1206 ret = i915_mutex_lock_interruptible(dev); 1207 if (ret != 0) 1208 return (ret); 1209 1210 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1211 if (&obj->base == NULL) { 1212 ret = -ENOENT; 1213 goto unlock; 1214 } 1215 1216 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1217 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1218 if (ret == -EINVAL) 1219 ret = 0; 1220 } else 1221 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1222 1223 drm_gem_object_unreference(&obj->base); 1224unlock: 1225 DRM_UNLOCK(dev); 1226 return (ret); 1227} 1228 1229int 1230i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1231 struct drm_file *file) 1232{ 1233 struct drm_i915_gem_sw_finish *args; 1234 struct drm_i915_gem_object *obj; 1235 int ret; 1236 1237 args = data; 1238 ret = 0; 1239 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1240 return (ENODEV); 1241 ret = i915_mutex_lock_interruptible(dev); 1242 if (ret != 0) 1243 return (ret); 1244 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1245 if (&obj->base == NULL) { 1246 ret = -ENOENT; 1247 goto unlock; 1248 } 1249 if (obj->pin_count != 0) 1250 i915_gem_object_flush_cpu_write_domain(obj); 1251 drm_gem_object_unreference(&obj->base); 1252unlock: 1253 DRM_UNLOCK(dev); 1254 return (ret); 1255} 1256 1257int 1258i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1259 struct drm_file *file) 1260{ 1261 struct drm_i915_gem_mmap *args; 1262 struct drm_gem_object *obj; 1263 struct proc *p; 1264 vm_map_t map; 1265 vm_offset_t addr; 1266 vm_size_t size; 1267 int error, rv; 1268 1269 args = data; 1270 1271 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1272 return (-ENODEV); 1273 1274 obj = drm_gem_object_lookup(dev, file, args->handle); 1275 if (obj == NULL) 1276 return (-ENOENT); 1277 error = 0; 1278 if (args->size == 0) 1279 goto out; 1280 p = curproc; 1281 map = &p->p_vmspace->vm_map; 1282 size = round_page(args->size); 1283 PROC_LOCK(p); 1284 if (map->size + size > lim_cur(p, RLIMIT_VMEM)) { 1285 PROC_UNLOCK(p); 1286 error = ENOMEM; 1287 goto out; 1288 } 1289 PROC_UNLOCK(p); 1290 1291 addr = 0; 1292 vm_object_reference(obj->vm_obj); 1293 DRM_UNLOCK(dev); 1294 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1295 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1296 VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1297 if (rv != KERN_SUCCESS) { 1298 vm_object_deallocate(obj->vm_obj); 1299 error = -vm_mmap_to_errno(rv); 1300 } else { 1301 args->addr_ptr = (uint64_t)addr; 1302 } 1303 DRM_LOCK(dev); 1304out: 1305 drm_gem_object_unreference(obj); 1306 return (error); 1307} 1308 1309static int 1310i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1311 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1312{ 1313 1314 *color = 0; /* XXXKIB */ 1315 return (0); 1316} 1317 1318int i915_intr_pf; 1319 1320static int 1321i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1322 vm_page_t *mres) 1323{ 1324 struct drm_gem_object *gem_obj; 1325 struct drm_i915_gem_object *obj; 1326 struct drm_device *dev; 1327 drm_i915_private_t *dev_priv; 1328 vm_page_t m, oldm; 1329 int cause, ret; 1330 bool write; 1331 1332 gem_obj = vm_obj->handle; 1333 obj = to_intel_bo(gem_obj); 1334 dev = obj->base.dev; 1335 dev_priv = dev->dev_private; 1336#if 0 1337 write = (prot & VM_PROT_WRITE) != 0; 1338#else 1339 write = true; 1340#endif 1341 vm_object_pip_add(vm_obj, 1); 1342 1343 /* 1344 * Remove the placeholder page inserted by vm_fault() from the 1345 * object before dropping the object lock. If 1346 * i915_gem_release_mmap() is active in parallel on this gem 1347 * object, then it owns the drm device sx and might find the 1348 * placeholder already. Then, since the page is busy, 1349 * i915_gem_release_mmap() sleeps waiting for the busy state 1350 * of the page cleared. We will be not able to acquire drm 1351 * device lock until i915_gem_release_mmap() is able to make a 1352 * progress. 1353 */ 1354 if (*mres != NULL) { 1355 oldm = *mres; 1356 vm_page_lock(oldm); 1357 vm_page_remove(oldm); 1358 vm_page_unlock(oldm); 1359 *mres = NULL; 1360 } else 1361 oldm = NULL; 1362 VM_OBJECT_WUNLOCK(vm_obj); 1363retry: 1364 cause = ret = 0; 1365 m = NULL; 1366 1367 if (i915_intr_pf) { 1368 ret = i915_mutex_lock_interruptible(dev); 1369 if (ret != 0) { 1370 cause = 10; 1371 goto out; 1372 } 1373 } else 1374 DRM_LOCK(dev); 1375 1376 /* 1377 * Since the object lock was dropped, other thread might have 1378 * faulted on the same GTT address and instantiated the 1379 * mapping for the page. Recheck. 1380 */ 1381 VM_OBJECT_WLOCK(vm_obj); 1382 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1383 if (m != NULL) { 1384 if (vm_page_busied(m)) { 1385 DRM_UNLOCK(dev); 1386 vm_page_lock(m); 1387 VM_OBJECT_WUNLOCK(vm_obj); 1388 vm_page_busy_sleep(m, "915pee"); 1389 goto retry; 1390 } 1391 goto have_page; 1392 } else 1393 VM_OBJECT_WUNLOCK(vm_obj); 1394 1395 /* Now bind it into the GTT if needed */ 1396 if (!obj->map_and_fenceable) { 1397 ret = i915_gem_object_unbind(obj); 1398 if (ret != 0) { 1399 cause = 20; 1400 goto unlock; 1401 } 1402 } 1403 if (!obj->gtt_space) { 1404 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1405 if (ret != 0) { 1406 cause = 30; 1407 goto unlock; 1408 } 1409 1410 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1411 if (ret != 0) { 1412 cause = 40; 1413 goto unlock; 1414 } 1415 } 1416 1417 if (obj->tiling_mode == I915_TILING_NONE) 1418 ret = i915_gem_object_put_fence(obj); 1419 else 1420 ret = i915_gem_object_get_fence(obj, NULL); 1421 if (ret != 0) { 1422 cause = 50; 1423 goto unlock; 1424 } 1425 1426 if (i915_gem_object_is_inactive(obj)) 1427 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1428 1429 obj->fault_mappable = true; 1430 VM_OBJECT_WLOCK(vm_obj); 1431 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 1432 offset); 1433 if (m == NULL) { 1434 VM_OBJECT_WUNLOCK(vm_obj); 1435 cause = 60; 1436 ret = -EFAULT; 1437 goto unlock; 1438 } 1439 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1440 ("not fictitious %p", m)); 1441 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1442 1443 if (vm_page_busied(m)) { 1444 DRM_UNLOCK(dev); 1445 vm_page_lock(m); 1446 VM_OBJECT_WUNLOCK(vm_obj); 1447 vm_page_busy_sleep(m, "915pbs"); 1448 goto retry; 1449 } 1450 if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { 1451 DRM_UNLOCK(dev); 1452 VM_OBJECT_WUNLOCK(vm_obj); 1453 VM_WAIT; 1454 goto retry; 1455 } 1456 m->valid = VM_PAGE_BITS_ALL; 1457have_page: 1458 *mres = m; 1459 vm_page_xbusy(m); 1460 1461 CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 1462 m->phys_addr); 1463 DRM_UNLOCK(dev); 1464 if (oldm != NULL) { 1465 vm_page_lock(oldm); 1466 vm_page_free(oldm); 1467 vm_page_unlock(oldm); 1468 } 1469 vm_object_pip_wakeup(vm_obj); 1470 return (VM_PAGER_OK); 1471 1472unlock: 1473 DRM_UNLOCK(dev); 1474out: 1475 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1476 CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, 1477 -ret, cause); 1478 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1479 kern_yield(PRI_USER); 1480 goto retry; 1481 } 1482 VM_OBJECT_WLOCK(vm_obj); 1483 vm_object_pip_wakeup(vm_obj); 1484 return (VM_PAGER_ERROR); 1485} 1486 1487static void 1488i915_gem_pager_dtor(void *handle) 1489{ 1490 struct drm_gem_object *obj; 1491 struct drm_device *dev; 1492 1493 obj = handle; 1494 dev = obj->dev; 1495 1496 DRM_LOCK(dev); 1497 drm_gem_free_mmap_offset(obj); 1498 i915_gem_release_mmap(to_intel_bo(obj)); 1499 drm_gem_object_unreference(obj); 1500 DRM_UNLOCK(dev); 1501} 1502 1503struct cdev_pager_ops i915_gem_pager_ops = { 1504 .cdev_pg_fault = i915_gem_pager_fault, 1505 .cdev_pg_ctor = i915_gem_pager_ctor, 1506 .cdev_pg_dtor = i915_gem_pager_dtor 1507}; 1508 1509int 1510i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1511 uint32_t handle, uint64_t *offset) 1512{ 1513 struct drm_i915_private *dev_priv; 1514 struct drm_i915_gem_object *obj; 1515 int ret; 1516 1517 if (!(dev->driver->driver_features & DRIVER_GEM)) 1518 return (-ENODEV); 1519 1520 dev_priv = dev->dev_private; 1521 1522 ret = i915_mutex_lock_interruptible(dev); 1523 if (ret != 0) 1524 return (ret); 1525 1526 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1527 if (&obj->base == NULL) { 1528 ret = -ENOENT; 1529 goto unlock; 1530 } 1531 1532 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1533 ret = -E2BIG; 1534 goto out; 1535 } 1536 1537 if (obj->madv != I915_MADV_WILLNEED) { 1538 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1539 ret = -EINVAL; 1540 goto out; 1541 } 1542 1543 ret = drm_gem_create_mmap_offset(&obj->base); 1544 if (ret != 0) 1545 goto out; 1546 1547 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1548 DRM_GEM_MAPPING_KEY; 1549out: 1550 drm_gem_object_unreference(&obj->base); 1551unlock: 1552 DRM_UNLOCK(dev); 1553 return (ret); 1554} 1555 1556int 1557i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1558 struct drm_file *file) 1559{ 1560 struct drm_i915_private *dev_priv; 1561 struct drm_i915_gem_mmap_gtt *args; 1562 1563 dev_priv = dev->dev_private; 1564 args = data; 1565 1566 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1567} 1568 1569struct drm_i915_gem_object * 1570i915_gem_alloc_object(struct drm_device *dev, size_t size) 1571{ 1572 struct drm_i915_private *dev_priv; 1573 struct drm_i915_gem_object *obj; 1574 1575 dev_priv = dev->dev_private; 1576 1577 obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1578 1579 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1580 free(obj, DRM_I915_GEM); 1581 return (NULL); 1582 } 1583 1584 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1585 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1586 1587 if (HAS_LLC(dev)) 1588 obj->cache_level = I915_CACHE_LLC; 1589 else 1590 obj->cache_level = I915_CACHE_NONE; 1591 obj->base.driver_private = NULL; 1592 obj->fence_reg = I915_FENCE_REG_NONE; 1593 INIT_LIST_HEAD(&obj->mm_list); 1594 INIT_LIST_HEAD(&obj->gtt_list); 1595 INIT_LIST_HEAD(&obj->ring_list); 1596 INIT_LIST_HEAD(&obj->exec_list); 1597 INIT_LIST_HEAD(&obj->gpu_write_list); 1598 obj->madv = I915_MADV_WILLNEED; 1599 /* Avoid an unnecessary call to unbind on the first bind. */ 1600 obj->map_and_fenceable = true; 1601 1602 i915_gem_info_add_obj(dev_priv, size); 1603 1604 return (obj); 1605} 1606 1607void 1608i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1609{ 1610 1611 /* If we don't have a page list set up, then we're not pinned 1612 * to GPU, and we can ignore the cache flush because it'll happen 1613 * again at bind time. 1614 */ 1615 if (obj->pages == NULL) 1616 return; 1617 1618 /* If the GPU is snooping the contents of the CPU cache, 1619 * we do not need to manually clear the CPU cache lines. However, 1620 * the caches are only snooped when the render cache is 1621 * flushed/invalidated. As we always have to emit invalidations 1622 * and flushes when moving into and out of the RENDER domain, correct 1623 * snooping behaviour occurs naturally as the result of our domain 1624 * tracking. 1625 */ 1626 if (obj->cache_level != I915_CACHE_NONE) 1627 return; 1628 1629 CTR1(KTR_DRM, "object_clflush %p", obj); 1630 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1631} 1632 1633static void 1634i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1635{ 1636 uint32_t old_write_domain; 1637 1638 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1639 return; 1640 1641 i915_gem_clflush_object(obj); 1642 intel_gtt_chipset_flush(); 1643 old_write_domain = obj->base.write_domain; 1644 obj->base.write_domain = 0; 1645 1646 CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 1647 obj->base.read_domains, old_write_domain); 1648} 1649 1650static int 1651i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1652{ 1653 1654 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1655 return (0); 1656 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1657} 1658 1659static void 1660i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1661{ 1662 uint32_t old_write_domain; 1663 1664 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1665 return; 1666 1667 wmb(); 1668 1669 old_write_domain = obj->base.write_domain; 1670 obj->base.write_domain = 0; 1671 1672 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 1673 obj->base.read_domains, old_write_domain); 1674} 1675 1676int 1677i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1678{ 1679 uint32_t old_write_domain, old_read_domains; 1680 int ret; 1681 1682 if (obj->gtt_space == NULL) 1683 return (-EINVAL); 1684 1685 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1686 return 0; 1687 1688 ret = i915_gem_object_flush_gpu_write_domain(obj); 1689 if (ret != 0) 1690 return (ret); 1691 1692 if (obj->pending_gpu_write || write) { 1693 ret = i915_gem_object_wait_rendering(obj); 1694 if (ret != 0) 1695 return (ret); 1696 } 1697 1698 i915_gem_object_flush_cpu_write_domain(obj); 1699 1700 old_write_domain = obj->base.write_domain; 1701 old_read_domains = obj->base.read_domains; 1702 1703 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1704 ("In GTT write domain")); 1705 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1706 if (write) { 1707 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1708 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1709 obj->dirty = 1; 1710 } 1711 1712 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 1713 old_read_domains, old_write_domain); 1714 return (0); 1715} 1716 1717int 1718i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1719 enum i915_cache_level cache_level) 1720{ 1721 struct drm_device *dev; 1722 drm_i915_private_t *dev_priv; 1723 int ret; 1724 1725 if (obj->cache_level == cache_level) 1726 return 0; 1727 1728 if (obj->pin_count) { 1729 DRM_DEBUG("can not change the cache level of pinned objects\n"); 1730 return (-EBUSY); 1731 } 1732 1733 dev = obj->base.dev; 1734 dev_priv = dev->dev_private; 1735 if (obj->gtt_space) { 1736 ret = i915_gem_object_finish_gpu(obj); 1737 if (ret != 0) 1738 return (ret); 1739 1740 i915_gem_object_finish_gtt(obj); 1741 1742 /* Before SandyBridge, you could not use tiling or fence 1743 * registers with snooped memory, so relinquish any fences 1744 * currently pointing to our region in the aperture. 1745 */ 1746 if (INTEL_INFO(obj->base.dev)->gen < 6) { 1747 ret = i915_gem_object_put_fence(obj); 1748 if (ret != 0) 1749 return (ret); 1750 } 1751 1752 i915_gem_gtt_rebind_object(obj, cache_level); 1753 if (obj->has_aliasing_ppgtt_mapping) 1754 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1755 obj, cache_level); 1756 } 1757 1758 if (cache_level == I915_CACHE_NONE) { 1759 u32 old_read_domains, old_write_domain; 1760 1761 /* If we're coming from LLC cached, then we haven't 1762 * actually been tracking whether the data is in the 1763 * CPU cache or not, since we only allow one bit set 1764 * in obj->write_domain and have been skipping the clflushes. 1765 * Just set it to the CPU cache for now. 1766 */ 1767 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1768 ("obj %p in CPU write domain", obj)); 1769 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1770 ("obj %p in CPU read domain", obj)); 1771 1772 old_read_domains = obj->base.read_domains; 1773 old_write_domain = obj->base.write_domain; 1774 1775 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1776 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1777 1778 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 1779 obj, old_read_domains, old_write_domain); 1780 } 1781 1782 obj->cache_level = cache_level; 1783 return (0); 1784} 1785 1786int 1787i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1788 u32 alignment, struct intel_ring_buffer *pipelined) 1789{ 1790 u32 old_read_domains, old_write_domain; 1791 int ret; 1792 1793 ret = i915_gem_object_flush_gpu_write_domain(obj); 1794 if (ret != 0) 1795 return (ret); 1796 1797 if (pipelined != obj->ring) { 1798 ret = i915_gem_object_wait_rendering(obj); 1799 if (ret == -ERESTART || ret == -EINTR) 1800 return (ret); 1801 } 1802 1803 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1804 if (ret != 0) 1805 return (ret); 1806 1807 ret = i915_gem_object_pin(obj, alignment, true); 1808 if (ret != 0) 1809 return (ret); 1810 1811 i915_gem_object_flush_cpu_write_domain(obj); 1812 1813 old_write_domain = obj->base.write_domain; 1814 old_read_domains = obj->base.read_domains; 1815 1816 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1817 ("obj %p in GTT write domain", obj)); 1818 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1819 1820 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 1821 obj, old_read_domains, obj->base.write_domain); 1822 return (0); 1823} 1824 1825int 1826i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1827{ 1828 int ret; 1829 1830 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1831 return (0); 1832 1833 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1834 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1835 if (ret != 0) 1836 return (ret); 1837 } 1838 1839 ret = i915_gem_object_wait_rendering(obj); 1840 if (ret != 0) 1841 return (ret); 1842 1843 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1844 1845 return (0); 1846} 1847 1848static int 1849i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1850{ 1851 uint32_t old_write_domain, old_read_domains; 1852 int ret; 1853 1854 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1855 return 0; 1856 1857 ret = i915_gem_object_flush_gpu_write_domain(obj); 1858 if (ret != 0) 1859 return (ret); 1860 1861 ret = i915_gem_object_wait_rendering(obj); 1862 if (ret != 0) 1863 return (ret); 1864 1865 i915_gem_object_flush_gtt_write_domain(obj); 1866 i915_gem_object_set_to_full_cpu_read_domain(obj); 1867 1868 old_write_domain = obj->base.write_domain; 1869 old_read_domains = obj->base.read_domains; 1870 1871 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1872 i915_gem_clflush_object(obj); 1873 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1874 } 1875 1876 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1877 ("In cpu write domain")); 1878 1879 if (write) { 1880 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1881 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1882 } 1883 1884 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 1885 old_read_domains, old_write_domain); 1886 return (0); 1887} 1888 1889static void 1890i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1891{ 1892 int i; 1893 1894 if (obj->page_cpu_valid == NULL) 1895 return; 1896 1897 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1898 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1899 if (obj->page_cpu_valid[i] != 0) 1900 continue; 1901 drm_clflush_pages(obj->pages + i, 1); 1902 } 1903 } 1904 1905 free(obj->page_cpu_valid, DRM_I915_GEM); 1906 obj->page_cpu_valid = NULL; 1907} 1908 1909static int 1910i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1911 uint64_t offset, uint64_t size) 1912{ 1913 uint32_t old_read_domains; 1914 int i, ret; 1915 1916 if (offset == 0 && size == obj->base.size) 1917 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1918 1919 ret = i915_gem_object_flush_gpu_write_domain(obj); 1920 if (ret != 0) 1921 return (ret); 1922 ret = i915_gem_object_wait_rendering(obj); 1923 if (ret != 0) 1924 return (ret); 1925 1926 i915_gem_object_flush_gtt_write_domain(obj); 1927 1928 if (obj->page_cpu_valid == NULL && 1929 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1930 return (0); 1931 1932 if (obj->page_cpu_valid == NULL) { 1933 obj->page_cpu_valid = malloc(obj->base.size / PAGE_SIZE, 1934 DRM_I915_GEM, M_WAITOK | M_ZERO); 1935 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1936 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1937 1938 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1939 i++) { 1940 if (obj->page_cpu_valid[i]) 1941 continue; 1942 drm_clflush_pages(obj->pages + i, 1); 1943 obj->page_cpu_valid[i] = 1; 1944 } 1945 1946 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1947 ("In gpu write domain")); 1948 1949 old_read_domains = obj->base.read_domains; 1950 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1951 1952 CTR3(KTR_DRM, "object_change_domain set_cpu_read %p %x %x", obj, 1953 old_read_domains, obj->base.write_domain); 1954 return (0); 1955} 1956 1957static uint32_t 1958i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1959{ 1960 uint32_t gtt_size; 1961 1962 if (INTEL_INFO(dev)->gen >= 4 || 1963 tiling_mode == I915_TILING_NONE) 1964 return (size); 1965 1966 /* Previous chips need a power-of-two fence region when tiling */ 1967 if (INTEL_INFO(dev)->gen == 3) 1968 gtt_size = 1024*1024; 1969 else 1970 gtt_size = 512*1024; 1971 1972 while (gtt_size < size) 1973 gtt_size <<= 1; 1974 1975 return (gtt_size); 1976} 1977 1978/** 1979 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1980 * @obj: object to check 1981 * 1982 * Return the required GTT alignment for an object, taking into account 1983 * potential fence register mapping. 1984 */ 1985static uint32_t 1986i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1987 int tiling_mode) 1988{ 1989 1990 /* 1991 * Minimum alignment is 4k (GTT page size), but might be greater 1992 * if a fence register is needed for the object. 1993 */ 1994 if (INTEL_INFO(dev)->gen >= 4 || 1995 tiling_mode == I915_TILING_NONE) 1996 return (4096); 1997 1998 /* 1999 * Previous chips need to be aligned to the size of the smallest 2000 * fence register that can contain the object. 2001 */ 2002 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2003} 2004 2005uint32_t 2006i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 2007 int tiling_mode) 2008{ 2009 2010 if (tiling_mode == I915_TILING_NONE) 2011 return (4096); 2012 2013 /* 2014 * Minimum alignment is 4k (GTT page size) for sane hw. 2015 */ 2016 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 2017 return (4096); 2018 2019 /* 2020 * Previous hardware however needs to be aligned to a power-of-two 2021 * tile height. The simplest method for determining this is to reuse 2022 * the power-of-tile object size. 2023 */ 2024 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2025} 2026 2027static int 2028i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2029 unsigned alignment, bool map_and_fenceable) 2030{ 2031 struct drm_device *dev; 2032 struct drm_i915_private *dev_priv; 2033 struct drm_mm_node *free_space; 2034 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2035 bool mappable, fenceable; 2036 int ret; 2037 2038 dev = obj->base.dev; 2039 dev_priv = dev->dev_private; 2040 2041 if (obj->madv != I915_MADV_WILLNEED) { 2042 DRM_ERROR("Attempting to bind a purgeable object\n"); 2043 return (-EINVAL); 2044 } 2045 2046 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2047 obj->tiling_mode); 2048 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2049 obj->tiling_mode); 2050 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2051 obj->base.size, obj->tiling_mode); 2052 if (alignment == 0) 2053 alignment = map_and_fenceable ? fence_alignment : 2054 unfenced_alignment; 2055 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2056 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2057 return (-EINVAL); 2058 } 2059 2060 size = map_and_fenceable ? fence_size : obj->base.size; 2061 2062 /* If the object is bigger than the entire aperture, reject it early 2063 * before evicting everything in a vain attempt to find space. 2064 */ 2065 if (obj->base.size > (map_and_fenceable ? 2066 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2067 DRM_ERROR( 2068"Attempting to bind an object larger than the aperture\n"); 2069 return (-E2BIG); 2070 } 2071 2072 search_free: 2073 if (map_and_fenceable) 2074 free_space = drm_mm_search_free_in_range( 2075 &dev_priv->mm.gtt_space, size, alignment, 0, 2076 dev_priv->mm.gtt_mappable_end, 0); 2077 else 2078 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2079 size, alignment, 0); 2080 if (free_space != NULL) { 2081 if (map_and_fenceable) 2082 obj->gtt_space = drm_mm_get_block_range_generic( 2083 free_space, size, alignment, 0, 2084 dev_priv->mm.gtt_mappable_end, 1); 2085 else 2086 obj->gtt_space = drm_mm_get_block_generic(free_space, 2087 size, alignment, 1); 2088 } 2089 if (obj->gtt_space == NULL) { 2090 ret = i915_gem_evict_something(dev, size, alignment, 2091 map_and_fenceable); 2092 if (ret != 0) 2093 return (ret); 2094 goto search_free; 2095 } 2096 ret = i915_gem_object_get_pages_gtt(obj, 0); 2097 if (ret != 0) { 2098 drm_mm_put_block(obj->gtt_space); 2099 obj->gtt_space = NULL; 2100 /* 2101 * i915_gem_object_get_pages_gtt() cannot return 2102 * ENOMEM, since we use vm_page_grab(). 2103 */ 2104 return (ret); 2105 } 2106 2107 ret = i915_gem_gtt_bind_object(obj); 2108 if (ret != 0) { 2109 i915_gem_object_put_pages_gtt(obj); 2110 drm_mm_put_block(obj->gtt_space); 2111 obj->gtt_space = NULL; 2112 if (i915_gem_evict_everything(dev, false)) 2113 return (ret); 2114 goto search_free; 2115 } 2116 2117 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2118 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2119 2120 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2121 ("Object in gpu read domain")); 2122 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2123 ("Object in gpu write domain")); 2124 2125 obj->gtt_offset = obj->gtt_space->start; 2126 2127 fenceable = 2128 obj->gtt_space->size == fence_size && 2129 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2130 2131 mappable = 2132 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2133 obj->map_and_fenceable = mappable && fenceable; 2134 2135 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 2136 obj->base.size, map_and_fenceable); 2137 return (0); 2138} 2139 2140static void 2141i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2142{ 2143 u32 old_write_domain, old_read_domains; 2144 2145 /* Act a barrier for all accesses through the GTT */ 2146 mb(); 2147 2148 /* Force a pagefault for domain tracking on next user access */ 2149 i915_gem_release_mmap(obj); 2150 2151 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2152 return; 2153 2154 old_read_domains = obj->base.read_domains; 2155 old_write_domain = obj->base.write_domain; 2156 2157 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2158 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2159 2160 CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2161 obj, old_read_domains, old_write_domain); 2162} 2163 2164int 2165i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2166{ 2167 drm_i915_private_t *dev_priv; 2168 int ret; 2169 2170 dev_priv = obj->base.dev->dev_private; 2171 ret = 0; 2172 if (obj->gtt_space == NULL) 2173 return (0); 2174 if (obj->pin_count != 0) { 2175 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2176 return (-EINVAL); 2177 } 2178 2179 ret = i915_gem_object_finish_gpu(obj); 2180 if (ret == -ERESTART || ret == -EINTR) 2181 return (ret); 2182 2183 i915_gem_object_finish_gtt(obj); 2184 2185 if (ret == 0) 2186 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2187 if (ret == -ERESTART || ret == -EINTR) 2188 return (ret); 2189 if (ret != 0) { 2190 i915_gem_clflush_object(obj); 2191 obj->base.read_domains = obj->base.write_domain = 2192 I915_GEM_DOMAIN_CPU; 2193 } 2194 2195 ret = i915_gem_object_put_fence(obj); 2196 if (ret == -ERESTART) 2197 return (ret); 2198 2199 i915_gem_gtt_unbind_object(obj); 2200 if (obj->has_aliasing_ppgtt_mapping) { 2201 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2202 obj->has_aliasing_ppgtt_mapping = 0; 2203 } 2204 i915_gem_object_put_pages_gtt(obj); 2205 2206 list_del_init(&obj->gtt_list); 2207 list_del_init(&obj->mm_list); 2208 obj->map_and_fenceable = true; 2209 2210 drm_mm_put_block(obj->gtt_space); 2211 obj->gtt_space = NULL; 2212 obj->gtt_offset = 0; 2213 2214 if (i915_gem_object_is_purgeable(obj)) 2215 i915_gem_object_truncate(obj); 2216 CTR1(KTR_DRM, "object_unbind %p", obj); 2217 2218 return (ret); 2219} 2220 2221static int 2222i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2223 int flags) 2224{ 2225 struct drm_device *dev; 2226 vm_object_t vm_obj; 2227 vm_page_t m; 2228 int page_count, i, j; 2229 2230 dev = obj->base.dev; 2231 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2232 page_count = obj->base.size / PAGE_SIZE; 2233 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2234 M_WAITOK); 2235 vm_obj = obj->base.vm_obj; 2236 VM_OBJECT_WLOCK(vm_obj); 2237 for (i = 0; i < page_count; i++) { 2238 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2239 goto failed; 2240 } 2241 VM_OBJECT_WUNLOCK(vm_obj); 2242 if (i915_gem_object_needs_bit17_swizzle(obj)) 2243 i915_gem_object_do_bit_17_swizzle(obj); 2244 return (0); 2245 2246failed: 2247 for (j = 0; j < i; j++) { 2248 m = obj->pages[j]; 2249 vm_page_lock(m); 2250 vm_page_unwire(m, PQ_INACTIVE); 2251 vm_page_unlock(m); 2252 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2253 } 2254 VM_OBJECT_WUNLOCK(vm_obj); 2255 free(obj->pages, DRM_I915_GEM); 2256 obj->pages = NULL; 2257 return (-EIO); 2258} 2259 2260#define GEM_PARANOID_CHECK_GTT 0 2261#if GEM_PARANOID_CHECK_GTT 2262static void 2263i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2264 int page_count) 2265{ 2266 struct drm_i915_private *dev_priv; 2267 vm_paddr_t pa; 2268 unsigned long start, end; 2269 u_int i; 2270 int j; 2271 2272 dev_priv = dev->dev_private; 2273 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2274 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2275 for (i = start; i < end; i++) { 2276 pa = intel_gtt_read_pte_paddr(i); 2277 for (j = 0; j < page_count; j++) { 2278 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2279 panic("Page %p in GTT pte index %d pte %x", 2280 ma[i], i, intel_gtt_read_pte(i)); 2281 } 2282 } 2283 } 2284} 2285#endif 2286 2287static void 2288i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2289{ 2290 vm_page_t m; 2291 int page_count, i; 2292 2293 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2294 2295 if (obj->tiling_mode != I915_TILING_NONE) 2296 i915_gem_object_save_bit_17_swizzle(obj); 2297 if (obj->madv == I915_MADV_DONTNEED) 2298 obj->dirty = 0; 2299 page_count = obj->base.size / PAGE_SIZE; 2300 VM_OBJECT_WLOCK(obj->base.vm_obj); 2301#if GEM_PARANOID_CHECK_GTT 2302 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2303#endif 2304 for (i = 0; i < page_count; i++) { 2305 m = obj->pages[i]; 2306 if (obj->dirty) 2307 vm_page_dirty(m); 2308 if (obj->madv == I915_MADV_WILLNEED) 2309 vm_page_reference(m); 2310 vm_page_lock(m); 2311 vm_page_unwire(obj->pages[i], PQ_ACTIVE); 2312 vm_page_unlock(m); 2313 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2314 } 2315 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 2316 obj->dirty = 0; 2317 free(obj->pages, DRM_I915_GEM); 2318 obj->pages = NULL; 2319} 2320 2321void 2322i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2323{ 2324 vm_object_t devobj; 2325 vm_page_t m; 2326 int i, page_count; 2327 2328 if (!obj->fault_mappable) 2329 return; 2330 2331 CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 2332 OFF_TO_IDX(obj->base.size)); 2333 devobj = cdev_pager_lookup(obj); 2334 if (devobj != NULL) { 2335 page_count = OFF_TO_IDX(obj->base.size); 2336 2337 VM_OBJECT_WLOCK(devobj); 2338retry: 2339 for (i = 0; i < page_count; i++) { 2340 m = vm_page_lookup(devobj, i); 2341 if (m == NULL) 2342 continue; 2343 if (vm_page_sleep_if_busy(m, "915unm")) 2344 goto retry; 2345 cdev_pager_free_page(devobj, m); 2346 } 2347 VM_OBJECT_WUNLOCK(devobj); 2348 vm_object_deallocate(devobj); 2349 } 2350 2351 obj->fault_mappable = false; 2352} 2353 2354int 2355i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2356{ 2357 int ret; 2358 2359 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2360 ("In GPU write domain")); 2361 2362 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, 2363 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, 2364 obj->active, obj->last_rendering_seqno); 2365 if (obj->active) { 2366 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2367 true); 2368 if (ret != 0) 2369 return (ret); 2370 } 2371 return (0); 2372} 2373 2374void 2375i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2376 struct intel_ring_buffer *ring, uint32_t seqno) 2377{ 2378 struct drm_device *dev = obj->base.dev; 2379 struct drm_i915_private *dev_priv = dev->dev_private; 2380 struct drm_i915_fence_reg *reg; 2381 2382 obj->ring = ring; 2383 KASSERT(ring != NULL, ("NULL ring")); 2384 2385 /* Add a reference if we're newly entering the active list. */ 2386 if (!obj->active) { 2387 drm_gem_object_reference(&obj->base); 2388 obj->active = 1; 2389 } 2390 2391 /* Move from whatever list we were on to the tail of execution. */ 2392 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2393 list_move_tail(&obj->ring_list, &ring->active_list); 2394 2395 obj->last_rendering_seqno = seqno; 2396 if (obj->fenced_gpu_access) { 2397 obj->last_fenced_seqno = seqno; 2398 obj->last_fenced_ring = ring; 2399 2400 /* Bump MRU to take account of the delayed flush */ 2401 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2402 reg = &dev_priv->fence_regs[obj->fence_reg]; 2403 list_move_tail(®->lru_list, 2404 &dev_priv->mm.fence_list); 2405 } 2406 } 2407} 2408 2409static void 2410i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2411{ 2412 list_del_init(&obj->ring_list); 2413 obj->last_rendering_seqno = 0; 2414 obj->last_fenced_seqno = 0; 2415} 2416 2417static void 2418i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2419{ 2420 struct drm_device *dev = obj->base.dev; 2421 drm_i915_private_t *dev_priv = dev->dev_private; 2422 2423 KASSERT(obj->active, ("Object not active")); 2424 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2425 2426 i915_gem_object_move_off_active(obj); 2427} 2428 2429static void 2430i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2431{ 2432 struct drm_device *dev = obj->base.dev; 2433 struct drm_i915_private *dev_priv = dev->dev_private; 2434 2435 if (obj->pin_count != 0) 2436 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2437 else 2438 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2439 2440 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2441 KASSERT(obj->active, ("Object not active")); 2442 obj->ring = NULL; 2443 obj->last_fenced_ring = NULL; 2444 2445 i915_gem_object_move_off_active(obj); 2446 obj->fenced_gpu_access = false; 2447 2448 obj->active = 0; 2449 obj->pending_gpu_write = false; 2450 drm_gem_object_unreference(&obj->base); 2451 2452#if 1 2453 KIB_NOTYET(); 2454#else 2455 WARN_ON(i915_verify_lists(dev)); 2456#endif 2457} 2458 2459static void 2460i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2461{ 2462 vm_object_t vm_obj; 2463 2464 vm_obj = obj->base.vm_obj; 2465 VM_OBJECT_WLOCK(vm_obj); 2466 vm_object_page_remove(vm_obj, 0, 0, false); 2467 VM_OBJECT_WUNLOCK(vm_obj); 2468 obj->madv = I915_MADV_PURGED_INTERNAL; 2469} 2470 2471static inline int 2472i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2473{ 2474 2475 return (obj->madv == I915_MADV_DONTNEED); 2476} 2477 2478static void 2479i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2480 uint32_t flush_domains) 2481{ 2482 struct drm_i915_gem_object *obj, *next; 2483 uint32_t old_write_domain; 2484 2485 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2486 gpu_write_list) { 2487 if (obj->base.write_domain & flush_domains) { 2488 old_write_domain = obj->base.write_domain; 2489 obj->base.write_domain = 0; 2490 list_del_init(&obj->gpu_write_list); 2491 i915_gem_object_move_to_active(obj, ring, 2492 i915_gem_next_request_seqno(ring)); 2493 2494 CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", 2495 obj, obj->base.read_domains, old_write_domain); 2496 } 2497 } 2498} 2499 2500static int 2501i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2502{ 2503 drm_i915_private_t *dev_priv; 2504 2505 dev_priv = obj->base.dev->dev_private; 2506 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2507 obj->tiling_mode != I915_TILING_NONE); 2508} 2509 2510static vm_page_t 2511i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2512{ 2513 vm_page_t m; 2514 int rv; 2515 2516 VM_OBJECT_ASSERT_WLOCKED(object); 2517 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 2518 if (m->valid != VM_PAGE_BITS_ALL) { 2519 if (vm_pager_has_page(object, pindex, NULL, NULL)) { 2520 rv = vm_pager_get_pages(object, &m, 1, 0); 2521 m = vm_page_lookup(object, pindex); 2522 if (m == NULL) 2523 return (NULL); 2524 if (rv != VM_PAGER_OK) { 2525 vm_page_lock(m); 2526 vm_page_free(m); 2527 vm_page_unlock(m); 2528 return (NULL); 2529 } 2530 } else { 2531 pmap_zero_page(m); 2532 m->valid = VM_PAGE_BITS_ALL; 2533 m->dirty = 0; 2534 } 2535 } 2536 vm_page_lock(m); 2537 vm_page_wire(m); 2538 vm_page_unlock(m); 2539 vm_page_xunbusy(m); 2540 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2541 return (m); 2542} 2543 2544int 2545i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2546 uint32_t flush_domains) 2547{ 2548 int ret; 2549 2550 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2551 return 0; 2552 2553 CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, 2554 flush_domains); 2555 ret = ring->flush(ring, invalidate_domains, flush_domains); 2556 if (ret) 2557 return ret; 2558 2559 if (flush_domains & I915_GEM_GPU_DOMAINS) 2560 i915_gem_process_flushing_list(ring, flush_domains); 2561 return 0; 2562} 2563 2564static int 2565i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2566{ 2567 int ret; 2568 2569 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2570 return 0; 2571 2572 if (!list_empty(&ring->gpu_write_list)) { 2573 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2574 I915_GEM_GPU_DOMAINS); 2575 if (ret != 0) 2576 return ret; 2577 } 2578 2579 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2580 do_retire)); 2581} 2582 2583int 2584i915_gpu_idle(struct drm_device *dev, bool do_retire) 2585{ 2586 drm_i915_private_t *dev_priv = dev->dev_private; 2587 int ret, i; 2588 2589 /* Flush everything onto the inactive list. */ 2590 for (i = 0; i < I915_NUM_RINGS; i++) { 2591 ret = i915_ring_idle(&dev_priv->rings[i], do_retire); 2592 if (ret) 2593 return ret; 2594 } 2595 2596 return 0; 2597} 2598 2599int 2600i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2601{ 2602 drm_i915_private_t *dev_priv; 2603 struct drm_i915_gem_request *request; 2604 uint32_t ier; 2605 int flags, ret; 2606 bool recovery_complete; 2607 2608 KASSERT(seqno != 0, ("Zero seqno")); 2609 2610 dev_priv = ring->dev->dev_private; 2611 ret = 0; 2612 2613 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 2614 /* Give the error handler a chance to run. */ 2615 mtx_lock(&dev_priv->error_completion_lock); 2616 recovery_complete = (&dev_priv->error_completion) > 0; 2617 mtx_unlock(&dev_priv->error_completion_lock); 2618 return (recovery_complete ? -EIO : -EAGAIN); 2619 } 2620 2621 if (seqno == ring->outstanding_lazy_request) { 2622 request = malloc(sizeof(*request), DRM_I915_GEM, 2623 M_WAITOK | M_ZERO); 2624 if (request == NULL) 2625 return (-ENOMEM); 2626 2627 ret = i915_add_request(ring, NULL, request); 2628 if (ret != 0) { 2629 free(request, DRM_I915_GEM); 2630 return (ret); 2631 } 2632 2633 seqno = request->seqno; 2634 } 2635 2636 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2637 if (HAS_PCH_SPLIT(ring->dev)) 2638 ier = I915_READ(DEIER) | I915_READ(GTIER); 2639 else 2640 ier = I915_READ(IER); 2641 if (!ier) { 2642 DRM_ERROR("something (likely vbetool) disabled " 2643 "interrupts, re-enabling\n"); 2644 ring->dev->driver->irq_preinstall(ring->dev); 2645 ring->dev->driver->irq_postinstall(ring->dev); 2646 } 2647 2648 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 2649 2650 ring->waiting_seqno = seqno; 2651 mtx_lock(&ring->irq_lock); 2652 if (ring->irq_get(ring)) { 2653 flags = dev_priv->mm.interruptible ? PCATCH : 0; 2654 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2655 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 2656 ret == 0) { 2657 ret = -msleep(ring, &ring->irq_lock, flags, 2658 "915gwr", 0); 2659 } 2660 ring->irq_put(ring); 2661 mtx_unlock(&ring->irq_lock); 2662 } else { 2663 mtx_unlock(&ring->irq_lock); 2664 if (_intel_wait_for(ring->dev, 2665 i915_seqno_passed(ring->get_seqno(ring), seqno) || 2666 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 2667 0, "i915wrq") != 0) 2668 ret = -EBUSY; 2669 } 2670 ring->waiting_seqno = 0; 2671 2672 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, 2673 ret); 2674 } 2675 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 2676 ret = -EAGAIN; 2677 2678 /* Directly dispatch request retiring. While we have the work queue 2679 * to handle this, the waiter on a request often wants an associated 2680 * buffer to have made it to the inactive list, and we would need 2681 * a separate wait queue to handle that. 2682 */ 2683 if (ret == 0 && do_retire) 2684 i915_gem_retire_requests_ring(ring); 2685 2686 return (ret); 2687} 2688 2689static u32 2690i915_gem_get_seqno(struct drm_device *dev) 2691{ 2692 drm_i915_private_t *dev_priv = dev->dev_private; 2693 u32 seqno = dev_priv->next_seqno; 2694 2695 /* reserve 0 for non-seqno */ 2696 if (++dev_priv->next_seqno == 0) 2697 dev_priv->next_seqno = 1; 2698 2699 return seqno; 2700} 2701 2702u32 2703i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2704{ 2705 if (ring->outstanding_lazy_request == 0) 2706 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2707 2708 return ring->outstanding_lazy_request; 2709} 2710 2711int 2712i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2713 struct drm_i915_gem_request *request) 2714{ 2715 drm_i915_private_t *dev_priv; 2716 struct drm_i915_file_private *file_priv; 2717 uint32_t seqno; 2718 u32 request_ring_position; 2719 int was_empty; 2720 int ret; 2721 2722 KASSERT(request != NULL, ("NULL request in add")); 2723 DRM_LOCK_ASSERT(ring->dev); 2724 dev_priv = ring->dev->dev_private; 2725 2726 seqno = i915_gem_next_request_seqno(ring); 2727 request_ring_position = intel_ring_get_tail(ring); 2728 2729 ret = ring->add_request(ring, &seqno); 2730 if (ret != 0) 2731 return ret; 2732 2733 CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); 2734 2735 request->seqno = seqno; 2736 request->ring = ring; 2737 request->tail = request_ring_position; 2738 request->emitted_jiffies = ticks; 2739 was_empty = list_empty(&ring->request_list); 2740 list_add_tail(&request->list, &ring->request_list); 2741 2742 if (file != NULL) { 2743 file_priv = file->driver_priv; 2744 2745 mtx_lock(&file_priv->mm.lck); 2746 request->file_priv = file_priv; 2747 list_add_tail(&request->client_list, 2748 &file_priv->mm.request_list); 2749 mtx_unlock(&file_priv->mm.lck); 2750 } 2751 2752 ring->outstanding_lazy_request = 0; 2753 2754 if (!dev_priv->mm.suspended) { 2755 if (i915_enable_hangcheck) { 2756 callout_schedule(&dev_priv->hangcheck_timer, 2757 DRM_I915_HANGCHECK_PERIOD); 2758 } 2759 if (was_empty) 2760 taskqueue_enqueue_timeout(dev_priv->tq, 2761 &dev_priv->mm.retire_task, hz); 2762 } 2763 return (0); 2764} 2765 2766static inline void 2767i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2768{ 2769 struct drm_i915_file_private *file_priv = request->file_priv; 2770 2771 if (!file_priv) 2772 return; 2773 2774 DRM_LOCK_ASSERT(request->ring->dev); 2775 2776 mtx_lock(&file_priv->mm.lck); 2777 if (request->file_priv != NULL) { 2778 list_del(&request->client_list); 2779 request->file_priv = NULL; 2780 } 2781 mtx_unlock(&file_priv->mm.lck); 2782} 2783 2784void 2785i915_gem_release(struct drm_device *dev, struct drm_file *file) 2786{ 2787 struct drm_i915_file_private *file_priv; 2788 struct drm_i915_gem_request *request; 2789 2790 file_priv = file->driver_priv; 2791 2792 /* Clean up our request list when the client is going away, so that 2793 * later retire_requests won't dereference our soon-to-be-gone 2794 * file_priv. 2795 */ 2796 mtx_lock(&file_priv->mm.lck); 2797 while (!list_empty(&file_priv->mm.request_list)) { 2798 request = list_first_entry(&file_priv->mm.request_list, 2799 struct drm_i915_gem_request, 2800 client_list); 2801 list_del(&request->client_list); 2802 request->file_priv = NULL; 2803 } 2804 mtx_unlock(&file_priv->mm.lck); 2805} 2806 2807static void 2808i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2809 struct intel_ring_buffer *ring) 2810{ 2811 2812 if (ring->dev != NULL) 2813 DRM_LOCK_ASSERT(ring->dev); 2814 2815 while (!list_empty(&ring->request_list)) { 2816 struct drm_i915_gem_request *request; 2817 2818 request = list_first_entry(&ring->request_list, 2819 struct drm_i915_gem_request, list); 2820 2821 list_del(&request->list); 2822 i915_gem_request_remove_from_client(request); 2823 free(request, DRM_I915_GEM); 2824 } 2825 2826 while (!list_empty(&ring->active_list)) { 2827 struct drm_i915_gem_object *obj; 2828 2829 obj = list_first_entry(&ring->active_list, 2830 struct drm_i915_gem_object, ring_list); 2831 2832 obj->base.write_domain = 0; 2833 list_del_init(&obj->gpu_write_list); 2834 i915_gem_object_move_to_inactive(obj); 2835 } 2836} 2837 2838static void 2839i915_gem_reset_fences(struct drm_device *dev) 2840{ 2841 struct drm_i915_private *dev_priv = dev->dev_private; 2842 int i; 2843 2844 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2845 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2846 struct drm_i915_gem_object *obj = reg->obj; 2847 2848 if (!obj) 2849 continue; 2850 2851 if (obj->tiling_mode) 2852 i915_gem_release_mmap(obj); 2853 2854 reg->obj->fence_reg = I915_FENCE_REG_NONE; 2855 reg->obj->fenced_gpu_access = false; 2856 reg->obj->last_fenced_seqno = 0; 2857 reg->obj->last_fenced_ring = NULL; 2858 i915_gem_clear_fence_reg(dev, reg); 2859 } 2860} 2861 2862void 2863i915_gem_reset(struct drm_device *dev) 2864{ 2865 struct drm_i915_private *dev_priv = dev->dev_private; 2866 struct drm_i915_gem_object *obj; 2867 int i; 2868 2869 for (i = 0; i < I915_NUM_RINGS; i++) 2870 i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); 2871 2872 /* Remove anything from the flushing lists. The GPU cache is likely 2873 * to be lost on reset along with the data, so simply move the 2874 * lost bo to the inactive list. 2875 */ 2876 while (!list_empty(&dev_priv->mm.flushing_list)) { 2877 obj = list_first_entry(&dev_priv->mm.flushing_list, 2878 struct drm_i915_gem_object, 2879 mm_list); 2880 2881 obj->base.write_domain = 0; 2882 list_del_init(&obj->gpu_write_list); 2883 i915_gem_object_move_to_inactive(obj); 2884 } 2885 2886 /* Move everything out of the GPU domains to ensure we do any 2887 * necessary invalidation upon reuse. 2888 */ 2889 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2890 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2891 } 2892 2893 /* The fence registers are invalidated so clear them out */ 2894 i915_gem_reset_fences(dev); 2895} 2896 2897/** 2898 * This function clears the request list as sequence numbers are passed. 2899 */ 2900void 2901i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2902{ 2903 uint32_t seqno; 2904 int i; 2905 2906 if (list_empty(&ring->request_list)) 2907 return; 2908 2909 seqno = ring->get_seqno(ring); 2910 CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 2911 2912 for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2913 if (seqno >= ring->sync_seqno[i]) 2914 ring->sync_seqno[i] = 0; 2915 2916 while (!list_empty(&ring->request_list)) { 2917 struct drm_i915_gem_request *request; 2918 2919 request = list_first_entry(&ring->request_list, 2920 struct drm_i915_gem_request, 2921 list); 2922 2923 if (!i915_seqno_passed(seqno, request->seqno)) 2924 break; 2925 2926 CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 2927 ring->name, seqno); 2928 ring->last_retired_head = request->tail; 2929 2930 list_del(&request->list); 2931 i915_gem_request_remove_from_client(request); 2932 free(request, DRM_I915_GEM); 2933 } 2934 2935 /* Move any buffers on the active list that are no longer referenced 2936 * by the ringbuffer to the flushing/inactive lists as appropriate. 2937 */ 2938 while (!list_empty(&ring->active_list)) { 2939 struct drm_i915_gem_object *obj; 2940 2941 obj = list_first_entry(&ring->active_list, 2942 struct drm_i915_gem_object, 2943 ring_list); 2944 2945 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2946 break; 2947 2948 if (obj->base.write_domain != 0) 2949 i915_gem_object_move_to_flushing(obj); 2950 else 2951 i915_gem_object_move_to_inactive(obj); 2952 } 2953 2954 if (ring->trace_irq_seqno && 2955 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2956 mtx_lock(&ring->irq_lock); 2957 ring->irq_put(ring); 2958 mtx_unlock(&ring->irq_lock); 2959 ring->trace_irq_seqno = 0; 2960 } 2961} 2962 2963void 2964i915_gem_retire_requests(struct drm_device *dev) 2965{ 2966 drm_i915_private_t *dev_priv = dev->dev_private; 2967 struct drm_i915_gem_object *obj, *next; 2968 int i; 2969 2970 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2971 list_for_each_entry_safe(obj, next, 2972 &dev_priv->mm.deferred_free_list, mm_list) 2973 i915_gem_free_object_tail(obj); 2974 } 2975 2976 for (i = 0; i < I915_NUM_RINGS; i++) 2977 i915_gem_retire_requests_ring(&dev_priv->rings[i]); 2978} 2979 2980static int 2981sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2982 struct intel_ring_buffer *pipelined) 2983{ 2984 struct drm_device *dev = obj->base.dev; 2985 drm_i915_private_t *dev_priv = dev->dev_private; 2986 u32 size = obj->gtt_space->size; 2987 int regnum = obj->fence_reg; 2988 uint64_t val; 2989 2990 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2991 0xfffff000) << 32; 2992 val |= obj->gtt_offset & 0xfffff000; 2993 val |= (uint64_t)((obj->stride / 128) - 1) << 2994 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2995 2996 if (obj->tiling_mode == I915_TILING_Y) 2997 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2998 val |= I965_FENCE_REG_VALID; 2999 3000 if (pipelined) { 3001 int ret = intel_ring_begin(pipelined, 6); 3002 if (ret) 3003 return ret; 3004 3005 intel_ring_emit(pipelined, MI_NOOP); 3006 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 3007 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 3008 intel_ring_emit(pipelined, (u32)val); 3009 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 3010 intel_ring_emit(pipelined, (u32)(val >> 32)); 3011 intel_ring_advance(pipelined); 3012 } else 3013 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 3014 3015 return 0; 3016} 3017 3018static int 3019i965_write_fence_reg(struct drm_i915_gem_object *obj, 3020 struct intel_ring_buffer *pipelined) 3021{ 3022 struct drm_device *dev = obj->base.dev; 3023 drm_i915_private_t *dev_priv = dev->dev_private; 3024 u32 size = obj->gtt_space->size; 3025 int regnum = obj->fence_reg; 3026 uint64_t val; 3027 3028 val = (uint64_t)((obj->gtt_offset + size - 4096) & 3029 0xfffff000) << 32; 3030 val |= obj->gtt_offset & 0xfffff000; 3031 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 3032 if (obj->tiling_mode == I915_TILING_Y) 3033 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3034 val |= I965_FENCE_REG_VALID; 3035 3036 if (pipelined) { 3037 int ret = intel_ring_begin(pipelined, 6); 3038 if (ret) 3039 return ret; 3040 3041 intel_ring_emit(pipelined, MI_NOOP); 3042 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 3043 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 3044 intel_ring_emit(pipelined, (u32)val); 3045 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 3046 intel_ring_emit(pipelined, (u32)(val >> 32)); 3047 intel_ring_advance(pipelined); 3048 } else 3049 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 3050 3051 return 0; 3052} 3053 3054static int 3055i915_write_fence_reg(struct drm_i915_gem_object *obj, 3056 struct intel_ring_buffer *pipelined) 3057{ 3058 struct drm_device *dev = obj->base.dev; 3059 drm_i915_private_t *dev_priv = dev->dev_private; 3060 u32 size = obj->gtt_space->size; 3061 u32 fence_reg, val, pitch_val; 3062 int tile_width; 3063 3064 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3065 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3066 printf( 3067"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3068 obj->gtt_offset, obj->map_and_fenceable, size); 3069 return -EINVAL; 3070 } 3071 3072 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3073 tile_width = 128; 3074 else 3075 tile_width = 512; 3076 3077 /* Note: pitch better be a power of two tile widths */ 3078 pitch_val = obj->stride / tile_width; 3079 pitch_val = ffs(pitch_val) - 1; 3080 3081 val = obj->gtt_offset; 3082 if (obj->tiling_mode == I915_TILING_Y) 3083 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3084 val |= I915_FENCE_SIZE_BITS(size); 3085 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3086 val |= I830_FENCE_REG_VALID; 3087 3088 fence_reg = obj->fence_reg; 3089 if (fence_reg < 8) 3090 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3091 else 3092 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3093 3094 if (pipelined) { 3095 int ret = intel_ring_begin(pipelined, 4); 3096 if (ret) 3097 return ret; 3098 3099 intel_ring_emit(pipelined, MI_NOOP); 3100 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3101 intel_ring_emit(pipelined, fence_reg); 3102 intel_ring_emit(pipelined, val); 3103 intel_ring_advance(pipelined); 3104 } else 3105 I915_WRITE(fence_reg, val); 3106 3107 return 0; 3108} 3109 3110static int 3111i830_write_fence_reg(struct drm_i915_gem_object *obj, 3112 struct intel_ring_buffer *pipelined) 3113{ 3114 struct drm_device *dev = obj->base.dev; 3115 drm_i915_private_t *dev_priv = dev->dev_private; 3116 u32 size = obj->gtt_space->size; 3117 int regnum = obj->fence_reg; 3118 uint32_t val; 3119 uint32_t pitch_val; 3120 3121 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3122 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3123 printf( 3124"object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3125 obj->gtt_offset, size); 3126 return -EINVAL; 3127 } 3128 3129 pitch_val = obj->stride / 128; 3130 pitch_val = ffs(pitch_val) - 1; 3131 3132 val = obj->gtt_offset; 3133 if (obj->tiling_mode == I915_TILING_Y) 3134 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3135 val |= I830_FENCE_SIZE_BITS(size); 3136 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3137 val |= I830_FENCE_REG_VALID; 3138 3139 if (pipelined) { 3140 int ret = intel_ring_begin(pipelined, 4); 3141 if (ret) 3142 return ret; 3143 3144 intel_ring_emit(pipelined, MI_NOOP); 3145 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3146 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 3147 intel_ring_emit(pipelined, val); 3148 intel_ring_advance(pipelined); 3149 } else 3150 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 3151 3152 return 0; 3153} 3154 3155static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 3156{ 3157 return i915_seqno_passed(ring->get_seqno(ring), seqno); 3158} 3159 3160static int 3161i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 3162 struct intel_ring_buffer *pipelined) 3163{ 3164 int ret; 3165 3166 if (obj->fenced_gpu_access) { 3167 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3168 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3169 obj->base.write_domain); 3170 if (ret) 3171 return ret; 3172 } 3173 3174 obj->fenced_gpu_access = false; 3175 } 3176 3177 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3178 if (!ring_passed_seqno(obj->last_fenced_ring, 3179 obj->last_fenced_seqno)) { 3180 ret = i915_wait_request(obj->last_fenced_ring, 3181 obj->last_fenced_seqno, 3182 true); 3183 if (ret) 3184 return ret; 3185 } 3186 3187 obj->last_fenced_seqno = 0; 3188 obj->last_fenced_ring = NULL; 3189 } 3190 3191 /* Ensure that all CPU reads are completed before installing a fence 3192 * and all writes before removing the fence. 3193 */ 3194 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3195 mb(); 3196 3197 return 0; 3198} 3199 3200int 3201i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3202{ 3203 int ret; 3204 3205 if (obj->tiling_mode) 3206 i915_gem_release_mmap(obj); 3207 3208 ret = i915_gem_object_flush_fence(obj, NULL); 3209 if (ret) 3210 return ret; 3211 3212 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3213 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3214 3215 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3216 printf("%s: pin_count %d\n", __func__, 3217 dev_priv->fence_regs[obj->fence_reg].pin_count); 3218 i915_gem_clear_fence_reg(obj->base.dev, 3219 &dev_priv->fence_regs[obj->fence_reg]); 3220 3221 obj->fence_reg = I915_FENCE_REG_NONE; 3222 } 3223 3224 return 0; 3225} 3226 3227static struct drm_i915_fence_reg * 3228i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3229{ 3230 struct drm_i915_private *dev_priv = dev->dev_private; 3231 struct drm_i915_fence_reg *reg, *first, *avail; 3232 int i; 3233 3234 /* First try to find a free reg */ 3235 avail = NULL; 3236 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3237 reg = &dev_priv->fence_regs[i]; 3238 if (!reg->obj) 3239 return reg; 3240 3241 if (!reg->pin_count) 3242 avail = reg; 3243 } 3244 3245 if (avail == NULL) 3246 return NULL; 3247 3248 /* None available, try to steal one or wait for a user to finish */ 3249 avail = first = NULL; 3250 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3251 if (reg->pin_count) 3252 continue; 3253 3254 if (first == NULL) 3255 first = reg; 3256 3257 if (!pipelined || 3258 !reg->obj->last_fenced_ring || 3259 reg->obj->last_fenced_ring == pipelined) { 3260 avail = reg; 3261 break; 3262 } 3263 } 3264 3265 if (avail == NULL) 3266 avail = first; 3267 3268 return avail; 3269} 3270 3271int 3272i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3273 struct intel_ring_buffer *pipelined) 3274{ 3275 struct drm_device *dev = obj->base.dev; 3276 struct drm_i915_private *dev_priv = dev->dev_private; 3277 struct drm_i915_fence_reg *reg; 3278 int ret; 3279 3280 pipelined = NULL; 3281 ret = 0; 3282 3283 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3284 reg = &dev_priv->fence_regs[obj->fence_reg]; 3285 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3286 3287 if (obj->tiling_changed) { 3288 ret = i915_gem_object_flush_fence(obj, pipelined); 3289 if (ret) 3290 return ret; 3291 3292 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3293 pipelined = NULL; 3294 3295 if (pipelined) { 3296 reg->setup_seqno = 3297 i915_gem_next_request_seqno(pipelined); 3298 obj->last_fenced_seqno = reg->setup_seqno; 3299 obj->last_fenced_ring = pipelined; 3300 } 3301 3302 goto update; 3303 } 3304 3305 if (!pipelined) { 3306 if (reg->setup_seqno) { 3307 if (!ring_passed_seqno(obj->last_fenced_ring, 3308 reg->setup_seqno)) { 3309 ret = i915_wait_request( 3310 obj->last_fenced_ring, 3311 reg->setup_seqno, 3312 true); 3313 if (ret) 3314 return ret; 3315 } 3316 3317 reg->setup_seqno = 0; 3318 } 3319 } else if (obj->last_fenced_ring && 3320 obj->last_fenced_ring != pipelined) { 3321 ret = i915_gem_object_flush_fence(obj, pipelined); 3322 if (ret) 3323 return ret; 3324 } 3325 3326 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3327 pipelined = NULL; 3328 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3329 3330 if (obj->tiling_changed) { 3331 if (pipelined) { 3332 reg->setup_seqno = 3333 i915_gem_next_request_seqno(pipelined); 3334 obj->last_fenced_seqno = reg->setup_seqno; 3335 obj->last_fenced_ring = pipelined; 3336 } 3337 goto update; 3338 } 3339 3340 return 0; 3341 } 3342 3343 reg = i915_find_fence_reg(dev, pipelined); 3344 if (reg == NULL) 3345 return -EDEADLK; 3346 3347 ret = i915_gem_object_flush_fence(obj, pipelined); 3348 if (ret) 3349 return ret; 3350 3351 if (reg->obj) { 3352 struct drm_i915_gem_object *old = reg->obj; 3353 3354 drm_gem_object_reference(&old->base); 3355 3356 if (old->tiling_mode) 3357 i915_gem_release_mmap(old); 3358 3359 ret = i915_gem_object_flush_fence(old, pipelined); 3360 if (ret) { 3361 drm_gem_object_unreference(&old->base); 3362 return ret; 3363 } 3364 3365 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3366 pipelined = NULL; 3367 3368 old->fence_reg = I915_FENCE_REG_NONE; 3369 old->last_fenced_ring = pipelined; 3370 old->last_fenced_seqno = 3371 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3372 3373 drm_gem_object_unreference(&old->base); 3374 } else if (obj->last_fenced_seqno == 0) 3375 pipelined = NULL; 3376 3377 reg->obj = obj; 3378 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3379 obj->fence_reg = reg - dev_priv->fence_regs; 3380 obj->last_fenced_ring = pipelined; 3381 3382 reg->setup_seqno = 3383 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3384 obj->last_fenced_seqno = reg->setup_seqno; 3385 3386update: 3387 obj->tiling_changed = false; 3388 switch (INTEL_INFO(dev)->gen) { 3389 case 7: 3390 case 6: 3391 ret = sandybridge_write_fence_reg(obj, pipelined); 3392 break; 3393 case 5: 3394 case 4: 3395 ret = i965_write_fence_reg(obj, pipelined); 3396 break; 3397 case 3: 3398 ret = i915_write_fence_reg(obj, pipelined); 3399 break; 3400 case 2: 3401 ret = i830_write_fence_reg(obj, pipelined); 3402 break; 3403 } 3404 3405 return ret; 3406} 3407 3408static void 3409i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3410{ 3411 drm_i915_private_t *dev_priv = dev->dev_private; 3412 uint32_t fence_reg = reg - dev_priv->fence_regs; 3413 3414 switch (INTEL_INFO(dev)->gen) { 3415 case 7: 3416 case 6: 3417 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3418 break; 3419 case 5: 3420 case 4: 3421 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3422 break; 3423 case 3: 3424 if (fence_reg >= 8) 3425 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3426 else 3427 case 2: 3428 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3429 3430 I915_WRITE(fence_reg, 0); 3431 break; 3432 } 3433 3434 list_del_init(®->lru_list); 3435 reg->obj = NULL; 3436 reg->setup_seqno = 0; 3437 reg->pin_count = 0; 3438} 3439 3440int 3441i915_gem_init_object(struct drm_gem_object *obj) 3442{ 3443 3444 printf("i915_gem_init_object called\n"); 3445 return (0); 3446} 3447 3448static bool 3449i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3450{ 3451 3452 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3453} 3454 3455static void 3456i915_gem_retire_task_handler(void *arg, int pending) 3457{ 3458 drm_i915_private_t *dev_priv; 3459 struct drm_device *dev; 3460 bool idle; 3461 int i; 3462 3463 dev_priv = arg; 3464 dev = dev_priv->dev; 3465 3466 /* Come back later if the device is busy... */ 3467 if (!sx_try_xlock(&dev->dev_struct_lock)) { 3468 taskqueue_enqueue_timeout(dev_priv->tq, 3469 &dev_priv->mm.retire_task, hz); 3470 return; 3471 } 3472 3473 CTR0(KTR_DRM, "retire_task"); 3474 3475 i915_gem_retire_requests(dev); 3476 3477 /* Send a periodic flush down the ring so we don't hold onto GEM 3478 * objects indefinitely. 3479 */ 3480 idle = true; 3481 for (i = 0; i < I915_NUM_RINGS; i++) { 3482 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 3483 3484 if (!list_empty(&ring->gpu_write_list)) { 3485 struct drm_i915_gem_request *request; 3486 int ret; 3487 3488 ret = i915_gem_flush_ring(ring, 3489 0, I915_GEM_GPU_DOMAINS); 3490 request = malloc(sizeof(*request), DRM_I915_GEM, 3491 M_WAITOK | M_ZERO); 3492 if (ret || request == NULL || 3493 i915_add_request(ring, NULL, request)) 3494 free(request, DRM_I915_GEM); 3495 } 3496 3497 idle &= list_empty(&ring->request_list); 3498 } 3499 3500 if (!dev_priv->mm.suspended && !idle) 3501 taskqueue_enqueue_timeout(dev_priv->tq, 3502 &dev_priv->mm.retire_task, hz); 3503 3504 DRM_UNLOCK(dev); 3505} 3506 3507void 3508i915_gem_lastclose(struct drm_device *dev) 3509{ 3510 int ret; 3511 3512 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3513 return; 3514 3515 ret = i915_gem_idle(dev); 3516 if (ret != 0) 3517 DRM_ERROR("failed to idle hardware: %d\n", ret); 3518} 3519 3520static int 3521i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3522{ 3523 drm_i915_private_t *dev_priv; 3524 struct drm_i915_gem_phys_object *phys_obj; 3525 int ret; 3526 3527 dev_priv = dev->dev_private; 3528 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3529 return (0); 3530 3531 phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3532 M_WAITOK | M_ZERO); 3533 3534 phys_obj->id = id; 3535 3536 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3537 if (phys_obj->handle == NULL) { 3538 ret = -ENOMEM; 3539 goto free_obj; 3540 } 3541 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3542 size / PAGE_SIZE, PAT_WRITE_COMBINING); 3543 3544 dev_priv->mm.phys_objs[id - 1] = phys_obj; 3545 3546 return (0); 3547 3548free_obj: 3549 free(phys_obj, DRM_I915_GEM); 3550 return (ret); 3551} 3552 3553static void 3554i915_gem_free_phys_object(struct drm_device *dev, int id) 3555{ 3556 drm_i915_private_t *dev_priv; 3557 struct drm_i915_gem_phys_object *phys_obj; 3558 3559 dev_priv = dev->dev_private; 3560 if (dev_priv->mm.phys_objs[id - 1] == NULL) 3561 return; 3562 3563 phys_obj = dev_priv->mm.phys_objs[id - 1]; 3564 if (phys_obj->cur_obj != NULL) 3565 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3566 3567 drm_pci_free(dev, phys_obj->handle); 3568 free(phys_obj, DRM_I915_GEM); 3569 dev_priv->mm.phys_objs[id - 1] = NULL; 3570} 3571 3572void 3573i915_gem_free_all_phys_object(struct drm_device *dev) 3574{ 3575 int i; 3576 3577 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3578 i915_gem_free_phys_object(dev, i); 3579} 3580 3581void 3582i915_gem_detach_phys_object(struct drm_device *dev, 3583 struct drm_i915_gem_object *obj) 3584{ 3585 vm_page_t m; 3586 struct sf_buf *sf; 3587 char *vaddr, *dst; 3588 int i, page_count; 3589 3590 if (obj->phys_obj == NULL) 3591 return; 3592 vaddr = obj->phys_obj->handle->vaddr; 3593 3594 page_count = obj->base.size / PAGE_SIZE; 3595 VM_OBJECT_WLOCK(obj->base.vm_obj); 3596 for (i = 0; i < page_count; i++) { 3597 m = i915_gem_wire_page(obj->base.vm_obj, i); 3598 if (m == NULL) 3599 continue; /* XXX */ 3600 3601 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3602 sf = sf_buf_alloc(m, 0); 3603 if (sf != NULL) { 3604 dst = (char *)sf_buf_kva(sf); 3605 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3606 sf_buf_free(sf); 3607 } 3608 drm_clflush_pages(&m, 1); 3609 3610 VM_OBJECT_WLOCK(obj->base.vm_obj); 3611 vm_page_reference(m); 3612 vm_page_lock(m); 3613 vm_page_dirty(m); 3614 vm_page_unwire(m, PQ_INACTIVE); 3615 vm_page_unlock(m); 3616 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3617 } 3618 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3619 intel_gtt_chipset_flush(); 3620 3621 obj->phys_obj->cur_obj = NULL; 3622 obj->phys_obj = NULL; 3623} 3624 3625int 3626i915_gem_attach_phys_object(struct drm_device *dev, 3627 struct drm_i915_gem_object *obj, int id, int align) 3628{ 3629 drm_i915_private_t *dev_priv; 3630 vm_page_t m; 3631 struct sf_buf *sf; 3632 char *dst, *src; 3633 int i, page_count, ret; 3634 3635 if (id > I915_MAX_PHYS_OBJECT) 3636 return (-EINVAL); 3637 3638 if (obj->phys_obj != NULL) { 3639 if (obj->phys_obj->id == id) 3640 return (0); 3641 i915_gem_detach_phys_object(dev, obj); 3642 } 3643 3644 dev_priv = dev->dev_private; 3645 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3646 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3647 if (ret != 0) { 3648 DRM_ERROR("failed to init phys object %d size: %zu\n", 3649 id, obj->base.size); 3650 return (ret); 3651 } 3652 } 3653 3654 /* bind to the object */ 3655 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3656 obj->phys_obj->cur_obj = obj; 3657 3658 page_count = obj->base.size / PAGE_SIZE; 3659 3660 VM_OBJECT_WLOCK(obj->base.vm_obj); 3661 ret = 0; 3662 for (i = 0; i < page_count; i++) { 3663 m = i915_gem_wire_page(obj->base.vm_obj, i); 3664 if (m == NULL) { 3665 ret = -EIO; 3666 break; 3667 } 3668 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3669 sf = sf_buf_alloc(m, 0); 3670 src = (char *)sf_buf_kva(sf); 3671 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3672 memcpy(dst, src, PAGE_SIZE); 3673 sf_buf_free(sf); 3674 3675 VM_OBJECT_WLOCK(obj->base.vm_obj); 3676 3677 vm_page_reference(m); 3678 vm_page_lock(m); 3679 vm_page_unwire(m, PQ_INACTIVE); 3680 vm_page_unlock(m); 3681 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3682 } 3683 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3684 3685 return (0); 3686} 3687 3688static int 3689i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3690 uint64_t data_ptr, uint64_t offset, uint64_t size, 3691 struct drm_file *file_priv) 3692{ 3693 char *user_data, *vaddr; 3694 int ret; 3695 3696 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3697 user_data = (char *)(uintptr_t)data_ptr; 3698 3699 if (copyin_nofault(user_data, vaddr, size) != 0) { 3700 /* The physical object once assigned is fixed for the lifetime 3701 * of the obj, so we can safely drop the lock and continue 3702 * to access vaddr. 3703 */ 3704 DRM_UNLOCK(dev); 3705 ret = -copyin(user_data, vaddr, size); 3706 DRM_LOCK(dev); 3707 if (ret != 0) 3708 return (ret); 3709 } 3710 3711 intel_gtt_chipset_flush(); 3712 return (0); 3713} 3714 3715static int 3716i915_gpu_is_active(struct drm_device *dev) 3717{ 3718 drm_i915_private_t *dev_priv; 3719 3720 dev_priv = dev->dev_private; 3721 return (!list_empty(&dev_priv->mm.flushing_list) || 3722 !list_empty(&dev_priv->mm.active_list)); 3723} 3724 3725static void 3726i915_gem_lowmem(void *arg) 3727{ 3728 struct drm_device *dev; 3729 struct drm_i915_private *dev_priv; 3730 struct drm_i915_gem_object *obj, *next; 3731 int cnt, cnt_fail, cnt_total; 3732 3733 dev = arg; 3734 dev_priv = dev->dev_private; 3735 3736 if (!sx_try_xlock(&dev->dev_struct_lock)) 3737 return; 3738 3739 CTR0(KTR_DRM, "gem_lowmem"); 3740 3741rescan: 3742 /* first scan for clean buffers */ 3743 i915_gem_retire_requests(dev); 3744 3745 cnt_total = cnt_fail = cnt = 0; 3746 3747 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3748 mm_list) { 3749 if (i915_gem_object_is_purgeable(obj)) { 3750 if (i915_gem_object_unbind(obj) != 0) 3751 cnt_total++; 3752 } else 3753 cnt_total++; 3754 } 3755 3756 /* second pass, evict/count anything still on the inactive list */ 3757 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3758 mm_list) { 3759 if (i915_gem_object_unbind(obj) == 0) 3760 cnt++; 3761 else 3762 cnt_fail++; 3763 } 3764 3765 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3766 /* 3767 * We are desperate for pages, so as a last resort, wait 3768 * for the GPU to finish and discard whatever we can. 3769 * This has a dramatic impact to reduce the number of 3770 * OOM-killer events whilst running the GPU aggressively. 3771 */ 3772 if (i915_gpu_idle(dev, true) == 0) 3773 goto rescan; 3774 } 3775 DRM_UNLOCK(dev); 3776} 3777 3778void 3779i915_gem_unload(struct drm_device *dev) 3780{ 3781 struct drm_i915_private *dev_priv; 3782 3783 dev_priv = dev->dev_private; 3784 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3785} 3786