i915_gem.c revision 251452
1/*- 2 * Copyright �� 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 */ 53 54#include <sys/cdefs.h> 55__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 251452 2013-06-06 06:17:20Z alc $"); 56 57#include <dev/drm2/drmP.h> 58#include <dev/drm2/drm.h> 59#include <dev/drm2/i915/i915_drm.h> 60#include <dev/drm2/i915/i915_drv.h> 61#include <dev/drm2/i915/intel_drv.h> 62#include <dev/drm2/i915/intel_ringbuffer.h> 63#include <sys/resourcevar.h> 64#include <sys/sched.h> 65#include <sys/sf_buf.h> 66 67static void i915_gem_object_flush_cpu_write_domain( 68 struct drm_i915_gem_object *obj); 69static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 70 int tiling_mode); 71static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 72 uint32_t size, int tiling_mode); 73static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 74 unsigned alignment, bool map_and_fenceable); 75static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 76 int flags); 77static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 78static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 79 bool write); 80static void i915_gem_object_set_to_full_cpu_read_domain( 81 struct drm_i915_gem_object *obj); 82static int i915_gem_object_set_cpu_read_domain_range( 83 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 84static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 85static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 86static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 87static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 88static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 89static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 90static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 91 uint32_t flush_domains); 92static void i915_gem_clear_fence_reg(struct drm_device *dev, 93 struct drm_i915_fence_reg *reg); 94static void i915_gem_reset_fences(struct drm_device *dev); 95static void i915_gem_retire_task_handler(void *arg, int pending); 96static int i915_gem_phys_pwrite(struct drm_device *dev, 97 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 98 uint64_t size, struct drm_file *file_priv); 99static void i915_gem_lowmem(void *arg); 100 101MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 102long i915_gem_wired_pages_cnt; 103 104static void 105i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 106{ 107 108 dev_priv->mm.object_count++; 109 dev_priv->mm.object_memory += size; 110} 111 112static void 113i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 114{ 115 116 dev_priv->mm.object_count--; 117 dev_priv->mm.object_memory -= size; 118} 119 120static int 121i915_gem_wait_for_error(struct drm_device *dev) 122{ 123 struct drm_i915_private *dev_priv; 124 int ret; 125 126 dev_priv = dev->dev_private; 127 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 128 return (0); 129 130 mtx_lock(&dev_priv->error_completion_lock); 131 while (dev_priv->error_completion == 0) { 132 ret = -msleep(&dev_priv->error_completion, 133 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 134 if (ret != 0) { 135 mtx_unlock(&dev_priv->error_completion_lock); 136 return (ret); 137 } 138 } 139 mtx_unlock(&dev_priv->error_completion_lock); 140 141 if (atomic_read(&dev_priv->mm.wedged)) { 142 mtx_lock(&dev_priv->error_completion_lock); 143 dev_priv->error_completion++; 144 mtx_unlock(&dev_priv->error_completion_lock); 145 } 146 return (0); 147} 148 149int 150i915_mutex_lock_interruptible(struct drm_device *dev) 151{ 152 struct drm_i915_private *dev_priv; 153 int ret; 154 155 dev_priv = dev->dev_private; 156 ret = i915_gem_wait_for_error(dev); 157 if (ret != 0) 158 return (ret); 159 160 /* 161 * interruptible shall it be. might indeed be if dev_lock is 162 * changed to sx 163 */ 164 ret = sx_xlock_sig(&dev->dev_struct_lock); 165 if (ret != 0) 166 return (-ret); 167 168 return (0); 169} 170 171 172static void 173i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 174{ 175 struct drm_device *dev; 176 drm_i915_private_t *dev_priv; 177 int ret; 178 179 dev = obj->base.dev; 180 dev_priv = dev->dev_private; 181 182 ret = i915_gem_object_unbind(obj); 183 if (ret == -ERESTART) { 184 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 185 return; 186 } 187 188 CTR1(KTR_DRM, "object_destroy_tail %p", obj); 189 drm_gem_free_mmap_offset(&obj->base); 190 drm_gem_object_release(&obj->base); 191 i915_gem_info_remove_obj(dev_priv, obj->base.size); 192 193 free(obj->page_cpu_valid, DRM_I915_GEM); 194 free(obj->bit_17, DRM_I915_GEM); 195 free(obj, DRM_I915_GEM); 196} 197 198void 199i915_gem_free_object(struct drm_gem_object *gem_obj) 200{ 201 struct drm_i915_gem_object *obj; 202 struct drm_device *dev; 203 204 obj = to_intel_bo(gem_obj); 205 dev = obj->base.dev; 206 207 while (obj->pin_count > 0) 208 i915_gem_object_unpin(obj); 209 210 if (obj->phys_obj != NULL) 211 i915_gem_detach_phys_object(dev, obj); 212 213 i915_gem_free_object_tail(obj); 214} 215 216static void 217init_ring_lists(struct intel_ring_buffer *ring) 218{ 219 220 INIT_LIST_HEAD(&ring->active_list); 221 INIT_LIST_HEAD(&ring->request_list); 222 INIT_LIST_HEAD(&ring->gpu_write_list); 223} 224 225void 226i915_gem_load(struct drm_device *dev) 227{ 228 drm_i915_private_t *dev_priv; 229 int i; 230 231 dev_priv = dev->dev_private; 232 233 INIT_LIST_HEAD(&dev_priv->mm.active_list); 234 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 235 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 236 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 237 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 238 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 239 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 240 for (i = 0; i < I915_NUM_RINGS; i++) 241 init_ring_lists(&dev_priv->rings[i]); 242 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 243 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 244 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 245 i915_gem_retire_task_handler, dev_priv); 246 dev_priv->error_completion = 0; 247 248 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 249 if (IS_GEN3(dev)) { 250 u32 tmp = I915_READ(MI_ARB_STATE); 251 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { 252 /* 253 * arb state is a masked write, so set bit + 254 * bit in mask. 255 */ 256 tmp = MI_ARB_C3_LP_WRITE_ENABLE | 257 (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); 258 I915_WRITE(MI_ARB_STATE, tmp); 259 } 260 } 261 262 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 263 264 /* Old X drivers will take 0-2 for front, back, depth buffers */ 265 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 266 dev_priv->fence_reg_start = 3; 267 268 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 269 IS_G33(dev)) 270 dev_priv->num_fence_regs = 16; 271 else 272 dev_priv->num_fence_regs = 8; 273 274 /* Initialize fence registers to zero */ 275 for (i = 0; i < dev_priv->num_fence_regs; i++) { 276 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); 277 } 278 i915_gem_detect_bit_6_swizzle(dev); 279 dev_priv->mm.interruptible = true; 280 281 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 282 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 283} 284 285int 286i915_gem_do_init(struct drm_device *dev, unsigned long start, 287 unsigned long mappable_end, unsigned long end) 288{ 289 drm_i915_private_t *dev_priv; 290 unsigned long mappable; 291 int error; 292 293 dev_priv = dev->dev_private; 294 mappable = min(end, mappable_end) - start; 295 296 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 297 298 dev_priv->mm.gtt_start = start; 299 dev_priv->mm.gtt_mappable_end = mappable_end; 300 dev_priv->mm.gtt_end = end; 301 dev_priv->mm.gtt_total = end - start; 302 dev_priv->mm.mappable_gtt_total = mappable; 303 304 /* Take over this portion of the GTT */ 305 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 306 device_printf(dev->device, 307 "taking over the fictitious range 0x%lx-0x%lx\n", 308 dev->agp->base + start, dev->agp->base + start + mappable); 309 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 310 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 311 return (error); 312} 313 314int 315i915_gem_init_ioctl(struct drm_device *dev, void *data, 316 struct drm_file *file) 317{ 318 struct drm_i915_gem_init *args; 319 drm_i915_private_t *dev_priv; 320 321 dev_priv = dev->dev_private; 322 args = data; 323 324 if (args->gtt_start >= args->gtt_end || 325 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 326 return (-EINVAL); 327 328 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) 329 return (-EBUSY); 330 /* 331 * XXXKIB. The second-time initialization should be guarded 332 * against. 333 */ 334 return (i915_gem_do_init(dev, args->gtt_start, args->gtt_end, 335 args->gtt_end)); 336} 337 338int 339i915_gem_idle(struct drm_device *dev) 340{ 341 drm_i915_private_t *dev_priv; 342 int ret; 343 344 dev_priv = dev->dev_private; 345 if (dev_priv->mm.suspended) 346 return (0); 347 348 ret = i915_gpu_idle(dev, true); 349 if (ret != 0) 350 return (ret); 351 352 /* Under UMS, be paranoid and evict. */ 353 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 354 ret = i915_gem_evict_inactive(dev, false); 355 if (ret != 0) 356 return ret; 357 } 358 359 i915_gem_reset_fences(dev); 360 361 /* Hack! Don't let anybody do execbuf while we don't control the chip. 362 * We need to replace this with a semaphore, or something. 363 * And not confound mm.suspended! 364 */ 365 dev_priv->mm.suspended = 1; 366 callout_stop(&dev_priv->hangcheck_timer); 367 368 i915_kernel_lost_context(dev); 369 i915_gem_cleanup_ringbuffer(dev); 370 371 /* Cancel the retire work handler, which should be idle now. */ 372 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 373 return (ret); 374} 375 376void 377i915_gem_init_swizzling(struct drm_device *dev) 378{ 379 drm_i915_private_t *dev_priv; 380 381 dev_priv = dev->dev_private; 382 383 if (INTEL_INFO(dev)->gen < 5 || 384 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 385 return; 386 387 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 388 DISP_TILE_SURFACE_SWIZZLING); 389 390 if (IS_GEN5(dev)) 391 return; 392 393 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 394 if (IS_GEN6(dev)) 395 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); 396 else 397 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); 398} 399 400void 401i915_gem_init_ppgtt(struct drm_device *dev) 402{ 403 drm_i915_private_t *dev_priv; 404 struct i915_hw_ppgtt *ppgtt; 405 uint32_t pd_offset, pd_entry; 406 vm_paddr_t pt_addr; 407 struct intel_ring_buffer *ring; 408 u_int first_pd_entry_in_global_pt, i; 409 410 dev_priv = dev->dev_private; 411 ppgtt = dev_priv->mm.aliasing_ppgtt; 412 if (ppgtt == NULL) 413 return; 414 415 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 416 for (i = 0; i < ppgtt->num_pd_entries; i++) { 417 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 418 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 419 pd_entry |= GEN6_PDE_VALID; 420 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 421 } 422 intel_gtt_read_pte(first_pd_entry_in_global_pt); 423 424 pd_offset = ppgtt->pd_offset; 425 pd_offset /= 64; /* in cachelines, */ 426 pd_offset <<= 16; 427 428 if (INTEL_INFO(dev)->gen == 6) { 429 uint32_t ecochk = I915_READ(GAM_ECOCHK); 430 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 431 ECOCHK_PPGTT_CACHE64B); 432 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 433 } else if (INTEL_INFO(dev)->gen >= 7) { 434 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 435 /* GFX_MODE is per-ring on gen7+ */ 436 } 437 438 for (i = 0; i < I915_NUM_RINGS; i++) { 439 ring = &dev_priv->rings[i]; 440 441 if (INTEL_INFO(dev)->gen >= 7) 442 I915_WRITE(RING_MODE_GEN7(ring), 443 GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 444 445 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 446 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 447 } 448} 449 450int 451i915_gem_init_hw(struct drm_device *dev) 452{ 453 drm_i915_private_t *dev_priv; 454 int ret; 455 456 dev_priv = dev->dev_private; 457 458 i915_gem_init_swizzling(dev); 459 460 ret = intel_init_render_ring_buffer(dev); 461 if (ret != 0) 462 return (ret); 463 464 if (HAS_BSD(dev)) { 465 ret = intel_init_bsd_ring_buffer(dev); 466 if (ret != 0) 467 goto cleanup_render_ring; 468 } 469 470 if (HAS_BLT(dev)) { 471 ret = intel_init_blt_ring_buffer(dev); 472 if (ret != 0) 473 goto cleanup_bsd_ring; 474 } 475 476 dev_priv->next_seqno = 1; 477 i915_gem_init_ppgtt(dev); 478 return (0); 479 480cleanup_bsd_ring: 481 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 482cleanup_render_ring: 483 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 484 return (ret); 485} 486 487int 488i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 489 struct drm_file *file) 490{ 491 struct drm_i915_private *dev_priv; 492 struct drm_i915_gem_get_aperture *args; 493 struct drm_i915_gem_object *obj; 494 size_t pinned; 495 496 dev_priv = dev->dev_private; 497 args = data; 498 499 if (!(dev->driver->driver_features & DRIVER_GEM)) 500 return (-ENODEV); 501 502 pinned = 0; 503 DRM_LOCK(dev); 504 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 505 pinned += obj->gtt_space->size; 506 DRM_UNLOCK(dev); 507 508 args->aper_size = dev_priv->mm.gtt_total; 509 args->aper_available_size = args->aper_size - pinned; 510 511 return (0); 512} 513 514int 515i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 516 bool map_and_fenceable) 517{ 518 struct drm_device *dev; 519 struct drm_i915_private *dev_priv; 520 int ret; 521 522 dev = obj->base.dev; 523 dev_priv = dev->dev_private; 524 525 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 526 ("Max pin count")); 527 528 if (obj->gtt_space != NULL) { 529 if ((alignment && obj->gtt_offset & (alignment - 1)) || 530 (map_and_fenceable && !obj->map_and_fenceable)) { 531 DRM_DEBUG("bo is already pinned with incorrect alignment:" 532 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 533 " obj->map_and_fenceable=%d\n", 534 obj->gtt_offset, alignment, 535 map_and_fenceable, 536 obj->map_and_fenceable); 537 ret = i915_gem_object_unbind(obj); 538 if (ret != 0) 539 return (ret); 540 } 541 } 542 543 if (obj->gtt_space == NULL) { 544 ret = i915_gem_object_bind_to_gtt(obj, alignment, 545 map_and_fenceable); 546 if (ret) 547 return (ret); 548 } 549 550 if (obj->pin_count++ == 0 && !obj->active) 551 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 552 obj->pin_mappable |= map_and_fenceable; 553 554#if 1 555 KIB_NOTYET(); 556#else 557 WARN_ON(i915_verify_lists(dev)); 558#endif 559 return (0); 560} 561 562void 563i915_gem_object_unpin(struct drm_i915_gem_object *obj) 564{ 565 struct drm_device *dev; 566 drm_i915_private_t *dev_priv; 567 568 dev = obj->base.dev; 569 dev_priv = dev->dev_private; 570 571#if 1 572 KIB_NOTYET(); 573#else 574 WARN_ON(i915_verify_lists(dev)); 575#endif 576 577 KASSERT(obj->pin_count != 0, ("zero pin count")); 578 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 579 580 if (--obj->pin_count == 0) { 581 if (!obj->active) 582 list_move_tail(&obj->mm_list, 583 &dev_priv->mm.inactive_list); 584 obj->pin_mappable = false; 585 } 586#if 1 587 KIB_NOTYET(); 588#else 589 WARN_ON(i915_verify_lists(dev)); 590#endif 591} 592 593int 594i915_gem_pin_ioctl(struct drm_device *dev, void *data, 595 struct drm_file *file) 596{ 597 struct drm_i915_gem_pin *args; 598 struct drm_i915_gem_object *obj; 599 struct drm_gem_object *gobj; 600 int ret; 601 602 args = data; 603 604 ret = i915_mutex_lock_interruptible(dev); 605 if (ret != 0) 606 return ret; 607 608 gobj = drm_gem_object_lookup(dev, file, args->handle); 609 if (gobj == NULL) { 610 ret = -ENOENT; 611 goto unlock; 612 } 613 obj = to_intel_bo(gobj); 614 615 if (obj->madv != I915_MADV_WILLNEED) { 616 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 617 ret = -EINVAL; 618 goto out; 619 } 620 621 if (obj->pin_filp != NULL && obj->pin_filp != file) { 622 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 623 args->handle); 624 ret = -EINVAL; 625 goto out; 626 } 627 628 obj->user_pin_count++; 629 obj->pin_filp = file; 630 if (obj->user_pin_count == 1) { 631 ret = i915_gem_object_pin(obj, args->alignment, true); 632 if (ret != 0) 633 goto out; 634 } 635 636 /* XXX - flush the CPU caches for pinned objects 637 * as the X server doesn't manage domains yet 638 */ 639 i915_gem_object_flush_cpu_write_domain(obj); 640 args->offset = obj->gtt_offset; 641out: 642 drm_gem_object_unreference(&obj->base); 643unlock: 644 DRM_UNLOCK(dev); 645 return (ret); 646} 647 648int 649i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 650 struct drm_file *file) 651{ 652 struct drm_i915_gem_pin *args; 653 struct drm_i915_gem_object *obj; 654 int ret; 655 656 args = data; 657 ret = i915_mutex_lock_interruptible(dev); 658 if (ret != 0) 659 return (ret); 660 661 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 662 if (&obj->base == NULL) { 663 ret = -ENOENT; 664 goto unlock; 665 } 666 667 if (obj->pin_filp != file) { 668 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 669 args->handle); 670 ret = -EINVAL; 671 goto out; 672 } 673 obj->user_pin_count--; 674 if (obj->user_pin_count == 0) { 675 obj->pin_filp = NULL; 676 i915_gem_object_unpin(obj); 677 } 678 679out: 680 drm_gem_object_unreference(&obj->base); 681unlock: 682 DRM_UNLOCK(dev); 683 return (ret); 684} 685 686int 687i915_gem_busy_ioctl(struct drm_device *dev, void *data, 688 struct drm_file *file) 689{ 690 struct drm_i915_gem_busy *args; 691 struct drm_i915_gem_object *obj; 692 struct drm_i915_gem_request *request; 693 int ret; 694 695 args = data; 696 697 ret = i915_mutex_lock_interruptible(dev); 698 if (ret != 0) 699 return ret; 700 701 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 702 if (&obj->base == NULL) { 703 ret = -ENOENT; 704 goto unlock; 705 } 706 707 args->busy = obj->active; 708 if (args->busy) { 709 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 710 ret = i915_gem_flush_ring(obj->ring, 711 0, obj->base.write_domain); 712 } else if (obj->ring->outstanding_lazy_request == 713 obj->last_rendering_seqno) { 714 request = malloc(sizeof(*request), DRM_I915_GEM, 715 M_WAITOK | M_ZERO); 716 ret = i915_add_request(obj->ring, NULL, request); 717 if (ret != 0) 718 free(request, DRM_I915_GEM); 719 } 720 721 i915_gem_retire_requests_ring(obj->ring); 722 args->busy = obj->active; 723 } 724 725 drm_gem_object_unreference(&obj->base); 726unlock: 727 DRM_UNLOCK(dev); 728 return (ret); 729} 730 731static int 732i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 733{ 734 struct drm_i915_private *dev_priv; 735 struct drm_i915_file_private *file_priv; 736 unsigned long recent_enough; 737 struct drm_i915_gem_request *request; 738 struct intel_ring_buffer *ring; 739 u32 seqno; 740 int ret; 741 742 dev_priv = dev->dev_private; 743 if (atomic_read(&dev_priv->mm.wedged)) 744 return (-EIO); 745 746 file_priv = file->driver_priv; 747 recent_enough = ticks - (20 * hz / 1000); 748 ring = NULL; 749 seqno = 0; 750 751 mtx_lock(&file_priv->mm.lck); 752 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 753 if (time_after_eq(request->emitted_jiffies, recent_enough)) 754 break; 755 ring = request->ring; 756 seqno = request->seqno; 757 } 758 mtx_unlock(&file_priv->mm.lck); 759 if (seqno == 0) 760 return (0); 761 762 ret = 0; 763 mtx_lock(&ring->irq_lock); 764 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 765 if (ring->irq_get(ring)) { 766 while (ret == 0 && 767 !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 768 atomic_read(&dev_priv->mm.wedged))) 769 ret = -msleep(ring, &ring->irq_lock, PCATCH, 770 "915thr", 0); 771 ring->irq_put(ring); 772 if (ret == 0 && atomic_read(&dev_priv->mm.wedged)) 773 ret = -EIO; 774 } else if (_intel_wait_for(dev, 775 i915_seqno_passed(ring->get_seqno(ring), seqno) || 776 atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 777 ret = -EBUSY; 778 } 779 } 780 mtx_unlock(&ring->irq_lock); 781 782 if (ret == 0) 783 taskqueue_enqueue_timeout(dev_priv->tq, 784 &dev_priv->mm.retire_task, 0); 785 786 return (ret); 787} 788 789int 790i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 791 struct drm_file *file_priv) 792{ 793 794 return (i915_gem_ring_throttle(dev, file_priv)); 795} 796 797int 798i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 799 struct drm_file *file_priv) 800{ 801 struct drm_i915_gem_madvise *args; 802 struct drm_i915_gem_object *obj; 803 int ret; 804 805 args = data; 806 switch (args->madv) { 807 case I915_MADV_DONTNEED: 808 case I915_MADV_WILLNEED: 809 break; 810 default: 811 return (-EINVAL); 812 } 813 814 ret = i915_mutex_lock_interruptible(dev); 815 if (ret != 0) 816 return (ret); 817 818 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 819 if (&obj->base == NULL) { 820 ret = -ENOENT; 821 goto unlock; 822 } 823 824 if (obj->pin_count != 0) { 825 ret = -EINVAL; 826 goto out; 827 } 828 829 if (obj->madv != I915_MADV_PURGED_INTERNAL) 830 obj->madv = args->madv; 831 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 832 i915_gem_object_truncate(obj); 833 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 834 835out: 836 drm_gem_object_unreference(&obj->base); 837unlock: 838 DRM_UNLOCK(dev); 839 return (ret); 840} 841 842void 843i915_gem_cleanup_ringbuffer(struct drm_device *dev) 844{ 845 drm_i915_private_t *dev_priv; 846 int i; 847 848 dev_priv = dev->dev_private; 849 for (i = 0; i < I915_NUM_RINGS; i++) 850 intel_cleanup_ring_buffer(&dev_priv->rings[i]); 851} 852 853int 854i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 855 struct drm_file *file_priv) 856{ 857 drm_i915_private_t *dev_priv; 858 int ret, i; 859 860 if (drm_core_check_feature(dev, DRIVER_MODESET)) 861 return (0); 862 dev_priv = dev->dev_private; 863 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 864 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 865 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 866 } 867 868 dev_priv->mm.suspended = 0; 869 870 ret = i915_gem_init_hw(dev); 871 if (ret != 0) { 872 return (ret); 873 } 874 875 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 876 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 877 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 878 for (i = 0; i < I915_NUM_RINGS; i++) { 879 KASSERT(list_empty(&dev_priv->rings[i].active_list), 880 ("ring %d active list", i)); 881 KASSERT(list_empty(&dev_priv->rings[i].request_list), 882 ("ring %d request list", i)); 883 } 884 885 DRM_UNLOCK(dev); 886 ret = drm_irq_install(dev); 887 DRM_LOCK(dev); 888 if (ret) 889 goto cleanup_ringbuffer; 890 891 return (0); 892 893cleanup_ringbuffer: 894 i915_gem_cleanup_ringbuffer(dev); 895 dev_priv->mm.suspended = 1; 896 897 return (ret); 898} 899 900int 901i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 902 struct drm_file *file_priv) 903{ 904 905 if (drm_core_check_feature(dev, DRIVER_MODESET)) 906 return 0; 907 908 drm_irq_uninstall(dev); 909 return (i915_gem_idle(dev)); 910} 911 912int 913i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 914 uint32_t *handle_p) 915{ 916 struct drm_i915_gem_object *obj; 917 uint32_t handle; 918 int ret; 919 920 size = roundup(size, PAGE_SIZE); 921 if (size == 0) 922 return (-EINVAL); 923 924 obj = i915_gem_alloc_object(dev, size); 925 if (obj == NULL) 926 return (-ENOMEM); 927 928 handle = 0; 929 ret = drm_gem_handle_create(file, &obj->base, &handle); 930 if (ret != 0) { 931 drm_gem_object_release(&obj->base); 932 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 933 free(obj, DRM_I915_GEM); 934 return (-ret); 935 } 936 937 /* drop reference from allocate - handle holds it now */ 938 drm_gem_object_unreference(&obj->base); 939 CTR2(KTR_DRM, "object_create %p %x", obj, size); 940 *handle_p = handle; 941 return (0); 942} 943 944int 945i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 946 struct drm_mode_create_dumb *args) 947{ 948 949 /* have to work out size/pitch and return them */ 950 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 951 args->size = args->pitch * args->height; 952 return (i915_gem_create(file, dev, args->size, &args->handle)); 953} 954 955int 956i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 957 uint32_t handle) 958{ 959 960 return (drm_gem_handle_delete(file, handle)); 961} 962 963int 964i915_gem_create_ioctl(struct drm_device *dev, void *data, 965 struct drm_file *file) 966{ 967 struct drm_i915_gem_create *args = data; 968 969 return (i915_gem_create(file, dev, args->size, &args->handle)); 970} 971 972static int 973i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 974 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 975 struct drm_file *file) 976{ 977 vm_object_t vm_obj; 978 vm_page_t m; 979 struct sf_buf *sf; 980 vm_offset_t mkva; 981 vm_pindex_t obj_pi; 982 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 983 984 if (obj->gtt_offset != 0 && rw == UIO_READ) 985 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 986 else 987 do_bit17_swizzling = 0; 988 989 obj->dirty = 1; 990 vm_obj = obj->base.vm_obj; 991 ret = 0; 992 993 VM_OBJECT_WLOCK(vm_obj); 994 vm_object_pip_add(vm_obj, 1); 995 while (size > 0) { 996 obj_pi = OFF_TO_IDX(offset); 997 obj_po = offset & PAGE_MASK; 998 999 m = i915_gem_wire_page(vm_obj, obj_pi); 1000 VM_OBJECT_WUNLOCK(vm_obj); 1001 1002 sched_pin(); 1003 sf = sf_buf_alloc(m, SFB_CPUPRIVATE); 1004 mkva = sf_buf_kva(sf); 1005 length = min(size, PAGE_SIZE - obj_po); 1006 while (length > 0) { 1007 if (do_bit17_swizzling && 1008 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 1009 cnt = roundup2(obj_po + 1, 64); 1010 cnt = min(cnt - obj_po, length); 1011 swizzled_po = obj_po ^ 64; 1012 } else { 1013 cnt = length; 1014 swizzled_po = obj_po; 1015 } 1016 if (rw == UIO_READ) 1017 ret = -copyout_nofault( 1018 (char *)mkva + swizzled_po, 1019 (void *)(uintptr_t)data_ptr, cnt); 1020 else 1021 ret = -copyin_nofault( 1022 (void *)(uintptr_t)data_ptr, 1023 (char *)mkva + swizzled_po, cnt); 1024 if (ret != 0) 1025 break; 1026 data_ptr += cnt; 1027 size -= cnt; 1028 length -= cnt; 1029 offset += cnt; 1030 obj_po += cnt; 1031 } 1032 sf_buf_free(sf); 1033 sched_unpin(); 1034 VM_OBJECT_WLOCK(vm_obj); 1035 if (rw == UIO_WRITE) 1036 vm_page_dirty(m); 1037 vm_page_reference(m); 1038 vm_page_lock(m); 1039 vm_page_unwire(m, 1); 1040 vm_page_unlock(m); 1041 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1042 1043 if (ret != 0) 1044 break; 1045 } 1046 vm_object_pip_wakeup(vm_obj); 1047 VM_OBJECT_WUNLOCK(vm_obj); 1048 1049 return (ret); 1050} 1051 1052static int 1053i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 1054 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 1055{ 1056 vm_offset_t mkva; 1057 vm_pindex_t obj_pi; 1058 int obj_po, ret; 1059 1060 obj_pi = OFF_TO_IDX(offset); 1061 obj_po = offset & PAGE_MASK; 1062 1063 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 1064 IDX_TO_OFF(obj_pi), size, PAT_WRITE_COMBINING); 1065 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva + 1066 obj_po, size); 1067 pmap_unmapdev(mkva, size); 1068 return (ret); 1069} 1070 1071static int 1072i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 1073 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 1074{ 1075 struct drm_i915_gem_object *obj; 1076 vm_page_t *ma; 1077 vm_offset_t start, end; 1078 int npages, ret; 1079 1080 if (size == 0) 1081 return (0); 1082 start = trunc_page(data_ptr); 1083 end = round_page(data_ptr + size); 1084 npages = howmany(end - start, PAGE_SIZE); 1085 ma = malloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 1086 M_ZERO); 1087 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 1088 (vm_offset_t)data_ptr, size, 1089 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 1090 if (npages == -1) { 1091 ret = -EFAULT; 1092 goto free_ma; 1093 } 1094 1095 ret = i915_mutex_lock_interruptible(dev); 1096 if (ret != 0) 1097 goto unlocked; 1098 1099 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1100 if (&obj->base == NULL) { 1101 ret = -ENOENT; 1102 goto unlock; 1103 } 1104 if (offset > obj->base.size || size > obj->base.size - offset) { 1105 ret = -EINVAL; 1106 goto out; 1107 } 1108 1109 if (rw == UIO_READ) { 1110 CTR3(KTR_DRM, "object_pread %p %jx %jx", obj, offset, size); 1111 ret = i915_gem_object_set_cpu_read_domain_range(obj, 1112 offset, size); 1113 if (ret != 0) 1114 goto out; 1115 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1116 UIO_READ, file); 1117 } else { 1118 if (obj->phys_obj) { 1119 CTR3(KTR_DRM, "object_phys_write %p %jx %jx", obj, 1120 offset, size); 1121 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1122 size, file); 1123 } else if (obj->gtt_space && 1124 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1125 CTR3(KTR_DRM, "object_gtt_write %p %jx %jx", obj, 1126 offset, size); 1127 ret = i915_gem_object_pin(obj, 0, true); 1128 if (ret != 0) 1129 goto out; 1130 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1131 if (ret != 0) 1132 goto out_unpin; 1133 ret = i915_gem_object_put_fence(obj); 1134 if (ret != 0) 1135 goto out_unpin; 1136 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1137 offset, file); 1138out_unpin: 1139 i915_gem_object_unpin(obj); 1140 } else { 1141 CTR3(KTR_DRM, "object_pwrite %p %jx %jx", obj, 1142 offset, size); 1143 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1144 if (ret != 0) 1145 goto out; 1146 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1147 UIO_WRITE, file); 1148 } 1149 } 1150out: 1151 drm_gem_object_unreference(&obj->base); 1152unlock: 1153 DRM_UNLOCK(dev); 1154unlocked: 1155 vm_page_unhold_pages(ma, npages); 1156free_ma: 1157 free(ma, DRM_I915_GEM); 1158 return (ret); 1159} 1160 1161int 1162i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1163{ 1164 struct drm_i915_gem_pread *args; 1165 1166 args = data; 1167 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1168 args->offset, UIO_READ, file)); 1169} 1170 1171int 1172i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1173{ 1174 struct drm_i915_gem_pwrite *args; 1175 1176 args = data; 1177 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1178 args->offset, UIO_WRITE, file)); 1179} 1180 1181int 1182i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1183 struct drm_file *file) 1184{ 1185 struct drm_i915_gem_set_domain *args; 1186 struct drm_i915_gem_object *obj; 1187 uint32_t read_domains; 1188 uint32_t write_domain; 1189 int ret; 1190 1191 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1192 return (-ENODEV); 1193 1194 args = data; 1195 read_domains = args->read_domains; 1196 write_domain = args->write_domain; 1197 1198 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1199 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1200 (write_domain != 0 && read_domains != write_domain)) 1201 return (-EINVAL); 1202 1203 ret = i915_mutex_lock_interruptible(dev); 1204 if (ret != 0) 1205 return (ret); 1206 1207 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1208 if (&obj->base == NULL) { 1209 ret = -ENOENT; 1210 goto unlock; 1211 } 1212 1213 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1214 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1215 if (ret == -EINVAL) 1216 ret = 0; 1217 } else 1218 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1219 1220 drm_gem_object_unreference(&obj->base); 1221unlock: 1222 DRM_UNLOCK(dev); 1223 return (ret); 1224} 1225 1226int 1227i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1228 struct drm_file *file) 1229{ 1230 struct drm_i915_gem_sw_finish *args; 1231 struct drm_i915_gem_object *obj; 1232 int ret; 1233 1234 args = data; 1235 ret = 0; 1236 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1237 return (ENODEV); 1238 ret = i915_mutex_lock_interruptible(dev); 1239 if (ret != 0) 1240 return (ret); 1241 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1242 if (&obj->base == NULL) { 1243 ret = -ENOENT; 1244 goto unlock; 1245 } 1246 if (obj->pin_count != 0) 1247 i915_gem_object_flush_cpu_write_domain(obj); 1248 drm_gem_object_unreference(&obj->base); 1249unlock: 1250 DRM_UNLOCK(dev); 1251 return (ret); 1252} 1253 1254int 1255i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1256 struct drm_file *file) 1257{ 1258 struct drm_i915_gem_mmap *args; 1259 struct drm_gem_object *obj; 1260 struct proc *p; 1261 vm_map_t map; 1262 vm_offset_t addr; 1263 vm_size_t size; 1264 int error, rv; 1265 1266 args = data; 1267 1268 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1269 return (-ENODEV); 1270 1271 obj = drm_gem_object_lookup(dev, file, args->handle); 1272 if (obj == NULL) 1273 return (-ENOENT); 1274 error = 0; 1275 if (args->size == 0) 1276 goto out; 1277 p = curproc; 1278 map = &p->p_vmspace->vm_map; 1279 size = round_page(args->size); 1280 PROC_LOCK(p); 1281 if (map->size + size > lim_cur(p, RLIMIT_VMEM)) { 1282 PROC_UNLOCK(p); 1283 error = ENOMEM; 1284 goto out; 1285 } 1286 PROC_UNLOCK(p); 1287 1288 addr = 0; 1289 vm_object_reference(obj->vm_obj); 1290 DRM_UNLOCK(dev); 1291 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 1292 VMFS_ANY_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1293 VM_PROT_READ | VM_PROT_WRITE, MAP_SHARED); 1294 if (rv != KERN_SUCCESS) { 1295 vm_object_deallocate(obj->vm_obj); 1296 error = -vm_mmap_to_errno(rv); 1297 } else { 1298 args->addr_ptr = (uint64_t)addr; 1299 } 1300 DRM_LOCK(dev); 1301out: 1302 drm_gem_object_unreference(obj); 1303 return (error); 1304} 1305 1306static int 1307i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1308 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1309{ 1310 1311 *color = 0; /* XXXKIB */ 1312 return (0); 1313} 1314 1315int i915_intr_pf; 1316 1317static int 1318i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1319 vm_page_t *mres) 1320{ 1321 struct drm_gem_object *gem_obj; 1322 struct drm_i915_gem_object *obj; 1323 struct drm_device *dev; 1324 drm_i915_private_t *dev_priv; 1325 vm_page_t m, oldm; 1326 int cause, ret; 1327 bool write; 1328 1329 gem_obj = vm_obj->handle; 1330 obj = to_intel_bo(gem_obj); 1331 dev = obj->base.dev; 1332 dev_priv = dev->dev_private; 1333#if 0 1334 write = (prot & VM_PROT_WRITE) != 0; 1335#else 1336 write = true; 1337#endif 1338 vm_object_pip_add(vm_obj, 1); 1339 1340 /* 1341 * Remove the placeholder page inserted by vm_fault() from the 1342 * object before dropping the object lock. If 1343 * i915_gem_release_mmap() is active in parallel on this gem 1344 * object, then it owns the drm device sx and might find the 1345 * placeholder already. Then, since the page is busy, 1346 * i915_gem_release_mmap() sleeps waiting for the busy state 1347 * of the page cleared. We will be not able to acquire drm 1348 * device lock until i915_gem_release_mmap() is able to make a 1349 * progress. 1350 */ 1351 if (*mres != NULL) { 1352 oldm = *mres; 1353 vm_page_lock(oldm); 1354 vm_page_remove(oldm); 1355 vm_page_unlock(oldm); 1356 *mres = NULL; 1357 } else 1358 oldm = NULL; 1359retry: 1360 VM_OBJECT_WUNLOCK(vm_obj); 1361unlocked_vmobj: 1362 cause = ret = 0; 1363 m = NULL; 1364 1365 1366 if (i915_intr_pf) { 1367 ret = i915_mutex_lock_interruptible(dev); 1368 if (ret != 0) { 1369 cause = 10; 1370 goto out; 1371 } 1372 } else 1373 DRM_LOCK(dev); 1374 1375 /* Now bind it into the GTT if needed */ 1376 if (!obj->map_and_fenceable) { 1377 ret = i915_gem_object_unbind(obj); 1378 if (ret != 0) { 1379 cause = 20; 1380 goto unlock; 1381 } 1382 } 1383 if (!obj->gtt_space) { 1384 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1385 if (ret != 0) { 1386 cause = 30; 1387 goto unlock; 1388 } 1389 1390 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1391 if (ret != 0) { 1392 cause = 40; 1393 goto unlock; 1394 } 1395 } 1396 1397 if (obj->tiling_mode == I915_TILING_NONE) 1398 ret = i915_gem_object_put_fence(obj); 1399 else 1400 ret = i915_gem_object_get_fence(obj, NULL); 1401 if (ret != 0) { 1402 cause = 50; 1403 goto unlock; 1404 } 1405 1406 if (i915_gem_object_is_inactive(obj)) 1407 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1408 1409 obj->fault_mappable = true; 1410 VM_OBJECT_WLOCK(vm_obj); 1411 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 1412 offset); 1413 if (m == NULL) { 1414 cause = 60; 1415 ret = -EFAULT; 1416 goto unlock; 1417 } 1418 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1419 ("not fictitious %p", m)); 1420 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1421 1422 if ((m->flags & VPO_BUSY) != 0) { 1423 DRM_UNLOCK(dev); 1424 vm_page_sleep(m, "915pbs"); 1425 goto retry; 1426 } 1427 m->valid = VM_PAGE_BITS_ALL; 1428 *mres = m; 1429 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); 1430 vm_page_busy(m); 1431 1432 CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 1433 m->phys_addr); 1434 DRM_UNLOCK(dev); 1435 if (oldm != NULL) { 1436 vm_page_lock(oldm); 1437 vm_page_free(oldm); 1438 vm_page_unlock(oldm); 1439 } 1440 vm_object_pip_wakeup(vm_obj); 1441 return (VM_PAGER_OK); 1442 1443unlock: 1444 DRM_UNLOCK(dev); 1445out: 1446 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1447 CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, 1448 -ret, cause); 1449 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1450 kern_yield(PRI_USER); 1451 goto unlocked_vmobj; 1452 } 1453 VM_OBJECT_WLOCK(vm_obj); 1454 vm_object_pip_wakeup(vm_obj); 1455 return (VM_PAGER_ERROR); 1456} 1457 1458static void 1459i915_gem_pager_dtor(void *handle) 1460{ 1461 struct drm_gem_object *obj; 1462 struct drm_device *dev; 1463 1464 obj = handle; 1465 dev = obj->dev; 1466 1467 DRM_LOCK(dev); 1468 drm_gem_free_mmap_offset(obj); 1469 i915_gem_release_mmap(to_intel_bo(obj)); 1470 drm_gem_object_unreference(obj); 1471 DRM_UNLOCK(dev); 1472} 1473 1474struct cdev_pager_ops i915_gem_pager_ops = { 1475 .cdev_pg_fault = i915_gem_pager_fault, 1476 .cdev_pg_ctor = i915_gem_pager_ctor, 1477 .cdev_pg_dtor = i915_gem_pager_dtor 1478}; 1479 1480int 1481i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1482 uint32_t handle, uint64_t *offset) 1483{ 1484 struct drm_i915_private *dev_priv; 1485 struct drm_i915_gem_object *obj; 1486 int ret; 1487 1488 if (!(dev->driver->driver_features & DRIVER_GEM)) 1489 return (-ENODEV); 1490 1491 dev_priv = dev->dev_private; 1492 1493 ret = i915_mutex_lock_interruptible(dev); 1494 if (ret != 0) 1495 return (ret); 1496 1497 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1498 if (&obj->base == NULL) { 1499 ret = -ENOENT; 1500 goto unlock; 1501 } 1502 1503 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1504 ret = -E2BIG; 1505 goto out; 1506 } 1507 1508 if (obj->madv != I915_MADV_WILLNEED) { 1509 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1510 ret = -EINVAL; 1511 goto out; 1512 } 1513 1514 ret = drm_gem_create_mmap_offset(&obj->base); 1515 if (ret != 0) 1516 goto out; 1517 1518 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1519 DRM_GEM_MAPPING_KEY; 1520out: 1521 drm_gem_object_unreference(&obj->base); 1522unlock: 1523 DRM_UNLOCK(dev); 1524 return (ret); 1525} 1526 1527int 1528i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1529 struct drm_file *file) 1530{ 1531 struct drm_i915_private *dev_priv; 1532 struct drm_i915_gem_mmap_gtt *args; 1533 1534 dev_priv = dev->dev_private; 1535 args = data; 1536 1537 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1538} 1539 1540struct drm_i915_gem_object * 1541i915_gem_alloc_object(struct drm_device *dev, size_t size) 1542{ 1543 struct drm_i915_private *dev_priv; 1544 struct drm_i915_gem_object *obj; 1545 1546 dev_priv = dev->dev_private; 1547 1548 obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1549 1550 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1551 free(obj, DRM_I915_GEM); 1552 return (NULL); 1553 } 1554 1555 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1556 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1557 1558 if (HAS_LLC(dev)) 1559 obj->cache_level = I915_CACHE_LLC; 1560 else 1561 obj->cache_level = I915_CACHE_NONE; 1562 obj->base.driver_private = NULL; 1563 obj->fence_reg = I915_FENCE_REG_NONE; 1564 INIT_LIST_HEAD(&obj->mm_list); 1565 INIT_LIST_HEAD(&obj->gtt_list); 1566 INIT_LIST_HEAD(&obj->ring_list); 1567 INIT_LIST_HEAD(&obj->exec_list); 1568 INIT_LIST_HEAD(&obj->gpu_write_list); 1569 obj->madv = I915_MADV_WILLNEED; 1570 /* Avoid an unnecessary call to unbind on the first bind. */ 1571 obj->map_and_fenceable = true; 1572 1573 i915_gem_info_add_obj(dev_priv, size); 1574 1575 return (obj); 1576} 1577 1578void 1579i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1580{ 1581 1582 /* If we don't have a page list set up, then we're not pinned 1583 * to GPU, and we can ignore the cache flush because it'll happen 1584 * again at bind time. 1585 */ 1586 if (obj->pages == NULL) 1587 return; 1588 1589 /* If the GPU is snooping the contents of the CPU cache, 1590 * we do not need to manually clear the CPU cache lines. However, 1591 * the caches are only snooped when the render cache is 1592 * flushed/invalidated. As we always have to emit invalidations 1593 * and flushes when moving into and out of the RENDER domain, correct 1594 * snooping behaviour occurs naturally as the result of our domain 1595 * tracking. 1596 */ 1597 if (obj->cache_level != I915_CACHE_NONE) 1598 return; 1599 1600 CTR1(KTR_DRM, "object_clflush %p", obj); 1601 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1602} 1603 1604static void 1605i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1606{ 1607 uint32_t old_write_domain; 1608 1609 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1610 return; 1611 1612 i915_gem_clflush_object(obj); 1613 intel_gtt_chipset_flush(); 1614 old_write_domain = obj->base.write_domain; 1615 obj->base.write_domain = 0; 1616 1617 CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 1618 obj->base.read_domains, old_write_domain); 1619} 1620 1621static int 1622i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1623{ 1624 1625 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1626 return (0); 1627 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1628} 1629 1630static void 1631i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1632{ 1633 uint32_t old_write_domain; 1634 1635 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1636 return; 1637 1638 wmb(); 1639 1640 old_write_domain = obj->base.write_domain; 1641 obj->base.write_domain = 0; 1642 1643 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 1644 obj->base.read_domains, old_write_domain); 1645} 1646 1647int 1648i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1649{ 1650 uint32_t old_write_domain, old_read_domains; 1651 int ret; 1652 1653 if (obj->gtt_space == NULL) 1654 return (-EINVAL); 1655 1656 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1657 return 0; 1658 1659 ret = i915_gem_object_flush_gpu_write_domain(obj); 1660 if (ret != 0) 1661 return (ret); 1662 1663 if (obj->pending_gpu_write || write) { 1664 ret = i915_gem_object_wait_rendering(obj); 1665 if (ret != 0) 1666 return (ret); 1667 } 1668 1669 i915_gem_object_flush_cpu_write_domain(obj); 1670 1671 old_write_domain = obj->base.write_domain; 1672 old_read_domains = obj->base.read_domains; 1673 1674 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1675 ("In GTT write domain")); 1676 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1677 if (write) { 1678 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1679 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1680 obj->dirty = 1; 1681 } 1682 1683 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 1684 old_read_domains, old_write_domain); 1685 return (0); 1686} 1687 1688int 1689i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1690 enum i915_cache_level cache_level) 1691{ 1692 struct drm_device *dev; 1693 drm_i915_private_t *dev_priv; 1694 int ret; 1695 1696 if (obj->cache_level == cache_level) 1697 return 0; 1698 1699 if (obj->pin_count) { 1700 DRM_DEBUG("can not change the cache level of pinned objects\n"); 1701 return (-EBUSY); 1702 } 1703 1704 dev = obj->base.dev; 1705 dev_priv = dev->dev_private; 1706 if (obj->gtt_space) { 1707 ret = i915_gem_object_finish_gpu(obj); 1708 if (ret != 0) 1709 return (ret); 1710 1711 i915_gem_object_finish_gtt(obj); 1712 1713 /* Before SandyBridge, you could not use tiling or fence 1714 * registers with snooped memory, so relinquish any fences 1715 * currently pointing to our region in the aperture. 1716 */ 1717 if (INTEL_INFO(obj->base.dev)->gen < 6) { 1718 ret = i915_gem_object_put_fence(obj); 1719 if (ret != 0) 1720 return (ret); 1721 } 1722 1723 i915_gem_gtt_rebind_object(obj, cache_level); 1724 if (obj->has_aliasing_ppgtt_mapping) 1725 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1726 obj, cache_level); 1727 } 1728 1729 if (cache_level == I915_CACHE_NONE) { 1730 u32 old_read_domains, old_write_domain; 1731 1732 /* If we're coming from LLC cached, then we haven't 1733 * actually been tracking whether the data is in the 1734 * CPU cache or not, since we only allow one bit set 1735 * in obj->write_domain and have been skipping the clflushes. 1736 * Just set it to the CPU cache for now. 1737 */ 1738 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1739 ("obj %p in CPU write domain", obj)); 1740 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1741 ("obj %p in CPU read domain", obj)); 1742 1743 old_read_domains = obj->base.read_domains; 1744 old_write_domain = obj->base.write_domain; 1745 1746 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1747 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1748 1749 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 1750 obj, old_read_domains, old_write_domain); 1751 } 1752 1753 obj->cache_level = cache_level; 1754 return (0); 1755} 1756 1757int 1758i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1759 u32 alignment, struct intel_ring_buffer *pipelined) 1760{ 1761 u32 old_read_domains, old_write_domain; 1762 int ret; 1763 1764 ret = i915_gem_object_flush_gpu_write_domain(obj); 1765 if (ret != 0) 1766 return (ret); 1767 1768 if (pipelined != obj->ring) { 1769 ret = i915_gem_object_wait_rendering(obj); 1770 if (ret == -ERESTART || ret == -EINTR) 1771 return (ret); 1772 } 1773 1774 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1775 if (ret != 0) 1776 return (ret); 1777 1778 ret = i915_gem_object_pin(obj, alignment, true); 1779 if (ret != 0) 1780 return (ret); 1781 1782 i915_gem_object_flush_cpu_write_domain(obj); 1783 1784 old_write_domain = obj->base.write_domain; 1785 old_read_domains = obj->base.read_domains; 1786 1787 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1788 ("obj %p in GTT write domain", obj)); 1789 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1790 1791 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 1792 obj, old_read_domains, obj->base.write_domain); 1793 return (0); 1794} 1795 1796int 1797i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1798{ 1799 int ret; 1800 1801 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1802 return (0); 1803 1804 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1805 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1806 if (ret != 0) 1807 return (ret); 1808 } 1809 1810 ret = i915_gem_object_wait_rendering(obj); 1811 if (ret != 0) 1812 return (ret); 1813 1814 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1815 1816 return (0); 1817} 1818 1819static int 1820i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1821{ 1822 uint32_t old_write_domain, old_read_domains; 1823 int ret; 1824 1825 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1826 return 0; 1827 1828 ret = i915_gem_object_flush_gpu_write_domain(obj); 1829 if (ret != 0) 1830 return (ret); 1831 1832 ret = i915_gem_object_wait_rendering(obj); 1833 if (ret != 0) 1834 return (ret); 1835 1836 i915_gem_object_flush_gtt_write_domain(obj); 1837 i915_gem_object_set_to_full_cpu_read_domain(obj); 1838 1839 old_write_domain = obj->base.write_domain; 1840 old_read_domains = obj->base.read_domains; 1841 1842 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1843 i915_gem_clflush_object(obj); 1844 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1845 } 1846 1847 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1848 ("In cpu write domain")); 1849 1850 if (write) { 1851 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1852 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1853 } 1854 1855 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 1856 old_read_domains, old_write_domain); 1857 return (0); 1858} 1859 1860static void 1861i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1862{ 1863 int i; 1864 1865 if (obj->page_cpu_valid == NULL) 1866 return; 1867 1868 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1869 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1870 if (obj->page_cpu_valid[i] != 0) 1871 continue; 1872 drm_clflush_pages(obj->pages + i, 1); 1873 } 1874 } 1875 1876 free(obj->page_cpu_valid, DRM_I915_GEM); 1877 obj->page_cpu_valid = NULL; 1878} 1879 1880static int 1881i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1882 uint64_t offset, uint64_t size) 1883{ 1884 uint32_t old_read_domains; 1885 int i, ret; 1886 1887 if (offset == 0 && size == obj->base.size) 1888 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1889 1890 ret = i915_gem_object_flush_gpu_write_domain(obj); 1891 if (ret != 0) 1892 return (ret); 1893 ret = i915_gem_object_wait_rendering(obj); 1894 if (ret != 0) 1895 return (ret); 1896 1897 i915_gem_object_flush_gtt_write_domain(obj); 1898 1899 if (obj->page_cpu_valid == NULL && 1900 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1901 return (0); 1902 1903 if (obj->page_cpu_valid == NULL) { 1904 obj->page_cpu_valid = malloc(obj->base.size / PAGE_SIZE, 1905 DRM_I915_GEM, M_WAITOK | M_ZERO); 1906 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1907 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1908 1909 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1910 i++) { 1911 if (obj->page_cpu_valid[i]) 1912 continue; 1913 drm_clflush_pages(obj->pages + i, 1); 1914 obj->page_cpu_valid[i] = 1; 1915 } 1916 1917 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1918 ("In gpu write domain")); 1919 1920 old_read_domains = obj->base.read_domains; 1921 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1922 1923 CTR3(KTR_DRM, "object_change_domain set_cpu_read %p %x %x", obj, 1924 old_read_domains, obj->base.write_domain); 1925 return (0); 1926} 1927 1928static uint32_t 1929i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1930{ 1931 uint32_t gtt_size; 1932 1933 if (INTEL_INFO(dev)->gen >= 4 || 1934 tiling_mode == I915_TILING_NONE) 1935 return (size); 1936 1937 /* Previous chips need a power-of-two fence region when tiling */ 1938 if (INTEL_INFO(dev)->gen == 3) 1939 gtt_size = 1024*1024; 1940 else 1941 gtt_size = 512*1024; 1942 1943 while (gtt_size < size) 1944 gtt_size <<= 1; 1945 1946 return (gtt_size); 1947} 1948 1949/** 1950 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1951 * @obj: object to check 1952 * 1953 * Return the required GTT alignment for an object, taking into account 1954 * potential fence register mapping. 1955 */ 1956static uint32_t 1957i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1958 int tiling_mode) 1959{ 1960 1961 /* 1962 * Minimum alignment is 4k (GTT page size), but might be greater 1963 * if a fence register is needed for the object. 1964 */ 1965 if (INTEL_INFO(dev)->gen >= 4 || 1966 tiling_mode == I915_TILING_NONE) 1967 return (4096); 1968 1969 /* 1970 * Previous chips need to be aligned to the size of the smallest 1971 * fence register that can contain the object. 1972 */ 1973 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 1974} 1975 1976uint32_t 1977i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 1978 int tiling_mode) 1979{ 1980 1981 if (tiling_mode == I915_TILING_NONE) 1982 return (4096); 1983 1984 /* 1985 * Minimum alignment is 4k (GTT page size) for sane hw. 1986 */ 1987 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 1988 return (4096); 1989 1990 /* 1991 * Previous hardware however needs to be aligned to a power-of-two 1992 * tile height. The simplest method for determining this is to reuse 1993 * the power-of-tile object size. 1994 */ 1995 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 1996} 1997 1998static int 1999i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2000 unsigned alignment, bool map_and_fenceable) 2001{ 2002 struct drm_device *dev; 2003 struct drm_i915_private *dev_priv; 2004 struct drm_mm_node *free_space; 2005 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2006 bool mappable, fenceable; 2007 int ret; 2008 2009 dev = obj->base.dev; 2010 dev_priv = dev->dev_private; 2011 2012 if (obj->madv != I915_MADV_WILLNEED) { 2013 DRM_ERROR("Attempting to bind a purgeable object\n"); 2014 return (-EINVAL); 2015 } 2016 2017 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2018 obj->tiling_mode); 2019 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2020 obj->tiling_mode); 2021 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2022 obj->base.size, obj->tiling_mode); 2023 if (alignment == 0) 2024 alignment = map_and_fenceable ? fence_alignment : 2025 unfenced_alignment; 2026 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2027 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2028 return (-EINVAL); 2029 } 2030 2031 size = map_and_fenceable ? fence_size : obj->base.size; 2032 2033 /* If the object is bigger than the entire aperture, reject it early 2034 * before evicting everything in a vain attempt to find space. 2035 */ 2036 if (obj->base.size > (map_and_fenceable ? 2037 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2038 DRM_ERROR( 2039"Attempting to bind an object larger than the aperture\n"); 2040 return (-E2BIG); 2041 } 2042 2043 search_free: 2044 if (map_and_fenceable) 2045 free_space = drm_mm_search_free_in_range( 2046 &dev_priv->mm.gtt_space, size, alignment, 0, 2047 dev_priv->mm.gtt_mappable_end, 0); 2048 else 2049 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2050 size, alignment, 0); 2051 if (free_space != NULL) { 2052 if (map_and_fenceable) 2053 obj->gtt_space = drm_mm_get_block_range_generic( 2054 free_space, size, alignment, 0, 2055 dev_priv->mm.gtt_mappable_end, 1); 2056 else 2057 obj->gtt_space = drm_mm_get_block_generic(free_space, 2058 size, alignment, 1); 2059 } 2060 if (obj->gtt_space == NULL) { 2061 ret = i915_gem_evict_something(dev, size, alignment, 2062 map_and_fenceable); 2063 if (ret != 0) 2064 return (ret); 2065 goto search_free; 2066 } 2067 ret = i915_gem_object_get_pages_gtt(obj, 0); 2068 if (ret != 0) { 2069 drm_mm_put_block(obj->gtt_space); 2070 obj->gtt_space = NULL; 2071 /* 2072 * i915_gem_object_get_pages_gtt() cannot return 2073 * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY) 2074 * (which does not support operation without a flag 2075 * anyway). 2076 */ 2077 return (ret); 2078 } 2079 2080 ret = i915_gem_gtt_bind_object(obj); 2081 if (ret != 0) { 2082 i915_gem_object_put_pages_gtt(obj); 2083 drm_mm_put_block(obj->gtt_space); 2084 obj->gtt_space = NULL; 2085 if (i915_gem_evict_everything(dev, false)) 2086 return (ret); 2087 goto search_free; 2088 } 2089 2090 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2091 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2092 2093 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2094 ("Object in gpu read domain")); 2095 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2096 ("Object in gpu write domain")); 2097 2098 obj->gtt_offset = obj->gtt_space->start; 2099 2100 fenceable = 2101 obj->gtt_space->size == fence_size && 2102 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2103 2104 mappable = 2105 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2106 obj->map_and_fenceable = mappable && fenceable; 2107 2108 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 2109 obj->base.size, map_and_fenceable); 2110 return (0); 2111} 2112 2113static void 2114i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2115{ 2116 u32 old_write_domain, old_read_domains; 2117 2118 /* Act a barrier for all accesses through the GTT */ 2119 mb(); 2120 2121 /* Force a pagefault for domain tracking on next user access */ 2122 i915_gem_release_mmap(obj); 2123 2124 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2125 return; 2126 2127 old_read_domains = obj->base.read_domains; 2128 old_write_domain = obj->base.write_domain; 2129 2130 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2131 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2132 2133 CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2134 obj, old_read_domains, old_write_domain); 2135} 2136 2137int 2138i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2139{ 2140 drm_i915_private_t *dev_priv; 2141 int ret; 2142 2143 dev_priv = obj->base.dev->dev_private; 2144 ret = 0; 2145 if (obj->gtt_space == NULL) 2146 return (0); 2147 if (obj->pin_count != 0) { 2148 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2149 return (-EINVAL); 2150 } 2151 2152 ret = i915_gem_object_finish_gpu(obj); 2153 if (ret == -ERESTART || ret == -EINTR) 2154 return (ret); 2155 2156 i915_gem_object_finish_gtt(obj); 2157 2158 if (ret == 0) 2159 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2160 if (ret == -ERESTART || ret == -EINTR) 2161 return (ret); 2162 if (ret != 0) { 2163 i915_gem_clflush_object(obj); 2164 obj->base.read_domains = obj->base.write_domain = 2165 I915_GEM_DOMAIN_CPU; 2166 } 2167 2168 ret = i915_gem_object_put_fence(obj); 2169 if (ret == -ERESTART) 2170 return (ret); 2171 2172 i915_gem_gtt_unbind_object(obj); 2173 if (obj->has_aliasing_ppgtt_mapping) { 2174 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2175 obj->has_aliasing_ppgtt_mapping = 0; 2176 } 2177 i915_gem_object_put_pages_gtt(obj); 2178 2179 list_del_init(&obj->gtt_list); 2180 list_del_init(&obj->mm_list); 2181 obj->map_and_fenceable = true; 2182 2183 drm_mm_put_block(obj->gtt_space); 2184 obj->gtt_space = NULL; 2185 obj->gtt_offset = 0; 2186 2187 if (i915_gem_object_is_purgeable(obj)) 2188 i915_gem_object_truncate(obj); 2189 CTR1(KTR_DRM, "object_unbind %p", obj); 2190 2191 return (ret); 2192} 2193 2194static int 2195i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2196 int flags) 2197{ 2198 struct drm_device *dev; 2199 vm_object_t vm_obj; 2200 vm_page_t m; 2201 int page_count, i, j; 2202 2203 dev = obj->base.dev; 2204 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2205 page_count = obj->base.size / PAGE_SIZE; 2206 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2207 M_WAITOK); 2208 vm_obj = obj->base.vm_obj; 2209 VM_OBJECT_WLOCK(vm_obj); 2210 for (i = 0; i < page_count; i++) { 2211 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2212 goto failed; 2213 } 2214 VM_OBJECT_WUNLOCK(vm_obj); 2215 if (i915_gem_object_needs_bit17_swizzle(obj)) 2216 i915_gem_object_do_bit_17_swizzle(obj); 2217 return (0); 2218 2219failed: 2220 for (j = 0; j < i; j++) { 2221 m = obj->pages[j]; 2222 vm_page_lock(m); 2223 vm_page_unwire(m, 0); 2224 vm_page_unlock(m); 2225 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2226 } 2227 VM_OBJECT_WUNLOCK(vm_obj); 2228 free(obj->pages, DRM_I915_GEM); 2229 obj->pages = NULL; 2230 return (-EIO); 2231} 2232 2233#define GEM_PARANOID_CHECK_GTT 0 2234#if GEM_PARANOID_CHECK_GTT 2235static void 2236i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2237 int page_count) 2238{ 2239 struct drm_i915_private *dev_priv; 2240 vm_paddr_t pa; 2241 unsigned long start, end; 2242 u_int i; 2243 int j; 2244 2245 dev_priv = dev->dev_private; 2246 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2247 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2248 for (i = start; i < end; i++) { 2249 pa = intel_gtt_read_pte_paddr(i); 2250 for (j = 0; j < page_count; j++) { 2251 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2252 panic("Page %p in GTT pte index %d pte %x", 2253 ma[i], i, intel_gtt_read_pte(i)); 2254 } 2255 } 2256 } 2257} 2258#endif 2259 2260static void 2261i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2262{ 2263 vm_page_t m; 2264 int page_count, i; 2265 2266 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2267 2268 if (obj->tiling_mode != I915_TILING_NONE) 2269 i915_gem_object_save_bit_17_swizzle(obj); 2270 if (obj->madv == I915_MADV_DONTNEED) 2271 obj->dirty = 0; 2272 page_count = obj->base.size / PAGE_SIZE; 2273 VM_OBJECT_WLOCK(obj->base.vm_obj); 2274#if GEM_PARANOID_CHECK_GTT 2275 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2276#endif 2277 for (i = 0; i < page_count; i++) { 2278 m = obj->pages[i]; 2279 if (obj->dirty) 2280 vm_page_dirty(m); 2281 if (obj->madv == I915_MADV_WILLNEED) 2282 vm_page_reference(m); 2283 vm_page_lock(m); 2284 vm_page_unwire(obj->pages[i], 1); 2285 vm_page_unlock(m); 2286 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2287 } 2288 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 2289 obj->dirty = 0; 2290 free(obj->pages, DRM_I915_GEM); 2291 obj->pages = NULL; 2292} 2293 2294void 2295i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2296{ 2297 vm_object_t devobj; 2298 vm_page_t m; 2299 int i, page_count; 2300 2301 if (!obj->fault_mappable) 2302 return; 2303 2304 CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 2305 OFF_TO_IDX(obj->base.size)); 2306 devobj = cdev_pager_lookup(obj); 2307 if (devobj != NULL) { 2308 page_count = OFF_TO_IDX(obj->base.size); 2309 2310 VM_OBJECT_WLOCK(devobj); 2311retry: 2312 for (i = 0; i < page_count; i++) { 2313 m = vm_page_lookup(devobj, i); 2314 if (m == NULL) 2315 continue; 2316 if (vm_page_sleep_if_busy(m, true, "915unm")) 2317 goto retry; 2318 cdev_pager_free_page(devobj, m); 2319 } 2320 VM_OBJECT_WUNLOCK(devobj); 2321 vm_object_deallocate(devobj); 2322 } 2323 2324 obj->fault_mappable = false; 2325} 2326 2327int 2328i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2329{ 2330 int ret; 2331 2332 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2333 ("In GPU write domain")); 2334 2335 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, 2336 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, 2337 obj->active, obj->last_rendering_seqno); 2338 if (obj->active) { 2339 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2340 true); 2341 if (ret != 0) 2342 return (ret); 2343 } 2344 return (0); 2345} 2346 2347void 2348i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2349 struct intel_ring_buffer *ring, uint32_t seqno) 2350{ 2351 struct drm_device *dev = obj->base.dev; 2352 struct drm_i915_private *dev_priv = dev->dev_private; 2353 struct drm_i915_fence_reg *reg; 2354 2355 obj->ring = ring; 2356 KASSERT(ring != NULL, ("NULL ring")); 2357 2358 /* Add a reference if we're newly entering the active list. */ 2359 if (!obj->active) { 2360 drm_gem_object_reference(&obj->base); 2361 obj->active = 1; 2362 } 2363 2364 /* Move from whatever list we were on to the tail of execution. */ 2365 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2366 list_move_tail(&obj->ring_list, &ring->active_list); 2367 2368 obj->last_rendering_seqno = seqno; 2369 if (obj->fenced_gpu_access) { 2370 obj->last_fenced_seqno = seqno; 2371 obj->last_fenced_ring = ring; 2372 2373 /* Bump MRU to take account of the delayed flush */ 2374 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2375 reg = &dev_priv->fence_regs[obj->fence_reg]; 2376 list_move_tail(®->lru_list, 2377 &dev_priv->mm.fence_list); 2378 } 2379 } 2380} 2381 2382static void 2383i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2384{ 2385 list_del_init(&obj->ring_list); 2386 obj->last_rendering_seqno = 0; 2387 obj->last_fenced_seqno = 0; 2388} 2389 2390static void 2391i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2392{ 2393 struct drm_device *dev = obj->base.dev; 2394 drm_i915_private_t *dev_priv = dev->dev_private; 2395 2396 KASSERT(obj->active, ("Object not active")); 2397 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2398 2399 i915_gem_object_move_off_active(obj); 2400} 2401 2402static void 2403i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2404{ 2405 struct drm_device *dev = obj->base.dev; 2406 struct drm_i915_private *dev_priv = dev->dev_private; 2407 2408 if (obj->pin_count != 0) 2409 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2410 else 2411 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2412 2413 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2414 KASSERT(obj->active, ("Object not active")); 2415 obj->ring = NULL; 2416 obj->last_fenced_ring = NULL; 2417 2418 i915_gem_object_move_off_active(obj); 2419 obj->fenced_gpu_access = false; 2420 2421 obj->active = 0; 2422 obj->pending_gpu_write = false; 2423 drm_gem_object_unreference(&obj->base); 2424 2425#if 1 2426 KIB_NOTYET(); 2427#else 2428 WARN_ON(i915_verify_lists(dev)); 2429#endif 2430} 2431 2432static void 2433i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2434{ 2435 vm_object_t vm_obj; 2436 2437 vm_obj = obj->base.vm_obj; 2438 VM_OBJECT_WLOCK(vm_obj); 2439 vm_object_page_remove(vm_obj, 0, 0, false); 2440 VM_OBJECT_WUNLOCK(vm_obj); 2441 obj->madv = I915_MADV_PURGED_INTERNAL; 2442} 2443 2444static inline int 2445i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2446{ 2447 2448 return (obj->madv == I915_MADV_DONTNEED); 2449} 2450 2451static void 2452i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2453 uint32_t flush_domains) 2454{ 2455 struct drm_i915_gem_object *obj, *next; 2456 uint32_t old_write_domain; 2457 2458 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2459 gpu_write_list) { 2460 if (obj->base.write_domain & flush_domains) { 2461 old_write_domain = obj->base.write_domain; 2462 obj->base.write_domain = 0; 2463 list_del_init(&obj->gpu_write_list); 2464 i915_gem_object_move_to_active(obj, ring, 2465 i915_gem_next_request_seqno(ring)); 2466 2467 CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", 2468 obj, obj->base.read_domains, old_write_domain); 2469 } 2470 } 2471} 2472 2473static int 2474i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2475{ 2476 drm_i915_private_t *dev_priv; 2477 2478 dev_priv = obj->base.dev->dev_private; 2479 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2480 obj->tiling_mode != I915_TILING_NONE); 2481} 2482 2483static vm_page_t 2484i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2485{ 2486 vm_page_t m; 2487 int rv; 2488 2489 VM_OBJECT_ASSERT_WLOCKED(object); 2490 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | 2491 VM_ALLOC_RETRY); 2492 if (m->valid != VM_PAGE_BITS_ALL) { 2493 vm_page_busy(m); 2494 if (vm_pager_has_page(object, pindex, NULL, NULL)) { 2495 rv = vm_pager_get_pages(object, &m, 1, 0); 2496 m = vm_page_lookup(object, pindex); 2497 if (m == NULL) 2498 return (NULL); 2499 if (rv != VM_PAGER_OK) { 2500 vm_page_lock(m); 2501 vm_page_free(m); 2502 vm_page_unlock(m); 2503 return (NULL); 2504 } 2505 } else { 2506 pmap_zero_page(m); 2507 m->valid = VM_PAGE_BITS_ALL; 2508 m->dirty = 0; 2509 } 2510 vm_page_wakeup(m); 2511 } 2512 vm_page_lock(m); 2513 vm_page_wire(m); 2514 vm_page_unlock(m); 2515 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2516 return (m); 2517} 2518 2519int 2520i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2521 uint32_t flush_domains) 2522{ 2523 int ret; 2524 2525 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2526 return 0; 2527 2528 CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, 2529 flush_domains); 2530 ret = ring->flush(ring, invalidate_domains, flush_domains); 2531 if (ret) 2532 return ret; 2533 2534 if (flush_domains & I915_GEM_GPU_DOMAINS) 2535 i915_gem_process_flushing_list(ring, flush_domains); 2536 return 0; 2537} 2538 2539static int 2540i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2541{ 2542 int ret; 2543 2544 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2545 return 0; 2546 2547 if (!list_empty(&ring->gpu_write_list)) { 2548 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2549 I915_GEM_GPU_DOMAINS); 2550 if (ret != 0) 2551 return ret; 2552 } 2553 2554 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2555 do_retire)); 2556} 2557 2558int 2559i915_gpu_idle(struct drm_device *dev, bool do_retire) 2560{ 2561 drm_i915_private_t *dev_priv = dev->dev_private; 2562 int ret, i; 2563 2564 /* Flush everything onto the inactive list. */ 2565 for (i = 0; i < I915_NUM_RINGS; i++) { 2566 ret = i915_ring_idle(&dev_priv->rings[i], do_retire); 2567 if (ret) 2568 return ret; 2569 } 2570 2571 return 0; 2572} 2573 2574int 2575i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2576{ 2577 drm_i915_private_t *dev_priv; 2578 struct drm_i915_gem_request *request; 2579 uint32_t ier; 2580 int flags, ret; 2581 bool recovery_complete; 2582 2583 KASSERT(seqno != 0, ("Zero seqno")); 2584 2585 dev_priv = ring->dev->dev_private; 2586 ret = 0; 2587 2588 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 2589 /* Give the error handler a chance to run. */ 2590 mtx_lock(&dev_priv->error_completion_lock); 2591 recovery_complete = (&dev_priv->error_completion) > 0; 2592 mtx_unlock(&dev_priv->error_completion_lock); 2593 return (recovery_complete ? -EIO : -EAGAIN); 2594 } 2595 2596 if (seqno == ring->outstanding_lazy_request) { 2597 request = malloc(sizeof(*request), DRM_I915_GEM, 2598 M_WAITOK | M_ZERO); 2599 if (request == NULL) 2600 return (-ENOMEM); 2601 2602 ret = i915_add_request(ring, NULL, request); 2603 if (ret != 0) { 2604 free(request, DRM_I915_GEM); 2605 return (ret); 2606 } 2607 2608 seqno = request->seqno; 2609 } 2610 2611 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2612 if (HAS_PCH_SPLIT(ring->dev)) 2613 ier = I915_READ(DEIER) | I915_READ(GTIER); 2614 else 2615 ier = I915_READ(IER); 2616 if (!ier) { 2617 DRM_ERROR("something (likely vbetool) disabled " 2618 "interrupts, re-enabling\n"); 2619 ring->dev->driver->irq_preinstall(ring->dev); 2620 ring->dev->driver->irq_postinstall(ring->dev); 2621 } 2622 2623 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 2624 2625 ring->waiting_seqno = seqno; 2626 mtx_lock(&ring->irq_lock); 2627 if (ring->irq_get(ring)) { 2628 flags = dev_priv->mm.interruptible ? PCATCH : 0; 2629 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2630 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 2631 ret == 0) { 2632 ret = -msleep(ring, &ring->irq_lock, flags, 2633 "915gwr", 0); 2634 } 2635 ring->irq_put(ring); 2636 mtx_unlock(&ring->irq_lock); 2637 } else { 2638 mtx_unlock(&ring->irq_lock); 2639 if (_intel_wait_for(ring->dev, 2640 i915_seqno_passed(ring->get_seqno(ring), seqno) || 2641 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 2642 0, "i915wrq") != 0) 2643 ret = -EBUSY; 2644 } 2645 ring->waiting_seqno = 0; 2646 2647 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, 2648 ret); 2649 } 2650 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 2651 ret = -EAGAIN; 2652 2653 /* Directly dispatch request retiring. While we have the work queue 2654 * to handle this, the waiter on a request often wants an associated 2655 * buffer to have made it to the inactive list, and we would need 2656 * a separate wait queue to handle that. 2657 */ 2658 if (ret == 0 && do_retire) 2659 i915_gem_retire_requests_ring(ring); 2660 2661 return (ret); 2662} 2663 2664static u32 2665i915_gem_get_seqno(struct drm_device *dev) 2666{ 2667 drm_i915_private_t *dev_priv = dev->dev_private; 2668 u32 seqno = dev_priv->next_seqno; 2669 2670 /* reserve 0 for non-seqno */ 2671 if (++dev_priv->next_seqno == 0) 2672 dev_priv->next_seqno = 1; 2673 2674 return seqno; 2675} 2676 2677u32 2678i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2679{ 2680 if (ring->outstanding_lazy_request == 0) 2681 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2682 2683 return ring->outstanding_lazy_request; 2684} 2685 2686int 2687i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2688 struct drm_i915_gem_request *request) 2689{ 2690 drm_i915_private_t *dev_priv; 2691 struct drm_i915_file_private *file_priv; 2692 uint32_t seqno; 2693 u32 request_ring_position; 2694 int was_empty; 2695 int ret; 2696 2697 KASSERT(request != NULL, ("NULL request in add")); 2698 DRM_LOCK_ASSERT(ring->dev); 2699 dev_priv = ring->dev->dev_private; 2700 2701 seqno = i915_gem_next_request_seqno(ring); 2702 request_ring_position = intel_ring_get_tail(ring); 2703 2704 ret = ring->add_request(ring, &seqno); 2705 if (ret != 0) 2706 return ret; 2707 2708 CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); 2709 2710 request->seqno = seqno; 2711 request->ring = ring; 2712 request->tail = request_ring_position; 2713 request->emitted_jiffies = ticks; 2714 was_empty = list_empty(&ring->request_list); 2715 list_add_tail(&request->list, &ring->request_list); 2716 2717 if (file != NULL) { 2718 file_priv = file->driver_priv; 2719 2720 mtx_lock(&file_priv->mm.lck); 2721 request->file_priv = file_priv; 2722 list_add_tail(&request->client_list, 2723 &file_priv->mm.request_list); 2724 mtx_unlock(&file_priv->mm.lck); 2725 } 2726 2727 ring->outstanding_lazy_request = 0; 2728 2729 if (!dev_priv->mm.suspended) { 2730 if (i915_enable_hangcheck) { 2731 callout_schedule(&dev_priv->hangcheck_timer, 2732 DRM_I915_HANGCHECK_PERIOD); 2733 } 2734 if (was_empty) 2735 taskqueue_enqueue_timeout(dev_priv->tq, 2736 &dev_priv->mm.retire_task, hz); 2737 } 2738 return (0); 2739} 2740 2741static inline void 2742i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2743{ 2744 struct drm_i915_file_private *file_priv = request->file_priv; 2745 2746 if (!file_priv) 2747 return; 2748 2749 DRM_LOCK_ASSERT(request->ring->dev); 2750 2751 mtx_lock(&file_priv->mm.lck); 2752 if (request->file_priv != NULL) { 2753 list_del(&request->client_list); 2754 request->file_priv = NULL; 2755 } 2756 mtx_unlock(&file_priv->mm.lck); 2757} 2758 2759void 2760i915_gem_release(struct drm_device *dev, struct drm_file *file) 2761{ 2762 struct drm_i915_file_private *file_priv; 2763 struct drm_i915_gem_request *request; 2764 2765 file_priv = file->driver_priv; 2766 2767 /* Clean up our request list when the client is going away, so that 2768 * later retire_requests won't dereference our soon-to-be-gone 2769 * file_priv. 2770 */ 2771 mtx_lock(&file_priv->mm.lck); 2772 while (!list_empty(&file_priv->mm.request_list)) { 2773 request = list_first_entry(&file_priv->mm.request_list, 2774 struct drm_i915_gem_request, 2775 client_list); 2776 list_del(&request->client_list); 2777 request->file_priv = NULL; 2778 } 2779 mtx_unlock(&file_priv->mm.lck); 2780} 2781 2782static void 2783i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2784 struct intel_ring_buffer *ring) 2785{ 2786 2787 if (ring->dev != NULL) 2788 DRM_LOCK_ASSERT(ring->dev); 2789 2790 while (!list_empty(&ring->request_list)) { 2791 struct drm_i915_gem_request *request; 2792 2793 request = list_first_entry(&ring->request_list, 2794 struct drm_i915_gem_request, list); 2795 2796 list_del(&request->list); 2797 i915_gem_request_remove_from_client(request); 2798 free(request, DRM_I915_GEM); 2799 } 2800 2801 while (!list_empty(&ring->active_list)) { 2802 struct drm_i915_gem_object *obj; 2803 2804 obj = list_first_entry(&ring->active_list, 2805 struct drm_i915_gem_object, ring_list); 2806 2807 obj->base.write_domain = 0; 2808 list_del_init(&obj->gpu_write_list); 2809 i915_gem_object_move_to_inactive(obj); 2810 } 2811} 2812 2813static void 2814i915_gem_reset_fences(struct drm_device *dev) 2815{ 2816 struct drm_i915_private *dev_priv = dev->dev_private; 2817 int i; 2818 2819 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2820 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2821 struct drm_i915_gem_object *obj = reg->obj; 2822 2823 if (!obj) 2824 continue; 2825 2826 if (obj->tiling_mode) 2827 i915_gem_release_mmap(obj); 2828 2829 reg->obj->fence_reg = I915_FENCE_REG_NONE; 2830 reg->obj->fenced_gpu_access = false; 2831 reg->obj->last_fenced_seqno = 0; 2832 reg->obj->last_fenced_ring = NULL; 2833 i915_gem_clear_fence_reg(dev, reg); 2834 } 2835} 2836 2837void 2838i915_gem_reset(struct drm_device *dev) 2839{ 2840 struct drm_i915_private *dev_priv = dev->dev_private; 2841 struct drm_i915_gem_object *obj; 2842 int i; 2843 2844 for (i = 0; i < I915_NUM_RINGS; i++) 2845 i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); 2846 2847 /* Remove anything from the flushing lists. The GPU cache is likely 2848 * to be lost on reset along with the data, so simply move the 2849 * lost bo to the inactive list. 2850 */ 2851 while (!list_empty(&dev_priv->mm.flushing_list)) { 2852 obj = list_first_entry(&dev_priv->mm.flushing_list, 2853 struct drm_i915_gem_object, 2854 mm_list); 2855 2856 obj->base.write_domain = 0; 2857 list_del_init(&obj->gpu_write_list); 2858 i915_gem_object_move_to_inactive(obj); 2859 } 2860 2861 /* Move everything out of the GPU domains to ensure we do any 2862 * necessary invalidation upon reuse. 2863 */ 2864 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2865 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2866 } 2867 2868 /* The fence registers are invalidated so clear them out */ 2869 i915_gem_reset_fences(dev); 2870} 2871 2872/** 2873 * This function clears the request list as sequence numbers are passed. 2874 */ 2875void 2876i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2877{ 2878 uint32_t seqno; 2879 int i; 2880 2881 if (list_empty(&ring->request_list)) 2882 return; 2883 2884 seqno = ring->get_seqno(ring); 2885 CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 2886 2887 for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2888 if (seqno >= ring->sync_seqno[i]) 2889 ring->sync_seqno[i] = 0; 2890 2891 while (!list_empty(&ring->request_list)) { 2892 struct drm_i915_gem_request *request; 2893 2894 request = list_first_entry(&ring->request_list, 2895 struct drm_i915_gem_request, 2896 list); 2897 2898 if (!i915_seqno_passed(seqno, request->seqno)) 2899 break; 2900 2901 CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 2902 ring->name, seqno); 2903 ring->last_retired_head = request->tail; 2904 2905 list_del(&request->list); 2906 i915_gem_request_remove_from_client(request); 2907 free(request, DRM_I915_GEM); 2908 } 2909 2910 /* Move any buffers on the active list that are no longer referenced 2911 * by the ringbuffer to the flushing/inactive lists as appropriate. 2912 */ 2913 while (!list_empty(&ring->active_list)) { 2914 struct drm_i915_gem_object *obj; 2915 2916 obj = list_first_entry(&ring->active_list, 2917 struct drm_i915_gem_object, 2918 ring_list); 2919 2920 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2921 break; 2922 2923 if (obj->base.write_domain != 0) 2924 i915_gem_object_move_to_flushing(obj); 2925 else 2926 i915_gem_object_move_to_inactive(obj); 2927 } 2928 2929 if (ring->trace_irq_seqno && 2930 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2931 mtx_lock(&ring->irq_lock); 2932 ring->irq_put(ring); 2933 mtx_unlock(&ring->irq_lock); 2934 ring->trace_irq_seqno = 0; 2935 } 2936} 2937 2938void 2939i915_gem_retire_requests(struct drm_device *dev) 2940{ 2941 drm_i915_private_t *dev_priv = dev->dev_private; 2942 struct drm_i915_gem_object *obj, *next; 2943 int i; 2944 2945 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2946 list_for_each_entry_safe(obj, next, 2947 &dev_priv->mm.deferred_free_list, mm_list) 2948 i915_gem_free_object_tail(obj); 2949 } 2950 2951 for (i = 0; i < I915_NUM_RINGS; i++) 2952 i915_gem_retire_requests_ring(&dev_priv->rings[i]); 2953} 2954 2955static int 2956sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2957 struct intel_ring_buffer *pipelined) 2958{ 2959 struct drm_device *dev = obj->base.dev; 2960 drm_i915_private_t *dev_priv = dev->dev_private; 2961 u32 size = obj->gtt_space->size; 2962 int regnum = obj->fence_reg; 2963 uint64_t val; 2964 2965 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2966 0xfffff000) << 32; 2967 val |= obj->gtt_offset & 0xfffff000; 2968 val |= (uint64_t)((obj->stride / 128) - 1) << 2969 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2970 2971 if (obj->tiling_mode == I915_TILING_Y) 2972 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2973 val |= I965_FENCE_REG_VALID; 2974 2975 if (pipelined) { 2976 int ret = intel_ring_begin(pipelined, 6); 2977 if (ret) 2978 return ret; 2979 2980 intel_ring_emit(pipelined, MI_NOOP); 2981 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2982 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 2983 intel_ring_emit(pipelined, (u32)val); 2984 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 2985 intel_ring_emit(pipelined, (u32)(val >> 32)); 2986 intel_ring_advance(pipelined); 2987 } else 2988 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 2989 2990 return 0; 2991} 2992 2993static int 2994i965_write_fence_reg(struct drm_i915_gem_object *obj, 2995 struct intel_ring_buffer *pipelined) 2996{ 2997 struct drm_device *dev = obj->base.dev; 2998 drm_i915_private_t *dev_priv = dev->dev_private; 2999 u32 size = obj->gtt_space->size; 3000 int regnum = obj->fence_reg; 3001 uint64_t val; 3002 3003 val = (uint64_t)((obj->gtt_offset + size - 4096) & 3004 0xfffff000) << 32; 3005 val |= obj->gtt_offset & 0xfffff000; 3006 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 3007 if (obj->tiling_mode == I915_TILING_Y) 3008 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3009 val |= I965_FENCE_REG_VALID; 3010 3011 if (pipelined) { 3012 int ret = intel_ring_begin(pipelined, 6); 3013 if (ret) 3014 return ret; 3015 3016 intel_ring_emit(pipelined, MI_NOOP); 3017 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 3018 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 3019 intel_ring_emit(pipelined, (u32)val); 3020 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 3021 intel_ring_emit(pipelined, (u32)(val >> 32)); 3022 intel_ring_advance(pipelined); 3023 } else 3024 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 3025 3026 return 0; 3027} 3028 3029static int 3030i915_write_fence_reg(struct drm_i915_gem_object *obj, 3031 struct intel_ring_buffer *pipelined) 3032{ 3033 struct drm_device *dev = obj->base.dev; 3034 drm_i915_private_t *dev_priv = dev->dev_private; 3035 u32 size = obj->gtt_space->size; 3036 u32 fence_reg, val, pitch_val; 3037 int tile_width; 3038 3039 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3040 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3041 printf( 3042"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3043 obj->gtt_offset, obj->map_and_fenceable, size); 3044 return -EINVAL; 3045 } 3046 3047 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3048 tile_width = 128; 3049 else 3050 tile_width = 512; 3051 3052 /* Note: pitch better be a power of two tile widths */ 3053 pitch_val = obj->stride / tile_width; 3054 pitch_val = ffs(pitch_val) - 1; 3055 3056 val = obj->gtt_offset; 3057 if (obj->tiling_mode == I915_TILING_Y) 3058 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3059 val |= I915_FENCE_SIZE_BITS(size); 3060 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3061 val |= I830_FENCE_REG_VALID; 3062 3063 fence_reg = obj->fence_reg; 3064 if (fence_reg < 8) 3065 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3066 else 3067 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3068 3069 if (pipelined) { 3070 int ret = intel_ring_begin(pipelined, 4); 3071 if (ret) 3072 return ret; 3073 3074 intel_ring_emit(pipelined, MI_NOOP); 3075 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3076 intel_ring_emit(pipelined, fence_reg); 3077 intel_ring_emit(pipelined, val); 3078 intel_ring_advance(pipelined); 3079 } else 3080 I915_WRITE(fence_reg, val); 3081 3082 return 0; 3083} 3084 3085static int 3086i830_write_fence_reg(struct drm_i915_gem_object *obj, 3087 struct intel_ring_buffer *pipelined) 3088{ 3089 struct drm_device *dev = obj->base.dev; 3090 drm_i915_private_t *dev_priv = dev->dev_private; 3091 u32 size = obj->gtt_space->size; 3092 int regnum = obj->fence_reg; 3093 uint32_t val; 3094 uint32_t pitch_val; 3095 3096 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3097 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3098 printf( 3099"object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3100 obj->gtt_offset, size); 3101 return -EINVAL; 3102 } 3103 3104 pitch_val = obj->stride / 128; 3105 pitch_val = ffs(pitch_val) - 1; 3106 3107 val = obj->gtt_offset; 3108 if (obj->tiling_mode == I915_TILING_Y) 3109 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3110 val |= I830_FENCE_SIZE_BITS(size); 3111 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3112 val |= I830_FENCE_REG_VALID; 3113 3114 if (pipelined) { 3115 int ret = intel_ring_begin(pipelined, 4); 3116 if (ret) 3117 return ret; 3118 3119 intel_ring_emit(pipelined, MI_NOOP); 3120 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3121 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 3122 intel_ring_emit(pipelined, val); 3123 intel_ring_advance(pipelined); 3124 } else 3125 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 3126 3127 return 0; 3128} 3129 3130static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 3131{ 3132 return i915_seqno_passed(ring->get_seqno(ring), seqno); 3133} 3134 3135static int 3136i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 3137 struct intel_ring_buffer *pipelined) 3138{ 3139 int ret; 3140 3141 if (obj->fenced_gpu_access) { 3142 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3143 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3144 obj->base.write_domain); 3145 if (ret) 3146 return ret; 3147 } 3148 3149 obj->fenced_gpu_access = false; 3150 } 3151 3152 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3153 if (!ring_passed_seqno(obj->last_fenced_ring, 3154 obj->last_fenced_seqno)) { 3155 ret = i915_wait_request(obj->last_fenced_ring, 3156 obj->last_fenced_seqno, 3157 true); 3158 if (ret) 3159 return ret; 3160 } 3161 3162 obj->last_fenced_seqno = 0; 3163 obj->last_fenced_ring = NULL; 3164 } 3165 3166 /* Ensure that all CPU reads are completed before installing a fence 3167 * and all writes before removing the fence. 3168 */ 3169 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3170 mb(); 3171 3172 return 0; 3173} 3174 3175int 3176i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3177{ 3178 int ret; 3179 3180 if (obj->tiling_mode) 3181 i915_gem_release_mmap(obj); 3182 3183 ret = i915_gem_object_flush_fence(obj, NULL); 3184 if (ret) 3185 return ret; 3186 3187 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3188 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3189 3190 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3191 printf("%s: pin_count %d\n", __func__, 3192 dev_priv->fence_regs[obj->fence_reg].pin_count); 3193 i915_gem_clear_fence_reg(obj->base.dev, 3194 &dev_priv->fence_regs[obj->fence_reg]); 3195 3196 obj->fence_reg = I915_FENCE_REG_NONE; 3197 } 3198 3199 return 0; 3200} 3201 3202static struct drm_i915_fence_reg * 3203i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3204{ 3205 struct drm_i915_private *dev_priv = dev->dev_private; 3206 struct drm_i915_fence_reg *reg, *first, *avail; 3207 int i; 3208 3209 /* First try to find a free reg */ 3210 avail = NULL; 3211 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3212 reg = &dev_priv->fence_regs[i]; 3213 if (!reg->obj) 3214 return reg; 3215 3216 if (!reg->pin_count) 3217 avail = reg; 3218 } 3219 3220 if (avail == NULL) 3221 return NULL; 3222 3223 /* None available, try to steal one or wait for a user to finish */ 3224 avail = first = NULL; 3225 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3226 if (reg->pin_count) 3227 continue; 3228 3229 if (first == NULL) 3230 first = reg; 3231 3232 if (!pipelined || 3233 !reg->obj->last_fenced_ring || 3234 reg->obj->last_fenced_ring == pipelined) { 3235 avail = reg; 3236 break; 3237 } 3238 } 3239 3240 if (avail == NULL) 3241 avail = first; 3242 3243 return avail; 3244} 3245 3246int 3247i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3248 struct intel_ring_buffer *pipelined) 3249{ 3250 struct drm_device *dev = obj->base.dev; 3251 struct drm_i915_private *dev_priv = dev->dev_private; 3252 struct drm_i915_fence_reg *reg; 3253 int ret; 3254 3255 pipelined = NULL; 3256 ret = 0; 3257 3258 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3259 reg = &dev_priv->fence_regs[obj->fence_reg]; 3260 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3261 3262 if (obj->tiling_changed) { 3263 ret = i915_gem_object_flush_fence(obj, pipelined); 3264 if (ret) 3265 return ret; 3266 3267 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3268 pipelined = NULL; 3269 3270 if (pipelined) { 3271 reg->setup_seqno = 3272 i915_gem_next_request_seqno(pipelined); 3273 obj->last_fenced_seqno = reg->setup_seqno; 3274 obj->last_fenced_ring = pipelined; 3275 } 3276 3277 goto update; 3278 } 3279 3280 if (!pipelined) { 3281 if (reg->setup_seqno) { 3282 if (!ring_passed_seqno(obj->last_fenced_ring, 3283 reg->setup_seqno)) { 3284 ret = i915_wait_request( 3285 obj->last_fenced_ring, 3286 reg->setup_seqno, 3287 true); 3288 if (ret) 3289 return ret; 3290 } 3291 3292 reg->setup_seqno = 0; 3293 } 3294 } else if (obj->last_fenced_ring && 3295 obj->last_fenced_ring != pipelined) { 3296 ret = i915_gem_object_flush_fence(obj, pipelined); 3297 if (ret) 3298 return ret; 3299 } 3300 3301 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3302 pipelined = NULL; 3303 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3304 3305 if (obj->tiling_changed) { 3306 if (pipelined) { 3307 reg->setup_seqno = 3308 i915_gem_next_request_seqno(pipelined); 3309 obj->last_fenced_seqno = reg->setup_seqno; 3310 obj->last_fenced_ring = pipelined; 3311 } 3312 goto update; 3313 } 3314 3315 return 0; 3316 } 3317 3318 reg = i915_find_fence_reg(dev, pipelined); 3319 if (reg == NULL) 3320 return -EDEADLK; 3321 3322 ret = i915_gem_object_flush_fence(obj, pipelined); 3323 if (ret) 3324 return ret; 3325 3326 if (reg->obj) { 3327 struct drm_i915_gem_object *old = reg->obj; 3328 3329 drm_gem_object_reference(&old->base); 3330 3331 if (old->tiling_mode) 3332 i915_gem_release_mmap(old); 3333 3334 ret = i915_gem_object_flush_fence(old, pipelined); 3335 if (ret) { 3336 drm_gem_object_unreference(&old->base); 3337 return ret; 3338 } 3339 3340 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3341 pipelined = NULL; 3342 3343 old->fence_reg = I915_FENCE_REG_NONE; 3344 old->last_fenced_ring = pipelined; 3345 old->last_fenced_seqno = 3346 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3347 3348 drm_gem_object_unreference(&old->base); 3349 } else if (obj->last_fenced_seqno == 0) 3350 pipelined = NULL; 3351 3352 reg->obj = obj; 3353 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3354 obj->fence_reg = reg - dev_priv->fence_regs; 3355 obj->last_fenced_ring = pipelined; 3356 3357 reg->setup_seqno = 3358 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3359 obj->last_fenced_seqno = reg->setup_seqno; 3360 3361update: 3362 obj->tiling_changed = false; 3363 switch (INTEL_INFO(dev)->gen) { 3364 case 7: 3365 case 6: 3366 ret = sandybridge_write_fence_reg(obj, pipelined); 3367 break; 3368 case 5: 3369 case 4: 3370 ret = i965_write_fence_reg(obj, pipelined); 3371 break; 3372 case 3: 3373 ret = i915_write_fence_reg(obj, pipelined); 3374 break; 3375 case 2: 3376 ret = i830_write_fence_reg(obj, pipelined); 3377 break; 3378 } 3379 3380 return ret; 3381} 3382 3383static void 3384i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3385{ 3386 drm_i915_private_t *dev_priv = dev->dev_private; 3387 uint32_t fence_reg = reg - dev_priv->fence_regs; 3388 3389 switch (INTEL_INFO(dev)->gen) { 3390 case 7: 3391 case 6: 3392 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3393 break; 3394 case 5: 3395 case 4: 3396 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3397 break; 3398 case 3: 3399 if (fence_reg >= 8) 3400 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3401 else 3402 case 2: 3403 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3404 3405 I915_WRITE(fence_reg, 0); 3406 break; 3407 } 3408 3409 list_del_init(®->lru_list); 3410 reg->obj = NULL; 3411 reg->setup_seqno = 0; 3412 reg->pin_count = 0; 3413} 3414 3415int 3416i915_gem_init_object(struct drm_gem_object *obj) 3417{ 3418 3419 printf("i915_gem_init_object called\n"); 3420 return (0); 3421} 3422 3423static bool 3424i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3425{ 3426 3427 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3428} 3429 3430static void 3431i915_gem_retire_task_handler(void *arg, int pending) 3432{ 3433 drm_i915_private_t *dev_priv; 3434 struct drm_device *dev; 3435 bool idle; 3436 int i; 3437 3438 dev_priv = arg; 3439 dev = dev_priv->dev; 3440 3441 /* Come back later if the device is busy... */ 3442 if (!sx_try_xlock(&dev->dev_struct_lock)) { 3443 taskqueue_enqueue_timeout(dev_priv->tq, 3444 &dev_priv->mm.retire_task, hz); 3445 return; 3446 } 3447 3448 CTR0(KTR_DRM, "retire_task"); 3449 3450 i915_gem_retire_requests(dev); 3451 3452 /* Send a periodic flush down the ring so we don't hold onto GEM 3453 * objects indefinitely. 3454 */ 3455 idle = true; 3456 for (i = 0; i < I915_NUM_RINGS; i++) { 3457 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 3458 3459 if (!list_empty(&ring->gpu_write_list)) { 3460 struct drm_i915_gem_request *request; 3461 int ret; 3462 3463 ret = i915_gem_flush_ring(ring, 3464 0, I915_GEM_GPU_DOMAINS); 3465 request = malloc(sizeof(*request), DRM_I915_GEM, 3466 M_WAITOK | M_ZERO); 3467 if (ret || request == NULL || 3468 i915_add_request(ring, NULL, request)) 3469 free(request, DRM_I915_GEM); 3470 } 3471 3472 idle &= list_empty(&ring->request_list); 3473 } 3474 3475 if (!dev_priv->mm.suspended && !idle) 3476 taskqueue_enqueue_timeout(dev_priv->tq, 3477 &dev_priv->mm.retire_task, hz); 3478 3479 DRM_UNLOCK(dev); 3480} 3481 3482void 3483i915_gem_lastclose(struct drm_device *dev) 3484{ 3485 int ret; 3486 3487 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3488 return; 3489 3490 ret = i915_gem_idle(dev); 3491 if (ret != 0) 3492 DRM_ERROR("failed to idle hardware: %d\n", ret); 3493} 3494 3495static int 3496i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3497{ 3498 drm_i915_private_t *dev_priv; 3499 struct drm_i915_gem_phys_object *phys_obj; 3500 int ret; 3501 3502 dev_priv = dev->dev_private; 3503 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3504 return (0); 3505 3506 phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3507 M_WAITOK | M_ZERO); 3508 3509 phys_obj->id = id; 3510 3511 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3512 if (phys_obj->handle == NULL) { 3513 ret = -ENOMEM; 3514 goto free_obj; 3515 } 3516 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3517 size / PAGE_SIZE, PAT_WRITE_COMBINING); 3518 3519 dev_priv->mm.phys_objs[id - 1] = phys_obj; 3520 3521 return (0); 3522 3523free_obj: 3524 free(phys_obj, DRM_I915_GEM); 3525 return (ret); 3526} 3527 3528static void 3529i915_gem_free_phys_object(struct drm_device *dev, int id) 3530{ 3531 drm_i915_private_t *dev_priv; 3532 struct drm_i915_gem_phys_object *phys_obj; 3533 3534 dev_priv = dev->dev_private; 3535 if (dev_priv->mm.phys_objs[id - 1] == NULL) 3536 return; 3537 3538 phys_obj = dev_priv->mm.phys_objs[id - 1]; 3539 if (phys_obj->cur_obj != NULL) 3540 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3541 3542 drm_pci_free(dev, phys_obj->handle); 3543 free(phys_obj, DRM_I915_GEM); 3544 dev_priv->mm.phys_objs[id - 1] = NULL; 3545} 3546 3547void 3548i915_gem_free_all_phys_object(struct drm_device *dev) 3549{ 3550 int i; 3551 3552 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3553 i915_gem_free_phys_object(dev, i); 3554} 3555 3556void 3557i915_gem_detach_phys_object(struct drm_device *dev, 3558 struct drm_i915_gem_object *obj) 3559{ 3560 vm_page_t m; 3561 struct sf_buf *sf; 3562 char *vaddr, *dst; 3563 int i, page_count; 3564 3565 if (obj->phys_obj == NULL) 3566 return; 3567 vaddr = obj->phys_obj->handle->vaddr; 3568 3569 page_count = obj->base.size / PAGE_SIZE; 3570 VM_OBJECT_WLOCK(obj->base.vm_obj); 3571 for (i = 0; i < page_count; i++) { 3572 m = i915_gem_wire_page(obj->base.vm_obj, i); 3573 if (m == NULL) 3574 continue; /* XXX */ 3575 3576 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3577 sf = sf_buf_alloc(m, 0); 3578 if (sf != NULL) { 3579 dst = (char *)sf_buf_kva(sf); 3580 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3581 sf_buf_free(sf); 3582 } 3583 drm_clflush_pages(&m, 1); 3584 3585 VM_OBJECT_WLOCK(obj->base.vm_obj); 3586 vm_page_reference(m); 3587 vm_page_lock(m); 3588 vm_page_dirty(m); 3589 vm_page_unwire(m, 0); 3590 vm_page_unlock(m); 3591 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3592 } 3593 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3594 intel_gtt_chipset_flush(); 3595 3596 obj->phys_obj->cur_obj = NULL; 3597 obj->phys_obj = NULL; 3598} 3599 3600int 3601i915_gem_attach_phys_object(struct drm_device *dev, 3602 struct drm_i915_gem_object *obj, int id, int align) 3603{ 3604 drm_i915_private_t *dev_priv; 3605 vm_page_t m; 3606 struct sf_buf *sf; 3607 char *dst, *src; 3608 int i, page_count, ret; 3609 3610 if (id > I915_MAX_PHYS_OBJECT) 3611 return (-EINVAL); 3612 3613 if (obj->phys_obj != NULL) { 3614 if (obj->phys_obj->id == id) 3615 return (0); 3616 i915_gem_detach_phys_object(dev, obj); 3617 } 3618 3619 dev_priv = dev->dev_private; 3620 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3621 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3622 if (ret != 0) { 3623 DRM_ERROR("failed to init phys object %d size: %zu\n", 3624 id, obj->base.size); 3625 return (ret); 3626 } 3627 } 3628 3629 /* bind to the object */ 3630 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3631 obj->phys_obj->cur_obj = obj; 3632 3633 page_count = obj->base.size / PAGE_SIZE; 3634 3635 VM_OBJECT_WLOCK(obj->base.vm_obj); 3636 ret = 0; 3637 for (i = 0; i < page_count; i++) { 3638 m = i915_gem_wire_page(obj->base.vm_obj, i); 3639 if (m == NULL) { 3640 ret = -EIO; 3641 break; 3642 } 3643 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3644 sf = sf_buf_alloc(m, 0); 3645 src = (char *)sf_buf_kva(sf); 3646 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3647 memcpy(dst, src, PAGE_SIZE); 3648 sf_buf_free(sf); 3649 3650 VM_OBJECT_WLOCK(obj->base.vm_obj); 3651 3652 vm_page_reference(m); 3653 vm_page_lock(m); 3654 vm_page_unwire(m, 0); 3655 vm_page_unlock(m); 3656 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3657 } 3658 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3659 3660 return (0); 3661} 3662 3663static int 3664i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3665 uint64_t data_ptr, uint64_t offset, uint64_t size, 3666 struct drm_file *file_priv) 3667{ 3668 char *user_data, *vaddr; 3669 int ret; 3670 3671 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3672 user_data = (char *)(uintptr_t)data_ptr; 3673 3674 if (copyin_nofault(user_data, vaddr, size) != 0) { 3675 /* The physical object once assigned is fixed for the lifetime 3676 * of the obj, so we can safely drop the lock and continue 3677 * to access vaddr. 3678 */ 3679 DRM_UNLOCK(dev); 3680 ret = -copyin(user_data, vaddr, size); 3681 DRM_LOCK(dev); 3682 if (ret != 0) 3683 return (ret); 3684 } 3685 3686 intel_gtt_chipset_flush(); 3687 return (0); 3688} 3689 3690static int 3691i915_gpu_is_active(struct drm_device *dev) 3692{ 3693 drm_i915_private_t *dev_priv; 3694 3695 dev_priv = dev->dev_private; 3696 return (!list_empty(&dev_priv->mm.flushing_list) || 3697 !list_empty(&dev_priv->mm.active_list)); 3698} 3699 3700static void 3701i915_gem_lowmem(void *arg) 3702{ 3703 struct drm_device *dev; 3704 struct drm_i915_private *dev_priv; 3705 struct drm_i915_gem_object *obj, *next; 3706 int cnt, cnt_fail, cnt_total; 3707 3708 dev = arg; 3709 dev_priv = dev->dev_private; 3710 3711 if (!sx_try_xlock(&dev->dev_struct_lock)) 3712 return; 3713 3714 CTR0(KTR_DRM, "gem_lowmem"); 3715 3716rescan: 3717 /* first scan for clean buffers */ 3718 i915_gem_retire_requests(dev); 3719 3720 cnt_total = cnt_fail = cnt = 0; 3721 3722 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3723 mm_list) { 3724 if (i915_gem_object_is_purgeable(obj)) { 3725 if (i915_gem_object_unbind(obj) != 0) 3726 cnt_total++; 3727 } else 3728 cnt_total++; 3729 } 3730 3731 /* second pass, evict/count anything still on the inactive list */ 3732 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3733 mm_list) { 3734 if (i915_gem_object_unbind(obj) == 0) 3735 cnt++; 3736 else 3737 cnt_fail++; 3738 } 3739 3740 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3741 /* 3742 * We are desperate for pages, so as a last resort, wait 3743 * for the GPU to finish and discard whatever we can. 3744 * This has a dramatic impact to reduce the number of 3745 * OOM-killer events whilst running the GPU aggressively. 3746 */ 3747 if (i915_gpu_idle(dev, true) == 0) 3748 goto rescan; 3749 } 3750 DRM_UNLOCK(dev); 3751} 3752 3753void 3754i915_gem_unload(struct drm_device *dev) 3755{ 3756 struct drm_i915_private *dev_priv; 3757 3758 dev_priv = dev->dev_private; 3759 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3760} 3761