i915_gem.c revision 280183
1/*- 2 * Copyright �� 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 */ 53 54#include <sys/cdefs.h> 55__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 280183 2015-03-17 18:50:33Z dumbbell $"); 56 57#include <dev/drm2/drmP.h> 58#include <dev/drm2/drm.h> 59#include <dev/drm2/i915/i915_drm.h> 60#include <dev/drm2/i915/i915_drv.h> 61#include <dev/drm2/i915/intel_drv.h> 62#include <dev/drm2/i915/intel_ringbuffer.h> 63#include <sys/resourcevar.h> 64#include <sys/sched.h> 65#include <sys/sf_buf.h> 66 67#include <vm/vm.h> 68#include <vm/vm_pageout.h> 69 70#include <machine/md_var.h> 71 72static void i915_gem_object_flush_cpu_write_domain( 73 struct drm_i915_gem_object *obj); 74static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 75 int tiling_mode); 76static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 77 uint32_t size, int tiling_mode); 78static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 79 unsigned alignment, bool map_and_fenceable); 80static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 81 int flags); 82static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 83static void i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, 84 off_t start, off_t end); 85static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 86 off_t start, off_t end); 87static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 88static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 89static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 90static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 91static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 92static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, 93 bool *fresh); 94static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 95 uint32_t flush_domains); 96static void i915_gem_reset_fences(struct drm_device *dev); 97static void i915_gem_retire_task_handler(void *arg, int pending); 98static void i915_gem_lowmem(void *arg); 99static void i915_gem_write_fence(struct drm_device *dev, int reg, 100 struct drm_i915_gem_object *obj); 101static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 102 bool interruptible); 103static int i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno); 104 105MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 106long i915_gem_wired_pages_cnt; 107 108static bool cpu_cache_is_coherent(struct drm_device *dev, 109 enum i915_cache_level level) 110{ 111 return HAS_LLC(dev) || level != I915_CACHE_NONE; 112} 113 114static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 115{ 116 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) 117 return true; 118 119 return obj->pin_display; 120} 121 122static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) 123{ 124 if (obj->tiling_mode) 125 i915_gem_release_mmap(obj); 126 127 /* As we do not have an associated fence register, we will force 128 * a tiling change if we ever need to acquire one. 129 */ 130 obj->fence_dirty = false; 131 obj->fence_reg = I915_FENCE_REG_NONE; 132} 133 134static void 135i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 136{ 137 138 dev_priv->mm.object_count++; 139 dev_priv->mm.object_memory += size; 140} 141 142static void 143i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 144{ 145 146 dev_priv->mm.object_count--; 147 dev_priv->mm.object_memory -= size; 148} 149 150static int 151i915_gem_wait_for_error(struct drm_device *dev) 152{ 153 struct drm_i915_private *dev_priv; 154 int ret; 155 156 dev_priv = dev->dev_private; 157 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 158 return (0); 159 160 mtx_lock(&dev_priv->error_completion_lock); 161 while (dev_priv->error_completion == 0) { 162 ret = -msleep(&dev_priv->error_completion, 163 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 164 if (ret == -ERESTART) 165 ret = -ERESTARTSYS; 166 if (ret != 0) { 167 mtx_unlock(&dev_priv->error_completion_lock); 168 return (ret); 169 } 170 } 171 mtx_unlock(&dev_priv->error_completion_lock); 172 173 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 174 mtx_lock(&dev_priv->error_completion_lock); 175 dev_priv->error_completion++; 176 mtx_unlock(&dev_priv->error_completion_lock); 177 } 178 return (0); 179} 180 181int 182i915_mutex_lock_interruptible(struct drm_device *dev) 183{ 184 struct drm_i915_private *dev_priv; 185 int ret; 186 187 dev_priv = dev->dev_private; 188 ret = i915_gem_wait_for_error(dev); 189 if (ret != 0) 190 return (ret); 191 192 /* 193 * interruptible shall it be. might indeed be if dev_lock is 194 * changed to sx 195 */ 196 ret = sx_xlock_sig(&dev->dev_struct_lock); 197 if (ret != 0) 198 return (-ret); 199 200 return (0); 201} 202 203 204void 205i915_gem_free_object(struct drm_gem_object *gem_obj) 206{ 207 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 208 struct drm_device *dev; 209 drm_i915_private_t *dev_priv; 210 211 dev = obj->base.dev; 212 dev_priv = dev->dev_private; 213 214 CTR1(KTR_DRM, "object_destroy_tail %p", obj); 215 216 if (obj->phys_obj) 217 i915_gem_detach_phys_object(dev, obj); 218 219 obj->pin_count = 0; 220 if (i915_gem_object_unbind(obj) == -ERESTARTSYS) { 221 bool was_interruptible; 222 223 was_interruptible = dev_priv->mm.interruptible; 224 dev_priv->mm.interruptible = false; 225 226 if (i915_gem_object_unbind(obj)) 227 printf("i915_gem_free_object: unbind\n"); 228 229 dev_priv->mm.interruptible = was_interruptible; 230 } 231 232 drm_gem_free_mmap_offset(&obj->base); 233 drm_gem_object_release(&obj->base); 234 i915_gem_info_remove_obj(dev_priv, obj->base.size); 235 236 free(obj->bit_17, DRM_I915_GEM); 237 free(obj, DRM_I915_GEM); 238} 239 240static void 241init_ring_lists(struct intel_ring_buffer *ring) 242{ 243 244 INIT_LIST_HEAD(&ring->active_list); 245 INIT_LIST_HEAD(&ring->request_list); 246 INIT_LIST_HEAD(&ring->gpu_write_list); 247} 248 249void 250i915_gem_load(struct drm_device *dev) 251{ 252 drm_i915_private_t *dev_priv; 253 int i; 254 255 dev_priv = dev->dev_private; 256 257 INIT_LIST_HEAD(&dev_priv->mm.active_list); 258 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 259 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 260 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 261 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 262 for (i = 0; i < I915_NUM_RINGS; i++) 263 init_ring_lists(&dev_priv->rings[i]); 264 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 265 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 266 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 267 i915_gem_retire_task_handler, dev_priv); 268 dev_priv->error_completion = 0; 269 270 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 271 if (IS_GEN3(dev)) { 272 I915_WRITE(MI_ARB_STATE, 273 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 274 } 275 276 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 277 278 /* Old X drivers will take 0-2 for front, back, depth buffers */ 279 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 280 dev_priv->fence_reg_start = 3; 281 282 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 283 IS_G33(dev)) 284 dev_priv->num_fence_regs = 16; 285 else 286 dev_priv->num_fence_regs = 8; 287 288 /* Initialize fence registers to zero */ 289 i915_gem_reset_fences(dev); 290 291 i915_gem_detect_bit_6_swizzle(dev); 292 dev_priv->mm.interruptible = true; 293 294 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 295 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 296} 297 298int 299i915_gem_init_ioctl(struct drm_device *dev, void *data, 300 struct drm_file *file) 301{ 302 struct drm_i915_gem_init *args; 303 drm_i915_private_t *dev_priv; 304 int error; 305 306 if (drm_core_check_feature(dev, DRIVER_MODESET)) 307 return -ENODEV; 308 309 dev_priv = dev->dev_private; 310 args = data; 311 312 if (args->gtt_start >= args->gtt_end || 313 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 314 return (-EINVAL); 315 316 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) 317 return (-EBUSY); 318 319 /* GEM with user mode setting was never supported on ilk and later. */ 320 if (INTEL_INFO(dev)->gen >= 5) 321 return -ENODEV; 322 323 /* 324 * XXXKIB. The second-time initialization should be guarded 325 * against. 326 */ 327 DRM_LOCK(dev); 328 error = i915_gem_init_global_gtt(dev, args->gtt_start, 329 args->gtt_end, args->gtt_end); 330 DRM_UNLOCK(dev); 331 return (error); 332} 333 334int 335i915_gem_idle(struct drm_device *dev) 336{ 337 drm_i915_private_t *dev_priv; 338 int ret; 339 340 DRM_LOCK(dev); 341 342 dev_priv = dev->dev_private; 343 if (dev_priv->mm.suspended) { 344 DRM_UNLOCK(dev); 345 return (0); 346 } 347 348 ret = i915_gpu_idle(dev); 349 if (ret != 0) { 350 DRM_UNLOCK(dev); 351 return (ret); 352 } 353 i915_gem_retire_requests(dev); 354 355 /* Under UMS, be paranoid and evict. */ 356 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 357 ret = i915_gem_evict_everything(dev, false); 358 if (ret != 0) { 359 DRM_UNLOCK(dev); 360 return ret; 361 } 362 } 363 364 i915_gem_reset_fences(dev); 365 366 /* Hack! Don't let anybody do execbuf while we don't control the chip. 367 * We need to replace this with a semaphore, or something. 368 * And not confound mm.suspended! 369 */ 370 dev_priv->mm.suspended = 1; 371 callout_stop(&dev_priv->hangcheck_timer); 372 373 i915_kernel_lost_context(dev); 374 i915_gem_cleanup_ringbuffer(dev); 375 376 DRM_UNLOCK(dev); 377 378 /* Cancel the retire work handler, which should be idle now. */ 379 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 380 return (ret); 381} 382 383void 384i915_gem_init_swizzling(struct drm_device *dev) 385{ 386 drm_i915_private_t *dev_priv; 387 388 dev_priv = dev->dev_private; 389 390 if (INTEL_INFO(dev)->gen < 5 || 391 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 392 return; 393 394 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 395 DISP_TILE_SURFACE_SWIZZLING); 396 397 if (IS_GEN5(dev)) 398 return; 399 400 401 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 402 if (IS_GEN6(dev)) 403 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 404 else 405 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 406} 407 408void i915_gem_init_ppgtt(struct drm_device *dev) 409{ 410 drm_i915_private_t *dev_priv; 411 struct i915_hw_ppgtt *ppgtt; 412 uint32_t pd_offset, pd_entry; 413 vm_paddr_t pt_addr; 414 struct intel_ring_buffer *ring; 415 u_int first_pd_entry_in_global_pt, i; 416 417 dev_priv = dev->dev_private; 418 ppgtt = dev_priv->mm.aliasing_ppgtt; 419 if (ppgtt == NULL) 420 return; 421 422 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 423 for (i = 0; i < ppgtt->num_pd_entries; i++) { 424 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 425 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 426 pd_entry |= GEN6_PDE_VALID; 427 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 428 } 429 intel_gtt_read_pte(first_pd_entry_in_global_pt); 430 431 pd_offset = ppgtt->pd_offset; 432 pd_offset /= 64; /* in cachelines, */ 433 pd_offset <<= 16; 434 435 if (INTEL_INFO(dev)->gen == 6) { 436 uint32_t ecochk, gab_ctl, ecobits; 437 438 ecobits = I915_READ(GAC_ECO_BITS); 439 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 440 441 gab_ctl = I915_READ(GAB_CTL); 442 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 443 444 ecochk = I915_READ(GAM_ECOCHK); 445 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 446 ECOCHK_PPGTT_CACHE64B); 447 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 448 } else if (INTEL_INFO(dev)->gen >= 7) { 449 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 450 /* GFX_MODE is per-ring on gen7+ */ 451 } 452 453 for_each_ring(ring, dev_priv, i) { 454 if (INTEL_INFO(dev)->gen >= 7) 455 I915_WRITE(RING_MODE_GEN7(ring), 456 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 457 458 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 459 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 460 } 461} 462 463int 464i915_gem_init_hw(struct drm_device *dev) 465{ 466 drm_i915_private_t *dev_priv; 467 int ret; 468 469 dev_priv = dev->dev_private; 470 471 i915_gem_init_swizzling(dev); 472 473 ret = intel_init_render_ring_buffer(dev); 474 if (ret != 0) 475 return (ret); 476 477 if (HAS_BSD(dev)) { 478 ret = intel_init_bsd_ring_buffer(dev); 479 if (ret != 0) 480 goto cleanup_render_ring; 481 } 482 483 if (HAS_BLT(dev)) { 484 ret = intel_init_blt_ring_buffer(dev); 485 if (ret != 0) 486 goto cleanup_bsd_ring; 487 } 488 489 dev_priv->next_seqno = 1; 490 i915_gem_context_init(dev); 491 i915_gem_init_ppgtt(dev); 492 return (0); 493 494cleanup_bsd_ring: 495 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 496cleanup_render_ring: 497 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 498 return (ret); 499} 500 501static bool 502intel_enable_ppgtt(struct drm_device *dev) 503{ 504 if (i915_enable_ppgtt >= 0) 505 return i915_enable_ppgtt; 506 507 /* Disable ppgtt on SNB if VT-d is on. */ 508 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled) 509 return false; 510 511 return true; 512} 513 514int i915_gem_init(struct drm_device *dev) 515{ 516 struct drm_i915_private *dev_priv = dev->dev_private; 517 unsigned long gtt_size, mappable_size; 518 int ret; 519 520 gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT; 521 mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT; 522 523 DRM_LOCK(dev); 524 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { 525 /* PPGTT pdes are stolen from global gtt ptes, so shrink the 526 * aperture accordingly when using aliasing ppgtt. */ 527 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; 528 529 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); 530 531 ret = i915_gem_init_aliasing_ppgtt(dev); 532 if (ret) { 533 DRM_UNLOCK(dev); 534 return ret; 535 } 536 } else { 537 /* Let GEM Manage all of the aperture. 538 * 539 * However, leave one page at the end still bound to the scratch 540 * page. There are a number of places where the hardware 541 * apparently prefetches past the end of the object, and we've 542 * seen multiple hangs with the GPU head pointer stuck in a 543 * batchbuffer bound at the last page of the aperture. One page 544 * should be enough to keep any prefetching inside of the 545 * aperture. 546 */ 547 i915_gem_init_global_gtt(dev, 0, mappable_size, 548 gtt_size); 549 } 550 551 ret = i915_gem_init_hw(dev); 552 DRM_UNLOCK(dev); 553 if (ret != 0) { 554 i915_gem_cleanup_aliasing_ppgtt(dev); 555 return (ret); 556 } 557 558 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ 559 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 560 dev_priv->dri1.allow_batchbuffer = 1; 561 return 0; 562} 563 564int 565i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 566 struct drm_file *file) 567{ 568 struct drm_i915_private *dev_priv; 569 struct drm_i915_gem_get_aperture *args; 570 struct drm_i915_gem_object *obj; 571 size_t pinned; 572 573 dev_priv = dev->dev_private; 574 args = data; 575 576 pinned = 0; 577 DRM_LOCK(dev); 578 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) 579 if (obj->pin_count) 580 pinned += obj->gtt_space->size; 581 DRM_UNLOCK(dev); 582 583 args->aper_size = dev_priv->mm.gtt_total; 584 args->aper_available_size = args->aper_size - pinned; 585 586 return (0); 587} 588 589int 590i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 591 bool map_and_fenceable) 592{ 593 int ret; 594 595 if (obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT) 596 return (-EBUSY); 597 598 if (obj->gtt_space != NULL) { 599 if ((alignment && obj->gtt_offset & (alignment - 1)) || 600 (map_and_fenceable && !obj->map_and_fenceable)) { 601 DRM_DEBUG("bo is already pinned with incorrect alignment:" 602 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 603 " obj->map_and_fenceable=%d\n", 604 obj->gtt_offset, alignment, 605 map_and_fenceable, 606 obj->map_and_fenceable); 607 ret = i915_gem_object_unbind(obj); 608 if (ret != 0) 609 return (ret); 610 } 611 } 612 613 if (obj->gtt_space == NULL) { 614 ret = i915_gem_object_bind_to_gtt(obj, alignment, 615 map_and_fenceable); 616 if (ret) 617 return (ret); 618 } 619 620 if (!obj->has_global_gtt_mapping && map_and_fenceable) 621 i915_gem_gtt_bind_object(obj, obj->cache_level); 622 623 obj->pin_count++; 624 obj->pin_mappable |= map_and_fenceable; 625 626 return 0; 627} 628 629void 630i915_gem_object_unpin(struct drm_i915_gem_object *obj) 631{ 632 633 KASSERT(obj->pin_count != 0, ("zero pin count")); 634 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 635 636 if (--obj->pin_count == 0) 637 obj->pin_mappable = false; 638} 639 640int 641i915_gem_pin_ioctl(struct drm_device *dev, void *data, 642 struct drm_file *file) 643{ 644 struct drm_i915_gem_pin *args; 645 struct drm_i915_gem_object *obj; 646 struct drm_gem_object *gobj; 647 int ret; 648 649 args = data; 650 651 ret = i915_mutex_lock_interruptible(dev); 652 if (ret != 0) 653 return ret; 654 655 gobj = drm_gem_object_lookup(dev, file, args->handle); 656 if (gobj == NULL) { 657 ret = -ENOENT; 658 goto unlock; 659 } 660 obj = to_intel_bo(gobj); 661 662 if (obj->madv != I915_MADV_WILLNEED) { 663 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 664 ret = -EINVAL; 665 goto out; 666 } 667 668 if (obj->pin_filp != NULL && obj->pin_filp != file) { 669 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 670 args->handle); 671 ret = -EINVAL; 672 goto out; 673 } 674 675 obj->user_pin_count++; 676 obj->pin_filp = file; 677 if (obj->user_pin_count == 1) { 678 ret = i915_gem_object_pin(obj, args->alignment, true); 679 if (ret != 0) 680 goto out; 681 } 682 683 /* XXX - flush the CPU caches for pinned objects 684 * as the X server doesn't manage domains yet 685 */ 686 i915_gem_object_flush_cpu_write_domain(obj); 687 args->offset = obj->gtt_offset; 688out: 689 drm_gem_object_unreference(&obj->base); 690unlock: 691 DRM_UNLOCK(dev); 692 return (ret); 693} 694 695int 696i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 697 struct drm_file *file) 698{ 699 struct drm_i915_gem_pin *args; 700 struct drm_i915_gem_object *obj; 701 int ret; 702 703 args = data; 704 ret = i915_mutex_lock_interruptible(dev); 705 if (ret != 0) 706 return (ret); 707 708 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 709 if (&obj->base == NULL) { 710 ret = -ENOENT; 711 goto unlock; 712 } 713 714 if (obj->pin_filp != file) { 715 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 716 args->handle); 717 ret = -EINVAL; 718 goto out; 719 } 720 obj->user_pin_count--; 721 if (obj->user_pin_count == 0) { 722 obj->pin_filp = NULL; 723 i915_gem_object_unpin(obj); 724 } 725 726out: 727 drm_gem_object_unreference(&obj->base); 728unlock: 729 DRM_UNLOCK(dev); 730 return (ret); 731} 732 733int 734i915_gem_busy_ioctl(struct drm_device *dev, void *data, 735 struct drm_file *file) 736{ 737 struct drm_i915_gem_busy *args; 738 struct drm_i915_gem_object *obj; 739 int ret; 740 741 args = data; 742 743 ret = i915_mutex_lock_interruptible(dev); 744 if (ret != 0) 745 return ret; 746 747 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 748 if (&obj->base == NULL) { 749 ret = -ENOENT; 750 goto unlock; 751 } 752 753 args->busy = obj->active; 754 if (args->busy) { 755 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 756 ret = i915_gem_flush_ring(obj->ring, 757 0, obj->base.write_domain); 758 } else { 759 ret = i915_gem_check_olr(obj->ring, 760 obj->last_rendering_seqno); 761 } 762 763 i915_gem_retire_requests_ring(obj->ring); 764 args->busy = obj->active; 765 } 766 767 drm_gem_object_unreference(&obj->base); 768unlock: 769 DRM_UNLOCK(dev); 770 return (ret); 771} 772 773static int 774i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 775{ 776 struct drm_i915_private *dev_priv; 777 struct drm_i915_file_private *file_priv; 778 unsigned long recent_enough; 779 struct drm_i915_gem_request *request; 780 struct intel_ring_buffer *ring; 781 u32 seqno; 782 int ret; 783 784 dev_priv = dev->dev_private; 785 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 786 return (-EIO); 787 788 file_priv = file->driver_priv; 789 recent_enough = ticks - (20 * hz / 1000); 790 ring = NULL; 791 seqno = 0; 792 793 mtx_lock(&file_priv->mm.lck); 794 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 795 if (time_after_eq(request->emitted_jiffies, recent_enough)) 796 break; 797 ring = request->ring; 798 seqno = request->seqno; 799 } 800 mtx_unlock(&file_priv->mm.lck); 801 if (seqno == 0) 802 return (0); 803 804 ret = __wait_seqno(ring, seqno, true); 805 if (ret == 0) 806 taskqueue_enqueue_timeout(dev_priv->tq, 807 &dev_priv->mm.retire_task, 0); 808 809 return (ret); 810} 811 812int 813i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 814 struct drm_file *file_priv) 815{ 816 817 return (i915_gem_ring_throttle(dev, file_priv)); 818} 819 820int 821i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 822 struct drm_file *file_priv) 823{ 824 struct drm_i915_gem_madvise *args; 825 struct drm_i915_gem_object *obj; 826 int ret; 827 828 args = data; 829 switch (args->madv) { 830 case I915_MADV_DONTNEED: 831 case I915_MADV_WILLNEED: 832 break; 833 default: 834 return (-EINVAL); 835 } 836 837 ret = i915_mutex_lock_interruptible(dev); 838 if (ret != 0) 839 return (ret); 840 841 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 842 if (&obj->base == NULL) { 843 ret = -ENOENT; 844 goto unlock; 845 } 846 847 if (obj->pin_count != 0) { 848 ret = -EINVAL; 849 goto out; 850 } 851 852 if (obj->madv != I915_MADV_PURGED_INTERNAL) 853 obj->madv = args->madv; 854 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 855 i915_gem_object_truncate(obj); 856 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 857 858out: 859 drm_gem_object_unreference(&obj->base); 860unlock: 861 DRM_UNLOCK(dev); 862 return (ret); 863} 864 865void 866i915_gem_cleanup_ringbuffer(struct drm_device *dev) 867{ 868 drm_i915_private_t *dev_priv; 869 struct intel_ring_buffer *ring; 870 int i; 871 872 dev_priv = dev->dev_private; 873 for_each_ring(ring, dev_priv, i) 874 intel_cleanup_ring_buffer(ring); 875} 876 877int 878i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 879 struct drm_file *file_priv) 880{ 881 drm_i915_private_t *dev_priv; 882 int ret; 883 884 if (drm_core_check_feature(dev, DRIVER_MODESET)) 885 return (0); 886 dev_priv = dev->dev_private; 887 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 888 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 889 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 890 } 891 892 DRM_LOCK(dev); 893 dev_priv->mm.suspended = 0; 894 895 ret = i915_gem_init_hw(dev); 896 if (ret != 0) { 897 DRM_UNLOCK(dev); 898 return (ret); 899 } 900 901 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 902 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 903 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 904 DRM_UNLOCK(dev); 905 906 ret = drm_irq_install(dev); 907 if (ret) 908 goto cleanup_ringbuffer; 909 910 return (0); 911 912cleanup_ringbuffer: 913 DRM_LOCK(dev); 914 i915_gem_cleanup_ringbuffer(dev); 915 dev_priv->mm.suspended = 1; 916 DRM_UNLOCK(dev); 917 918 return (ret); 919} 920 921int 922i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 923 struct drm_file *file_priv) 924{ 925 926 if (drm_core_check_feature(dev, DRIVER_MODESET)) 927 return 0; 928 929 drm_irq_uninstall(dev); 930 return (i915_gem_idle(dev)); 931} 932 933int 934i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 935 uint32_t *handle_p) 936{ 937 struct drm_i915_gem_object *obj; 938 uint32_t handle; 939 int ret; 940 941 size = roundup(size, PAGE_SIZE); 942 if (size == 0) 943 return (-EINVAL); 944 945 obj = i915_gem_alloc_object(dev, size); 946 if (obj == NULL) 947 return (-ENOMEM); 948 949 ret = drm_gem_handle_create(file, &obj->base, &handle); 950 if (ret != 0) { 951 drm_gem_object_release(&obj->base); 952 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 953 free(obj, DRM_I915_GEM); 954 return (ret); 955 } 956 957 /* drop reference from allocate - handle holds it now */ 958 drm_gem_object_unreference(&obj->base); 959 CTR2(KTR_DRM, "object_create %p %x", obj, size); 960 *handle_p = handle; 961 return (0); 962} 963 964int 965i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 966 struct drm_mode_create_dumb *args) 967{ 968 969 /* have to work out size/pitch and return them */ 970 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 971 args->size = args->pitch * args->height; 972 return (i915_gem_create(file, dev, args->size, &args->handle)); 973} 974 975int 976i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 977 uint32_t handle) 978{ 979 980 return (drm_gem_handle_delete(file, handle)); 981} 982 983int 984i915_gem_create_ioctl(struct drm_device *dev, void *data, 985 struct drm_file *file) 986{ 987 struct drm_i915_gem_create *args = data; 988 989 return (i915_gem_create(file, dev, args->size, &args->handle)); 990} 991 992#define __user 993#define __force 994#define __iomem 995#define to_user_ptr(x) ((void *)(uintptr_t)(x)) 996#define offset_in_page(x) ((x) & PAGE_MASK) 997#define page_to_phys(x) VM_PAGE_TO_PHYS(x) 998static inline int 999__copy_to_user_inatomic(void __user *to, const void *from, unsigned n) 1000{ 1001 return (copyout_nofault(from, to, n) != 0 ? n : 0); 1002} 1003static inline unsigned long 1004__copy_from_user_inatomic_nocache(void *to, const void __user *from, 1005 unsigned long n) 1006{ 1007 1008 /* 1009 * XXXKIB. Equivalent Linux function is implemented using 1010 * MOVNTI for aligned moves. For unaligned head and tail, 1011 * normal move is performed. As such, it is not incorrect, if 1012 * only somewhat slower, to use normal copyin. All uses 1013 * except shmem_pwrite_fast() have the destination mapped WC. 1014 */ 1015 return ((copyin_nofault(__DECONST(void *, from), to, n) != 0 ? n : 0)); 1016} 1017static inline int 1018fault_in_multipages_readable(const char __user *uaddr, int size) 1019{ 1020 char c; 1021 int ret = 0; 1022 const char __user *end = uaddr + size - 1; 1023 1024 if (unlikely(size == 0)) 1025 return ret; 1026 1027 while (uaddr <= end) { 1028 ret = -copyin(uaddr, &c, 1); 1029 if (ret != 0) 1030 return -EFAULT; 1031 uaddr += PAGE_SIZE; 1032 } 1033 1034 /* Check whether the range spilled into the next page. */ 1035 if (((unsigned long)uaddr & ~PAGE_MASK) == 1036 ((unsigned long)end & ~PAGE_MASK)) { 1037 ret = -copyin(end, &c, 1); 1038 } 1039 1040 return ret; 1041} 1042 1043static inline int 1044fault_in_multipages_writeable(char __user *uaddr, int size) 1045{ 1046 int ret = 0; 1047 char __user *end = uaddr + size - 1; 1048 1049 if (unlikely(size == 0)) 1050 return ret; 1051 1052 /* 1053 * Writing zeroes into userspace here is OK, because we know that if 1054 * the zero gets there, we'll be overwriting it. 1055 */ 1056 while (uaddr <= end) { 1057 ret = subyte(uaddr, 0); 1058 if (ret != 0) 1059 return -EFAULT; 1060 uaddr += PAGE_SIZE; 1061 } 1062 1063 /* Check whether the range spilled into the next page. */ 1064 if (((unsigned long)uaddr & ~PAGE_MASK) == 1065 ((unsigned long)end & ~PAGE_MASK)) 1066 ret = subyte(end, 0); 1067 1068 return ret; 1069} 1070 1071static inline int 1072__copy_to_user_swizzled(char __user *cpu_vaddr, 1073 const char *gpu_vaddr, int gpu_offset, 1074 int length) 1075{ 1076 int ret, cpu_offset = 0; 1077 1078 while (length > 0) { 1079 int cacheline_end = roundup2(gpu_offset + 1, 64); 1080 int this_length = min(cacheline_end - gpu_offset, length); 1081 int swizzled_gpu_offset = gpu_offset ^ 64; 1082 1083 ret = __copy_to_user(cpu_vaddr + cpu_offset, 1084 gpu_vaddr + swizzled_gpu_offset, 1085 this_length); 1086 if (ret) 1087 return ret + length; 1088 1089 cpu_offset += this_length; 1090 gpu_offset += this_length; 1091 length -= this_length; 1092 } 1093 1094 return 0; 1095} 1096 1097static inline int 1098__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 1099 const char __user *cpu_vaddr, 1100 int length) 1101{ 1102 int ret, cpu_offset = 0; 1103 1104 while (length > 0) { 1105 int cacheline_end = roundup2(gpu_offset + 1, 64); 1106 int this_length = min(cacheline_end - gpu_offset, length); 1107 int swizzled_gpu_offset = gpu_offset ^ 64; 1108 1109 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 1110 cpu_vaddr + cpu_offset, 1111 this_length); 1112 if (ret) 1113 return ret + length; 1114 1115 cpu_offset += this_length; 1116 gpu_offset += this_length; 1117 length -= this_length; 1118 } 1119 1120 return 0; 1121} 1122 1123static int 1124i915_gem_phys_pwrite(struct drm_device *dev, 1125 struct drm_i915_gem_object *obj, 1126 struct drm_i915_gem_pwrite *args, 1127 struct drm_file *file_priv) 1128{ 1129 void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset; 1130 char __user *user_data = to_user_ptr(args->data_ptr); 1131 1132 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 1133 unsigned long unwritten; 1134 1135 /* The physical object once assigned is fixed for the lifetime 1136 * of the obj, so we can safely drop the lock and continue 1137 * to access vaddr. 1138 */ 1139 DRM_UNLOCK(dev); 1140 unwritten = copy_from_user(vaddr, user_data, args->size); 1141 DRM_LOCK(dev); 1142 if (unwritten) 1143 return -EFAULT; 1144 } 1145 1146 i915_gem_chipset_flush(dev); 1147 return 0; 1148} 1149 1150/* Per-page copy function for the shmem pread fastpath. 1151 * Flushes invalid cachelines before reading the target if 1152 * needs_clflush is set. */ 1153static int 1154shmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length, 1155 char __user *user_data, 1156 bool page_do_bit17_swizzling, bool needs_clflush) 1157{ 1158 char *vaddr; 1159 struct sf_buf *sf; 1160 int ret; 1161 1162 if (unlikely(page_do_bit17_swizzling)) 1163 return -EINVAL; 1164 1165 sched_pin(); 1166 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 1167 if (sf == NULL) { 1168 sched_unpin(); 1169 return (-EFAULT); 1170 } 1171 vaddr = (char *)sf_buf_kva(sf); 1172 if (needs_clflush) 1173 drm_clflush_virt_range(vaddr + shmem_page_offset, 1174 page_length); 1175 ret = __copy_to_user_inatomic(user_data, 1176 vaddr + shmem_page_offset, 1177 page_length); 1178 sf_buf_free(sf); 1179 sched_unpin(); 1180 1181 return ret ? -EFAULT : 0; 1182} 1183 1184static void 1185shmem_clflush_swizzled_range(char *addr, unsigned long length, 1186 bool swizzled) 1187{ 1188 if (unlikely(swizzled)) { 1189 unsigned long start = (unsigned long) addr; 1190 unsigned long end = (unsigned long) addr + length; 1191 1192 /* For swizzling simply ensure that we always flush both 1193 * channels. Lame, but simple and it works. Swizzled 1194 * pwrite/pread is far from a hotpath - current userspace 1195 * doesn't use it at all. */ 1196 start = rounddown2(start, 128); 1197 end = roundup2(end, 128); 1198 1199 drm_clflush_virt_range((void *)start, end - start); 1200 } else { 1201 drm_clflush_virt_range(addr, length); 1202 } 1203 1204} 1205 1206/* Only difference to the fast-path function is that this can handle bit17 1207 * and uses non-atomic copy and kmap functions. */ 1208static int 1209shmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length, 1210 char __user *user_data, 1211 bool page_do_bit17_swizzling, bool needs_clflush) 1212{ 1213 char *vaddr; 1214 struct sf_buf *sf; 1215 int ret; 1216 1217 sf = sf_buf_alloc(page, 0); 1218 vaddr = (char *)sf_buf_kva(sf); 1219 if (needs_clflush) 1220 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1221 page_length, 1222 page_do_bit17_swizzling); 1223 1224 if (page_do_bit17_swizzling) 1225 ret = __copy_to_user_swizzled(user_data, 1226 vaddr, shmem_page_offset, 1227 page_length); 1228 else 1229 ret = __copy_to_user(user_data, 1230 vaddr + shmem_page_offset, 1231 page_length); 1232 sf_buf_free(sf); 1233 1234 return ret ? - EFAULT : 0; 1235} 1236 1237static int 1238i915_gem_shmem_pread(struct drm_device *dev, 1239 struct drm_i915_gem_object *obj, 1240 struct drm_i915_gem_pread *args, 1241 struct drm_file *file) 1242{ 1243 char __user *user_data; 1244 ssize_t remain, sremain; 1245 off_t offset, soffset; 1246 int shmem_page_offset, page_length, ret = 0; 1247 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 1248 int prefaulted = 0; 1249 int needs_clflush = 0; 1250 1251 user_data = to_user_ptr(args->data_ptr); 1252 sremain = remain = args->size; 1253 1254 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 1255 1256 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 1257 /* If we're not in the cpu read domain, set ourself into the gtt 1258 * read domain and manually flush cachelines (if required). This 1259 * optimizes for the case when the gpu will dirty the data 1260 * anyway again before the next pread happens. */ 1261 needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level); 1262 ret = i915_gem_object_set_to_gtt_domain(obj, false); 1263 if (ret) 1264 return ret; 1265 } 1266 1267 soffset = offset = args->offset; 1268 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain); 1269 if (ret) 1270 return ret; 1271 1272 i915_gem_object_pin_pages(obj); 1273 1274 VM_OBJECT_WLOCK(obj->base.vm_obj); 1275 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 1276 OFF_TO_IDX(offset));; page = vm_page_next(page)) { 1277 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 1278 1279 if (remain <= 0) 1280 break; 1281 1282 /* Operation in this page 1283 * 1284 * shmem_page_offset = offset within page in shmem file 1285 * page_length = bytes to copy for this page 1286 */ 1287 shmem_page_offset = offset_in_page(offset); 1288 page_length = remain; 1289 if ((shmem_page_offset + page_length) > PAGE_SIZE) 1290 page_length = PAGE_SIZE - shmem_page_offset; 1291 1292 page_do_bit17_swizzling = obj_do_bit17_swizzling && 1293 (page_to_phys(page) & (1 << 17)) != 0; 1294 1295 ret = shmem_pread_fast(page, shmem_page_offset, page_length, 1296 user_data, page_do_bit17_swizzling, 1297 needs_clflush); 1298 if (ret == 0) 1299 goto next_page; 1300 1301 DRM_UNLOCK(dev); 1302 1303 if (likely(!i915_prefault_disable) && !prefaulted) { 1304 ret = fault_in_multipages_writeable(user_data, remain); 1305 /* Userspace is tricking us, but we've already clobbered 1306 * its pages with the prefault and promised to write the 1307 * data up to the first fault. Hence ignore any errors 1308 * and just continue. */ 1309 (void)ret; 1310 prefaulted = 1; 1311 } 1312 1313 ret = shmem_pread_slow(page, shmem_page_offset, page_length, 1314 user_data, page_do_bit17_swizzling, 1315 needs_clflush); 1316 1317 DRM_LOCK(dev); 1318 1319next_page: 1320 vm_page_reference(page); 1321 1322 if (ret) 1323 goto out; 1324 1325 remain -= page_length; 1326 user_data += page_length; 1327 offset += page_length; 1328 VM_OBJECT_WLOCK(obj->base.vm_obj); 1329 } 1330 1331out: 1332 i915_gem_object_unpin_pages(obj); 1333 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain); 1334 1335 return ret; 1336} 1337 1338/** 1339 * Reads data from the object referenced by handle. 1340 * 1341 * On error, the contents of *data are undefined. 1342 */ 1343int 1344i915_gem_pread_ioctl(struct drm_device *dev, void *data, 1345 struct drm_file *file) 1346{ 1347 struct drm_i915_gem_pread *args = data; 1348 struct drm_i915_gem_object *obj; 1349 int ret = 0; 1350 1351 if (args->size == 0) 1352 return 0; 1353 1354 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE)) 1355 return -EFAULT; 1356 1357 ret = i915_mutex_lock_interruptible(dev); 1358 if (ret) 1359 return ret; 1360 1361 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1362 if (&obj->base == NULL) { 1363 ret = -ENOENT; 1364 goto unlock; 1365 } 1366 1367 /* Bounds check source. */ 1368 if (args->offset > obj->base.size || 1369 args->size > obj->base.size - args->offset) { 1370 ret = -EINVAL; 1371 goto out; 1372 } 1373 1374#if 1 1375 KIB_NOTYET(); 1376#else 1377 /* prime objects have no backing filp to GEM pread/pwrite 1378 * pages from. 1379 */ 1380 if (!obj->base.filp) { 1381 ret = -EINVAL; 1382 goto out; 1383 } 1384#endif 1385 1386 CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size); 1387 1388 ret = i915_gem_shmem_pread(dev, obj, args, file); 1389 1390out: 1391 drm_gem_object_unreference(&obj->base); 1392unlock: 1393 DRM_UNLOCK(dev); 1394 return ret; 1395} 1396 1397/* This is the fast write path which cannot handle 1398 * page faults in the source data 1399 */ 1400 1401static inline int 1402fast_user_write(struct drm_device *dev, 1403 off_t page_base, int page_offset, 1404 char __user *user_data, 1405 int length) 1406{ 1407 void __iomem *vaddr_atomic; 1408 void *vaddr; 1409 unsigned long unwritten; 1410 1411 vaddr_atomic = pmap_mapdev_attr(dev->agp->base + page_base, 1412 length, PAT_WRITE_COMBINING); 1413 /* We can use the cpu mem copy function because this is X86. */ 1414 vaddr = (char *)vaddr_atomic + page_offset; 1415 unwritten = __copy_from_user_inatomic_nocache(vaddr, 1416 user_data, length); 1417 pmap_unmapdev((vm_offset_t)vaddr_atomic, length); 1418 return unwritten; 1419} 1420 1421/** 1422 * This is the fast pwrite path, where we copy the data directly from the 1423 * user into the GTT, uncached. 1424 */ 1425static int 1426i915_gem_gtt_pwrite_fast(struct drm_device *dev, 1427 struct drm_i915_gem_object *obj, 1428 struct drm_i915_gem_pwrite *args, 1429 struct drm_file *file) 1430{ 1431 ssize_t remain; 1432 off_t offset, page_base; 1433 char __user *user_data; 1434 int page_offset, page_length, ret; 1435 1436 ret = i915_gem_object_pin(obj, 0, true); 1437 /* XXXKIB ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); */ 1438 if (ret != 0) 1439 goto out; 1440 1441 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1442 if (ret) 1443 goto out_unpin; 1444 1445 ret = i915_gem_object_put_fence(obj); 1446 if (ret) 1447 goto out_unpin; 1448 1449 user_data = to_user_ptr(args->data_ptr); 1450 remain = args->size; 1451 1452 offset = obj->gtt_offset + args->offset; 1453 1454 while (remain > 0) { 1455 /* Operation in this page 1456 * 1457 * page_base = page offset within aperture 1458 * page_offset = offset within page 1459 * page_length = bytes to copy for this page 1460 */ 1461 page_base = offset & ~PAGE_MASK; 1462 page_offset = offset_in_page(offset); 1463 page_length = remain; 1464 if ((page_offset + remain) > PAGE_SIZE) 1465 page_length = PAGE_SIZE - page_offset; 1466 1467 /* If we get a fault while copying data, then (presumably) our 1468 * source page isn't available. Return the error and we'll 1469 * retry in the slow path. 1470 */ 1471 if (fast_user_write(dev, page_base, 1472 page_offset, user_data, page_length)) { 1473 ret = -EFAULT; 1474 goto out_unpin; 1475 } 1476 1477 remain -= page_length; 1478 user_data += page_length; 1479 offset += page_length; 1480 } 1481 1482out_unpin: 1483 i915_gem_object_unpin(obj); 1484out: 1485 return ret; 1486} 1487 1488/* Per-page copy function for the shmem pwrite fastpath. 1489 * Flushes invalid cachelines before writing to the target if 1490 * needs_clflush_before is set and flushes out any written cachelines after 1491 * writing if needs_clflush is set. */ 1492static int 1493shmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length, 1494 char __user *user_data, 1495 bool page_do_bit17_swizzling, 1496 bool needs_clflush_before, 1497 bool needs_clflush_after) 1498{ 1499 char *vaddr; 1500 struct sf_buf *sf; 1501 int ret; 1502 1503 if (unlikely(page_do_bit17_swizzling)) 1504 return -EINVAL; 1505 1506 sched_pin(); 1507 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 1508 if (sf == NULL) { 1509 sched_unpin(); 1510 return (-EFAULT); 1511 } 1512 vaddr = (char *)sf_buf_kva(sf); 1513 if (needs_clflush_before) 1514 drm_clflush_virt_range(vaddr + shmem_page_offset, 1515 page_length); 1516 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, 1517 user_data, 1518 page_length); 1519 if (needs_clflush_after) 1520 drm_clflush_virt_range(vaddr + shmem_page_offset, 1521 page_length); 1522 sf_buf_free(sf); 1523 sched_unpin(); 1524 1525 return ret ? -EFAULT : 0; 1526} 1527 1528/* Only difference to the fast-path function is that this can handle bit17 1529 * and uses non-atomic copy and kmap functions. */ 1530static int 1531shmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length, 1532 char __user *user_data, 1533 bool page_do_bit17_swizzling, 1534 bool needs_clflush_before, 1535 bool needs_clflush_after) 1536{ 1537 char *vaddr; 1538 struct sf_buf *sf; 1539 int ret; 1540 1541 sf = sf_buf_alloc(page, 0); 1542 vaddr = (char *)sf_buf_kva(sf); 1543 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 1544 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1545 page_length, 1546 page_do_bit17_swizzling); 1547 if (page_do_bit17_swizzling) 1548 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 1549 user_data, 1550 page_length); 1551 else 1552 ret = __copy_from_user(vaddr + shmem_page_offset, 1553 user_data, 1554 page_length); 1555 if (needs_clflush_after) 1556 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1557 page_length, 1558 page_do_bit17_swizzling); 1559 sf_buf_free(sf); 1560 1561 return ret ? -EFAULT : 0; 1562} 1563 1564static int 1565i915_gem_shmem_pwrite(struct drm_device *dev, 1566 struct drm_i915_gem_object *obj, 1567 struct drm_i915_gem_pwrite *args, 1568 struct drm_file *file) 1569{ 1570 ssize_t remain, sremain; 1571 off_t offset, soffset; 1572 char __user *user_data; 1573 int shmem_page_offset, page_length, ret = 0; 1574 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 1575 int hit_slowpath = 0; 1576 int needs_clflush_after = 0; 1577 int needs_clflush_before = 0; 1578 1579 user_data = to_user_ptr(args->data_ptr); 1580 sremain = remain = args->size; 1581 1582 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 1583 1584 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1585 /* If we're not in the cpu write domain, set ourself into the gtt 1586 * write domain and manually flush cachelines (if required). This 1587 * optimizes for the case when the gpu will use the data 1588 * right away and we therefore have to clflush anyway. */ 1589 needs_clflush_after = cpu_write_needs_clflush(obj); 1590 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1591 if (ret) 1592 return ret; 1593 } 1594 /* Same trick applies to invalidate partially written cachelines read 1595 * before writing. */ 1596 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1597 needs_clflush_before = 1598 !cpu_cache_is_coherent(dev, obj->cache_level); 1599 1600 soffset = offset = args->offset; 1601 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain); 1602 if (ret) 1603 return ret; 1604 1605 i915_gem_object_pin_pages(obj); 1606 1607 obj->dirty = 1; 1608 1609 VM_OBJECT_WLOCK(obj->base.vm_obj); 1610 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 1611 OFF_TO_IDX(offset));; page = vm_page_next(page)) { 1612 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 1613 int partial_cacheline_write; 1614 1615 if (remain <= 0) 1616 break; 1617 1618 /* Operation in this page 1619 * 1620 * shmem_page_offset = offset within page in shmem file 1621 * page_length = bytes to copy for this page 1622 */ 1623 shmem_page_offset = offset_in_page(offset); 1624 1625 page_length = remain; 1626 if ((shmem_page_offset + page_length) > PAGE_SIZE) 1627 page_length = PAGE_SIZE - shmem_page_offset; 1628 1629 /* If we don't overwrite a cacheline completely we need to be 1630 * careful to have up-to-date data by first clflushing. Don't 1631 * overcomplicate things and flush the entire patch. */ 1632 partial_cacheline_write = needs_clflush_before && 1633 ((shmem_page_offset | page_length) 1634 & (cpu_clflush_line_size - 1)); 1635 1636 page_do_bit17_swizzling = obj_do_bit17_swizzling && 1637 (page_to_phys(page) & (1 << 17)) != 0; 1638 1639 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 1640 user_data, page_do_bit17_swizzling, 1641 partial_cacheline_write, 1642 needs_clflush_after); 1643 if (ret == 0) 1644 goto next_page; 1645 1646 hit_slowpath = 1; 1647 DRM_UNLOCK(dev); 1648 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 1649 user_data, page_do_bit17_swizzling, 1650 partial_cacheline_write, 1651 needs_clflush_after); 1652 1653 DRM_LOCK(dev); 1654 1655next_page: 1656 vm_page_dirty(page); 1657 vm_page_reference(page); 1658 1659 if (ret) 1660 goto out; 1661 1662 remain -= page_length; 1663 user_data += page_length; 1664 offset += page_length; 1665 VM_OBJECT_WLOCK(obj->base.vm_obj); 1666 } 1667 1668out: 1669 i915_gem_object_unpin_pages(obj); 1670 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain); 1671 1672 if (hit_slowpath) { 1673 /* 1674 * Fixup: Flush cpu caches in case we didn't flush the dirty 1675 * cachelines in-line while writing and the object moved 1676 * out of the cpu write domain while we've dropped the lock. 1677 */ 1678 if (!needs_clflush_after && 1679 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1680 i915_gem_clflush_object(obj); 1681 i915_gem_chipset_flush(dev); 1682 } 1683 } 1684 1685 if (needs_clflush_after) 1686 i915_gem_chipset_flush(dev); 1687 1688 return ret; 1689} 1690 1691/** 1692 * Writes data to the object referenced by handle. 1693 * 1694 * On error, the contents of the buffer that were to be modified are undefined. 1695 */ 1696int 1697i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 1698 struct drm_file *file) 1699{ 1700 struct drm_i915_gem_pwrite *args = data; 1701 struct drm_i915_gem_object *obj; 1702 int ret; 1703 1704 if (args->size == 0) 1705 return 0; 1706 1707 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ)) 1708 return -EFAULT; 1709 1710 if (likely(!i915_prefault_disable)) { 1711 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), 1712 args->size); 1713 if (ret) 1714 return -EFAULT; 1715 } 1716 1717 ret = i915_mutex_lock_interruptible(dev); 1718 if (ret) 1719 return ret; 1720 1721 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1722 if (&obj->base == NULL) { 1723 ret = -ENOENT; 1724 goto unlock; 1725 } 1726 1727 /* Bounds check destination. */ 1728 if (args->offset > obj->base.size || 1729 args->size > obj->base.size - args->offset) { 1730 ret = -EINVAL; 1731 goto out; 1732 } 1733 1734#if 1 1735 KIB_NOTYET(); 1736#else 1737 /* prime objects have no backing filp to GEM pread/pwrite 1738 * pages from. 1739 */ 1740 if (!obj->base.filp) { 1741 ret = -EINVAL; 1742 goto out; 1743 } 1744#endif 1745 1746 CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size); 1747 1748 ret = -EFAULT; 1749 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1750 * it would end up going through the fenced access, and we'll get 1751 * different detiling behavior between reading and writing. 1752 * pread/pwrite currently are reading and writing from the CPU 1753 * perspective, requiring manual detiling by the client. 1754 */ 1755 if (obj->phys_obj) { 1756 ret = i915_gem_phys_pwrite(dev, obj, args, file); 1757 goto out; 1758 } 1759 1760 if (obj->tiling_mode == I915_TILING_NONE && 1761 obj->base.write_domain != I915_GEM_DOMAIN_CPU && 1762 cpu_write_needs_clflush(obj)) { 1763 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); 1764 /* Note that the gtt paths might fail with non-page-backed user 1765 * pointers (e.g. gtt mappings when moving data between 1766 * textures). Fallback to the shmem path in that case. */ 1767 } 1768 1769 if (ret == -EFAULT || ret == -ENOSPC) 1770 ret = i915_gem_shmem_pwrite(dev, obj, args, file); 1771 1772out: 1773 drm_gem_object_unreference(&obj->base); 1774unlock: 1775 DRM_UNLOCK(dev); 1776 return ret; 1777} 1778#undef __user 1779#undef __force 1780#undef __iomem 1781#undef to_user_ptr 1782#undef offset_in_page 1783#undef page_to_phys 1784 1785int 1786i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1787 struct drm_file *file) 1788{ 1789 struct drm_i915_gem_set_domain *args; 1790 struct drm_i915_gem_object *obj; 1791 uint32_t read_domains; 1792 uint32_t write_domain; 1793 int ret; 1794 1795 args = data; 1796 read_domains = args->read_domains; 1797 write_domain = args->write_domain; 1798 1799 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1800 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1801 (write_domain != 0 && read_domains != write_domain)) 1802 return (-EINVAL); 1803 1804 ret = i915_mutex_lock_interruptible(dev); 1805 if (ret != 0) 1806 return (ret); 1807 1808 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1809 if (&obj->base == NULL) { 1810 ret = -ENOENT; 1811 goto unlock; 1812 } 1813 1814 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1815 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1816 if (ret == -EINVAL) 1817 ret = 0; 1818 } else 1819 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1820 1821 drm_gem_object_unreference(&obj->base); 1822unlock: 1823 DRM_UNLOCK(dev); 1824 return (ret); 1825} 1826 1827int 1828i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1829 struct drm_file *file) 1830{ 1831 struct drm_i915_gem_sw_finish *args; 1832 struct drm_i915_gem_object *obj; 1833 int ret; 1834 1835 args = data; 1836 1837 ret = i915_mutex_lock_interruptible(dev); 1838 if (ret != 0) 1839 return (ret); 1840 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1841 if (&obj->base == NULL) { 1842 ret = -ENOENT; 1843 goto unlock; 1844 } 1845 if (obj->pin_count != 0) 1846 i915_gem_object_flush_cpu_write_domain(obj); 1847 drm_gem_object_unreference(&obj->base); 1848unlock: 1849 DRM_UNLOCK(dev); 1850 return (ret); 1851} 1852 1853int 1854i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1855 struct drm_file *file) 1856{ 1857 struct drm_i915_gem_mmap *args; 1858 struct drm_gem_object *obj; 1859 struct proc *p; 1860 vm_map_t map; 1861 vm_offset_t addr; 1862 vm_size_t size; 1863 int error, rv; 1864 1865 args = data; 1866 1867 obj = drm_gem_object_lookup(dev, file, args->handle); 1868 if (obj == NULL) 1869 return (-ENOENT); 1870 error = 0; 1871 if (args->size == 0) 1872 goto out; 1873 p = curproc; 1874 map = &p->p_vmspace->vm_map; 1875 size = round_page(args->size); 1876 PROC_LOCK(p); 1877 if (map->size + size > lim_cur(p, RLIMIT_VMEM)) { 1878 PROC_UNLOCK(p); 1879 error = -ENOMEM; 1880 goto out; 1881 } 1882 PROC_UNLOCK(p); 1883 1884 addr = 0; 1885 vm_object_reference(obj->vm_obj); 1886 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1887 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1888 VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1889 if (rv != KERN_SUCCESS) { 1890 vm_object_deallocate(obj->vm_obj); 1891 error = -vm_mmap_to_errno(rv); 1892 } else { 1893 args->addr_ptr = (uint64_t)addr; 1894 } 1895out: 1896 drm_gem_object_unreference(obj); 1897 return (error); 1898} 1899 1900static int 1901i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1902 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1903{ 1904 1905 *color = 0; /* XXXKIB */ 1906 return (0); 1907} 1908 1909int i915_intr_pf; 1910 1911static int 1912i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1913 vm_page_t *mres) 1914{ 1915 struct drm_gem_object *gem_obj; 1916 struct drm_i915_gem_object *obj; 1917 struct drm_device *dev; 1918 drm_i915_private_t *dev_priv; 1919 vm_page_t m, oldm; 1920 int cause, ret; 1921 bool write; 1922 1923 gem_obj = vm_obj->handle; 1924 obj = to_intel_bo(gem_obj); 1925 dev = obj->base.dev; 1926 dev_priv = dev->dev_private; 1927#if 0 1928 write = (prot & VM_PROT_WRITE) != 0; 1929#else 1930 write = true; 1931#endif 1932 vm_object_pip_add(vm_obj, 1); 1933 1934 /* 1935 * Remove the placeholder page inserted by vm_fault() from the 1936 * object before dropping the object lock. If 1937 * i915_gem_release_mmap() is active in parallel on this gem 1938 * object, then it owns the drm device sx and might find the 1939 * placeholder already. Then, since the page is busy, 1940 * i915_gem_release_mmap() sleeps waiting for the busy state 1941 * of the page cleared. We will be not able to acquire drm 1942 * device lock until i915_gem_release_mmap() is able to make a 1943 * progress. 1944 */ 1945 if (*mres != NULL) { 1946 oldm = *mres; 1947 vm_page_lock(oldm); 1948 vm_page_remove(oldm); 1949 vm_page_unlock(oldm); 1950 *mres = NULL; 1951 } else 1952 oldm = NULL; 1953 VM_OBJECT_WUNLOCK(vm_obj); 1954retry: 1955 cause = ret = 0; 1956 m = NULL; 1957 1958 if (i915_intr_pf) { 1959 ret = i915_mutex_lock_interruptible(dev); 1960 if (ret != 0) { 1961 cause = 10; 1962 goto out; 1963 } 1964 } else 1965 DRM_LOCK(dev); 1966 1967 /* 1968 * Since the object lock was dropped, other thread might have 1969 * faulted on the same GTT address and instantiated the 1970 * mapping for the page. Recheck. 1971 */ 1972 VM_OBJECT_WLOCK(vm_obj); 1973 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1974 if (m != NULL) { 1975 if (vm_page_busied(m)) { 1976 DRM_UNLOCK(dev); 1977 vm_page_lock(m); 1978 VM_OBJECT_WUNLOCK(vm_obj); 1979 vm_page_busy_sleep(m, "915pee"); 1980 goto retry; 1981 } 1982 goto have_page; 1983 } else 1984 VM_OBJECT_WUNLOCK(vm_obj); 1985 1986 /* Now bind it into the GTT if needed */ 1987 if (!obj->map_and_fenceable) { 1988 ret = i915_gem_object_unbind(obj); 1989 if (ret != 0) { 1990 cause = 20; 1991 goto unlock; 1992 } 1993 } 1994 if (!obj->gtt_space) { 1995 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1996 if (ret != 0) { 1997 cause = 30; 1998 goto unlock; 1999 } 2000 2001 ret = i915_gem_object_set_to_gtt_domain(obj, write); 2002 if (ret != 0) { 2003 cause = 40; 2004 goto unlock; 2005 } 2006 } 2007 2008 if (!obj->has_global_gtt_mapping) 2009 i915_gem_gtt_bind_object(obj, obj->cache_level); 2010 2011 ret = i915_gem_object_get_fence(obj); 2012 if (ret != 0) { 2013 cause = 50; 2014 goto unlock; 2015 } 2016 2017 if (i915_gem_object_is_inactive(obj)) 2018 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2019 2020 obj->fault_mappable = true; 2021 VM_OBJECT_WLOCK(vm_obj); 2022 m = PHYS_TO_VM_PAGE(dev->agp->base + obj->gtt_offset + offset); 2023 KASSERT((m->flags & PG_FICTITIOUS) != 0, 2024 ("physical address %#jx not fictitious", 2025 (uintmax_t)(dev->agp->base + obj->gtt_offset + offset))); 2026 if (m == NULL) { 2027 VM_OBJECT_WUNLOCK(vm_obj); 2028 cause = 60; 2029 ret = -EFAULT; 2030 goto unlock; 2031 } 2032 KASSERT((m->flags & PG_FICTITIOUS) != 0, 2033 ("not fictitious %p", m)); 2034 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 2035 2036 if (vm_page_busied(m)) { 2037 DRM_UNLOCK(dev); 2038 vm_page_lock(m); 2039 VM_OBJECT_WUNLOCK(vm_obj); 2040 vm_page_busy_sleep(m, "915pbs"); 2041 goto retry; 2042 } 2043 if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { 2044 DRM_UNLOCK(dev); 2045 VM_OBJECT_WUNLOCK(vm_obj); 2046 VM_WAIT; 2047 goto retry; 2048 } 2049 m->valid = VM_PAGE_BITS_ALL; 2050have_page: 2051 *mres = m; 2052 vm_page_xbusy(m); 2053 2054 CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 2055 m->phys_addr); 2056 DRM_UNLOCK(dev); 2057 if (oldm != NULL) { 2058 vm_page_lock(oldm); 2059 vm_page_free(oldm); 2060 vm_page_unlock(oldm); 2061 } 2062 vm_object_pip_wakeup(vm_obj); 2063 return (VM_PAGER_OK); 2064 2065unlock: 2066 DRM_UNLOCK(dev); 2067out: 2068 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 2069 CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, 2070 -ret, cause); 2071 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 2072 kern_yield(PRI_USER); 2073 goto retry; 2074 } 2075 VM_OBJECT_WLOCK(vm_obj); 2076 vm_object_pip_wakeup(vm_obj); 2077 return (VM_PAGER_ERROR); 2078} 2079 2080static void 2081i915_gem_pager_dtor(void *handle) 2082{ 2083 struct drm_gem_object *obj; 2084 struct drm_device *dev; 2085 2086 obj = handle; 2087 dev = obj->dev; 2088 2089 DRM_LOCK(dev); 2090 drm_gem_free_mmap_offset(obj); 2091 i915_gem_release_mmap(to_intel_bo(obj)); 2092 drm_gem_object_unreference(obj); 2093 DRM_UNLOCK(dev); 2094} 2095 2096struct cdev_pager_ops i915_gem_pager_ops = { 2097 .cdev_pg_fault = i915_gem_pager_fault, 2098 .cdev_pg_ctor = i915_gem_pager_ctor, 2099 .cdev_pg_dtor = i915_gem_pager_dtor 2100}; 2101 2102int 2103i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 2104 uint32_t handle, uint64_t *offset) 2105{ 2106 struct drm_i915_private *dev_priv; 2107 struct drm_i915_gem_object *obj; 2108 int ret; 2109 2110 dev_priv = dev->dev_private; 2111 2112 ret = i915_mutex_lock_interruptible(dev); 2113 if (ret != 0) 2114 return (ret); 2115 2116 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 2117 if (&obj->base == NULL) { 2118 ret = -ENOENT; 2119 goto unlock; 2120 } 2121 2122 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 2123 ret = -E2BIG; 2124 goto out; 2125 } 2126 2127 if (obj->madv != I915_MADV_WILLNEED) { 2128 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 2129 ret = -EINVAL; 2130 goto out; 2131 } 2132 2133 ret = drm_gem_create_mmap_offset(&obj->base); 2134 if (ret != 0) 2135 goto out; 2136 2137 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 2138 DRM_GEM_MAPPING_KEY; 2139out: 2140 drm_gem_object_unreference(&obj->base); 2141unlock: 2142 DRM_UNLOCK(dev); 2143 return (ret); 2144} 2145 2146int 2147i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 2148 struct drm_file *file) 2149{ 2150 struct drm_i915_private *dev_priv; 2151 struct drm_i915_gem_mmap_gtt *args; 2152 2153 dev_priv = dev->dev_private; 2154 args = data; 2155 2156 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 2157} 2158 2159struct drm_i915_gem_object * 2160i915_gem_alloc_object(struct drm_device *dev, size_t size) 2161{ 2162 struct drm_i915_private *dev_priv; 2163 struct drm_i915_gem_object *obj; 2164 2165 dev_priv = dev->dev_private; 2166 2167 obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 2168 2169 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 2170 free(obj, DRM_I915_GEM); 2171 return (NULL); 2172 } 2173 2174 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2175 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2176 2177 if (HAS_LLC(dev)) 2178 obj->cache_level = I915_CACHE_LLC; 2179 else 2180 obj->cache_level = I915_CACHE_NONE; 2181 obj->base.driver_private = NULL; 2182 obj->fence_reg = I915_FENCE_REG_NONE; 2183 INIT_LIST_HEAD(&obj->mm_list); 2184 INIT_LIST_HEAD(&obj->gtt_list); 2185 INIT_LIST_HEAD(&obj->ring_list); 2186 INIT_LIST_HEAD(&obj->exec_list); 2187 INIT_LIST_HEAD(&obj->gpu_write_list); 2188 obj->madv = I915_MADV_WILLNEED; 2189 /* Avoid an unnecessary call to unbind on the first bind. */ 2190 obj->map_and_fenceable = true; 2191 2192 i915_gem_info_add_obj(dev_priv, size); 2193 2194 return (obj); 2195} 2196 2197void 2198i915_gem_clflush_object(struct drm_i915_gem_object *obj) 2199{ 2200 2201 /* If we don't have a page list set up, then we're not pinned 2202 * to GPU, and we can ignore the cache flush because it'll happen 2203 * again at bind time. 2204 */ 2205 if (obj->pages == NULL) 2206 return; 2207 2208 /* If the GPU is snooping the contents of the CPU cache, 2209 * we do not need to manually clear the CPU cache lines. However, 2210 * the caches are only snooped when the render cache is 2211 * flushed/invalidated. As we always have to emit invalidations 2212 * and flushes when moving into and out of the RENDER domain, correct 2213 * snooping behaviour occurs naturally as the result of our domain 2214 * tracking. 2215 */ 2216 if (obj->cache_level != I915_CACHE_NONE) 2217 return; 2218 2219 CTR1(KTR_DRM, "object_clflush %p", obj); 2220 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 2221} 2222 2223static void 2224i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 2225{ 2226 uint32_t old_write_domain; 2227 2228 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 2229 return; 2230 2231 i915_gem_clflush_object(obj); 2232 intel_gtt_chipset_flush(); 2233 old_write_domain = obj->base.write_domain; 2234 obj->base.write_domain = 0; 2235 2236 CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 2237 obj->base.read_domains, old_write_domain); 2238} 2239 2240static int 2241i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 2242{ 2243 2244 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 2245 return (0); 2246 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 2247} 2248 2249static void 2250i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 2251{ 2252 uint32_t old_write_domain; 2253 2254 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 2255 return; 2256 2257 wmb(); 2258 2259 old_write_domain = obj->base.write_domain; 2260 obj->base.write_domain = 0; 2261 2262 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 2263 obj->base.read_domains, old_write_domain); 2264} 2265 2266int 2267i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 2268{ 2269 drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 2270 uint32_t old_write_domain, old_read_domains; 2271 int ret; 2272 2273 if (obj->gtt_space == NULL) 2274 return (-EINVAL); 2275 2276 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 2277 return 0; 2278 2279 ret = i915_gem_object_flush_gpu_write_domain(obj); 2280 if (ret != 0) 2281 return (ret); 2282 2283 if (obj->pending_gpu_write || write) { 2284 ret = i915_gem_object_wait_rendering(obj); 2285 if (ret != 0) 2286 return (ret); 2287 } 2288 2289 i915_gem_object_flush_cpu_write_domain(obj); 2290 2291 old_write_domain = obj->base.write_domain; 2292 old_read_domains = obj->base.read_domains; 2293 2294 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 2295 ("In GTT write domain")); 2296 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 2297 if (write) { 2298 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 2299 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 2300 obj->dirty = 1; 2301 } 2302 2303 /* And bump the LRU for this access */ 2304 if (i915_gem_object_is_inactive(obj)) 2305 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2306 2307 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 2308 old_read_domains, old_write_domain); 2309 return (0); 2310} 2311 2312int 2313i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 2314 enum i915_cache_level cache_level) 2315{ 2316 struct drm_device *dev; 2317 drm_i915_private_t *dev_priv; 2318 int ret; 2319 2320 if (obj->cache_level == cache_level) 2321 return 0; 2322 2323 if (obj->pin_count) { 2324 DRM_DEBUG("can not change the cache level of pinned objects\n"); 2325 return (-EBUSY); 2326 } 2327 2328 dev = obj->base.dev; 2329 dev_priv = dev->dev_private; 2330 if (obj->gtt_space) { 2331 ret = i915_gem_object_finish_gpu(obj); 2332 if (ret != 0) 2333 return (ret); 2334 2335 i915_gem_object_finish_gtt(obj); 2336 2337 /* Before SandyBridge, you could not use tiling or fence 2338 * registers with snooped memory, so relinquish any fences 2339 * currently pointing to our region in the aperture. 2340 */ 2341 if (INTEL_INFO(obj->base.dev)->gen < 6) { 2342 ret = i915_gem_object_put_fence(obj); 2343 if (ret != 0) 2344 return (ret); 2345 } 2346 2347 if (obj->has_global_gtt_mapping) 2348 i915_gem_gtt_bind_object(obj, cache_level); 2349 if (obj->has_aliasing_ppgtt_mapping) 2350 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 2351 obj, cache_level); 2352 } 2353 2354 if (cache_level == I915_CACHE_NONE) { 2355 u32 old_read_domains, old_write_domain; 2356 2357 /* If we're coming from LLC cached, then we haven't 2358 * actually been tracking whether the data is in the 2359 * CPU cache or not, since we only allow one bit set 2360 * in obj->write_domain and have been skipping the clflushes. 2361 * Just set it to the CPU cache for now. 2362 */ 2363 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 2364 ("obj %p in CPU write domain", obj)); 2365 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 2366 ("obj %p in CPU read domain", obj)); 2367 2368 old_read_domains = obj->base.read_domains; 2369 old_write_domain = obj->base.write_domain; 2370 2371 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2372 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2373 2374 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 2375 obj, old_read_domains, old_write_domain); 2376 } 2377 2378 obj->cache_level = cache_level; 2379 return (0); 2380} 2381 2382static bool is_pin_display(struct drm_i915_gem_object *obj) 2383{ 2384 /* There are 3 sources that pin objects: 2385 * 1. The display engine (scanouts, sprites, cursors); 2386 * 2. Reservations for execbuffer; 2387 * 3. The user. 2388 * 2389 * We can ignore reservations as we hold the struct_mutex and 2390 * are only called outside of the reservation path. The user 2391 * can only increment pin_count once, and so if after 2392 * subtracting the potential reference by the user, any pin_count 2393 * remains, it must be due to another use by the display engine. 2394 */ 2395 return obj->pin_count - !!obj->user_pin_count; 2396} 2397 2398int 2399i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 2400 u32 alignment, struct intel_ring_buffer *pipelined) 2401{ 2402 u32 old_read_domains, old_write_domain; 2403 int ret; 2404 2405 ret = i915_gem_object_flush_gpu_write_domain(obj); 2406 if (ret != 0) 2407 return (ret); 2408 2409 if (pipelined != obj->ring) { 2410 ret = i915_gem_object_sync(obj, pipelined); 2411 if (ret) 2412 return (ret); 2413 } 2414 2415 obj->pin_display = true; 2416 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 2417 if (ret != 0) 2418 goto err_unpin_display; 2419 2420 ret = i915_gem_object_pin(obj, alignment, true); 2421 if (ret != 0) 2422 goto err_unpin_display; 2423 2424 i915_gem_object_flush_cpu_write_domain(obj); 2425 2426 old_write_domain = obj->base.write_domain; 2427 old_read_domains = obj->base.read_domains; 2428 2429 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 2430 ("obj %p in GTT write domain", obj)); 2431 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 2432 2433 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 2434 obj, old_read_domains, obj->base.write_domain); 2435 return (0); 2436 2437err_unpin_display: 2438 obj->pin_display = is_pin_display(obj); 2439 return ret; 2440} 2441 2442void 2443i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) 2444{ 2445 i915_gem_object_unpin(obj); 2446 obj->pin_display = is_pin_display(obj); 2447} 2448 2449int 2450i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 2451{ 2452 int ret; 2453 2454 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 2455 return (0); 2456 2457 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 2458 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 2459 if (ret != 0) 2460 return (ret); 2461 } 2462 2463 ret = i915_gem_object_wait_rendering(obj); 2464 if (ret != 0) 2465 return (ret); 2466 2467 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2468 2469 return (0); 2470} 2471 2472int 2473i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 2474{ 2475 uint32_t old_write_domain, old_read_domains; 2476 int ret; 2477 2478 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 2479 return 0; 2480 2481 ret = i915_gem_object_flush_gpu_write_domain(obj); 2482 if (ret != 0) 2483 return (ret); 2484 2485 if (write || obj->pending_gpu_write) { 2486 ret = i915_gem_object_wait_rendering(obj); 2487 if (ret != 0) 2488 return (ret); 2489 } 2490 2491 i915_gem_object_flush_gtt_write_domain(obj); 2492 2493 old_write_domain = obj->base.write_domain; 2494 old_read_domains = obj->base.read_domains; 2495 2496 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 2497 i915_gem_clflush_object(obj); 2498 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 2499 } 2500 2501 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 2502 ("In cpu write domain")); 2503 2504 if (write) { 2505 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 2506 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2507 } 2508 2509 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 2510 old_read_domains, old_write_domain); 2511 return (0); 2512} 2513 2514static uint32_t 2515i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 2516{ 2517 uint32_t gtt_size; 2518 2519 if (INTEL_INFO(dev)->gen >= 4 || 2520 tiling_mode == I915_TILING_NONE) 2521 return (size); 2522 2523 /* Previous chips need a power-of-two fence region when tiling */ 2524 if (INTEL_INFO(dev)->gen == 3) 2525 gtt_size = 1024*1024; 2526 else 2527 gtt_size = 512*1024; 2528 2529 while (gtt_size < size) 2530 gtt_size <<= 1; 2531 2532 return (gtt_size); 2533} 2534 2535/** 2536 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 2537 * @obj: object to check 2538 * 2539 * Return the required GTT alignment for an object, taking into account 2540 * potential fence register mapping. 2541 */ 2542static uint32_t 2543i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 2544 int tiling_mode) 2545{ 2546 2547 /* 2548 * Minimum alignment is 4k (GTT page size), but might be greater 2549 * if a fence register is needed for the object. 2550 */ 2551 if (INTEL_INFO(dev)->gen >= 4 || 2552 tiling_mode == I915_TILING_NONE) 2553 return (4096); 2554 2555 /* 2556 * Previous chips need to be aligned to the size of the smallest 2557 * fence register that can contain the object. 2558 */ 2559 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2560} 2561 2562uint32_t 2563i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 2564 int tiling_mode) 2565{ 2566 2567 if (tiling_mode == I915_TILING_NONE) 2568 return (4096); 2569 2570 /* 2571 * Minimum alignment is 4k (GTT page size) for sane hw. 2572 */ 2573 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 2574 return (4096); 2575 2576 /* 2577 * Previous hardware however needs to be aligned to a power-of-two 2578 * tile height. The simplest method for determining this is to reuse 2579 * the power-of-tile object size. 2580 */ 2581 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2582} 2583 2584static int 2585i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2586 unsigned alignment, bool map_and_fenceable) 2587{ 2588 struct drm_device *dev; 2589 struct drm_i915_private *dev_priv; 2590 struct drm_mm_node *free_space; 2591 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2592 bool mappable, fenceable; 2593 int ret; 2594 2595 dev = obj->base.dev; 2596 dev_priv = dev->dev_private; 2597 2598 if (obj->madv != I915_MADV_WILLNEED) { 2599 DRM_ERROR("Attempting to bind a purgeable object\n"); 2600 return (-EINVAL); 2601 } 2602 2603 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2604 obj->tiling_mode); 2605 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2606 obj->tiling_mode); 2607 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2608 obj->base.size, obj->tiling_mode); 2609 if (alignment == 0) 2610 alignment = map_and_fenceable ? fence_alignment : 2611 unfenced_alignment; 2612 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2613 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2614 return (-EINVAL); 2615 } 2616 2617 size = map_and_fenceable ? fence_size : obj->base.size; 2618 2619 /* If the object is bigger than the entire aperture, reject it early 2620 * before evicting everything in a vain attempt to find space. 2621 */ 2622 if (obj->base.size > (map_and_fenceable ? 2623 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2624 DRM_ERROR( 2625"Attempting to bind an object larger than the aperture\n"); 2626 return (-E2BIG); 2627 } 2628 2629 search_free: 2630 if (map_and_fenceable) 2631 free_space = drm_mm_search_free_in_range( 2632 &dev_priv->mm.gtt_space, size, alignment, 0, 2633 dev_priv->mm.gtt_mappable_end, 0); 2634 else 2635 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2636 size, alignment, 0); 2637 if (free_space != NULL) { 2638 if (map_and_fenceable) 2639 obj->gtt_space = drm_mm_get_block_range_generic( 2640 free_space, size, alignment, 0, 0, 2641 dev_priv->mm.gtt_mappable_end, 1); 2642 else 2643 obj->gtt_space = drm_mm_get_block_generic(free_space, 2644 size, alignment, 0, 1); 2645 } 2646 if (obj->gtt_space == NULL) { 2647 ret = i915_gem_evict_something(dev, size, alignment, 2648 map_and_fenceable); 2649 if (ret != 0) 2650 return (ret); 2651 goto search_free; 2652 } 2653 ret = i915_gem_object_get_pages_gtt(obj, 0); 2654 if (ret != 0) { 2655 drm_mm_put_block(obj->gtt_space); 2656 obj->gtt_space = NULL; 2657 /* 2658 * i915_gem_object_get_pages_gtt() cannot return 2659 * ENOMEM, since we use vm_page_grab(). 2660 */ 2661 return (ret); 2662 } 2663 2664 ret = i915_gem_gtt_prepare_object(obj); 2665 if (ret != 0) { 2666 i915_gem_object_put_pages_gtt(obj); 2667 drm_mm_put_block(obj->gtt_space); 2668 obj->gtt_space = NULL; 2669 if (i915_gem_evict_everything(dev, false)) 2670 return (ret); 2671 goto search_free; 2672 } 2673 2674 if (!dev_priv->mm.aliasing_ppgtt) 2675 i915_gem_gtt_bind_object(obj, obj->cache_level); 2676 2677 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2678 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2679 2680 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2681 ("Object in gpu read domain")); 2682 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2683 ("Object in gpu write domain")); 2684 2685 obj->gtt_offset = obj->gtt_space->start; 2686 2687 fenceable = 2688 obj->gtt_space->size == fence_size && 2689 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2690 2691 mappable = 2692 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2693 obj->map_and_fenceable = mappable && fenceable; 2694 2695 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 2696 obj->base.size, map_and_fenceable); 2697 return (0); 2698} 2699 2700int 2701i915_gem_object_sync(struct drm_i915_gem_object *obj, 2702 struct intel_ring_buffer *to) 2703{ 2704 struct intel_ring_buffer *from = obj->ring; 2705 u32 seqno; 2706 int ret, idx; 2707 2708 if (from == NULL || to == from) 2709 return 0; 2710 2711 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) 2712 return i915_gem_object_wait_rendering(obj); 2713 2714 idx = intel_ring_sync_index(from, to); 2715 2716 seqno = obj->last_rendering_seqno; 2717 if (seqno <= from->sync_seqno[idx]) 2718 return 0; 2719 2720 if (seqno == from->outstanding_lazy_request) { 2721 struct drm_i915_gem_request *request; 2722 2723 request = malloc(sizeof(*request), DRM_I915_GEM, 2724 M_WAITOK | M_ZERO); 2725 ret = i915_add_request(from, NULL, request); 2726 if (ret) { 2727 free(request, DRM_I915_GEM); 2728 return ret; 2729 } 2730 seqno = request->seqno; 2731 } 2732 2733 2734 ret = to->sync_to(to, from, seqno); 2735 if (!ret) 2736 from->sync_seqno[idx] = seqno; 2737 2738 return ret; 2739} 2740 2741static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2742{ 2743 u32 old_write_domain, old_read_domains; 2744 2745 /* Act a barrier for all accesses through the GTT */ 2746 mb(); 2747 2748 /* Force a pagefault for domain tracking on next user access */ 2749 i915_gem_release_mmap(obj); 2750 2751 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2752 return; 2753 2754 old_read_domains = obj->base.read_domains; 2755 old_write_domain = obj->base.write_domain; 2756 2757 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2758 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2759 2760 CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2761 obj, old_read_domains, old_write_domain); 2762} 2763 2764int 2765i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2766{ 2767 drm_i915_private_t *dev_priv; 2768 int ret; 2769 2770 dev_priv = obj->base.dev->dev_private; 2771 ret = 0; 2772 if (obj->gtt_space == NULL) 2773 return (0); 2774 if (obj->pin_count != 0) { 2775 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2776 return (-EINVAL); 2777 } 2778 2779 ret = i915_gem_object_finish_gpu(obj); 2780 if (ret == -ERESTARTSYS || ret == -EINTR) 2781 return (ret); 2782 2783 i915_gem_object_finish_gtt(obj); 2784 2785 if (ret == 0) 2786 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2787 if (ret == -ERESTARTSYS || ret == -EINTR) 2788 return (ret); 2789 if (ret != 0) { 2790 i915_gem_clflush_object(obj); 2791 obj->base.read_domains = obj->base.write_domain = 2792 I915_GEM_DOMAIN_CPU; 2793 } 2794 2795 ret = i915_gem_object_put_fence(obj); 2796 if (ret) 2797 return (ret); 2798 2799 if (obj->has_global_gtt_mapping) 2800 i915_gem_gtt_unbind_object(obj); 2801 if (obj->has_aliasing_ppgtt_mapping) { 2802 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2803 obj->has_aliasing_ppgtt_mapping = 0; 2804 } 2805 i915_gem_gtt_finish_object(obj); 2806 2807 i915_gem_object_put_pages_gtt(obj); 2808 2809 list_del_init(&obj->gtt_list); 2810 list_del_init(&obj->mm_list); 2811 obj->map_and_fenceable = true; 2812 2813 drm_mm_put_block(obj->gtt_space); 2814 obj->gtt_space = NULL; 2815 obj->gtt_offset = 0; 2816 2817 if (i915_gem_object_is_purgeable(obj)) 2818 i915_gem_object_truncate(obj); 2819 CTR1(KTR_DRM, "object_unbind %p", obj); 2820 2821 return (ret); 2822} 2823 2824static void 2825i915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj, 2826 vm_pindex_t si, vm_pindex_t ei) 2827{ 2828 vm_object_t vm_obj; 2829 vm_page_t m; 2830 vm_pindex_t i; 2831 2832 vm_obj = obj->base.vm_obj; 2833 VM_OBJECT_ASSERT_LOCKED(vm_obj); 2834 for (i = si, m = vm_page_lookup(vm_obj, i); i < ei; 2835 m = vm_page_next(m), i++) { 2836 KASSERT(m->pindex == i, ("pindex %jx %jx", 2837 (uintmax_t)m->pindex, (uintmax_t)i)); 2838 vm_page_lock(m); 2839 vm_page_unwire(m, PQ_INACTIVE); 2840 if (m->wire_count == 0) 2841 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2842 vm_page_unlock(m); 2843 } 2844} 2845 2846static void 2847i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj, 2848 off_t start, off_t end) 2849{ 2850 vm_object_t vm_obj; 2851 2852 vm_obj = obj->base.vm_obj; 2853 VM_OBJECT_WLOCK(vm_obj); 2854 i915_gem_object_put_pages_range_locked(obj, 2855 OFF_TO_IDX(trunc_page(start)), OFF_TO_IDX(round_page(end))); 2856 VM_OBJECT_WUNLOCK(vm_obj); 2857} 2858 2859static int 2860i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 2861 off_t start, off_t end) 2862{ 2863 vm_object_t vm_obj; 2864 vm_page_t m; 2865 vm_pindex_t si, ei, i; 2866 bool need_swizzle, fresh; 2867 2868 need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0; 2869 vm_obj = obj->base.vm_obj; 2870 si = OFF_TO_IDX(trunc_page(start)); 2871 ei = OFF_TO_IDX(round_page(end)); 2872 VM_OBJECT_WLOCK(vm_obj); 2873 for (i = si; i < ei; i++) { 2874 m = i915_gem_wire_page(vm_obj, i, &fresh); 2875 if (m == NULL) 2876 goto failed; 2877 if (need_swizzle && fresh) 2878 i915_gem_object_do_bit_17_swizzle_page(obj, m); 2879 } 2880 VM_OBJECT_WUNLOCK(vm_obj); 2881 return (0); 2882failed: 2883 i915_gem_object_put_pages_range_locked(obj, si, i); 2884 VM_OBJECT_WUNLOCK(vm_obj); 2885 return (-EIO); 2886} 2887 2888static int 2889i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2890 int flags) 2891{ 2892 struct drm_device *dev; 2893 vm_object_t vm_obj; 2894 vm_page_t m; 2895 vm_pindex_t i, page_count; 2896 int res; 2897 2898 dev = obj->base.dev; 2899 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2900 page_count = OFF_TO_IDX(obj->base.size); 2901 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2902 M_WAITOK); 2903 res = i915_gem_object_get_pages_range(obj, 0, obj->base.size); 2904 if (res != 0) { 2905 free(obj->pages, DRM_I915_GEM); 2906 obj->pages = NULL; 2907 return (res); 2908 } 2909 vm_obj = obj->base.vm_obj; 2910 VM_OBJECT_WLOCK(vm_obj); 2911 for (i = 0, m = vm_page_lookup(vm_obj, 0); i < page_count; 2912 i++, m = vm_page_next(m)) { 2913 KASSERT(m->pindex == i, ("pindex %jx %jx", 2914 (uintmax_t)m->pindex, (uintmax_t)i)); 2915 obj->pages[i] = m; 2916 } 2917 VM_OBJECT_WUNLOCK(vm_obj); 2918 return (0); 2919} 2920 2921#define GEM_PARANOID_CHECK_GTT 0 2922#if GEM_PARANOID_CHECK_GTT 2923static void 2924i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2925 int page_count) 2926{ 2927 struct drm_i915_private *dev_priv; 2928 vm_paddr_t pa; 2929 unsigned long start, end; 2930 u_int i; 2931 int j; 2932 2933 dev_priv = dev->dev_private; 2934 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2935 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2936 for (i = start; i < end; i++) { 2937 pa = intel_gtt_read_pte_paddr(i); 2938 for (j = 0; j < page_count; j++) { 2939 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2940 panic("Page %p in GTT pte index %d pte %x", 2941 ma[i], i, intel_gtt_read_pte(i)); 2942 } 2943 } 2944 } 2945} 2946#endif 2947 2948static void 2949i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2950{ 2951 vm_page_t m; 2952 int page_count, i; 2953 2954 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2955 2956 if (obj->tiling_mode != I915_TILING_NONE) 2957 i915_gem_object_save_bit_17_swizzle(obj); 2958 if (obj->madv == I915_MADV_DONTNEED) 2959 obj->dirty = 0; 2960 page_count = obj->base.size / PAGE_SIZE; 2961 VM_OBJECT_WLOCK(obj->base.vm_obj); 2962#if GEM_PARANOID_CHECK_GTT 2963 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2964#endif 2965 for (i = 0; i < page_count; i++) { 2966 m = obj->pages[i]; 2967 if (obj->dirty) 2968 vm_page_dirty(m); 2969 if (obj->madv == I915_MADV_WILLNEED) 2970 vm_page_reference(m); 2971 vm_page_lock(m); 2972 vm_page_unwire(obj->pages[i], PQ_ACTIVE); 2973 vm_page_unlock(m); 2974 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2975 } 2976 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 2977 obj->dirty = 0; 2978 free(obj->pages, DRM_I915_GEM); 2979 obj->pages = NULL; 2980} 2981 2982void 2983i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2984{ 2985 vm_object_t devobj; 2986 vm_page_t m; 2987 int i, page_count; 2988 2989 if (!obj->fault_mappable) 2990 return; 2991 2992 CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 2993 OFF_TO_IDX(obj->base.size)); 2994 devobj = cdev_pager_lookup(obj); 2995 if (devobj != NULL) { 2996 page_count = OFF_TO_IDX(obj->base.size); 2997 2998 VM_OBJECT_WLOCK(devobj); 2999retry: 3000 for (i = 0; i < page_count; i++) { 3001 m = vm_page_lookup(devobj, i); 3002 if (m == NULL) 3003 continue; 3004 if (vm_page_sleep_if_busy(m, "915unm")) 3005 goto retry; 3006 cdev_pager_free_page(devobj, m); 3007 } 3008 VM_OBJECT_WUNLOCK(devobj); 3009 vm_object_deallocate(devobj); 3010 } 3011 3012 obj->fault_mappable = false; 3013} 3014 3015int 3016i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 3017{ 3018 int ret; 3019 3020 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 3021 ("In GPU write domain")); 3022 3023 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, 3024 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, 3025 obj->active, obj->last_rendering_seqno); 3026 if (obj->active) { 3027 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno); 3028 if (ret != 0) 3029 return (ret); 3030 i915_gem_retire_requests_ring(obj->ring); 3031 } 3032 return (0); 3033} 3034 3035void 3036i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 3037 struct intel_ring_buffer *ring, uint32_t seqno) 3038{ 3039 struct drm_device *dev = obj->base.dev; 3040 struct drm_i915_private *dev_priv = dev->dev_private; 3041 struct drm_i915_fence_reg *reg; 3042 3043 obj->ring = ring; 3044 KASSERT(ring != NULL, ("NULL ring")); 3045 3046 /* Add a reference if we're newly entering the active list. */ 3047 if (!obj->active) { 3048 drm_gem_object_reference(&obj->base); 3049 obj->active = 1; 3050 } 3051 3052 /* Move from whatever list we were on to the tail of execution. */ 3053 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 3054 list_move_tail(&obj->ring_list, &ring->active_list); 3055 3056 obj->last_rendering_seqno = seqno; 3057 if (obj->fenced_gpu_access) { 3058 obj->last_fenced_seqno = seqno; 3059 3060 /* Bump MRU to take account of the delayed flush */ 3061 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3062 reg = &dev_priv->fence_regs[obj->fence_reg]; 3063 list_move_tail(®->lru_list, 3064 &dev_priv->mm.fence_list); 3065 } 3066 } 3067} 3068 3069static void 3070i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 3071{ 3072 list_del_init(&obj->ring_list); 3073 obj->last_rendering_seqno = 0; 3074 obj->last_fenced_seqno = 0; 3075} 3076 3077static void 3078i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 3079{ 3080 struct drm_device *dev = obj->base.dev; 3081 drm_i915_private_t *dev_priv = dev->dev_private; 3082 3083 KASSERT(obj->active, ("Object not active")); 3084 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 3085 3086 i915_gem_object_move_off_active(obj); 3087} 3088 3089static void 3090i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 3091{ 3092 struct drm_device *dev = obj->base.dev; 3093 struct drm_i915_private *dev_priv = dev->dev_private; 3094 3095 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3096 3097 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 3098 KASSERT(obj->active, ("Object not active")); 3099 obj->ring = NULL; 3100 3101 i915_gem_object_move_off_active(obj); 3102 obj->fenced_gpu_access = false; 3103 3104 obj->active = 0; 3105 obj->pending_gpu_write = false; 3106 drm_gem_object_unreference(&obj->base); 3107 3108#if 1 3109 KIB_NOTYET(); 3110#else 3111 WARN_ON(i915_verify_lists(dev)); 3112#endif 3113} 3114 3115static void 3116i915_gem_object_truncate(struct drm_i915_gem_object *obj) 3117{ 3118 vm_object_t vm_obj; 3119 3120 vm_obj = obj->base.vm_obj; 3121 VM_OBJECT_WLOCK(vm_obj); 3122 vm_object_page_remove(vm_obj, 0, 0, false); 3123 VM_OBJECT_WUNLOCK(vm_obj); 3124 drm_gem_free_mmap_offset(&obj->base); 3125 obj->madv = I915_MADV_PURGED_INTERNAL; 3126} 3127 3128static inline int 3129i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 3130{ 3131 3132 return (obj->madv == I915_MADV_DONTNEED); 3133} 3134 3135static void 3136i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 3137 uint32_t flush_domains) 3138{ 3139 struct drm_i915_gem_object *obj, *next; 3140 uint32_t old_write_domain; 3141 3142 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 3143 gpu_write_list) { 3144 if (obj->base.write_domain & flush_domains) { 3145 old_write_domain = obj->base.write_domain; 3146 obj->base.write_domain = 0; 3147 list_del_init(&obj->gpu_write_list); 3148 i915_gem_object_move_to_active(obj, ring, 3149 i915_gem_next_request_seqno(ring)); 3150 3151 CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", 3152 obj, obj->base.read_domains, old_write_domain); 3153 } 3154 } 3155} 3156 3157static int 3158i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 3159{ 3160 drm_i915_private_t *dev_priv; 3161 3162 dev_priv = obj->base.dev->dev_private; 3163 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 3164 obj->tiling_mode != I915_TILING_NONE); 3165} 3166 3167static vm_page_t 3168i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh) 3169{ 3170 vm_page_t m; 3171 int rv; 3172 3173 VM_OBJECT_ASSERT_WLOCKED(object); 3174 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 3175 if (m->valid != VM_PAGE_BITS_ALL) { 3176 if (vm_pager_has_page(object, pindex, NULL, NULL)) { 3177 rv = vm_pager_get_pages(object, &m, 1, 0); 3178 m = vm_page_lookup(object, pindex); 3179 if (m == NULL) 3180 return (NULL); 3181 if (rv != VM_PAGER_OK) { 3182 vm_page_lock(m); 3183 vm_page_free(m); 3184 vm_page_unlock(m); 3185 return (NULL); 3186 } 3187 if (fresh != NULL) 3188 *fresh = true; 3189 } else { 3190 pmap_zero_page(m); 3191 m->valid = VM_PAGE_BITS_ALL; 3192 m->dirty = 0; 3193 if (fresh != NULL) 3194 *fresh = false; 3195 } 3196 } else if (fresh != NULL) { 3197 *fresh = false; 3198 } 3199 vm_page_lock(m); 3200 vm_page_wire(m); 3201 vm_page_unlock(m); 3202 vm_page_xunbusy(m); 3203 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 3204 return (m); 3205} 3206 3207int 3208i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 3209 uint32_t flush_domains) 3210{ 3211 int ret; 3212 3213 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 3214 return 0; 3215 3216 CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, 3217 flush_domains); 3218 ret = ring->flush(ring, invalidate_domains, flush_domains); 3219 if (ret) 3220 return ret; 3221 3222 if (flush_domains & I915_GEM_GPU_DOMAINS) 3223 i915_gem_process_flushing_list(ring, flush_domains); 3224 return 0; 3225} 3226 3227static int 3228i915_ring_idle(struct intel_ring_buffer *ring) 3229{ 3230 int ret; 3231 3232 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 3233 return 0; 3234 3235 if (!list_empty(&ring->gpu_write_list)) { 3236 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 3237 I915_GEM_GPU_DOMAINS); 3238 if (ret != 0) 3239 return ret; 3240 } 3241 3242 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring))); 3243} 3244 3245int 3246i915_gpu_idle(struct drm_device *dev) 3247{ 3248 drm_i915_private_t *dev_priv = dev->dev_private; 3249 struct intel_ring_buffer *ring; 3250 int ret, i; 3251 3252 /* Flush everything onto the inactive list. */ 3253 for_each_ring(ring, dev_priv, i) { 3254 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); 3255 if (ret) 3256 return ret; 3257 3258 ret = i915_ring_idle(ring); 3259 if (ret) 3260 return ret; 3261 3262 /* Is the device fubar? */ 3263 if (!list_empty(&ring->gpu_write_list)) 3264 return -EBUSY; 3265 } 3266 3267 return 0; 3268} 3269 3270static int 3271i915_gem_check_wedge(struct drm_i915_private *dev_priv) 3272{ 3273 DRM_LOCK_ASSERT(dev_priv->dev); 3274 3275 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 3276 bool recovery_complete; 3277 /* Give the error handler a chance to run. */ 3278 mtx_lock(&dev_priv->error_completion_lock); 3279 recovery_complete = (&dev_priv->error_completion) > 0; 3280 mtx_unlock(&dev_priv->error_completion_lock); 3281 return (recovery_complete ? -EIO : -EAGAIN); 3282 } 3283 3284 return 0; 3285} 3286 3287/* 3288 * Compare seqno against outstanding lazy request. Emit a request if they are 3289 * equal. 3290 */ 3291static int 3292i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) 3293{ 3294 int ret = 0; 3295 3296 DRM_LOCK_ASSERT(ring->dev); 3297 3298 if (seqno == ring->outstanding_lazy_request) { 3299 struct drm_i915_gem_request *request; 3300 3301 request = malloc(sizeof(*request), DRM_I915_GEM, 3302 M_WAITOK | M_ZERO); 3303 3304 ret = i915_add_request(ring, NULL, request); 3305 if (ret != 0) { 3306 free(request, DRM_I915_GEM); 3307 return (ret); 3308 } 3309 3310 MPASS(seqno == request->seqno); 3311 } 3312 return ret; 3313} 3314 3315static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 3316 bool interruptible) 3317{ 3318 drm_i915_private_t *dev_priv = ring->dev->dev_private; 3319 int ret = 0, flags; 3320 3321 if (i915_seqno_passed(ring->get_seqno(ring), seqno)) 3322 return 0; 3323 3324 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 3325 3326 mtx_lock(&dev_priv->irq_lock); 3327 if (!ring->irq_get(ring)) { 3328 mtx_unlock(&dev_priv->irq_lock); 3329 return (-ENODEV); 3330 } 3331 3332 flags = interruptible ? PCATCH : 0; 3333 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 3334 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 3335 ret == 0) { 3336 ret = -msleep(ring, &dev_priv->irq_lock, flags, "915gwr", 0); 3337 if (ret == -ERESTART) 3338 ret = -ERESTARTSYS; 3339 } 3340 ring->irq_put(ring); 3341 mtx_unlock(&dev_priv->irq_lock); 3342 3343 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, ret); 3344 3345 return ret; 3346} 3347 3348int 3349i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno) 3350{ 3351 drm_i915_private_t *dev_priv; 3352 int ret; 3353 3354 KASSERT(seqno != 0, ("Zero seqno")); 3355 3356 dev_priv = ring->dev->dev_private; 3357 ret = 0; 3358 3359 ret = i915_gem_check_wedge(dev_priv); 3360 if (ret) 3361 return ret; 3362 3363 ret = i915_gem_check_olr(ring, seqno); 3364 if (ret) 3365 return ret; 3366 3367 ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible); 3368 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 3369 ret = -EAGAIN; 3370 3371 return (ret); 3372} 3373 3374static u32 3375i915_gem_get_seqno(struct drm_device *dev) 3376{ 3377 drm_i915_private_t *dev_priv = dev->dev_private; 3378 u32 seqno = dev_priv->next_seqno; 3379 3380 /* reserve 0 for non-seqno */ 3381 if (++dev_priv->next_seqno == 0) 3382 dev_priv->next_seqno = 1; 3383 3384 return seqno; 3385} 3386 3387u32 3388i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 3389{ 3390 if (ring->outstanding_lazy_request == 0) 3391 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 3392 3393 return ring->outstanding_lazy_request; 3394} 3395 3396int 3397i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 3398 struct drm_i915_gem_request *request) 3399{ 3400 drm_i915_private_t *dev_priv; 3401 struct drm_i915_file_private *file_priv; 3402 uint32_t seqno; 3403 u32 request_ring_position; 3404 int was_empty; 3405 int ret; 3406 3407 KASSERT(request != NULL, ("NULL request in add")); 3408 DRM_LOCK_ASSERT(ring->dev); 3409 dev_priv = ring->dev->dev_private; 3410 3411 seqno = i915_gem_next_request_seqno(ring); 3412 request_ring_position = intel_ring_get_tail(ring); 3413 3414 ret = ring->add_request(ring, &seqno); 3415 if (ret != 0) 3416 return ret; 3417 3418 CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); 3419 3420 request->seqno = seqno; 3421 request->ring = ring; 3422 request->tail = request_ring_position; 3423 request->emitted_jiffies = ticks; 3424 was_empty = list_empty(&ring->request_list); 3425 list_add_tail(&request->list, &ring->request_list); 3426 3427 if (file != NULL) { 3428 file_priv = file->driver_priv; 3429 3430 mtx_lock(&file_priv->mm.lck); 3431 request->file_priv = file_priv; 3432 list_add_tail(&request->client_list, 3433 &file_priv->mm.request_list); 3434 mtx_unlock(&file_priv->mm.lck); 3435 } 3436 3437 ring->outstanding_lazy_request = 0; 3438 3439 if (!dev_priv->mm.suspended) { 3440 if (i915_enable_hangcheck) { 3441 callout_schedule(&dev_priv->hangcheck_timer, 3442 DRM_I915_HANGCHECK_PERIOD); 3443 } 3444 if (was_empty) 3445 taskqueue_enqueue_timeout(dev_priv->tq, 3446 &dev_priv->mm.retire_task, hz); 3447 } 3448 return (0); 3449} 3450 3451static inline void 3452i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 3453{ 3454 struct drm_i915_file_private *file_priv = request->file_priv; 3455 3456 if (!file_priv) 3457 return; 3458 3459 DRM_LOCK_ASSERT(request->ring->dev); 3460 3461 mtx_lock(&file_priv->mm.lck); 3462 if (request->file_priv != NULL) { 3463 list_del(&request->client_list); 3464 request->file_priv = NULL; 3465 } 3466 mtx_unlock(&file_priv->mm.lck); 3467} 3468 3469void 3470i915_gem_release(struct drm_device *dev, struct drm_file *file) 3471{ 3472 struct drm_i915_file_private *file_priv; 3473 struct drm_i915_gem_request *request; 3474 3475 file_priv = file->driver_priv; 3476 3477 /* Clean up our request list when the client is going away, so that 3478 * later retire_requests won't dereference our soon-to-be-gone 3479 * file_priv. 3480 */ 3481 mtx_lock(&file_priv->mm.lck); 3482 while (!list_empty(&file_priv->mm.request_list)) { 3483 request = list_first_entry(&file_priv->mm.request_list, 3484 struct drm_i915_gem_request, 3485 client_list); 3486 list_del(&request->client_list); 3487 request->file_priv = NULL; 3488 } 3489 mtx_unlock(&file_priv->mm.lck); 3490} 3491 3492static void 3493i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 3494 struct intel_ring_buffer *ring) 3495{ 3496 3497 if (ring->dev != NULL) 3498 DRM_LOCK_ASSERT(ring->dev); 3499 3500 while (!list_empty(&ring->request_list)) { 3501 struct drm_i915_gem_request *request; 3502 3503 request = list_first_entry(&ring->request_list, 3504 struct drm_i915_gem_request, list); 3505 3506 list_del(&request->list); 3507 i915_gem_request_remove_from_client(request); 3508 free(request, DRM_I915_GEM); 3509 } 3510 3511 while (!list_empty(&ring->active_list)) { 3512 struct drm_i915_gem_object *obj; 3513 3514 obj = list_first_entry(&ring->active_list, 3515 struct drm_i915_gem_object, ring_list); 3516 3517 obj->base.write_domain = 0; 3518 list_del_init(&obj->gpu_write_list); 3519 i915_gem_object_move_to_inactive(obj); 3520 } 3521} 3522 3523static void 3524i915_gem_reset_fences(struct drm_device *dev) 3525{ 3526 struct drm_i915_private *dev_priv = dev->dev_private; 3527 int i; 3528 3529 for (i = 0; i < dev_priv->num_fence_regs; i++) { 3530 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 3531 3532 i915_gem_write_fence(dev, i, NULL); 3533 3534 if (reg->obj) 3535 i915_gem_object_fence_lost(reg->obj); 3536 3537 reg->pin_count = 0; 3538 reg->obj = NULL; 3539 INIT_LIST_HEAD(®->lru_list); 3540 } 3541 3542 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 3543} 3544 3545void 3546i915_gem_reset(struct drm_device *dev) 3547{ 3548 struct drm_i915_private *dev_priv = dev->dev_private; 3549 struct drm_i915_gem_object *obj; 3550 struct intel_ring_buffer *ring; 3551 int i; 3552 3553 for_each_ring(ring, dev_priv, i) 3554 i915_gem_reset_ring_lists(dev_priv, ring); 3555 3556 /* Remove anything from the flushing lists. The GPU cache is likely 3557 * to be lost on reset along with the data, so simply move the 3558 * lost bo to the inactive list. 3559 */ 3560 while (!list_empty(&dev_priv->mm.flushing_list)) { 3561 obj = list_first_entry(&dev_priv->mm.flushing_list, 3562 struct drm_i915_gem_object, 3563 mm_list); 3564 3565 obj->base.write_domain = 0; 3566 list_del_init(&obj->gpu_write_list); 3567 i915_gem_object_move_to_inactive(obj); 3568 } 3569 3570 /* Move everything out of the GPU domains to ensure we do any 3571 * necessary invalidation upon reuse. 3572 */ 3573 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 3574 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 3575 } 3576 3577 /* The fence registers are invalidated so clear them out */ 3578 i915_gem_reset_fences(dev); 3579} 3580 3581/** 3582 * This function clears the request list as sequence numbers are passed. 3583 */ 3584void 3585i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 3586{ 3587 uint32_t seqno; 3588 int i; 3589 3590 if (list_empty(&ring->request_list)) 3591 return; 3592 3593 seqno = ring->get_seqno(ring); 3594 CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 3595 3596 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) 3597 if (seqno >= ring->sync_seqno[i]) 3598 ring->sync_seqno[i] = 0; 3599 3600 while (!list_empty(&ring->request_list)) { 3601 struct drm_i915_gem_request *request; 3602 3603 request = list_first_entry(&ring->request_list, 3604 struct drm_i915_gem_request, 3605 list); 3606 3607 if (!i915_seqno_passed(seqno, request->seqno)) 3608 break; 3609 3610 CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 3611 ring->name, seqno); 3612 ring->last_retired_head = request->tail; 3613 3614 list_del(&request->list); 3615 i915_gem_request_remove_from_client(request); 3616 free(request, DRM_I915_GEM); 3617 } 3618 3619 /* Move any buffers on the active list that are no longer referenced 3620 * by the ringbuffer to the flushing/inactive lists as appropriate. 3621 */ 3622 while (!list_empty(&ring->active_list)) { 3623 struct drm_i915_gem_object *obj; 3624 3625 obj = list_first_entry(&ring->active_list, 3626 struct drm_i915_gem_object, 3627 ring_list); 3628 3629 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 3630 break; 3631 3632 if (obj->base.write_domain != 0) 3633 i915_gem_object_move_to_flushing(obj); 3634 else 3635 i915_gem_object_move_to_inactive(obj); 3636 } 3637 3638 if (ring->trace_irq_seqno && 3639 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 3640 struct drm_i915_private *dev_priv = ring->dev->dev_private; 3641 mtx_lock(&dev_priv->irq_lock); 3642 ring->irq_put(ring); 3643 mtx_unlock(&dev_priv->irq_lock); 3644 ring->trace_irq_seqno = 0; 3645 } 3646} 3647 3648void 3649i915_gem_retire_requests(struct drm_device *dev) 3650{ 3651 drm_i915_private_t *dev_priv = dev->dev_private; 3652 struct intel_ring_buffer *ring; 3653 int i; 3654 3655 for_each_ring(ring, dev_priv, i) 3656 i915_gem_retire_requests_ring(ring); 3657} 3658 3659static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, 3660 struct drm_i915_gem_object *obj) 3661{ 3662 drm_i915_private_t *dev_priv = dev->dev_private; 3663 uint64_t val; 3664 3665 if (obj) { 3666 u32 size = obj->gtt_space->size; 3667 3668 val = (uint64_t)((obj->gtt_offset + size - 4096) & 3669 0xfffff000) << 32; 3670 val |= obj->gtt_offset & 0xfffff000; 3671 val |= (uint64_t)((obj->stride / 128) - 1) << 3672 SANDYBRIDGE_FENCE_PITCH_SHIFT; 3673 3674 if (obj->tiling_mode == I915_TILING_Y) 3675 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3676 val |= I965_FENCE_REG_VALID; 3677 } else 3678 val = 0; 3679 3680 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); 3681 POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); 3682} 3683 3684static void i965_write_fence_reg(struct drm_device *dev, int reg, 3685 struct drm_i915_gem_object *obj) 3686{ 3687 drm_i915_private_t *dev_priv = dev->dev_private; 3688 uint64_t val; 3689 3690 if (obj) { 3691 u32 size = obj->gtt_space->size; 3692 3693 val = (uint64_t)((obj->gtt_offset + size - 4096) & 3694 0xfffff000) << 32; 3695 val |= obj->gtt_offset & 0xfffff000; 3696 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 3697 if (obj->tiling_mode == I915_TILING_Y) 3698 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3699 val |= I965_FENCE_REG_VALID; 3700 } else 3701 val = 0; 3702 3703 I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); 3704 POSTING_READ(FENCE_REG_965_0 + reg * 8); 3705} 3706 3707static void i915_write_fence_reg(struct drm_device *dev, int reg, 3708 struct drm_i915_gem_object *obj) 3709{ 3710 drm_i915_private_t *dev_priv = dev->dev_private; 3711 u32 val; 3712 3713 if (obj) { 3714 u32 size = obj->gtt_space->size; 3715 int pitch_val; 3716 int tile_width; 3717 3718 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3719 (size & -size) != size || 3720 (obj->gtt_offset & (size - 1))) 3721 printf( 3722 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3723 obj->gtt_offset, obj->map_and_fenceable, size); 3724 3725 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3726 tile_width = 128; 3727 else 3728 tile_width = 512; 3729 3730 /* Note: pitch better be a power of two tile widths */ 3731 pitch_val = obj->stride / tile_width; 3732 pitch_val = ffs(pitch_val) - 1; 3733 3734 val = obj->gtt_offset; 3735 if (obj->tiling_mode == I915_TILING_Y) 3736 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3737 val |= I915_FENCE_SIZE_BITS(size); 3738 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3739 val |= I830_FENCE_REG_VALID; 3740 } else 3741 val = 0; 3742 3743 if (reg < 8) 3744 reg = FENCE_REG_830_0 + reg * 4; 3745 else 3746 reg = FENCE_REG_945_8 + (reg - 8) * 4; 3747 3748 I915_WRITE(reg, val); 3749 POSTING_READ(reg); 3750} 3751 3752static void i830_write_fence_reg(struct drm_device *dev, int reg, 3753 struct drm_i915_gem_object *obj) 3754{ 3755 drm_i915_private_t *dev_priv = dev->dev_private; 3756 uint32_t val; 3757 3758 if (obj) { 3759 u32 size = obj->gtt_space->size; 3760 uint32_t pitch_val; 3761 3762 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3763 (size & -size) != size || 3764 (obj->gtt_offset & (size - 1))) 3765 printf( 3766 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3767 obj->gtt_offset, size); 3768 3769 pitch_val = obj->stride / 128; 3770 pitch_val = ffs(pitch_val) - 1; 3771 3772 val = obj->gtt_offset; 3773 if (obj->tiling_mode == I915_TILING_Y) 3774 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3775 val |= I830_FENCE_SIZE_BITS(size); 3776 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3777 val |= I830_FENCE_REG_VALID; 3778 } else 3779 val = 0; 3780 3781 I915_WRITE(FENCE_REG_830_0 + reg * 4, val); 3782 POSTING_READ(FENCE_REG_830_0 + reg * 4); 3783} 3784 3785static void i915_gem_write_fence(struct drm_device *dev, int reg, 3786 struct drm_i915_gem_object *obj) 3787{ 3788 switch (INTEL_INFO(dev)->gen) { 3789 case 7: 3790 case 6: sandybridge_write_fence_reg(dev, reg, obj); break; 3791 case 5: 3792 case 4: i965_write_fence_reg(dev, reg, obj); break; 3793 case 3: i915_write_fence_reg(dev, reg, obj); break; 3794 case 2: i830_write_fence_reg(dev, reg, obj); break; 3795 default: break; 3796 } 3797} 3798 3799static inline int fence_number(struct drm_i915_private *dev_priv, 3800 struct drm_i915_fence_reg *fence) 3801{ 3802 return fence - dev_priv->fence_regs; 3803} 3804 3805static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 3806 struct drm_i915_fence_reg *fence, 3807 bool enable) 3808{ 3809 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3810 int reg = fence_number(dev_priv, fence); 3811 3812 i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); 3813 3814 if (enable) { 3815 obj->fence_reg = reg; 3816 fence->obj = obj; 3817 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); 3818 } else { 3819 obj->fence_reg = I915_FENCE_REG_NONE; 3820 fence->obj = NULL; 3821 list_del_init(&fence->lru_list); 3822 } 3823} 3824 3825static int 3826i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) 3827{ 3828 int ret; 3829 3830 if (obj->fenced_gpu_access) { 3831 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3832 ret = i915_gem_flush_ring(obj->ring, 3833 0, obj->base.write_domain); 3834 if (ret) 3835 return ret; 3836 } 3837 3838 obj->fenced_gpu_access = false; 3839 } 3840 3841 if (obj->last_fenced_seqno) { 3842 ret = i915_wait_request(obj->ring, 3843 obj->last_fenced_seqno); 3844 if (ret) 3845 return ret; 3846 3847 obj->last_fenced_seqno = 0; 3848 } 3849 3850 /* Ensure that all CPU reads are completed before installing a fence 3851 * and all writes before removing the fence. 3852 */ 3853 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3854 mb(); 3855 3856 return 0; 3857} 3858 3859int 3860i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3861{ 3862 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3863 int ret; 3864 3865 ret = i915_gem_object_flush_fence(obj); 3866 if (ret) 3867 return ret; 3868 3869 if (obj->fence_reg == I915_FENCE_REG_NONE) 3870 return 0; 3871 3872 i915_gem_object_update_fence(obj, 3873 &dev_priv->fence_regs[obj->fence_reg], 3874 false); 3875 i915_gem_object_fence_lost(obj); 3876 3877 return 0; 3878} 3879 3880static struct drm_i915_fence_reg * 3881i915_find_fence_reg(struct drm_device *dev) 3882{ 3883 struct drm_i915_private *dev_priv = dev->dev_private; 3884 struct drm_i915_fence_reg *reg, *avail; 3885 int i; 3886 3887 /* First try to find a free reg */ 3888 avail = NULL; 3889 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3890 reg = &dev_priv->fence_regs[i]; 3891 if (!reg->obj) 3892 return reg; 3893 3894 if (!reg->pin_count) 3895 avail = reg; 3896 } 3897 3898 if (avail == NULL) 3899 return NULL; 3900 3901 /* None available, try to steal one or wait for a user to finish */ 3902 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3903 if (reg->pin_count) 3904 continue; 3905 3906 return reg; 3907 } 3908 3909 return NULL; 3910} 3911 3912int 3913i915_gem_object_get_fence(struct drm_i915_gem_object *obj) 3914{ 3915 struct drm_device *dev = obj->base.dev; 3916 struct drm_i915_private *dev_priv = dev->dev_private; 3917 bool enable = obj->tiling_mode != I915_TILING_NONE; 3918 struct drm_i915_fence_reg *reg; 3919 int ret; 3920 3921 /* Have we updated the tiling parameters upon the object and so 3922 * will need to serialise the write to the associated fence register? 3923 */ 3924 if (obj->fence_dirty) { 3925 ret = i915_gem_object_flush_fence(obj); 3926 if (ret) 3927 return ret; 3928 } 3929 3930 ret = 0; 3931 3932 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3933 reg = &dev_priv->fence_regs[obj->fence_reg]; 3934 if (!obj->fence_dirty) { 3935 list_move_tail(®->lru_list, 3936 &dev_priv->mm.fence_list); 3937 return 0; 3938 } 3939 } else if (enable) { 3940 reg = i915_find_fence_reg(dev); 3941 if (reg == NULL) 3942 return -EDEADLK; 3943 3944 if (reg->obj) { 3945 struct drm_i915_gem_object *old = reg->obj; 3946 3947 ret = i915_gem_object_flush_fence(old); 3948 if (ret) 3949 return ret; 3950 3951 i915_gem_object_fence_lost(old); 3952 } 3953 } else 3954 return 0; 3955 3956 i915_gem_object_update_fence(obj, reg, enable); 3957 obj->fence_dirty = false; 3958 3959 return 0; 3960} 3961 3962int 3963i915_gem_init_object(struct drm_gem_object *obj) 3964{ 3965 3966 printf("i915_gem_init_object called\n"); 3967 return (0); 3968} 3969 3970static bool 3971i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3972{ 3973 3974 return !obj->active; 3975} 3976 3977static void 3978i915_gem_retire_task_handler(void *arg, int pending) 3979{ 3980 drm_i915_private_t *dev_priv; 3981 struct drm_device *dev; 3982 struct intel_ring_buffer *ring; 3983 bool idle; 3984 int i; 3985 3986 dev_priv = arg; 3987 dev = dev_priv->dev; 3988 3989 /* Come back later if the device is busy... */ 3990 if (!sx_try_xlock(&dev->dev_struct_lock)) { 3991 taskqueue_enqueue_timeout(dev_priv->tq, 3992 &dev_priv->mm.retire_task, hz); 3993 return; 3994 } 3995 3996 CTR0(KTR_DRM, "retire_task"); 3997 3998 i915_gem_retire_requests(dev); 3999 4000 /* Send a periodic flush down the ring so we don't hold onto GEM 4001 * objects indefinitely. 4002 */ 4003 idle = true; 4004 for_each_ring(ring, dev_priv, i) { 4005 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 4006 4007 if (!list_empty(&ring->gpu_write_list)) { 4008 struct drm_i915_gem_request *request; 4009 int ret; 4010 4011 ret = i915_gem_flush_ring(ring, 4012 0, I915_GEM_GPU_DOMAINS); 4013 request = malloc(sizeof(*request), DRM_I915_GEM, 4014 M_WAITOK | M_ZERO); 4015 if (ret || request == NULL || 4016 i915_add_request(ring, NULL, request)) 4017 free(request, DRM_I915_GEM); 4018 } 4019 4020 idle &= list_empty(&ring->request_list); 4021 } 4022 4023 if (!dev_priv->mm.suspended && !idle) 4024 taskqueue_enqueue_timeout(dev_priv->tq, 4025 &dev_priv->mm.retire_task, hz); 4026 4027 DRM_UNLOCK(dev); 4028} 4029 4030void 4031i915_gem_lastclose(struct drm_device *dev) 4032{ 4033 int ret; 4034 4035 if (drm_core_check_feature(dev, DRIVER_MODESET)) 4036 return; 4037 4038 ret = i915_gem_idle(dev); 4039 if (ret != 0) 4040 DRM_ERROR("failed to idle hardware: %d\n", ret); 4041} 4042 4043static int 4044i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 4045{ 4046 drm_i915_private_t *dev_priv; 4047 struct drm_i915_gem_phys_object *phys_obj; 4048 int ret; 4049 4050 dev_priv = dev->dev_private; 4051 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 4052 return (0); 4053 4054 phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 4055 M_WAITOK | M_ZERO); 4056 4057 phys_obj->id = id; 4058 4059 phys_obj->handle = drm_pci_alloc(dev, size, align, BUS_SPACE_MAXADDR); 4060 if (phys_obj->handle == NULL) { 4061 ret = -ENOMEM; 4062 goto free_obj; 4063 } 4064 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 4065 size / PAGE_SIZE, PAT_WRITE_COMBINING); 4066 4067 dev_priv->mm.phys_objs[id - 1] = phys_obj; 4068 4069 return (0); 4070 4071free_obj: 4072 free(phys_obj, DRM_I915_GEM); 4073 return (ret); 4074} 4075 4076static void 4077i915_gem_free_phys_object(struct drm_device *dev, int id) 4078{ 4079 drm_i915_private_t *dev_priv; 4080 struct drm_i915_gem_phys_object *phys_obj; 4081 4082 dev_priv = dev->dev_private; 4083 if (dev_priv->mm.phys_objs[id - 1] == NULL) 4084 return; 4085 4086 phys_obj = dev_priv->mm.phys_objs[id - 1]; 4087 if (phys_obj->cur_obj != NULL) 4088 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 4089 4090 drm_pci_free(dev, phys_obj->handle); 4091 free(phys_obj, DRM_I915_GEM); 4092 dev_priv->mm.phys_objs[id - 1] = NULL; 4093} 4094 4095void 4096i915_gem_free_all_phys_object(struct drm_device *dev) 4097{ 4098 int i; 4099 4100 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 4101 i915_gem_free_phys_object(dev, i); 4102} 4103 4104void 4105i915_gem_detach_phys_object(struct drm_device *dev, 4106 struct drm_i915_gem_object *obj) 4107{ 4108 vm_page_t m; 4109 struct sf_buf *sf; 4110 char *vaddr, *dst; 4111 int i, page_count; 4112 4113 if (obj->phys_obj == NULL) 4114 return; 4115 vaddr = obj->phys_obj->handle->vaddr; 4116 4117 page_count = obj->base.size / PAGE_SIZE; 4118 VM_OBJECT_WLOCK(obj->base.vm_obj); 4119 for (i = 0; i < page_count; i++) { 4120 m = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4121 if (m == NULL) 4122 continue; /* XXX */ 4123 4124 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4125 sf = sf_buf_alloc(m, 0); 4126 if (sf != NULL) { 4127 dst = (char *)sf_buf_kva(sf); 4128 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 4129 sf_buf_free(sf); 4130 } 4131 drm_clflush_pages(&m, 1); 4132 4133 VM_OBJECT_WLOCK(obj->base.vm_obj); 4134 vm_page_reference(m); 4135 vm_page_lock(m); 4136 vm_page_dirty(m); 4137 vm_page_unwire(m, PQ_INACTIVE); 4138 vm_page_unlock(m); 4139 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4140 } 4141 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4142 intel_gtt_chipset_flush(); 4143 4144 obj->phys_obj->cur_obj = NULL; 4145 obj->phys_obj = NULL; 4146} 4147 4148int 4149i915_gem_attach_phys_object(struct drm_device *dev, 4150 struct drm_i915_gem_object *obj, int id, int align) 4151{ 4152 drm_i915_private_t *dev_priv; 4153 vm_page_t m; 4154 struct sf_buf *sf; 4155 char *dst, *src; 4156 int i, page_count, ret; 4157 4158 if (id > I915_MAX_PHYS_OBJECT) 4159 return (-EINVAL); 4160 4161 if (obj->phys_obj != NULL) { 4162 if (obj->phys_obj->id == id) 4163 return (0); 4164 i915_gem_detach_phys_object(dev, obj); 4165 } 4166 4167 dev_priv = dev->dev_private; 4168 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 4169 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 4170 if (ret != 0) { 4171 DRM_ERROR("failed to init phys object %d size: %zu\n", 4172 id, obj->base.size); 4173 return (ret); 4174 } 4175 } 4176 4177 /* bind to the object */ 4178 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4179 obj->phys_obj->cur_obj = obj; 4180 4181 page_count = obj->base.size / PAGE_SIZE; 4182 4183 VM_OBJECT_WLOCK(obj->base.vm_obj); 4184 ret = 0; 4185 for (i = 0; i < page_count; i++) { 4186 m = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4187 if (m == NULL) { 4188 ret = -EIO; 4189 break; 4190 } 4191 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4192 sf = sf_buf_alloc(m, 0); 4193 src = (char *)sf_buf_kva(sf); 4194 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 4195 memcpy(dst, src, PAGE_SIZE); 4196 sf_buf_free(sf); 4197 4198 VM_OBJECT_WLOCK(obj->base.vm_obj); 4199 4200 vm_page_reference(m); 4201 vm_page_lock(m); 4202 vm_page_unwire(m, PQ_INACTIVE); 4203 vm_page_unlock(m); 4204 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4205 } 4206 VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4207 4208 return (ret); 4209} 4210 4211static int 4212i915_gpu_is_active(struct drm_device *dev) 4213{ 4214 drm_i915_private_t *dev_priv; 4215 4216 dev_priv = dev->dev_private; 4217 return (!list_empty(&dev_priv->mm.flushing_list) || 4218 !list_empty(&dev_priv->mm.active_list)); 4219} 4220 4221static void 4222i915_gem_lowmem(void *arg) 4223{ 4224 struct drm_device *dev; 4225 struct drm_i915_private *dev_priv; 4226 struct drm_i915_gem_object *obj, *next; 4227 int cnt, cnt_fail, cnt_total; 4228 4229 dev = arg; 4230 dev_priv = dev->dev_private; 4231 4232 if (!sx_try_xlock(&dev->dev_struct_lock)) 4233 return; 4234 4235 CTR0(KTR_DRM, "gem_lowmem"); 4236 4237rescan: 4238 /* first scan for clean buffers */ 4239 i915_gem_retire_requests(dev); 4240 4241 cnt_total = cnt_fail = cnt = 0; 4242 4243 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 4244 mm_list) { 4245 if (i915_gem_object_is_purgeable(obj)) { 4246 if (i915_gem_object_unbind(obj) != 0) 4247 cnt_total++; 4248 } else 4249 cnt_total++; 4250 } 4251 4252 /* second pass, evict/count anything still on the inactive list */ 4253 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 4254 mm_list) { 4255 if (i915_gem_object_unbind(obj) == 0) 4256 cnt++; 4257 else 4258 cnt_fail++; 4259 } 4260 4261 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 4262 /* 4263 * We are desperate for pages, so as a last resort, wait 4264 * for the GPU to finish and discard whatever we can. 4265 * This has a dramatic impact to reduce the number of 4266 * OOM-killer events whilst running the GPU aggressively. 4267 */ 4268 if (i915_gpu_idle(dev) == 0) 4269 goto rescan; 4270 } 4271 DRM_UNLOCK(dev); 4272} 4273 4274void 4275i915_gem_unload(struct drm_device *dev) 4276{ 4277 struct drm_i915_private *dev_priv; 4278 4279 dev_priv = dev->dev_private; 4280 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 4281} 4282