i915_gem.c revision 269634
1235783Skib/*- 2235783Skib * Copyright �� 2008 Intel Corporation 3235783Skib * 4235783Skib * Permission is hereby granted, free of charge, to any person obtaining a 5235783Skib * copy of this software and associated documentation files (the "Software"), 6235783Skib * to deal in the Software without restriction, including without limitation 7235783Skib * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8235783Skib * and/or sell copies of the Software, and to permit persons to whom the 9235783Skib * Software is furnished to do so, subject to the following conditions: 10235783Skib * 11235783Skib * The above copyright notice and this permission notice (including the next 12235783Skib * paragraph) shall be included in all copies or substantial portions of the 13235783Skib * Software. 14235783Skib * 15235783Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16235783Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17235783Skib * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18235783Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19235783Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20235783Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21235783Skib * IN THE SOFTWARE. 22235783Skib * 23235783Skib * Authors: 24235783Skib * Eric Anholt <eric@anholt.net> 25235783Skib * 26235783Skib * Copyright (c) 2011 The FreeBSD Foundation 27235783Skib * All rights reserved. 28235783Skib * 29235783Skib * This software was developed by Konstantin Belousov under sponsorship from 30235783Skib * the FreeBSD Foundation. 31235783Skib * 32235783Skib * Redistribution and use in source and binary forms, with or without 33235783Skib * modification, are permitted provided that the following conditions 34235783Skib * are met: 35235783Skib * 1. Redistributions of source code must retain the above copyright 36235783Skib * notice, this list of conditions and the following disclaimer. 37235783Skib * 2. Redistributions in binary form must reproduce the above copyright 38235783Skib * notice, this list of conditions and the following disclaimer in the 39235783Skib * documentation and/or other materials provided with the distribution. 40235783Skib * 41235783Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42235783Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43235783Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44235783Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45235783Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46235783Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47235783Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48235783Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49235783Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50235783Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51235783Skib * SUCH DAMAGE. 52235783Skib */ 53235783Skib 54235783Skib#include <sys/cdefs.h> 55235783Skib__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 269634 2014-08-06 17:45:59Z royger $"); 56235783Skib 57235783Skib#include <dev/drm2/drmP.h> 58235783Skib#include <dev/drm2/drm.h> 59235783Skib#include <dev/drm2/i915/i915_drm.h> 60235783Skib#include <dev/drm2/i915/i915_drv.h> 61235783Skib#include <dev/drm2/i915/intel_drv.h> 62235783Skib#include <dev/drm2/i915/intel_ringbuffer.h> 63235783Skib#include <sys/resourcevar.h> 64235783Skib#include <sys/sched.h> 65235783Skib#include <sys/sf_buf.h> 66235783Skib 67254141Sattilio#include <vm/vm.h> 68254141Sattilio#include <vm/vm_pageout.h> 69254141Sattilio 70235783Skibstatic void i915_gem_object_flush_cpu_write_domain( 71235783Skib struct drm_i915_gem_object *obj); 72235783Skibstatic uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 73235783Skib int tiling_mode); 74235783Skibstatic uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 75235783Skib uint32_t size, int tiling_mode); 76235783Skibstatic int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 77235783Skib unsigned alignment, bool map_and_fenceable); 78235783Skibstatic int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 79235783Skib int flags); 80235783Skibstatic void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 81235783Skibstatic int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 82235783Skib bool write); 83235783Skibstatic void i915_gem_object_set_to_full_cpu_read_domain( 84235783Skib struct drm_i915_gem_object *obj); 85235783Skibstatic int i915_gem_object_set_cpu_read_domain_range( 86235783Skib struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 87235783Skibstatic void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 88235783Skibstatic void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 89235783Skibstatic int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 90235783Skibstatic bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 91235783Skibstatic int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 92235783Skibstatic vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 93235783Skibstatic void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 94235783Skib uint32_t flush_domains); 95235783Skibstatic void i915_gem_clear_fence_reg(struct drm_device *dev, 96235783Skib struct drm_i915_fence_reg *reg); 97235783Skibstatic void i915_gem_reset_fences(struct drm_device *dev); 98235783Skibstatic void i915_gem_retire_task_handler(void *arg, int pending); 99235783Skibstatic int i915_gem_phys_pwrite(struct drm_device *dev, 100235783Skib struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 101235783Skib uint64_t size, struct drm_file *file_priv); 102235783Skibstatic void i915_gem_lowmem(void *arg); 103235783Skib 104235783SkibMALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 105235783Skiblong i915_gem_wired_pages_cnt; 106235783Skib 107235783Skibstatic void 108235783Skibi915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 109235783Skib{ 110235783Skib 111235783Skib dev_priv->mm.object_count++; 112235783Skib dev_priv->mm.object_memory += size; 113235783Skib} 114235783Skib 115235783Skibstatic void 116235783Skibi915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 117235783Skib{ 118235783Skib 119235783Skib dev_priv->mm.object_count--; 120235783Skib dev_priv->mm.object_memory -= size; 121235783Skib} 122235783Skib 123235783Skibstatic int 124235783Skibi915_gem_wait_for_error(struct drm_device *dev) 125235783Skib{ 126235783Skib struct drm_i915_private *dev_priv; 127235783Skib int ret; 128235783Skib 129235783Skib dev_priv = dev->dev_private; 130235783Skib if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 131235783Skib return (0); 132235783Skib 133235783Skib mtx_lock(&dev_priv->error_completion_lock); 134235783Skib while (dev_priv->error_completion == 0) { 135235783Skib ret = -msleep(&dev_priv->error_completion, 136235783Skib &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 137235783Skib if (ret != 0) { 138235783Skib mtx_unlock(&dev_priv->error_completion_lock); 139235783Skib return (ret); 140235783Skib } 141235783Skib } 142235783Skib mtx_unlock(&dev_priv->error_completion_lock); 143235783Skib 144255013Sjkim if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 145235783Skib mtx_lock(&dev_priv->error_completion_lock); 146235783Skib dev_priv->error_completion++; 147235783Skib mtx_unlock(&dev_priv->error_completion_lock); 148235783Skib } 149235783Skib return (0); 150235783Skib} 151235783Skib 152235783Skibint 153235783Skibi915_mutex_lock_interruptible(struct drm_device *dev) 154235783Skib{ 155235783Skib struct drm_i915_private *dev_priv; 156235783Skib int ret; 157235783Skib 158235783Skib dev_priv = dev->dev_private; 159235783Skib ret = i915_gem_wait_for_error(dev); 160235783Skib if (ret != 0) 161235783Skib return (ret); 162235783Skib 163235783Skib /* 164235783Skib * interruptible shall it be. might indeed be if dev_lock is 165235783Skib * changed to sx 166235783Skib */ 167235783Skib ret = sx_xlock_sig(&dev->dev_struct_lock); 168235783Skib if (ret != 0) 169235783Skib return (-ret); 170235783Skib 171235783Skib return (0); 172235783Skib} 173235783Skib 174235783Skib 175235783Skibstatic void 176235783Skibi915_gem_free_object_tail(struct drm_i915_gem_object *obj) 177235783Skib{ 178235783Skib struct drm_device *dev; 179235783Skib drm_i915_private_t *dev_priv; 180235783Skib int ret; 181235783Skib 182235783Skib dev = obj->base.dev; 183235783Skib dev_priv = dev->dev_private; 184235783Skib 185235783Skib ret = i915_gem_object_unbind(obj); 186235783Skib if (ret == -ERESTART) { 187235783Skib list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 188235783Skib return; 189235783Skib } 190235783Skib 191235783Skib CTR1(KTR_DRM, "object_destroy_tail %p", obj); 192235783Skib drm_gem_free_mmap_offset(&obj->base); 193235783Skib drm_gem_object_release(&obj->base); 194235783Skib i915_gem_info_remove_obj(dev_priv, obj->base.size); 195235783Skib 196235783Skib free(obj->page_cpu_valid, DRM_I915_GEM); 197235783Skib free(obj->bit_17, DRM_I915_GEM); 198235783Skib free(obj, DRM_I915_GEM); 199235783Skib} 200235783Skib 201235783Skibvoid 202235783Skibi915_gem_free_object(struct drm_gem_object *gem_obj) 203235783Skib{ 204235783Skib struct drm_i915_gem_object *obj; 205235783Skib struct drm_device *dev; 206235783Skib 207235783Skib obj = to_intel_bo(gem_obj); 208235783Skib dev = obj->base.dev; 209235783Skib 210235783Skib while (obj->pin_count > 0) 211235783Skib i915_gem_object_unpin(obj); 212235783Skib 213235783Skib if (obj->phys_obj != NULL) 214235783Skib i915_gem_detach_phys_object(dev, obj); 215235783Skib 216235783Skib i915_gem_free_object_tail(obj); 217235783Skib} 218235783Skib 219235783Skibstatic void 220235783Skibinit_ring_lists(struct intel_ring_buffer *ring) 221235783Skib{ 222235783Skib 223235783Skib INIT_LIST_HEAD(&ring->active_list); 224235783Skib INIT_LIST_HEAD(&ring->request_list); 225235783Skib INIT_LIST_HEAD(&ring->gpu_write_list); 226235783Skib} 227235783Skib 228235783Skibvoid 229235783Skibi915_gem_load(struct drm_device *dev) 230235783Skib{ 231235783Skib drm_i915_private_t *dev_priv; 232235783Skib int i; 233235783Skib 234235783Skib dev_priv = dev->dev_private; 235235783Skib 236235783Skib INIT_LIST_HEAD(&dev_priv->mm.active_list); 237235783Skib INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 238235783Skib INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 239235783Skib INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 240235783Skib INIT_LIST_HEAD(&dev_priv->mm.fence_list); 241235783Skib INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 242235783Skib INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 243235783Skib for (i = 0; i < I915_NUM_RINGS; i++) 244235783Skib init_ring_lists(&dev_priv->rings[i]); 245235783Skib for (i = 0; i < I915_MAX_NUM_FENCES; i++) 246235783Skib INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 247235783Skib TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 248235783Skib i915_gem_retire_task_handler, dev_priv); 249235783Skib dev_priv->error_completion = 0; 250235783Skib 251235783Skib /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 252235783Skib if (IS_GEN3(dev)) { 253235783Skib u32 tmp = I915_READ(MI_ARB_STATE); 254235783Skib if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { 255235783Skib /* 256235783Skib * arb state is a masked write, so set bit + 257235783Skib * bit in mask. 258235783Skib */ 259235783Skib tmp = MI_ARB_C3_LP_WRITE_ENABLE | 260235783Skib (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); 261235783Skib I915_WRITE(MI_ARB_STATE, tmp); 262235783Skib } 263235783Skib } 264235783Skib 265235783Skib dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 266235783Skib 267235783Skib /* Old X drivers will take 0-2 for front, back, depth buffers */ 268235783Skib if (!drm_core_check_feature(dev, DRIVER_MODESET)) 269235783Skib dev_priv->fence_reg_start = 3; 270235783Skib 271235783Skib if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 272235783Skib IS_G33(dev)) 273235783Skib dev_priv->num_fence_regs = 16; 274235783Skib else 275235783Skib dev_priv->num_fence_regs = 8; 276235783Skib 277235783Skib /* Initialize fence registers to zero */ 278235783Skib for (i = 0; i < dev_priv->num_fence_regs; i++) { 279235783Skib i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); 280235783Skib } 281235783Skib i915_gem_detect_bit_6_swizzle(dev); 282235783Skib dev_priv->mm.interruptible = true; 283235783Skib 284235783Skib dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 285235783Skib i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 286235783Skib} 287235783Skib 288235783Skibint 289235783Skibi915_gem_do_init(struct drm_device *dev, unsigned long start, 290235783Skib unsigned long mappable_end, unsigned long end) 291235783Skib{ 292235783Skib drm_i915_private_t *dev_priv; 293235783Skib unsigned long mappable; 294235783Skib int error; 295235783Skib 296235783Skib dev_priv = dev->dev_private; 297235783Skib mappable = min(end, mappable_end) - start; 298235783Skib 299235783Skib drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 300235783Skib 301235783Skib dev_priv->mm.gtt_start = start; 302235783Skib dev_priv->mm.gtt_mappable_end = mappable_end; 303235783Skib dev_priv->mm.gtt_end = end; 304235783Skib dev_priv->mm.gtt_total = end - start; 305235783Skib dev_priv->mm.mappable_gtt_total = mappable; 306235783Skib 307235783Skib /* Take over this portion of the GTT */ 308235783Skib intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 309235783Skib device_printf(dev->device, 310235783Skib "taking over the fictitious range 0x%lx-0x%lx\n", 311235783Skib dev->agp->base + start, dev->agp->base + start + mappable); 312235783Skib error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 313235783Skib dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 314235783Skib return (error); 315235783Skib} 316235783Skib 317235783Skibint 318235783Skibi915_gem_init_ioctl(struct drm_device *dev, void *data, 319235783Skib struct drm_file *file) 320235783Skib{ 321235783Skib struct drm_i915_gem_init *args; 322235783Skib drm_i915_private_t *dev_priv; 323235783Skib 324235783Skib dev_priv = dev->dev_private; 325235783Skib args = data; 326235783Skib 327235783Skib if (args->gtt_start >= args->gtt_end || 328235783Skib (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 329235783Skib return (-EINVAL); 330235783Skib 331235783Skib if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock)) 332235783Skib return (-EBUSY); 333235783Skib /* 334235783Skib * XXXKIB. The second-time initialization should be guarded 335235783Skib * against. 336235783Skib */ 337235783Skib return (i915_gem_do_init(dev, args->gtt_start, args->gtt_end, 338235783Skib args->gtt_end)); 339235783Skib} 340235783Skib 341235783Skibint 342235783Skibi915_gem_idle(struct drm_device *dev) 343235783Skib{ 344235783Skib drm_i915_private_t *dev_priv; 345235783Skib int ret; 346235783Skib 347235783Skib dev_priv = dev->dev_private; 348235783Skib if (dev_priv->mm.suspended) 349235783Skib return (0); 350235783Skib 351235783Skib ret = i915_gpu_idle(dev, true); 352235783Skib if (ret != 0) 353235783Skib return (ret); 354235783Skib 355235783Skib /* Under UMS, be paranoid and evict. */ 356235783Skib if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 357235783Skib ret = i915_gem_evict_inactive(dev, false); 358235783Skib if (ret != 0) 359235783Skib return ret; 360235783Skib } 361235783Skib 362235783Skib i915_gem_reset_fences(dev); 363235783Skib 364235783Skib /* Hack! Don't let anybody do execbuf while we don't control the chip. 365235783Skib * We need to replace this with a semaphore, or something. 366235783Skib * And not confound mm.suspended! 367235783Skib */ 368235783Skib dev_priv->mm.suspended = 1; 369235783Skib callout_stop(&dev_priv->hangcheck_timer); 370235783Skib 371235783Skib i915_kernel_lost_context(dev); 372235783Skib i915_gem_cleanup_ringbuffer(dev); 373235783Skib 374235783Skib /* Cancel the retire work handler, which should be idle now. */ 375235783Skib taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 376235783Skib return (ret); 377235783Skib} 378235783Skib 379235783Skibvoid 380235783Skibi915_gem_init_swizzling(struct drm_device *dev) 381235783Skib{ 382235783Skib drm_i915_private_t *dev_priv; 383235783Skib 384235783Skib dev_priv = dev->dev_private; 385235783Skib 386235783Skib if (INTEL_INFO(dev)->gen < 5 || 387235783Skib dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 388235783Skib return; 389235783Skib 390235783Skib I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 391235783Skib DISP_TILE_SURFACE_SWIZZLING); 392235783Skib 393235783Skib if (IS_GEN5(dev)) 394235783Skib return; 395235783Skib 396235783Skib I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 397235783Skib if (IS_GEN6(dev)) 398235783Skib I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); 399235783Skib else 400235783Skib I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); 401235783Skib} 402235783Skib 403235783Skibvoid 404235783Skibi915_gem_init_ppgtt(struct drm_device *dev) 405235783Skib{ 406235783Skib drm_i915_private_t *dev_priv; 407235783Skib struct i915_hw_ppgtt *ppgtt; 408235783Skib uint32_t pd_offset, pd_entry; 409235783Skib vm_paddr_t pt_addr; 410235783Skib struct intel_ring_buffer *ring; 411235783Skib u_int first_pd_entry_in_global_pt, i; 412235783Skib 413235783Skib dev_priv = dev->dev_private; 414235783Skib ppgtt = dev_priv->mm.aliasing_ppgtt; 415235783Skib if (ppgtt == NULL) 416235783Skib return; 417235783Skib 418235783Skib first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 419235783Skib for (i = 0; i < ppgtt->num_pd_entries; i++) { 420235783Skib pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 421235783Skib pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 422235783Skib pd_entry |= GEN6_PDE_VALID; 423235783Skib intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 424235783Skib } 425235783Skib intel_gtt_read_pte(first_pd_entry_in_global_pt); 426235783Skib 427235783Skib pd_offset = ppgtt->pd_offset; 428235783Skib pd_offset /= 64; /* in cachelines, */ 429235783Skib pd_offset <<= 16; 430235783Skib 431235783Skib if (INTEL_INFO(dev)->gen == 6) { 432235783Skib uint32_t ecochk = I915_READ(GAM_ECOCHK); 433235783Skib I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 434235783Skib ECOCHK_PPGTT_CACHE64B); 435235783Skib I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 436235783Skib } else if (INTEL_INFO(dev)->gen >= 7) { 437235783Skib I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 438235783Skib /* GFX_MODE is per-ring on gen7+ */ 439235783Skib } 440235783Skib 441235783Skib for (i = 0; i < I915_NUM_RINGS; i++) { 442235783Skib ring = &dev_priv->rings[i]; 443235783Skib 444235783Skib if (INTEL_INFO(dev)->gen >= 7) 445235783Skib I915_WRITE(RING_MODE_GEN7(ring), 446235783Skib GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 447235783Skib 448235783Skib I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 449235783Skib I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 450235783Skib } 451235783Skib} 452235783Skib 453235783Skibint 454235783Skibi915_gem_init_hw(struct drm_device *dev) 455235783Skib{ 456235783Skib drm_i915_private_t *dev_priv; 457235783Skib int ret; 458235783Skib 459235783Skib dev_priv = dev->dev_private; 460235783Skib 461235783Skib i915_gem_init_swizzling(dev); 462235783Skib 463235783Skib ret = intel_init_render_ring_buffer(dev); 464235783Skib if (ret != 0) 465235783Skib return (ret); 466235783Skib 467235783Skib if (HAS_BSD(dev)) { 468235783Skib ret = intel_init_bsd_ring_buffer(dev); 469235783Skib if (ret != 0) 470235783Skib goto cleanup_render_ring; 471235783Skib } 472235783Skib 473235783Skib if (HAS_BLT(dev)) { 474235783Skib ret = intel_init_blt_ring_buffer(dev); 475235783Skib if (ret != 0) 476235783Skib goto cleanup_bsd_ring; 477235783Skib } 478235783Skib 479235783Skib dev_priv->next_seqno = 1; 480235783Skib i915_gem_init_ppgtt(dev); 481235783Skib return (0); 482235783Skib 483235783Skibcleanup_bsd_ring: 484235783Skib intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 485235783Skibcleanup_render_ring: 486235783Skib intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 487235783Skib return (ret); 488235783Skib} 489235783Skib 490235783Skibint 491235783Skibi915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 492235783Skib struct drm_file *file) 493235783Skib{ 494235783Skib struct drm_i915_private *dev_priv; 495235783Skib struct drm_i915_gem_get_aperture *args; 496235783Skib struct drm_i915_gem_object *obj; 497235783Skib size_t pinned; 498235783Skib 499235783Skib dev_priv = dev->dev_private; 500235783Skib args = data; 501235783Skib 502235783Skib if (!(dev->driver->driver_features & DRIVER_GEM)) 503235783Skib return (-ENODEV); 504235783Skib 505235783Skib pinned = 0; 506235783Skib DRM_LOCK(dev); 507235783Skib list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 508235783Skib pinned += obj->gtt_space->size; 509235783Skib DRM_UNLOCK(dev); 510235783Skib 511235783Skib args->aper_size = dev_priv->mm.gtt_total; 512235783Skib args->aper_available_size = args->aper_size - pinned; 513235783Skib 514235783Skib return (0); 515235783Skib} 516235783Skib 517235783Skibint 518235783Skibi915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 519235783Skib bool map_and_fenceable) 520235783Skib{ 521235783Skib struct drm_device *dev; 522235783Skib struct drm_i915_private *dev_priv; 523235783Skib int ret; 524235783Skib 525235783Skib dev = obj->base.dev; 526235783Skib dev_priv = dev->dev_private; 527235783Skib 528235783Skib KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 529235783Skib ("Max pin count")); 530235783Skib 531235783Skib if (obj->gtt_space != NULL) { 532235783Skib if ((alignment && obj->gtt_offset & (alignment - 1)) || 533235783Skib (map_and_fenceable && !obj->map_and_fenceable)) { 534235783Skib DRM_DEBUG("bo is already pinned with incorrect alignment:" 535235783Skib " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 536235783Skib " obj->map_and_fenceable=%d\n", 537235783Skib obj->gtt_offset, alignment, 538235783Skib map_and_fenceable, 539235783Skib obj->map_and_fenceable); 540235783Skib ret = i915_gem_object_unbind(obj); 541235783Skib if (ret != 0) 542235783Skib return (ret); 543235783Skib } 544235783Skib } 545235783Skib 546235783Skib if (obj->gtt_space == NULL) { 547235783Skib ret = i915_gem_object_bind_to_gtt(obj, alignment, 548235783Skib map_and_fenceable); 549235783Skib if (ret) 550235783Skib return (ret); 551235783Skib } 552235783Skib 553235783Skib if (obj->pin_count++ == 0 && !obj->active) 554235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 555235783Skib obj->pin_mappable |= map_and_fenceable; 556235783Skib 557235783Skib#if 1 558235783Skib KIB_NOTYET(); 559235783Skib#else 560235783Skib WARN_ON(i915_verify_lists(dev)); 561235783Skib#endif 562235783Skib return (0); 563235783Skib} 564235783Skib 565235783Skibvoid 566235783Skibi915_gem_object_unpin(struct drm_i915_gem_object *obj) 567235783Skib{ 568235783Skib struct drm_device *dev; 569235783Skib drm_i915_private_t *dev_priv; 570235783Skib 571235783Skib dev = obj->base.dev; 572235783Skib dev_priv = dev->dev_private; 573235783Skib 574235783Skib#if 1 575235783Skib KIB_NOTYET(); 576235783Skib#else 577235783Skib WARN_ON(i915_verify_lists(dev)); 578235783Skib#endif 579235783Skib 580235783Skib KASSERT(obj->pin_count != 0, ("zero pin count")); 581235783Skib KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 582235783Skib 583235783Skib if (--obj->pin_count == 0) { 584235783Skib if (!obj->active) 585235783Skib list_move_tail(&obj->mm_list, 586235783Skib &dev_priv->mm.inactive_list); 587235783Skib obj->pin_mappable = false; 588235783Skib } 589235783Skib#if 1 590235783Skib KIB_NOTYET(); 591235783Skib#else 592235783Skib WARN_ON(i915_verify_lists(dev)); 593235783Skib#endif 594235783Skib} 595235783Skib 596235783Skibint 597235783Skibi915_gem_pin_ioctl(struct drm_device *dev, void *data, 598235783Skib struct drm_file *file) 599235783Skib{ 600235783Skib struct drm_i915_gem_pin *args; 601235783Skib struct drm_i915_gem_object *obj; 602235783Skib struct drm_gem_object *gobj; 603235783Skib int ret; 604235783Skib 605235783Skib args = data; 606235783Skib 607235783Skib ret = i915_mutex_lock_interruptible(dev); 608235783Skib if (ret != 0) 609235783Skib return ret; 610235783Skib 611235783Skib gobj = drm_gem_object_lookup(dev, file, args->handle); 612235783Skib if (gobj == NULL) { 613235783Skib ret = -ENOENT; 614235783Skib goto unlock; 615235783Skib } 616235783Skib obj = to_intel_bo(gobj); 617235783Skib 618235783Skib if (obj->madv != I915_MADV_WILLNEED) { 619235783Skib DRM_ERROR("Attempting to pin a purgeable buffer\n"); 620235783Skib ret = -EINVAL; 621235783Skib goto out; 622235783Skib } 623235783Skib 624235783Skib if (obj->pin_filp != NULL && obj->pin_filp != file) { 625235783Skib DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 626235783Skib args->handle); 627235783Skib ret = -EINVAL; 628235783Skib goto out; 629235783Skib } 630235783Skib 631235783Skib obj->user_pin_count++; 632235783Skib obj->pin_filp = file; 633235783Skib if (obj->user_pin_count == 1) { 634235783Skib ret = i915_gem_object_pin(obj, args->alignment, true); 635235783Skib if (ret != 0) 636235783Skib goto out; 637235783Skib } 638235783Skib 639235783Skib /* XXX - flush the CPU caches for pinned objects 640235783Skib * as the X server doesn't manage domains yet 641235783Skib */ 642235783Skib i915_gem_object_flush_cpu_write_domain(obj); 643235783Skib args->offset = obj->gtt_offset; 644235783Skibout: 645235783Skib drm_gem_object_unreference(&obj->base); 646235783Skibunlock: 647235783Skib DRM_UNLOCK(dev); 648235783Skib return (ret); 649235783Skib} 650235783Skib 651235783Skibint 652235783Skibi915_gem_unpin_ioctl(struct drm_device *dev, void *data, 653235783Skib struct drm_file *file) 654235783Skib{ 655235783Skib struct drm_i915_gem_pin *args; 656235783Skib struct drm_i915_gem_object *obj; 657235783Skib int ret; 658235783Skib 659235783Skib args = data; 660235783Skib ret = i915_mutex_lock_interruptible(dev); 661235783Skib if (ret != 0) 662235783Skib return (ret); 663235783Skib 664235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 665235783Skib if (&obj->base == NULL) { 666235783Skib ret = -ENOENT; 667235783Skib goto unlock; 668235783Skib } 669235783Skib 670235783Skib if (obj->pin_filp != file) { 671235783Skib DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 672235783Skib args->handle); 673235783Skib ret = -EINVAL; 674235783Skib goto out; 675235783Skib } 676235783Skib obj->user_pin_count--; 677235783Skib if (obj->user_pin_count == 0) { 678235783Skib obj->pin_filp = NULL; 679235783Skib i915_gem_object_unpin(obj); 680235783Skib } 681235783Skib 682235783Skibout: 683235783Skib drm_gem_object_unreference(&obj->base); 684235783Skibunlock: 685235783Skib DRM_UNLOCK(dev); 686235783Skib return (ret); 687235783Skib} 688235783Skib 689235783Skibint 690235783Skibi915_gem_busy_ioctl(struct drm_device *dev, void *data, 691235783Skib struct drm_file *file) 692235783Skib{ 693235783Skib struct drm_i915_gem_busy *args; 694235783Skib struct drm_i915_gem_object *obj; 695235783Skib struct drm_i915_gem_request *request; 696235783Skib int ret; 697235783Skib 698235783Skib args = data; 699235783Skib 700235783Skib ret = i915_mutex_lock_interruptible(dev); 701235783Skib if (ret != 0) 702235783Skib return ret; 703235783Skib 704235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 705235783Skib if (&obj->base == NULL) { 706235783Skib ret = -ENOENT; 707235783Skib goto unlock; 708235783Skib } 709235783Skib 710235783Skib args->busy = obj->active; 711235783Skib if (args->busy) { 712235783Skib if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 713235783Skib ret = i915_gem_flush_ring(obj->ring, 714235783Skib 0, obj->base.write_domain); 715235783Skib } else if (obj->ring->outstanding_lazy_request == 716235783Skib obj->last_rendering_seqno) { 717235783Skib request = malloc(sizeof(*request), DRM_I915_GEM, 718235783Skib M_WAITOK | M_ZERO); 719235783Skib ret = i915_add_request(obj->ring, NULL, request); 720235783Skib if (ret != 0) 721235783Skib free(request, DRM_I915_GEM); 722235783Skib } 723235783Skib 724235783Skib i915_gem_retire_requests_ring(obj->ring); 725235783Skib args->busy = obj->active; 726235783Skib } 727235783Skib 728235783Skib drm_gem_object_unreference(&obj->base); 729235783Skibunlock: 730235783Skib DRM_UNLOCK(dev); 731235783Skib return (ret); 732235783Skib} 733235783Skib 734235783Skibstatic int 735235783Skibi915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 736235783Skib{ 737235783Skib struct drm_i915_private *dev_priv; 738235783Skib struct drm_i915_file_private *file_priv; 739235783Skib unsigned long recent_enough; 740235783Skib struct drm_i915_gem_request *request; 741235783Skib struct intel_ring_buffer *ring; 742235783Skib u32 seqno; 743235783Skib int ret; 744235783Skib 745235783Skib dev_priv = dev->dev_private; 746255013Sjkim if (atomic_load_acq_int(&dev_priv->mm.wedged)) 747235783Skib return (-EIO); 748235783Skib 749235783Skib file_priv = file->driver_priv; 750235783Skib recent_enough = ticks - (20 * hz / 1000); 751235783Skib ring = NULL; 752235783Skib seqno = 0; 753235783Skib 754235783Skib mtx_lock(&file_priv->mm.lck); 755235783Skib list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 756235783Skib if (time_after_eq(request->emitted_jiffies, recent_enough)) 757235783Skib break; 758235783Skib ring = request->ring; 759235783Skib seqno = request->seqno; 760235783Skib } 761235783Skib mtx_unlock(&file_priv->mm.lck); 762235783Skib if (seqno == 0) 763235783Skib return (0); 764235783Skib 765235783Skib ret = 0; 766235783Skib mtx_lock(&ring->irq_lock); 767235783Skib if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 768235783Skib if (ring->irq_get(ring)) { 769235783Skib while (ret == 0 && 770235783Skib !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 771255013Sjkim atomic_load_acq_int(&dev_priv->mm.wedged))) 772235783Skib ret = -msleep(ring, &ring->irq_lock, PCATCH, 773235783Skib "915thr", 0); 774235783Skib ring->irq_put(ring); 775255013Sjkim if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged)) 776235783Skib ret = -EIO; 777235783Skib } else if (_intel_wait_for(dev, 778235783Skib i915_seqno_passed(ring->get_seqno(ring), seqno) || 779255013Sjkim atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 780235783Skib ret = -EBUSY; 781235783Skib } 782235783Skib } 783235783Skib mtx_unlock(&ring->irq_lock); 784235783Skib 785235783Skib if (ret == 0) 786235783Skib taskqueue_enqueue_timeout(dev_priv->tq, 787235783Skib &dev_priv->mm.retire_task, 0); 788235783Skib 789235783Skib return (ret); 790235783Skib} 791235783Skib 792235783Skibint 793235783Skibi915_gem_throttle_ioctl(struct drm_device *dev, void *data, 794235783Skib struct drm_file *file_priv) 795235783Skib{ 796235783Skib 797235783Skib return (i915_gem_ring_throttle(dev, file_priv)); 798235783Skib} 799235783Skib 800235783Skibint 801235783Skibi915_gem_madvise_ioctl(struct drm_device *dev, void *data, 802235783Skib struct drm_file *file_priv) 803235783Skib{ 804235783Skib struct drm_i915_gem_madvise *args; 805235783Skib struct drm_i915_gem_object *obj; 806235783Skib int ret; 807235783Skib 808235783Skib args = data; 809235783Skib switch (args->madv) { 810235783Skib case I915_MADV_DONTNEED: 811235783Skib case I915_MADV_WILLNEED: 812235783Skib break; 813235783Skib default: 814235783Skib return (-EINVAL); 815235783Skib } 816235783Skib 817235783Skib ret = i915_mutex_lock_interruptible(dev); 818235783Skib if (ret != 0) 819235783Skib return (ret); 820235783Skib 821235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 822235783Skib if (&obj->base == NULL) { 823235783Skib ret = -ENOENT; 824235783Skib goto unlock; 825235783Skib } 826235783Skib 827235783Skib if (obj->pin_count != 0) { 828235783Skib ret = -EINVAL; 829235783Skib goto out; 830235783Skib } 831235783Skib 832235783Skib if (obj->madv != I915_MADV_PURGED_INTERNAL) 833235783Skib obj->madv = args->madv; 834235783Skib if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 835235783Skib i915_gem_object_truncate(obj); 836235783Skib args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 837235783Skib 838235783Skibout: 839235783Skib drm_gem_object_unreference(&obj->base); 840235783Skibunlock: 841235783Skib DRM_UNLOCK(dev); 842235783Skib return (ret); 843235783Skib} 844235783Skib 845235783Skibvoid 846235783Skibi915_gem_cleanup_ringbuffer(struct drm_device *dev) 847235783Skib{ 848235783Skib drm_i915_private_t *dev_priv; 849235783Skib int i; 850235783Skib 851235783Skib dev_priv = dev->dev_private; 852235783Skib for (i = 0; i < I915_NUM_RINGS; i++) 853235783Skib intel_cleanup_ring_buffer(&dev_priv->rings[i]); 854235783Skib} 855235783Skib 856235783Skibint 857235783Skibi915_gem_entervt_ioctl(struct drm_device *dev, void *data, 858235783Skib struct drm_file *file_priv) 859235783Skib{ 860235783Skib drm_i915_private_t *dev_priv; 861235783Skib int ret, i; 862235783Skib 863235783Skib if (drm_core_check_feature(dev, DRIVER_MODESET)) 864235783Skib return (0); 865235783Skib dev_priv = dev->dev_private; 866235783Skib if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 867235783Skib DRM_ERROR("Reenabling wedged hardware, good luck\n"); 868235783Skib atomic_store_rel_int(&dev_priv->mm.wedged, 0); 869235783Skib } 870235783Skib 871235783Skib dev_priv->mm.suspended = 0; 872235783Skib 873235783Skib ret = i915_gem_init_hw(dev); 874235783Skib if (ret != 0) { 875235783Skib return (ret); 876235783Skib } 877235783Skib 878235783Skib KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 879235783Skib KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 880235783Skib KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 881235783Skib for (i = 0; i < I915_NUM_RINGS; i++) { 882235783Skib KASSERT(list_empty(&dev_priv->rings[i].active_list), 883235783Skib ("ring %d active list", i)); 884235783Skib KASSERT(list_empty(&dev_priv->rings[i].request_list), 885235783Skib ("ring %d request list", i)); 886235783Skib } 887235783Skib 888235783Skib DRM_UNLOCK(dev); 889235783Skib ret = drm_irq_install(dev); 890235783Skib DRM_LOCK(dev); 891235783Skib if (ret) 892235783Skib goto cleanup_ringbuffer; 893235783Skib 894235783Skib return (0); 895235783Skib 896235783Skibcleanup_ringbuffer: 897235783Skib i915_gem_cleanup_ringbuffer(dev); 898235783Skib dev_priv->mm.suspended = 1; 899235783Skib 900235783Skib return (ret); 901235783Skib} 902235783Skib 903235783Skibint 904235783Skibi915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 905235783Skib struct drm_file *file_priv) 906235783Skib{ 907235783Skib 908235783Skib if (drm_core_check_feature(dev, DRIVER_MODESET)) 909235783Skib return 0; 910235783Skib 911235783Skib drm_irq_uninstall(dev); 912235783Skib return (i915_gem_idle(dev)); 913235783Skib} 914235783Skib 915235783Skibint 916235783Skibi915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 917235783Skib uint32_t *handle_p) 918235783Skib{ 919235783Skib struct drm_i915_gem_object *obj; 920235783Skib uint32_t handle; 921235783Skib int ret; 922235783Skib 923235783Skib size = roundup(size, PAGE_SIZE); 924235783Skib if (size == 0) 925235783Skib return (-EINVAL); 926235783Skib 927235783Skib obj = i915_gem_alloc_object(dev, size); 928235783Skib if (obj == NULL) 929235783Skib return (-ENOMEM); 930235783Skib 931235783Skib handle = 0; 932235783Skib ret = drm_gem_handle_create(file, &obj->base, &handle); 933235783Skib if (ret != 0) { 934235783Skib drm_gem_object_release(&obj->base); 935235783Skib i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 936235783Skib free(obj, DRM_I915_GEM); 937235783Skib return (-ret); 938235783Skib } 939235783Skib 940235783Skib /* drop reference from allocate - handle holds it now */ 941235783Skib drm_gem_object_unreference(&obj->base); 942235783Skib CTR2(KTR_DRM, "object_create %p %x", obj, size); 943235783Skib *handle_p = handle; 944235783Skib return (0); 945235783Skib} 946235783Skib 947235783Skibint 948235783Skibi915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 949235783Skib struct drm_mode_create_dumb *args) 950235783Skib{ 951235783Skib 952235783Skib /* have to work out size/pitch and return them */ 953235783Skib args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 954235783Skib args->size = args->pitch * args->height; 955235783Skib return (i915_gem_create(file, dev, args->size, &args->handle)); 956235783Skib} 957235783Skib 958235783Skibint 959235783Skibi915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 960235783Skib uint32_t handle) 961235783Skib{ 962235783Skib 963235783Skib return (drm_gem_handle_delete(file, handle)); 964235783Skib} 965235783Skib 966235783Skibint 967235783Skibi915_gem_create_ioctl(struct drm_device *dev, void *data, 968235783Skib struct drm_file *file) 969235783Skib{ 970235783Skib struct drm_i915_gem_create *args = data; 971235783Skib 972235783Skib return (i915_gem_create(file, dev, args->size, &args->handle)); 973235783Skib} 974235783Skib 975235783Skibstatic int 976235783Skibi915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 977235783Skib uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 978235783Skib struct drm_file *file) 979235783Skib{ 980235783Skib vm_object_t vm_obj; 981235783Skib vm_page_t m; 982235783Skib struct sf_buf *sf; 983235783Skib vm_offset_t mkva; 984235783Skib vm_pindex_t obj_pi; 985235783Skib int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 986235783Skib 987235783Skib if (obj->gtt_offset != 0 && rw == UIO_READ) 988235783Skib do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 989235783Skib else 990235783Skib do_bit17_swizzling = 0; 991235783Skib 992235783Skib obj->dirty = 1; 993235783Skib vm_obj = obj->base.vm_obj; 994235783Skib ret = 0; 995235783Skib 996248084Sattilio VM_OBJECT_WLOCK(vm_obj); 997235783Skib vm_object_pip_add(vm_obj, 1); 998235783Skib while (size > 0) { 999235783Skib obj_pi = OFF_TO_IDX(offset); 1000235783Skib obj_po = offset & PAGE_MASK; 1001235783Skib 1002235783Skib m = i915_gem_wire_page(vm_obj, obj_pi); 1003248084Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1004235783Skib 1005235783Skib sched_pin(); 1006235783Skib sf = sf_buf_alloc(m, SFB_CPUPRIVATE); 1007235783Skib mkva = sf_buf_kva(sf); 1008235783Skib length = min(size, PAGE_SIZE - obj_po); 1009235783Skib while (length > 0) { 1010235783Skib if (do_bit17_swizzling && 1011235783Skib (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 1012235783Skib cnt = roundup2(obj_po + 1, 64); 1013235783Skib cnt = min(cnt - obj_po, length); 1014235783Skib swizzled_po = obj_po ^ 64; 1015235783Skib } else { 1016235783Skib cnt = length; 1017235783Skib swizzled_po = obj_po; 1018235783Skib } 1019235783Skib if (rw == UIO_READ) 1020235783Skib ret = -copyout_nofault( 1021235783Skib (char *)mkva + swizzled_po, 1022235783Skib (void *)(uintptr_t)data_ptr, cnt); 1023235783Skib else 1024235783Skib ret = -copyin_nofault( 1025235783Skib (void *)(uintptr_t)data_ptr, 1026235783Skib (char *)mkva + swizzled_po, cnt); 1027235783Skib if (ret != 0) 1028235783Skib break; 1029235783Skib data_ptr += cnt; 1030235783Skib size -= cnt; 1031235783Skib length -= cnt; 1032235783Skib offset += cnt; 1033235783Skib obj_po += cnt; 1034235783Skib } 1035235783Skib sf_buf_free(sf); 1036235783Skib sched_unpin(); 1037248084Sattilio VM_OBJECT_WLOCK(vm_obj); 1038235783Skib if (rw == UIO_WRITE) 1039235783Skib vm_page_dirty(m); 1040235783Skib vm_page_reference(m); 1041235783Skib vm_page_lock(m); 1042267548Sattilio vm_page_unwire(m, PQ_ACTIVE); 1043235783Skib vm_page_unlock(m); 1044235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1045235783Skib 1046235783Skib if (ret != 0) 1047235783Skib break; 1048235783Skib } 1049235783Skib vm_object_pip_wakeup(vm_obj); 1050248084Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1051235783Skib 1052235783Skib return (ret); 1053235783Skib} 1054235783Skib 1055235783Skibstatic int 1056235783Skibi915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 1057235783Skib uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 1058235783Skib{ 1059235783Skib vm_offset_t mkva; 1060235783Skib vm_pindex_t obj_pi; 1061235783Skib int obj_po, ret; 1062235783Skib 1063235783Skib obj_pi = OFF_TO_IDX(offset); 1064235783Skib obj_po = offset & PAGE_MASK; 1065235783Skib 1066235783Skib mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 1067235783Skib IDX_TO_OFF(obj_pi), size, PAT_WRITE_COMBINING); 1068235783Skib ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva + 1069235783Skib obj_po, size); 1070237131Skib pmap_unmapdev(mkva, size); 1071235783Skib return (ret); 1072235783Skib} 1073235783Skib 1074235783Skibstatic int 1075235783Skibi915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 1076235783Skib uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 1077235783Skib{ 1078235783Skib struct drm_i915_gem_object *obj; 1079235783Skib vm_page_t *ma; 1080235783Skib vm_offset_t start, end; 1081235783Skib int npages, ret; 1082235783Skib 1083235783Skib if (size == 0) 1084235783Skib return (0); 1085235783Skib start = trunc_page(data_ptr); 1086235783Skib end = round_page(data_ptr + size); 1087235783Skib npages = howmany(end - start, PAGE_SIZE); 1088235783Skib ma = malloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 1089235783Skib M_ZERO); 1090235783Skib npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 1091235783Skib (vm_offset_t)data_ptr, size, 1092235783Skib (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 1093235783Skib if (npages == -1) { 1094235783Skib ret = -EFAULT; 1095235783Skib goto free_ma; 1096235783Skib } 1097235783Skib 1098235783Skib ret = i915_mutex_lock_interruptible(dev); 1099235783Skib if (ret != 0) 1100235783Skib goto unlocked; 1101235783Skib 1102235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1103235783Skib if (&obj->base == NULL) { 1104235783Skib ret = -ENOENT; 1105235783Skib goto unlock; 1106235783Skib } 1107235783Skib if (offset > obj->base.size || size > obj->base.size - offset) { 1108235783Skib ret = -EINVAL; 1109235783Skib goto out; 1110235783Skib } 1111235783Skib 1112235783Skib if (rw == UIO_READ) { 1113235783Skib CTR3(KTR_DRM, "object_pread %p %jx %jx", obj, offset, size); 1114235783Skib ret = i915_gem_object_set_cpu_read_domain_range(obj, 1115235783Skib offset, size); 1116235783Skib if (ret != 0) 1117235783Skib goto out; 1118235783Skib ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1119235783Skib UIO_READ, file); 1120235783Skib } else { 1121235783Skib if (obj->phys_obj) { 1122235783Skib CTR3(KTR_DRM, "object_phys_write %p %jx %jx", obj, 1123235783Skib offset, size); 1124235783Skib ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1125235783Skib size, file); 1126235783Skib } else if (obj->gtt_space && 1127235783Skib obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1128235783Skib CTR3(KTR_DRM, "object_gtt_write %p %jx %jx", obj, 1129235783Skib offset, size); 1130235783Skib ret = i915_gem_object_pin(obj, 0, true); 1131235783Skib if (ret != 0) 1132235783Skib goto out; 1133235783Skib ret = i915_gem_object_set_to_gtt_domain(obj, true); 1134235783Skib if (ret != 0) 1135235783Skib goto out_unpin; 1136235783Skib ret = i915_gem_object_put_fence(obj); 1137235783Skib if (ret != 0) 1138235783Skib goto out_unpin; 1139235783Skib ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1140235783Skib offset, file); 1141235783Skibout_unpin: 1142235783Skib i915_gem_object_unpin(obj); 1143235783Skib } else { 1144235783Skib CTR3(KTR_DRM, "object_pwrite %p %jx %jx", obj, 1145235783Skib offset, size); 1146235783Skib ret = i915_gem_object_set_to_cpu_domain(obj, true); 1147235783Skib if (ret != 0) 1148235783Skib goto out; 1149235783Skib ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1150235783Skib UIO_WRITE, file); 1151235783Skib } 1152235783Skib } 1153235783Skibout: 1154235783Skib drm_gem_object_unreference(&obj->base); 1155235783Skibunlock: 1156235783Skib DRM_UNLOCK(dev); 1157235783Skibunlocked: 1158235783Skib vm_page_unhold_pages(ma, npages); 1159235783Skibfree_ma: 1160235783Skib free(ma, DRM_I915_GEM); 1161235783Skib return (ret); 1162235783Skib} 1163235783Skib 1164235783Skibint 1165235783Skibi915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1166235783Skib{ 1167235783Skib struct drm_i915_gem_pread *args; 1168235783Skib 1169235783Skib args = data; 1170235783Skib return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1171235783Skib args->offset, UIO_READ, file)); 1172235783Skib} 1173235783Skib 1174235783Skibint 1175235783Skibi915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1176235783Skib{ 1177235783Skib struct drm_i915_gem_pwrite *args; 1178235783Skib 1179235783Skib args = data; 1180235783Skib return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1181235783Skib args->offset, UIO_WRITE, file)); 1182235783Skib} 1183235783Skib 1184235783Skibint 1185235783Skibi915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1186235783Skib struct drm_file *file) 1187235783Skib{ 1188235783Skib struct drm_i915_gem_set_domain *args; 1189235783Skib struct drm_i915_gem_object *obj; 1190235783Skib uint32_t read_domains; 1191235783Skib uint32_t write_domain; 1192235783Skib int ret; 1193235783Skib 1194235783Skib if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1195235783Skib return (-ENODEV); 1196235783Skib 1197235783Skib args = data; 1198235783Skib read_domains = args->read_domains; 1199235783Skib write_domain = args->write_domain; 1200235783Skib 1201235783Skib if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1202235783Skib (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1203235783Skib (write_domain != 0 && read_domains != write_domain)) 1204235783Skib return (-EINVAL); 1205235783Skib 1206235783Skib ret = i915_mutex_lock_interruptible(dev); 1207235783Skib if (ret != 0) 1208235783Skib return (ret); 1209235783Skib 1210235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1211235783Skib if (&obj->base == NULL) { 1212235783Skib ret = -ENOENT; 1213235783Skib goto unlock; 1214235783Skib } 1215235783Skib 1216235783Skib if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1217235783Skib ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1218235783Skib if (ret == -EINVAL) 1219235783Skib ret = 0; 1220235783Skib } else 1221235783Skib ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1222235783Skib 1223235783Skib drm_gem_object_unreference(&obj->base); 1224235783Skibunlock: 1225235783Skib DRM_UNLOCK(dev); 1226235783Skib return (ret); 1227235783Skib} 1228235783Skib 1229235783Skibint 1230235783Skibi915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1231235783Skib struct drm_file *file) 1232235783Skib{ 1233235783Skib struct drm_i915_gem_sw_finish *args; 1234235783Skib struct drm_i915_gem_object *obj; 1235235783Skib int ret; 1236235783Skib 1237235783Skib args = data; 1238235783Skib ret = 0; 1239235783Skib if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1240235783Skib return (ENODEV); 1241235783Skib ret = i915_mutex_lock_interruptible(dev); 1242235783Skib if (ret != 0) 1243235783Skib return (ret); 1244235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1245235783Skib if (&obj->base == NULL) { 1246235783Skib ret = -ENOENT; 1247235783Skib goto unlock; 1248235783Skib } 1249235783Skib if (obj->pin_count != 0) 1250235783Skib i915_gem_object_flush_cpu_write_domain(obj); 1251235783Skib drm_gem_object_unreference(&obj->base); 1252235783Skibunlock: 1253235783Skib DRM_UNLOCK(dev); 1254235783Skib return (ret); 1255235783Skib} 1256235783Skib 1257235783Skibint 1258235783Skibi915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1259235783Skib struct drm_file *file) 1260235783Skib{ 1261235783Skib struct drm_i915_gem_mmap *args; 1262235783Skib struct drm_gem_object *obj; 1263235783Skib struct proc *p; 1264235783Skib vm_map_t map; 1265235783Skib vm_offset_t addr; 1266235783Skib vm_size_t size; 1267235783Skib int error, rv; 1268235783Skib 1269235783Skib args = data; 1270235783Skib 1271235783Skib if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1272235783Skib return (-ENODEV); 1273235783Skib 1274235783Skib obj = drm_gem_object_lookup(dev, file, args->handle); 1275235783Skib if (obj == NULL) 1276235783Skib return (-ENOENT); 1277235783Skib error = 0; 1278235783Skib if (args->size == 0) 1279235783Skib goto out; 1280235783Skib p = curproc; 1281235783Skib map = &p->p_vmspace->vm_map; 1282235783Skib size = round_page(args->size); 1283235783Skib PROC_LOCK(p); 1284235783Skib if (map->size + size > lim_cur(p, RLIMIT_VMEM)) { 1285235783Skib PROC_UNLOCK(p); 1286235783Skib error = ENOMEM; 1287235783Skib goto out; 1288235783Skib } 1289235783Skib PROC_UNLOCK(p); 1290235783Skib 1291235783Skib addr = 0; 1292235783Skib vm_object_reference(obj->vm_obj); 1293235783Skib DRM_UNLOCK(dev); 1294255426Sjhb rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1295253471Sjhb VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1296253497Skib VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1297235783Skib if (rv != KERN_SUCCESS) { 1298235783Skib vm_object_deallocate(obj->vm_obj); 1299235783Skib error = -vm_mmap_to_errno(rv); 1300235783Skib } else { 1301235783Skib args->addr_ptr = (uint64_t)addr; 1302235783Skib } 1303235783Skib DRM_LOCK(dev); 1304235783Skibout: 1305235783Skib drm_gem_object_unreference(obj); 1306235783Skib return (error); 1307235783Skib} 1308235783Skib 1309235783Skibstatic int 1310235783Skibi915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1311235783Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 1312235783Skib{ 1313235783Skib 1314235783Skib *color = 0; /* XXXKIB */ 1315235783Skib return (0); 1316235783Skib} 1317235783Skib 1318235783Skibint i915_intr_pf; 1319235783Skib 1320235783Skibstatic int 1321235783Skibi915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1322235783Skib vm_page_t *mres) 1323235783Skib{ 1324235783Skib struct drm_gem_object *gem_obj; 1325235783Skib struct drm_i915_gem_object *obj; 1326235783Skib struct drm_device *dev; 1327235783Skib drm_i915_private_t *dev_priv; 1328235783Skib vm_page_t m, oldm; 1329235783Skib int cause, ret; 1330235783Skib bool write; 1331235783Skib 1332235783Skib gem_obj = vm_obj->handle; 1333235783Skib obj = to_intel_bo(gem_obj); 1334235783Skib dev = obj->base.dev; 1335235783Skib dev_priv = dev->dev_private; 1336235783Skib#if 0 1337235783Skib write = (prot & VM_PROT_WRITE) != 0; 1338235783Skib#else 1339235783Skib write = true; 1340235783Skib#endif 1341235783Skib vm_object_pip_add(vm_obj, 1); 1342235783Skib 1343235783Skib /* 1344235783Skib * Remove the placeholder page inserted by vm_fault() from the 1345235783Skib * object before dropping the object lock. If 1346235783Skib * i915_gem_release_mmap() is active in parallel on this gem 1347235783Skib * object, then it owns the drm device sx and might find the 1348235783Skib * placeholder already. Then, since the page is busy, 1349235783Skib * i915_gem_release_mmap() sleeps waiting for the busy state 1350235783Skib * of the page cleared. We will be not able to acquire drm 1351235783Skib * device lock until i915_gem_release_mmap() is able to make a 1352235783Skib * progress. 1353235783Skib */ 1354235783Skib if (*mres != NULL) { 1355235783Skib oldm = *mres; 1356235783Skib vm_page_lock(oldm); 1357235783Skib vm_page_remove(oldm); 1358235783Skib vm_page_unlock(oldm); 1359235783Skib *mres = NULL; 1360235783Skib } else 1361235783Skib oldm = NULL; 1362254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1363235783Skibretry: 1364235783Skib cause = ret = 0; 1365235783Skib m = NULL; 1366235783Skib 1367235783Skib if (i915_intr_pf) { 1368235783Skib ret = i915_mutex_lock_interruptible(dev); 1369235783Skib if (ret != 0) { 1370235783Skib cause = 10; 1371235783Skib goto out; 1372235783Skib } 1373235783Skib } else 1374235783Skib DRM_LOCK(dev); 1375235783Skib 1376251960Skib /* 1377251960Skib * Since the object lock was dropped, other thread might have 1378251960Skib * faulted on the same GTT address and instantiated the 1379251960Skib * mapping for the page. Recheck. 1380251960Skib */ 1381251960Skib VM_OBJECT_WLOCK(vm_obj); 1382251960Skib m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1383251960Skib if (m != NULL) { 1384254138Sattilio if (vm_page_busied(m)) { 1385251960Skib DRM_UNLOCK(dev); 1386254138Sattilio vm_page_lock(m); 1387254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1388254138Sattilio vm_page_busy_sleep(m, "915pee"); 1389251960Skib goto retry; 1390251960Skib } 1391251960Skib goto have_page; 1392251960Skib } else 1393251960Skib VM_OBJECT_WUNLOCK(vm_obj); 1394251960Skib 1395235783Skib /* Now bind it into the GTT if needed */ 1396235783Skib if (!obj->map_and_fenceable) { 1397235783Skib ret = i915_gem_object_unbind(obj); 1398235783Skib if (ret != 0) { 1399235783Skib cause = 20; 1400235783Skib goto unlock; 1401235783Skib } 1402235783Skib } 1403235783Skib if (!obj->gtt_space) { 1404235783Skib ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1405235783Skib if (ret != 0) { 1406235783Skib cause = 30; 1407235783Skib goto unlock; 1408235783Skib } 1409235783Skib 1410235783Skib ret = i915_gem_object_set_to_gtt_domain(obj, write); 1411235783Skib if (ret != 0) { 1412235783Skib cause = 40; 1413235783Skib goto unlock; 1414235783Skib } 1415235783Skib } 1416235783Skib 1417235783Skib if (obj->tiling_mode == I915_TILING_NONE) 1418235783Skib ret = i915_gem_object_put_fence(obj); 1419235783Skib else 1420235783Skib ret = i915_gem_object_get_fence(obj, NULL); 1421235783Skib if (ret != 0) { 1422235783Skib cause = 50; 1423235783Skib goto unlock; 1424235783Skib } 1425235783Skib 1426235783Skib if (i915_gem_object_is_inactive(obj)) 1427235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1428235783Skib 1429235783Skib obj->fault_mappable = true; 1430248084Sattilio VM_OBJECT_WLOCK(vm_obj); 1431269634Sroyger m = PHYS_TO_VM_PAGE(dev->agp->base + obj->gtt_offset + offset); 1432269634Sroyger KASSERT((m->flags & PG_FICTITIOUS) != 0, 1433269634Sroyger ("physical address %#jx not fictitious", 1434269634Sroyger (uintmax_t)(dev->agp->base + obj->gtt_offset + offset))); 1435235783Skib if (m == NULL) { 1436265102Skib VM_OBJECT_WUNLOCK(vm_obj); 1437235783Skib cause = 60; 1438235783Skib ret = -EFAULT; 1439235783Skib goto unlock; 1440235783Skib } 1441235783Skib KASSERT((m->flags & PG_FICTITIOUS) != 0, 1442235783Skib ("not fictitious %p", m)); 1443235783Skib KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1444235783Skib 1445254138Sattilio if (vm_page_busied(m)) { 1446235783Skib DRM_UNLOCK(dev); 1447254138Sattilio vm_page_lock(m); 1448254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1449254138Sattilio vm_page_busy_sleep(m, "915pbs"); 1450235783Skib goto retry; 1451235783Skib } 1452254141Sattilio if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { 1453254141Sattilio DRM_UNLOCK(dev); 1454254141Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1455254141Sattilio VM_WAIT; 1456254141Sattilio goto retry; 1457254141Sattilio } 1458235783Skib m->valid = VM_PAGE_BITS_ALL; 1459251960Skibhave_page: 1460235783Skib *mres = m; 1461254138Sattilio vm_page_xbusy(m); 1462235783Skib 1463235783Skib CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 1464235783Skib m->phys_addr); 1465235783Skib DRM_UNLOCK(dev); 1466235783Skib if (oldm != NULL) { 1467235783Skib vm_page_lock(oldm); 1468235783Skib vm_page_free(oldm); 1469235783Skib vm_page_unlock(oldm); 1470235783Skib } 1471235783Skib vm_object_pip_wakeup(vm_obj); 1472235783Skib return (VM_PAGER_OK); 1473235783Skib 1474235783Skibunlock: 1475235783Skib DRM_UNLOCK(dev); 1476235783Skibout: 1477235783Skib KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1478235783Skib CTR5(KTR_DRM, "fault_fail %p %jx %x err %d %d", gem_obj, offset, prot, 1479235783Skib -ret, cause); 1480235783Skib if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1481235783Skib kern_yield(PRI_USER); 1482254138Sattilio goto retry; 1483235783Skib } 1484248084Sattilio VM_OBJECT_WLOCK(vm_obj); 1485235783Skib vm_object_pip_wakeup(vm_obj); 1486235783Skib return (VM_PAGER_ERROR); 1487235783Skib} 1488235783Skib 1489235783Skibstatic void 1490235783Skibi915_gem_pager_dtor(void *handle) 1491235783Skib{ 1492235783Skib struct drm_gem_object *obj; 1493235783Skib struct drm_device *dev; 1494235783Skib 1495235783Skib obj = handle; 1496235783Skib dev = obj->dev; 1497235783Skib 1498235783Skib DRM_LOCK(dev); 1499235783Skib drm_gem_free_mmap_offset(obj); 1500235783Skib i915_gem_release_mmap(to_intel_bo(obj)); 1501235783Skib drm_gem_object_unreference(obj); 1502235783Skib DRM_UNLOCK(dev); 1503235783Skib} 1504235783Skib 1505235783Skibstruct cdev_pager_ops i915_gem_pager_ops = { 1506235783Skib .cdev_pg_fault = i915_gem_pager_fault, 1507235783Skib .cdev_pg_ctor = i915_gem_pager_ctor, 1508235783Skib .cdev_pg_dtor = i915_gem_pager_dtor 1509235783Skib}; 1510235783Skib 1511235783Skibint 1512235783Skibi915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1513235783Skib uint32_t handle, uint64_t *offset) 1514235783Skib{ 1515235783Skib struct drm_i915_private *dev_priv; 1516235783Skib struct drm_i915_gem_object *obj; 1517235783Skib int ret; 1518235783Skib 1519235783Skib if (!(dev->driver->driver_features & DRIVER_GEM)) 1520235783Skib return (-ENODEV); 1521235783Skib 1522235783Skib dev_priv = dev->dev_private; 1523235783Skib 1524235783Skib ret = i915_mutex_lock_interruptible(dev); 1525235783Skib if (ret != 0) 1526235783Skib return (ret); 1527235783Skib 1528235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1529235783Skib if (&obj->base == NULL) { 1530235783Skib ret = -ENOENT; 1531235783Skib goto unlock; 1532235783Skib } 1533235783Skib 1534235783Skib if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1535235783Skib ret = -E2BIG; 1536235783Skib goto out; 1537235783Skib } 1538235783Skib 1539235783Skib if (obj->madv != I915_MADV_WILLNEED) { 1540235783Skib DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1541235783Skib ret = -EINVAL; 1542235783Skib goto out; 1543235783Skib } 1544235783Skib 1545235783Skib ret = drm_gem_create_mmap_offset(&obj->base); 1546235783Skib if (ret != 0) 1547235783Skib goto out; 1548235783Skib 1549235783Skib *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1550235783Skib DRM_GEM_MAPPING_KEY; 1551235783Skibout: 1552235783Skib drm_gem_object_unreference(&obj->base); 1553235783Skibunlock: 1554235783Skib DRM_UNLOCK(dev); 1555235783Skib return (ret); 1556235783Skib} 1557235783Skib 1558235783Skibint 1559235783Skibi915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1560235783Skib struct drm_file *file) 1561235783Skib{ 1562235783Skib struct drm_i915_private *dev_priv; 1563235783Skib struct drm_i915_gem_mmap_gtt *args; 1564235783Skib 1565235783Skib dev_priv = dev->dev_private; 1566235783Skib args = data; 1567235783Skib 1568235783Skib return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1569235783Skib} 1570235783Skib 1571235783Skibstruct drm_i915_gem_object * 1572235783Skibi915_gem_alloc_object(struct drm_device *dev, size_t size) 1573235783Skib{ 1574235783Skib struct drm_i915_private *dev_priv; 1575235783Skib struct drm_i915_gem_object *obj; 1576235783Skib 1577235783Skib dev_priv = dev->dev_private; 1578235783Skib 1579235783Skib obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1580235783Skib 1581235783Skib if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1582235783Skib free(obj, DRM_I915_GEM); 1583235783Skib return (NULL); 1584235783Skib } 1585235783Skib 1586235783Skib obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1587235783Skib obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1588235783Skib 1589235783Skib if (HAS_LLC(dev)) 1590235783Skib obj->cache_level = I915_CACHE_LLC; 1591235783Skib else 1592235783Skib obj->cache_level = I915_CACHE_NONE; 1593235783Skib obj->base.driver_private = NULL; 1594235783Skib obj->fence_reg = I915_FENCE_REG_NONE; 1595235783Skib INIT_LIST_HEAD(&obj->mm_list); 1596235783Skib INIT_LIST_HEAD(&obj->gtt_list); 1597235783Skib INIT_LIST_HEAD(&obj->ring_list); 1598235783Skib INIT_LIST_HEAD(&obj->exec_list); 1599235783Skib INIT_LIST_HEAD(&obj->gpu_write_list); 1600235783Skib obj->madv = I915_MADV_WILLNEED; 1601235783Skib /* Avoid an unnecessary call to unbind on the first bind. */ 1602235783Skib obj->map_and_fenceable = true; 1603235783Skib 1604235783Skib i915_gem_info_add_obj(dev_priv, size); 1605235783Skib 1606235783Skib return (obj); 1607235783Skib} 1608235783Skib 1609235783Skibvoid 1610235783Skibi915_gem_clflush_object(struct drm_i915_gem_object *obj) 1611235783Skib{ 1612235783Skib 1613235783Skib /* If we don't have a page list set up, then we're not pinned 1614235783Skib * to GPU, and we can ignore the cache flush because it'll happen 1615235783Skib * again at bind time. 1616235783Skib */ 1617235783Skib if (obj->pages == NULL) 1618235783Skib return; 1619235783Skib 1620235783Skib /* If the GPU is snooping the contents of the CPU cache, 1621235783Skib * we do not need to manually clear the CPU cache lines. However, 1622235783Skib * the caches are only snooped when the render cache is 1623235783Skib * flushed/invalidated. As we always have to emit invalidations 1624235783Skib * and flushes when moving into and out of the RENDER domain, correct 1625235783Skib * snooping behaviour occurs naturally as the result of our domain 1626235783Skib * tracking. 1627235783Skib */ 1628235783Skib if (obj->cache_level != I915_CACHE_NONE) 1629235783Skib return; 1630235783Skib 1631235783Skib CTR1(KTR_DRM, "object_clflush %p", obj); 1632235783Skib drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1633235783Skib} 1634235783Skib 1635235783Skibstatic void 1636235783Skibi915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1637235783Skib{ 1638235783Skib uint32_t old_write_domain; 1639235783Skib 1640235783Skib if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1641235783Skib return; 1642235783Skib 1643235783Skib i915_gem_clflush_object(obj); 1644235783Skib intel_gtt_chipset_flush(); 1645235783Skib old_write_domain = obj->base.write_domain; 1646235783Skib obj->base.write_domain = 0; 1647235783Skib 1648235783Skib CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 1649235783Skib obj->base.read_domains, old_write_domain); 1650235783Skib} 1651235783Skib 1652235783Skibstatic int 1653235783Skibi915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1654235783Skib{ 1655235783Skib 1656235783Skib if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1657235783Skib return (0); 1658235783Skib return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1659235783Skib} 1660235783Skib 1661235783Skibstatic void 1662235783Skibi915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1663235783Skib{ 1664235783Skib uint32_t old_write_domain; 1665235783Skib 1666235783Skib if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1667235783Skib return; 1668235783Skib 1669235783Skib wmb(); 1670235783Skib 1671235783Skib old_write_domain = obj->base.write_domain; 1672235783Skib obj->base.write_domain = 0; 1673235783Skib 1674235783Skib CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 1675235783Skib obj->base.read_domains, old_write_domain); 1676235783Skib} 1677235783Skib 1678235783Skibint 1679235783Skibi915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1680235783Skib{ 1681235783Skib uint32_t old_write_domain, old_read_domains; 1682235783Skib int ret; 1683235783Skib 1684235783Skib if (obj->gtt_space == NULL) 1685235783Skib return (-EINVAL); 1686235783Skib 1687235783Skib if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1688235783Skib return 0; 1689235783Skib 1690235783Skib ret = i915_gem_object_flush_gpu_write_domain(obj); 1691235783Skib if (ret != 0) 1692235783Skib return (ret); 1693235783Skib 1694235783Skib if (obj->pending_gpu_write || write) { 1695235783Skib ret = i915_gem_object_wait_rendering(obj); 1696235783Skib if (ret != 0) 1697235783Skib return (ret); 1698235783Skib } 1699235783Skib 1700235783Skib i915_gem_object_flush_cpu_write_domain(obj); 1701235783Skib 1702235783Skib old_write_domain = obj->base.write_domain; 1703235783Skib old_read_domains = obj->base.read_domains; 1704235783Skib 1705235783Skib KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1706235783Skib ("In GTT write domain")); 1707235783Skib obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1708235783Skib if (write) { 1709235783Skib obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1710235783Skib obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1711235783Skib obj->dirty = 1; 1712235783Skib } 1713235783Skib 1714235783Skib CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 1715235783Skib old_read_domains, old_write_domain); 1716235783Skib return (0); 1717235783Skib} 1718235783Skib 1719235783Skibint 1720235783Skibi915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1721235783Skib enum i915_cache_level cache_level) 1722235783Skib{ 1723235783Skib struct drm_device *dev; 1724235783Skib drm_i915_private_t *dev_priv; 1725235783Skib int ret; 1726235783Skib 1727235783Skib if (obj->cache_level == cache_level) 1728235783Skib return 0; 1729235783Skib 1730235783Skib if (obj->pin_count) { 1731235783Skib DRM_DEBUG("can not change the cache level of pinned objects\n"); 1732235783Skib return (-EBUSY); 1733235783Skib } 1734235783Skib 1735235783Skib dev = obj->base.dev; 1736235783Skib dev_priv = dev->dev_private; 1737235783Skib if (obj->gtt_space) { 1738235783Skib ret = i915_gem_object_finish_gpu(obj); 1739235783Skib if (ret != 0) 1740235783Skib return (ret); 1741235783Skib 1742235783Skib i915_gem_object_finish_gtt(obj); 1743235783Skib 1744235783Skib /* Before SandyBridge, you could not use tiling or fence 1745235783Skib * registers with snooped memory, so relinquish any fences 1746235783Skib * currently pointing to our region in the aperture. 1747235783Skib */ 1748235783Skib if (INTEL_INFO(obj->base.dev)->gen < 6) { 1749235783Skib ret = i915_gem_object_put_fence(obj); 1750235783Skib if (ret != 0) 1751235783Skib return (ret); 1752235783Skib } 1753235783Skib 1754235783Skib i915_gem_gtt_rebind_object(obj, cache_level); 1755235783Skib if (obj->has_aliasing_ppgtt_mapping) 1756235783Skib i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1757235783Skib obj, cache_level); 1758235783Skib } 1759235783Skib 1760235783Skib if (cache_level == I915_CACHE_NONE) { 1761235783Skib u32 old_read_domains, old_write_domain; 1762235783Skib 1763235783Skib /* If we're coming from LLC cached, then we haven't 1764235783Skib * actually been tracking whether the data is in the 1765235783Skib * CPU cache or not, since we only allow one bit set 1766235783Skib * in obj->write_domain and have been skipping the clflushes. 1767235783Skib * Just set it to the CPU cache for now. 1768235783Skib */ 1769235783Skib KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1770235783Skib ("obj %p in CPU write domain", obj)); 1771235783Skib KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1772235783Skib ("obj %p in CPU read domain", obj)); 1773235783Skib 1774235783Skib old_read_domains = obj->base.read_domains; 1775235783Skib old_write_domain = obj->base.write_domain; 1776235783Skib 1777235783Skib obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1778235783Skib obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1779235783Skib 1780235783Skib CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 1781235783Skib obj, old_read_domains, old_write_domain); 1782235783Skib } 1783235783Skib 1784235783Skib obj->cache_level = cache_level; 1785235783Skib return (0); 1786235783Skib} 1787235783Skib 1788235783Skibint 1789235783Skibi915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1790235783Skib u32 alignment, struct intel_ring_buffer *pipelined) 1791235783Skib{ 1792235783Skib u32 old_read_domains, old_write_domain; 1793235783Skib int ret; 1794235783Skib 1795235783Skib ret = i915_gem_object_flush_gpu_write_domain(obj); 1796235783Skib if (ret != 0) 1797235783Skib return (ret); 1798235783Skib 1799235783Skib if (pipelined != obj->ring) { 1800235783Skib ret = i915_gem_object_wait_rendering(obj); 1801235783Skib if (ret == -ERESTART || ret == -EINTR) 1802235783Skib return (ret); 1803235783Skib } 1804235783Skib 1805235783Skib ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1806235783Skib if (ret != 0) 1807235783Skib return (ret); 1808235783Skib 1809235783Skib ret = i915_gem_object_pin(obj, alignment, true); 1810235783Skib if (ret != 0) 1811235783Skib return (ret); 1812235783Skib 1813235783Skib i915_gem_object_flush_cpu_write_domain(obj); 1814235783Skib 1815235783Skib old_write_domain = obj->base.write_domain; 1816235783Skib old_read_domains = obj->base.read_domains; 1817235783Skib 1818235783Skib KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1819235783Skib ("obj %p in GTT write domain", obj)); 1820235783Skib obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1821235783Skib 1822235783Skib CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 1823235783Skib obj, old_read_domains, obj->base.write_domain); 1824235783Skib return (0); 1825235783Skib} 1826235783Skib 1827235783Skibint 1828235783Skibi915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1829235783Skib{ 1830235783Skib int ret; 1831235783Skib 1832235783Skib if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1833235783Skib return (0); 1834235783Skib 1835235783Skib if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1836235783Skib ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1837235783Skib if (ret != 0) 1838235783Skib return (ret); 1839235783Skib } 1840235783Skib 1841235783Skib ret = i915_gem_object_wait_rendering(obj); 1842235783Skib if (ret != 0) 1843235783Skib return (ret); 1844235783Skib 1845235783Skib obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1846235783Skib 1847235783Skib return (0); 1848235783Skib} 1849235783Skib 1850235783Skibstatic int 1851235783Skibi915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1852235783Skib{ 1853235783Skib uint32_t old_write_domain, old_read_domains; 1854235783Skib int ret; 1855235783Skib 1856235783Skib if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1857235783Skib return 0; 1858235783Skib 1859235783Skib ret = i915_gem_object_flush_gpu_write_domain(obj); 1860235783Skib if (ret != 0) 1861235783Skib return (ret); 1862235783Skib 1863235783Skib ret = i915_gem_object_wait_rendering(obj); 1864235783Skib if (ret != 0) 1865235783Skib return (ret); 1866235783Skib 1867235783Skib i915_gem_object_flush_gtt_write_domain(obj); 1868235783Skib i915_gem_object_set_to_full_cpu_read_domain(obj); 1869235783Skib 1870235783Skib old_write_domain = obj->base.write_domain; 1871235783Skib old_read_domains = obj->base.read_domains; 1872235783Skib 1873235783Skib if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1874235783Skib i915_gem_clflush_object(obj); 1875235783Skib obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1876235783Skib } 1877235783Skib 1878235783Skib KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1879235783Skib ("In cpu write domain")); 1880235783Skib 1881235783Skib if (write) { 1882235783Skib obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1883235783Skib obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1884235783Skib } 1885235783Skib 1886235783Skib CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 1887235783Skib old_read_domains, old_write_domain); 1888235783Skib return (0); 1889235783Skib} 1890235783Skib 1891235783Skibstatic void 1892235783Skibi915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1893235783Skib{ 1894235783Skib int i; 1895235783Skib 1896235783Skib if (obj->page_cpu_valid == NULL) 1897235783Skib return; 1898235783Skib 1899235783Skib if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1900235783Skib for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1901235783Skib if (obj->page_cpu_valid[i] != 0) 1902235783Skib continue; 1903235783Skib drm_clflush_pages(obj->pages + i, 1); 1904235783Skib } 1905235783Skib } 1906235783Skib 1907235783Skib free(obj->page_cpu_valid, DRM_I915_GEM); 1908235783Skib obj->page_cpu_valid = NULL; 1909235783Skib} 1910235783Skib 1911235783Skibstatic int 1912235783Skibi915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1913235783Skib uint64_t offset, uint64_t size) 1914235783Skib{ 1915235783Skib uint32_t old_read_domains; 1916235783Skib int i, ret; 1917235783Skib 1918235783Skib if (offset == 0 && size == obj->base.size) 1919235783Skib return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1920235783Skib 1921235783Skib ret = i915_gem_object_flush_gpu_write_domain(obj); 1922235783Skib if (ret != 0) 1923235783Skib return (ret); 1924235783Skib ret = i915_gem_object_wait_rendering(obj); 1925235783Skib if (ret != 0) 1926235783Skib return (ret); 1927235783Skib 1928235783Skib i915_gem_object_flush_gtt_write_domain(obj); 1929235783Skib 1930235783Skib if (obj->page_cpu_valid == NULL && 1931235783Skib (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1932235783Skib return (0); 1933235783Skib 1934235783Skib if (obj->page_cpu_valid == NULL) { 1935235783Skib obj->page_cpu_valid = malloc(obj->base.size / PAGE_SIZE, 1936235783Skib DRM_I915_GEM, M_WAITOK | M_ZERO); 1937235783Skib } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1938235783Skib memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1939235783Skib 1940235783Skib for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1941235783Skib i++) { 1942235783Skib if (obj->page_cpu_valid[i]) 1943235783Skib continue; 1944235783Skib drm_clflush_pages(obj->pages + i, 1); 1945235783Skib obj->page_cpu_valid[i] = 1; 1946235783Skib } 1947235783Skib 1948235783Skib KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1949235783Skib ("In gpu write domain")); 1950235783Skib 1951235783Skib old_read_domains = obj->base.read_domains; 1952235783Skib obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1953235783Skib 1954235783Skib CTR3(KTR_DRM, "object_change_domain set_cpu_read %p %x %x", obj, 1955235783Skib old_read_domains, obj->base.write_domain); 1956235783Skib return (0); 1957235783Skib} 1958235783Skib 1959235783Skibstatic uint32_t 1960235783Skibi915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1961235783Skib{ 1962235783Skib uint32_t gtt_size; 1963235783Skib 1964235783Skib if (INTEL_INFO(dev)->gen >= 4 || 1965235783Skib tiling_mode == I915_TILING_NONE) 1966235783Skib return (size); 1967235783Skib 1968235783Skib /* Previous chips need a power-of-two fence region when tiling */ 1969235783Skib if (INTEL_INFO(dev)->gen == 3) 1970235783Skib gtt_size = 1024*1024; 1971235783Skib else 1972235783Skib gtt_size = 512*1024; 1973235783Skib 1974235783Skib while (gtt_size < size) 1975235783Skib gtt_size <<= 1; 1976235783Skib 1977235783Skib return (gtt_size); 1978235783Skib} 1979235783Skib 1980235783Skib/** 1981235783Skib * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1982235783Skib * @obj: object to check 1983235783Skib * 1984235783Skib * Return the required GTT alignment for an object, taking into account 1985235783Skib * potential fence register mapping. 1986235783Skib */ 1987235783Skibstatic uint32_t 1988235783Skibi915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1989235783Skib int tiling_mode) 1990235783Skib{ 1991235783Skib 1992235783Skib /* 1993235783Skib * Minimum alignment is 4k (GTT page size), but might be greater 1994235783Skib * if a fence register is needed for the object. 1995235783Skib */ 1996235783Skib if (INTEL_INFO(dev)->gen >= 4 || 1997235783Skib tiling_mode == I915_TILING_NONE) 1998235783Skib return (4096); 1999235783Skib 2000235783Skib /* 2001235783Skib * Previous chips need to be aligned to the size of the smallest 2002235783Skib * fence register that can contain the object. 2003235783Skib */ 2004235783Skib return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2005235783Skib} 2006235783Skib 2007235783Skibuint32_t 2008235783Skibi915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 2009235783Skib int tiling_mode) 2010235783Skib{ 2011235783Skib 2012235783Skib if (tiling_mode == I915_TILING_NONE) 2013235783Skib return (4096); 2014235783Skib 2015235783Skib /* 2016235783Skib * Minimum alignment is 4k (GTT page size) for sane hw. 2017235783Skib */ 2018235783Skib if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 2019235783Skib return (4096); 2020235783Skib 2021235783Skib /* 2022235783Skib * Previous hardware however needs to be aligned to a power-of-two 2023235783Skib * tile height. The simplest method for determining this is to reuse 2024235783Skib * the power-of-tile object size. 2025235783Skib */ 2026235783Skib return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2027235783Skib} 2028235783Skib 2029235783Skibstatic int 2030235783Skibi915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2031235783Skib unsigned alignment, bool map_and_fenceable) 2032235783Skib{ 2033235783Skib struct drm_device *dev; 2034235783Skib struct drm_i915_private *dev_priv; 2035235783Skib struct drm_mm_node *free_space; 2036235783Skib uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2037235783Skib bool mappable, fenceable; 2038235783Skib int ret; 2039235783Skib 2040235783Skib dev = obj->base.dev; 2041235783Skib dev_priv = dev->dev_private; 2042235783Skib 2043235783Skib if (obj->madv != I915_MADV_WILLNEED) { 2044235783Skib DRM_ERROR("Attempting to bind a purgeable object\n"); 2045235783Skib return (-EINVAL); 2046235783Skib } 2047235783Skib 2048235783Skib fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2049235783Skib obj->tiling_mode); 2050235783Skib fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2051235783Skib obj->tiling_mode); 2052235783Skib unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2053235783Skib obj->base.size, obj->tiling_mode); 2054235783Skib if (alignment == 0) 2055235783Skib alignment = map_and_fenceable ? fence_alignment : 2056235783Skib unfenced_alignment; 2057235783Skib if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2058235783Skib DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2059235783Skib return (-EINVAL); 2060235783Skib } 2061235783Skib 2062235783Skib size = map_and_fenceable ? fence_size : obj->base.size; 2063235783Skib 2064235783Skib /* If the object is bigger than the entire aperture, reject it early 2065235783Skib * before evicting everything in a vain attempt to find space. 2066235783Skib */ 2067235783Skib if (obj->base.size > (map_and_fenceable ? 2068235783Skib dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2069235783Skib DRM_ERROR( 2070235783Skib"Attempting to bind an object larger than the aperture\n"); 2071235783Skib return (-E2BIG); 2072235783Skib } 2073235783Skib 2074235783Skib search_free: 2075235783Skib if (map_and_fenceable) 2076235783Skib free_space = drm_mm_search_free_in_range( 2077235783Skib &dev_priv->mm.gtt_space, size, alignment, 0, 2078235783Skib dev_priv->mm.gtt_mappable_end, 0); 2079235783Skib else 2080235783Skib free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2081235783Skib size, alignment, 0); 2082235783Skib if (free_space != NULL) { 2083235783Skib if (map_and_fenceable) 2084235783Skib obj->gtt_space = drm_mm_get_block_range_generic( 2085235783Skib free_space, size, alignment, 0, 2086235783Skib dev_priv->mm.gtt_mappable_end, 1); 2087235783Skib else 2088235783Skib obj->gtt_space = drm_mm_get_block_generic(free_space, 2089235783Skib size, alignment, 1); 2090235783Skib } 2091235783Skib if (obj->gtt_space == NULL) { 2092235783Skib ret = i915_gem_evict_something(dev, size, alignment, 2093235783Skib map_and_fenceable); 2094235783Skib if (ret != 0) 2095235783Skib return (ret); 2096235783Skib goto search_free; 2097235783Skib } 2098235783Skib ret = i915_gem_object_get_pages_gtt(obj, 0); 2099235783Skib if (ret != 0) { 2100235783Skib drm_mm_put_block(obj->gtt_space); 2101235783Skib obj->gtt_space = NULL; 2102235783Skib /* 2103235783Skib * i915_gem_object_get_pages_gtt() cannot return 2104254649Skib * ENOMEM, since we use vm_page_grab(). 2105235783Skib */ 2106235783Skib return (ret); 2107235783Skib } 2108235783Skib 2109235783Skib ret = i915_gem_gtt_bind_object(obj); 2110235783Skib if (ret != 0) { 2111235783Skib i915_gem_object_put_pages_gtt(obj); 2112235783Skib drm_mm_put_block(obj->gtt_space); 2113235783Skib obj->gtt_space = NULL; 2114235783Skib if (i915_gem_evict_everything(dev, false)) 2115235783Skib return (ret); 2116235783Skib goto search_free; 2117235783Skib } 2118235783Skib 2119235783Skib list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2120235783Skib list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2121235783Skib 2122235783Skib KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2123235783Skib ("Object in gpu read domain")); 2124235783Skib KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2125235783Skib ("Object in gpu write domain")); 2126235783Skib 2127235783Skib obj->gtt_offset = obj->gtt_space->start; 2128235783Skib 2129235783Skib fenceable = 2130235783Skib obj->gtt_space->size == fence_size && 2131235783Skib (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2132235783Skib 2133235783Skib mappable = 2134235783Skib obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2135235783Skib obj->map_and_fenceable = mappable && fenceable; 2136235783Skib 2137235783Skib CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 2138235783Skib obj->base.size, map_and_fenceable); 2139235783Skib return (0); 2140235783Skib} 2141235783Skib 2142235783Skibstatic void 2143235783Skibi915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2144235783Skib{ 2145235783Skib u32 old_write_domain, old_read_domains; 2146235783Skib 2147235783Skib /* Act a barrier for all accesses through the GTT */ 2148235783Skib mb(); 2149235783Skib 2150235783Skib /* Force a pagefault for domain tracking on next user access */ 2151235783Skib i915_gem_release_mmap(obj); 2152235783Skib 2153235783Skib if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2154235783Skib return; 2155235783Skib 2156235783Skib old_read_domains = obj->base.read_domains; 2157235783Skib old_write_domain = obj->base.write_domain; 2158235783Skib 2159235783Skib obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2160235783Skib obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2161235783Skib 2162235783Skib CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2163235783Skib obj, old_read_domains, old_write_domain); 2164235783Skib} 2165235783Skib 2166235783Skibint 2167235783Skibi915_gem_object_unbind(struct drm_i915_gem_object *obj) 2168235783Skib{ 2169235783Skib drm_i915_private_t *dev_priv; 2170235783Skib int ret; 2171235783Skib 2172235783Skib dev_priv = obj->base.dev->dev_private; 2173235783Skib ret = 0; 2174235783Skib if (obj->gtt_space == NULL) 2175235783Skib return (0); 2176235783Skib if (obj->pin_count != 0) { 2177235783Skib DRM_ERROR("Attempting to unbind pinned buffer\n"); 2178235783Skib return (-EINVAL); 2179235783Skib } 2180235783Skib 2181235783Skib ret = i915_gem_object_finish_gpu(obj); 2182235783Skib if (ret == -ERESTART || ret == -EINTR) 2183235783Skib return (ret); 2184235783Skib 2185235783Skib i915_gem_object_finish_gtt(obj); 2186235783Skib 2187235783Skib if (ret == 0) 2188235783Skib ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2189235783Skib if (ret == -ERESTART || ret == -EINTR) 2190235783Skib return (ret); 2191235783Skib if (ret != 0) { 2192235783Skib i915_gem_clflush_object(obj); 2193235783Skib obj->base.read_domains = obj->base.write_domain = 2194235783Skib I915_GEM_DOMAIN_CPU; 2195235783Skib } 2196235783Skib 2197235783Skib ret = i915_gem_object_put_fence(obj); 2198235783Skib if (ret == -ERESTART) 2199235783Skib return (ret); 2200235783Skib 2201235783Skib i915_gem_gtt_unbind_object(obj); 2202235783Skib if (obj->has_aliasing_ppgtt_mapping) { 2203235783Skib i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2204235783Skib obj->has_aliasing_ppgtt_mapping = 0; 2205235783Skib } 2206235783Skib i915_gem_object_put_pages_gtt(obj); 2207235783Skib 2208235783Skib list_del_init(&obj->gtt_list); 2209235783Skib list_del_init(&obj->mm_list); 2210235783Skib obj->map_and_fenceable = true; 2211235783Skib 2212235783Skib drm_mm_put_block(obj->gtt_space); 2213235783Skib obj->gtt_space = NULL; 2214235783Skib obj->gtt_offset = 0; 2215235783Skib 2216235783Skib if (i915_gem_object_is_purgeable(obj)) 2217235783Skib i915_gem_object_truncate(obj); 2218235783Skib CTR1(KTR_DRM, "object_unbind %p", obj); 2219235783Skib 2220235783Skib return (ret); 2221235783Skib} 2222235783Skib 2223235783Skibstatic int 2224235783Skibi915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2225235783Skib int flags) 2226235783Skib{ 2227235783Skib struct drm_device *dev; 2228235783Skib vm_object_t vm_obj; 2229235783Skib vm_page_t m; 2230235783Skib int page_count, i, j; 2231235783Skib 2232235783Skib dev = obj->base.dev; 2233235783Skib KASSERT(obj->pages == NULL, ("Obj already has pages")); 2234235783Skib page_count = obj->base.size / PAGE_SIZE; 2235235783Skib obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2236235783Skib M_WAITOK); 2237235783Skib vm_obj = obj->base.vm_obj; 2238248084Sattilio VM_OBJECT_WLOCK(vm_obj); 2239235783Skib for (i = 0; i < page_count; i++) { 2240235783Skib if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2241235783Skib goto failed; 2242235783Skib } 2243248084Sattilio VM_OBJECT_WUNLOCK(vm_obj); 2244235783Skib if (i915_gem_object_needs_bit17_swizzle(obj)) 2245235783Skib i915_gem_object_do_bit_17_swizzle(obj); 2246235783Skib return (0); 2247235783Skib 2248235783Skibfailed: 2249235783Skib for (j = 0; j < i; j++) { 2250235783Skib m = obj->pages[j]; 2251235783Skib vm_page_lock(m); 2252267548Sattilio vm_page_unwire(m, PQ_INACTIVE); 2253235783Skib vm_page_unlock(m); 2254235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2255235783Skib } 2256248084Sattilio VM_OBJECT_WUNLOCK(vm_obj); 2257235783Skib free(obj->pages, DRM_I915_GEM); 2258235783Skib obj->pages = NULL; 2259235783Skib return (-EIO); 2260235783Skib} 2261235783Skib 2262235783Skib#define GEM_PARANOID_CHECK_GTT 0 2263235783Skib#if GEM_PARANOID_CHECK_GTT 2264235783Skibstatic void 2265235783Skibi915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2266235783Skib int page_count) 2267235783Skib{ 2268235783Skib struct drm_i915_private *dev_priv; 2269235783Skib vm_paddr_t pa; 2270235783Skib unsigned long start, end; 2271235783Skib u_int i; 2272235783Skib int j; 2273235783Skib 2274235783Skib dev_priv = dev->dev_private; 2275235783Skib start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2276235783Skib end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2277235783Skib for (i = start; i < end; i++) { 2278235783Skib pa = intel_gtt_read_pte_paddr(i); 2279235783Skib for (j = 0; j < page_count; j++) { 2280235783Skib if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2281235783Skib panic("Page %p in GTT pte index %d pte %x", 2282235783Skib ma[i], i, intel_gtt_read_pte(i)); 2283235783Skib } 2284235783Skib } 2285235783Skib } 2286235783Skib} 2287235783Skib#endif 2288235783Skib 2289235783Skibstatic void 2290235783Skibi915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2291235783Skib{ 2292235783Skib vm_page_t m; 2293235783Skib int page_count, i; 2294235783Skib 2295235783Skib KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2296235783Skib 2297235783Skib if (obj->tiling_mode != I915_TILING_NONE) 2298235783Skib i915_gem_object_save_bit_17_swizzle(obj); 2299235783Skib if (obj->madv == I915_MADV_DONTNEED) 2300235783Skib obj->dirty = 0; 2301235783Skib page_count = obj->base.size / PAGE_SIZE; 2302248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 2303235783Skib#if GEM_PARANOID_CHECK_GTT 2304235783Skib i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2305235783Skib#endif 2306235783Skib for (i = 0; i < page_count; i++) { 2307235783Skib m = obj->pages[i]; 2308235783Skib if (obj->dirty) 2309235783Skib vm_page_dirty(m); 2310235783Skib if (obj->madv == I915_MADV_WILLNEED) 2311235783Skib vm_page_reference(m); 2312235783Skib vm_page_lock(m); 2313267548Sattilio vm_page_unwire(obj->pages[i], PQ_ACTIVE); 2314235783Skib vm_page_unlock(m); 2315235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2316235783Skib } 2317248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 2318235783Skib obj->dirty = 0; 2319235783Skib free(obj->pages, DRM_I915_GEM); 2320235783Skib obj->pages = NULL; 2321235783Skib} 2322235783Skib 2323235783Skibvoid 2324235783Skibi915_gem_release_mmap(struct drm_i915_gem_object *obj) 2325235783Skib{ 2326235783Skib vm_object_t devobj; 2327235783Skib vm_page_t m; 2328235783Skib int i, page_count; 2329235783Skib 2330235783Skib if (!obj->fault_mappable) 2331235783Skib return; 2332235783Skib 2333235783Skib CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 2334235783Skib OFF_TO_IDX(obj->base.size)); 2335235783Skib devobj = cdev_pager_lookup(obj); 2336235783Skib if (devobj != NULL) { 2337235783Skib page_count = OFF_TO_IDX(obj->base.size); 2338235783Skib 2339248084Sattilio VM_OBJECT_WLOCK(devobj); 2340235783Skibretry: 2341235783Skib for (i = 0; i < page_count; i++) { 2342235783Skib m = vm_page_lookup(devobj, i); 2343235783Skib if (m == NULL) 2344235783Skib continue; 2345254138Sattilio if (vm_page_sleep_if_busy(m, "915unm")) 2346235783Skib goto retry; 2347235783Skib cdev_pager_free_page(devobj, m); 2348235783Skib } 2349248084Sattilio VM_OBJECT_WUNLOCK(devobj); 2350235783Skib vm_object_deallocate(devobj); 2351235783Skib } 2352235783Skib 2353235783Skib obj->fault_mappable = false; 2354235783Skib} 2355235783Skib 2356235783Skibint 2357235783Skibi915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2358235783Skib{ 2359235783Skib int ret; 2360235783Skib 2361235783Skib KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2362235783Skib ("In GPU write domain")); 2363235783Skib 2364235783Skib CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj, 2365235783Skib obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset, 2366235783Skib obj->active, obj->last_rendering_seqno); 2367235783Skib if (obj->active) { 2368235783Skib ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2369235783Skib true); 2370235783Skib if (ret != 0) 2371235783Skib return (ret); 2372235783Skib } 2373235783Skib return (0); 2374235783Skib} 2375235783Skib 2376235783Skibvoid 2377235783Skibi915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2378235783Skib struct intel_ring_buffer *ring, uint32_t seqno) 2379235783Skib{ 2380235783Skib struct drm_device *dev = obj->base.dev; 2381235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2382235783Skib struct drm_i915_fence_reg *reg; 2383235783Skib 2384235783Skib obj->ring = ring; 2385235783Skib KASSERT(ring != NULL, ("NULL ring")); 2386235783Skib 2387235783Skib /* Add a reference if we're newly entering the active list. */ 2388235783Skib if (!obj->active) { 2389235783Skib drm_gem_object_reference(&obj->base); 2390235783Skib obj->active = 1; 2391235783Skib } 2392235783Skib 2393235783Skib /* Move from whatever list we were on to the tail of execution. */ 2394235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2395235783Skib list_move_tail(&obj->ring_list, &ring->active_list); 2396235783Skib 2397235783Skib obj->last_rendering_seqno = seqno; 2398235783Skib if (obj->fenced_gpu_access) { 2399235783Skib obj->last_fenced_seqno = seqno; 2400235783Skib obj->last_fenced_ring = ring; 2401235783Skib 2402235783Skib /* Bump MRU to take account of the delayed flush */ 2403235783Skib if (obj->fence_reg != I915_FENCE_REG_NONE) { 2404235783Skib reg = &dev_priv->fence_regs[obj->fence_reg]; 2405235783Skib list_move_tail(®->lru_list, 2406235783Skib &dev_priv->mm.fence_list); 2407235783Skib } 2408235783Skib } 2409235783Skib} 2410235783Skib 2411235783Skibstatic void 2412235783Skibi915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2413235783Skib{ 2414235783Skib list_del_init(&obj->ring_list); 2415235783Skib obj->last_rendering_seqno = 0; 2416235783Skib obj->last_fenced_seqno = 0; 2417235783Skib} 2418235783Skib 2419235783Skibstatic void 2420235783Skibi915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2421235783Skib{ 2422235783Skib struct drm_device *dev = obj->base.dev; 2423235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2424235783Skib 2425235783Skib KASSERT(obj->active, ("Object not active")); 2426235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2427235783Skib 2428235783Skib i915_gem_object_move_off_active(obj); 2429235783Skib} 2430235783Skib 2431235783Skibstatic void 2432235783Skibi915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2433235783Skib{ 2434235783Skib struct drm_device *dev = obj->base.dev; 2435235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2436235783Skib 2437235783Skib if (obj->pin_count != 0) 2438235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2439235783Skib else 2440235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2441235783Skib 2442235783Skib KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2443235783Skib KASSERT(obj->active, ("Object not active")); 2444235783Skib obj->ring = NULL; 2445235783Skib obj->last_fenced_ring = NULL; 2446235783Skib 2447235783Skib i915_gem_object_move_off_active(obj); 2448235783Skib obj->fenced_gpu_access = false; 2449235783Skib 2450235783Skib obj->active = 0; 2451235783Skib obj->pending_gpu_write = false; 2452235783Skib drm_gem_object_unreference(&obj->base); 2453235783Skib 2454235783Skib#if 1 2455235783Skib KIB_NOTYET(); 2456235783Skib#else 2457235783Skib WARN_ON(i915_verify_lists(dev)); 2458235783Skib#endif 2459235783Skib} 2460235783Skib 2461235783Skibstatic void 2462235783Skibi915_gem_object_truncate(struct drm_i915_gem_object *obj) 2463235783Skib{ 2464235783Skib vm_object_t vm_obj; 2465235783Skib 2466235783Skib vm_obj = obj->base.vm_obj; 2467248084Sattilio VM_OBJECT_WLOCK(vm_obj); 2468235783Skib vm_object_page_remove(vm_obj, 0, 0, false); 2469248084Sattilio VM_OBJECT_WUNLOCK(vm_obj); 2470235783Skib obj->madv = I915_MADV_PURGED_INTERNAL; 2471235783Skib} 2472235783Skib 2473235783Skibstatic inline int 2474235783Skibi915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2475235783Skib{ 2476235783Skib 2477235783Skib return (obj->madv == I915_MADV_DONTNEED); 2478235783Skib} 2479235783Skib 2480235783Skibstatic void 2481235783Skibi915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2482235783Skib uint32_t flush_domains) 2483235783Skib{ 2484235783Skib struct drm_i915_gem_object *obj, *next; 2485235783Skib uint32_t old_write_domain; 2486235783Skib 2487235783Skib list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2488235783Skib gpu_write_list) { 2489235783Skib if (obj->base.write_domain & flush_domains) { 2490235783Skib old_write_domain = obj->base.write_domain; 2491235783Skib obj->base.write_domain = 0; 2492235783Skib list_del_init(&obj->gpu_write_list); 2493235783Skib i915_gem_object_move_to_active(obj, ring, 2494235783Skib i915_gem_next_request_seqno(ring)); 2495235783Skib 2496235783Skib CTR3(KTR_DRM, "object_change_domain process_flush %p %x %x", 2497235783Skib obj, obj->base.read_domains, old_write_domain); 2498235783Skib } 2499235783Skib } 2500235783Skib} 2501235783Skib 2502235783Skibstatic int 2503235783Skibi915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2504235783Skib{ 2505235783Skib drm_i915_private_t *dev_priv; 2506235783Skib 2507235783Skib dev_priv = obj->base.dev->dev_private; 2508235783Skib return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2509235783Skib obj->tiling_mode != I915_TILING_NONE); 2510235783Skib} 2511235783Skib 2512235783Skibstatic vm_page_t 2513235783Skibi915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2514235783Skib{ 2515235783Skib vm_page_t m; 2516235783Skib int rv; 2517235783Skib 2518248084Sattilio VM_OBJECT_ASSERT_WLOCKED(object); 2519254649Skib m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 2520235783Skib if (m->valid != VM_PAGE_BITS_ALL) { 2521235783Skib if (vm_pager_has_page(object, pindex, NULL, NULL)) { 2522235783Skib rv = vm_pager_get_pages(object, &m, 1, 0); 2523235783Skib m = vm_page_lookup(object, pindex); 2524235783Skib if (m == NULL) 2525235783Skib return (NULL); 2526235783Skib if (rv != VM_PAGER_OK) { 2527235783Skib vm_page_lock(m); 2528235783Skib vm_page_free(m); 2529235783Skib vm_page_unlock(m); 2530235783Skib return (NULL); 2531235783Skib } 2532235783Skib } else { 2533235783Skib pmap_zero_page(m); 2534235783Skib m->valid = VM_PAGE_BITS_ALL; 2535235783Skib m->dirty = 0; 2536235783Skib } 2537235783Skib } 2538235783Skib vm_page_lock(m); 2539235783Skib vm_page_wire(m); 2540235783Skib vm_page_unlock(m); 2541254138Sattilio vm_page_xunbusy(m); 2542235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2543235783Skib return (m); 2544235783Skib} 2545235783Skib 2546235783Skibint 2547235783Skibi915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2548235783Skib uint32_t flush_domains) 2549235783Skib{ 2550235783Skib int ret; 2551235783Skib 2552235783Skib if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2553235783Skib return 0; 2554235783Skib 2555235783Skib CTR3(KTR_DRM, "ring_flush %s %x %x", ring->name, invalidate_domains, 2556235783Skib flush_domains); 2557235783Skib ret = ring->flush(ring, invalidate_domains, flush_domains); 2558235783Skib if (ret) 2559235783Skib return ret; 2560235783Skib 2561235783Skib if (flush_domains & I915_GEM_GPU_DOMAINS) 2562235783Skib i915_gem_process_flushing_list(ring, flush_domains); 2563235783Skib return 0; 2564235783Skib} 2565235783Skib 2566235783Skibstatic int 2567235783Skibi915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2568235783Skib{ 2569235783Skib int ret; 2570235783Skib 2571235783Skib if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2572235783Skib return 0; 2573235783Skib 2574235783Skib if (!list_empty(&ring->gpu_write_list)) { 2575235783Skib ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2576235783Skib I915_GEM_GPU_DOMAINS); 2577235783Skib if (ret != 0) 2578235783Skib return ret; 2579235783Skib } 2580235783Skib 2581235783Skib return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2582235783Skib do_retire)); 2583235783Skib} 2584235783Skib 2585235783Skibint 2586235783Skibi915_gpu_idle(struct drm_device *dev, bool do_retire) 2587235783Skib{ 2588235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2589235783Skib int ret, i; 2590235783Skib 2591235783Skib /* Flush everything onto the inactive list. */ 2592235783Skib for (i = 0; i < I915_NUM_RINGS; i++) { 2593235783Skib ret = i915_ring_idle(&dev_priv->rings[i], do_retire); 2594235783Skib if (ret) 2595235783Skib return ret; 2596235783Skib } 2597235783Skib 2598235783Skib return 0; 2599235783Skib} 2600235783Skib 2601235783Skibint 2602235783Skibi915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2603235783Skib{ 2604235783Skib drm_i915_private_t *dev_priv; 2605235783Skib struct drm_i915_gem_request *request; 2606235783Skib uint32_t ier; 2607235783Skib int flags, ret; 2608235783Skib bool recovery_complete; 2609235783Skib 2610235783Skib KASSERT(seqno != 0, ("Zero seqno")); 2611235783Skib 2612235783Skib dev_priv = ring->dev->dev_private; 2613235783Skib ret = 0; 2614235783Skib 2615235783Skib if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 2616235783Skib /* Give the error handler a chance to run. */ 2617235783Skib mtx_lock(&dev_priv->error_completion_lock); 2618235783Skib recovery_complete = (&dev_priv->error_completion) > 0; 2619235783Skib mtx_unlock(&dev_priv->error_completion_lock); 2620235783Skib return (recovery_complete ? -EIO : -EAGAIN); 2621235783Skib } 2622235783Skib 2623235783Skib if (seqno == ring->outstanding_lazy_request) { 2624235783Skib request = malloc(sizeof(*request), DRM_I915_GEM, 2625235783Skib M_WAITOK | M_ZERO); 2626235783Skib if (request == NULL) 2627235783Skib return (-ENOMEM); 2628235783Skib 2629235783Skib ret = i915_add_request(ring, NULL, request); 2630235783Skib if (ret != 0) { 2631235783Skib free(request, DRM_I915_GEM); 2632235783Skib return (ret); 2633235783Skib } 2634235783Skib 2635235783Skib seqno = request->seqno; 2636235783Skib } 2637235783Skib 2638235783Skib if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2639235783Skib if (HAS_PCH_SPLIT(ring->dev)) 2640235783Skib ier = I915_READ(DEIER) | I915_READ(GTIER); 2641235783Skib else 2642235783Skib ier = I915_READ(IER); 2643235783Skib if (!ier) { 2644235783Skib DRM_ERROR("something (likely vbetool) disabled " 2645235783Skib "interrupts, re-enabling\n"); 2646235783Skib ring->dev->driver->irq_preinstall(ring->dev); 2647235783Skib ring->dev->driver->irq_postinstall(ring->dev); 2648235783Skib } 2649235783Skib 2650235783Skib CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 2651235783Skib 2652235783Skib ring->waiting_seqno = seqno; 2653235783Skib mtx_lock(&ring->irq_lock); 2654235783Skib if (ring->irq_get(ring)) { 2655235783Skib flags = dev_priv->mm.interruptible ? PCATCH : 0; 2656235783Skib while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2657235783Skib && !atomic_load_acq_int(&dev_priv->mm.wedged) && 2658235783Skib ret == 0) { 2659235783Skib ret = -msleep(ring, &ring->irq_lock, flags, 2660235783Skib "915gwr", 0); 2661235783Skib } 2662235783Skib ring->irq_put(ring); 2663235783Skib mtx_unlock(&ring->irq_lock); 2664235783Skib } else { 2665235783Skib mtx_unlock(&ring->irq_lock); 2666235783Skib if (_intel_wait_for(ring->dev, 2667235783Skib i915_seqno_passed(ring->get_seqno(ring), seqno) || 2668235783Skib atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 2669235783Skib 0, "i915wrq") != 0) 2670235783Skib ret = -EBUSY; 2671235783Skib } 2672235783Skib ring->waiting_seqno = 0; 2673235783Skib 2674235783Skib CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, 2675235783Skib ret); 2676235783Skib } 2677235783Skib if (atomic_load_acq_int(&dev_priv->mm.wedged)) 2678235783Skib ret = -EAGAIN; 2679235783Skib 2680235783Skib /* Directly dispatch request retiring. While we have the work queue 2681235783Skib * to handle this, the waiter on a request often wants an associated 2682235783Skib * buffer to have made it to the inactive list, and we would need 2683235783Skib * a separate wait queue to handle that. 2684235783Skib */ 2685235783Skib if (ret == 0 && do_retire) 2686235783Skib i915_gem_retire_requests_ring(ring); 2687235783Skib 2688235783Skib return (ret); 2689235783Skib} 2690235783Skib 2691235783Skibstatic u32 2692235783Skibi915_gem_get_seqno(struct drm_device *dev) 2693235783Skib{ 2694235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2695235783Skib u32 seqno = dev_priv->next_seqno; 2696235783Skib 2697235783Skib /* reserve 0 for non-seqno */ 2698235783Skib if (++dev_priv->next_seqno == 0) 2699235783Skib dev_priv->next_seqno = 1; 2700235783Skib 2701235783Skib return seqno; 2702235783Skib} 2703235783Skib 2704235783Skibu32 2705235783Skibi915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2706235783Skib{ 2707235783Skib if (ring->outstanding_lazy_request == 0) 2708235783Skib ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2709235783Skib 2710235783Skib return ring->outstanding_lazy_request; 2711235783Skib} 2712235783Skib 2713235783Skibint 2714235783Skibi915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2715235783Skib struct drm_i915_gem_request *request) 2716235783Skib{ 2717235783Skib drm_i915_private_t *dev_priv; 2718235783Skib struct drm_i915_file_private *file_priv; 2719235783Skib uint32_t seqno; 2720235783Skib u32 request_ring_position; 2721235783Skib int was_empty; 2722235783Skib int ret; 2723235783Skib 2724235783Skib KASSERT(request != NULL, ("NULL request in add")); 2725235783Skib DRM_LOCK_ASSERT(ring->dev); 2726235783Skib dev_priv = ring->dev->dev_private; 2727235783Skib 2728235783Skib seqno = i915_gem_next_request_seqno(ring); 2729235783Skib request_ring_position = intel_ring_get_tail(ring); 2730235783Skib 2731235783Skib ret = ring->add_request(ring, &seqno); 2732235783Skib if (ret != 0) 2733235783Skib return ret; 2734235783Skib 2735235783Skib CTR2(KTR_DRM, "request_add %s %d", ring->name, seqno); 2736235783Skib 2737235783Skib request->seqno = seqno; 2738235783Skib request->ring = ring; 2739235783Skib request->tail = request_ring_position; 2740235783Skib request->emitted_jiffies = ticks; 2741235783Skib was_empty = list_empty(&ring->request_list); 2742235783Skib list_add_tail(&request->list, &ring->request_list); 2743235783Skib 2744235783Skib if (file != NULL) { 2745235783Skib file_priv = file->driver_priv; 2746235783Skib 2747235783Skib mtx_lock(&file_priv->mm.lck); 2748235783Skib request->file_priv = file_priv; 2749235783Skib list_add_tail(&request->client_list, 2750235783Skib &file_priv->mm.request_list); 2751235783Skib mtx_unlock(&file_priv->mm.lck); 2752235783Skib } 2753235783Skib 2754235783Skib ring->outstanding_lazy_request = 0; 2755235783Skib 2756235783Skib if (!dev_priv->mm.suspended) { 2757235783Skib if (i915_enable_hangcheck) { 2758235783Skib callout_schedule(&dev_priv->hangcheck_timer, 2759235783Skib DRM_I915_HANGCHECK_PERIOD); 2760235783Skib } 2761235783Skib if (was_empty) 2762235783Skib taskqueue_enqueue_timeout(dev_priv->tq, 2763235783Skib &dev_priv->mm.retire_task, hz); 2764235783Skib } 2765235783Skib return (0); 2766235783Skib} 2767235783Skib 2768235783Skibstatic inline void 2769235783Skibi915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2770235783Skib{ 2771235783Skib struct drm_i915_file_private *file_priv = request->file_priv; 2772235783Skib 2773235783Skib if (!file_priv) 2774235783Skib return; 2775235783Skib 2776235783Skib DRM_LOCK_ASSERT(request->ring->dev); 2777235783Skib 2778235783Skib mtx_lock(&file_priv->mm.lck); 2779235783Skib if (request->file_priv != NULL) { 2780235783Skib list_del(&request->client_list); 2781235783Skib request->file_priv = NULL; 2782235783Skib } 2783235783Skib mtx_unlock(&file_priv->mm.lck); 2784235783Skib} 2785235783Skib 2786235783Skibvoid 2787235783Skibi915_gem_release(struct drm_device *dev, struct drm_file *file) 2788235783Skib{ 2789235783Skib struct drm_i915_file_private *file_priv; 2790235783Skib struct drm_i915_gem_request *request; 2791235783Skib 2792235783Skib file_priv = file->driver_priv; 2793235783Skib 2794235783Skib /* Clean up our request list when the client is going away, so that 2795235783Skib * later retire_requests won't dereference our soon-to-be-gone 2796235783Skib * file_priv. 2797235783Skib */ 2798235783Skib mtx_lock(&file_priv->mm.lck); 2799235783Skib while (!list_empty(&file_priv->mm.request_list)) { 2800235783Skib request = list_first_entry(&file_priv->mm.request_list, 2801235783Skib struct drm_i915_gem_request, 2802235783Skib client_list); 2803235783Skib list_del(&request->client_list); 2804235783Skib request->file_priv = NULL; 2805235783Skib } 2806235783Skib mtx_unlock(&file_priv->mm.lck); 2807235783Skib} 2808235783Skib 2809235783Skibstatic void 2810235783Skibi915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2811235783Skib struct intel_ring_buffer *ring) 2812235783Skib{ 2813235783Skib 2814235783Skib if (ring->dev != NULL) 2815235783Skib DRM_LOCK_ASSERT(ring->dev); 2816235783Skib 2817235783Skib while (!list_empty(&ring->request_list)) { 2818235783Skib struct drm_i915_gem_request *request; 2819235783Skib 2820235783Skib request = list_first_entry(&ring->request_list, 2821235783Skib struct drm_i915_gem_request, list); 2822235783Skib 2823235783Skib list_del(&request->list); 2824235783Skib i915_gem_request_remove_from_client(request); 2825235783Skib free(request, DRM_I915_GEM); 2826235783Skib } 2827235783Skib 2828235783Skib while (!list_empty(&ring->active_list)) { 2829235783Skib struct drm_i915_gem_object *obj; 2830235783Skib 2831235783Skib obj = list_first_entry(&ring->active_list, 2832235783Skib struct drm_i915_gem_object, ring_list); 2833235783Skib 2834235783Skib obj->base.write_domain = 0; 2835235783Skib list_del_init(&obj->gpu_write_list); 2836235783Skib i915_gem_object_move_to_inactive(obj); 2837235783Skib } 2838235783Skib} 2839235783Skib 2840235783Skibstatic void 2841235783Skibi915_gem_reset_fences(struct drm_device *dev) 2842235783Skib{ 2843235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2844235783Skib int i; 2845235783Skib 2846235783Skib for (i = 0; i < dev_priv->num_fence_regs; i++) { 2847235783Skib struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2848235783Skib struct drm_i915_gem_object *obj = reg->obj; 2849235783Skib 2850235783Skib if (!obj) 2851235783Skib continue; 2852235783Skib 2853235783Skib if (obj->tiling_mode) 2854235783Skib i915_gem_release_mmap(obj); 2855235783Skib 2856235783Skib reg->obj->fence_reg = I915_FENCE_REG_NONE; 2857235783Skib reg->obj->fenced_gpu_access = false; 2858235783Skib reg->obj->last_fenced_seqno = 0; 2859235783Skib reg->obj->last_fenced_ring = NULL; 2860235783Skib i915_gem_clear_fence_reg(dev, reg); 2861235783Skib } 2862235783Skib} 2863235783Skib 2864235783Skibvoid 2865235783Skibi915_gem_reset(struct drm_device *dev) 2866235783Skib{ 2867235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2868235783Skib struct drm_i915_gem_object *obj; 2869235783Skib int i; 2870235783Skib 2871235783Skib for (i = 0; i < I915_NUM_RINGS; i++) 2872235783Skib i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); 2873235783Skib 2874235783Skib /* Remove anything from the flushing lists. The GPU cache is likely 2875235783Skib * to be lost on reset along with the data, so simply move the 2876235783Skib * lost bo to the inactive list. 2877235783Skib */ 2878235783Skib while (!list_empty(&dev_priv->mm.flushing_list)) { 2879235783Skib obj = list_first_entry(&dev_priv->mm.flushing_list, 2880235783Skib struct drm_i915_gem_object, 2881235783Skib mm_list); 2882235783Skib 2883235783Skib obj->base.write_domain = 0; 2884235783Skib list_del_init(&obj->gpu_write_list); 2885235783Skib i915_gem_object_move_to_inactive(obj); 2886235783Skib } 2887235783Skib 2888235783Skib /* Move everything out of the GPU domains to ensure we do any 2889235783Skib * necessary invalidation upon reuse. 2890235783Skib */ 2891235783Skib list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2892235783Skib obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2893235783Skib } 2894235783Skib 2895235783Skib /* The fence registers are invalidated so clear them out */ 2896235783Skib i915_gem_reset_fences(dev); 2897235783Skib} 2898235783Skib 2899235783Skib/** 2900235783Skib * This function clears the request list as sequence numbers are passed. 2901235783Skib */ 2902235783Skibvoid 2903235783Skibi915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2904235783Skib{ 2905235783Skib uint32_t seqno; 2906235783Skib int i; 2907235783Skib 2908235783Skib if (list_empty(&ring->request_list)) 2909235783Skib return; 2910235783Skib 2911235783Skib seqno = ring->get_seqno(ring); 2912235783Skib CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 2913235783Skib 2914235783Skib for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2915235783Skib if (seqno >= ring->sync_seqno[i]) 2916235783Skib ring->sync_seqno[i] = 0; 2917235783Skib 2918235783Skib while (!list_empty(&ring->request_list)) { 2919235783Skib struct drm_i915_gem_request *request; 2920235783Skib 2921235783Skib request = list_first_entry(&ring->request_list, 2922235783Skib struct drm_i915_gem_request, 2923235783Skib list); 2924235783Skib 2925235783Skib if (!i915_seqno_passed(seqno, request->seqno)) 2926235783Skib break; 2927235783Skib 2928235783Skib CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 2929235783Skib ring->name, seqno); 2930235783Skib ring->last_retired_head = request->tail; 2931235783Skib 2932235783Skib list_del(&request->list); 2933235783Skib i915_gem_request_remove_from_client(request); 2934235783Skib free(request, DRM_I915_GEM); 2935235783Skib } 2936235783Skib 2937235783Skib /* Move any buffers on the active list that are no longer referenced 2938235783Skib * by the ringbuffer to the flushing/inactive lists as appropriate. 2939235783Skib */ 2940235783Skib while (!list_empty(&ring->active_list)) { 2941235783Skib struct drm_i915_gem_object *obj; 2942235783Skib 2943235783Skib obj = list_first_entry(&ring->active_list, 2944235783Skib struct drm_i915_gem_object, 2945235783Skib ring_list); 2946235783Skib 2947235783Skib if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2948235783Skib break; 2949235783Skib 2950235783Skib if (obj->base.write_domain != 0) 2951235783Skib i915_gem_object_move_to_flushing(obj); 2952235783Skib else 2953235783Skib i915_gem_object_move_to_inactive(obj); 2954235783Skib } 2955235783Skib 2956235783Skib if (ring->trace_irq_seqno && 2957235783Skib i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2958235783Skib mtx_lock(&ring->irq_lock); 2959235783Skib ring->irq_put(ring); 2960235783Skib mtx_unlock(&ring->irq_lock); 2961235783Skib ring->trace_irq_seqno = 0; 2962235783Skib } 2963235783Skib} 2964235783Skib 2965235783Skibvoid 2966235783Skibi915_gem_retire_requests(struct drm_device *dev) 2967235783Skib{ 2968235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2969235783Skib struct drm_i915_gem_object *obj, *next; 2970235783Skib int i; 2971235783Skib 2972235783Skib if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2973235783Skib list_for_each_entry_safe(obj, next, 2974235783Skib &dev_priv->mm.deferred_free_list, mm_list) 2975235783Skib i915_gem_free_object_tail(obj); 2976235783Skib } 2977235783Skib 2978235783Skib for (i = 0; i < I915_NUM_RINGS; i++) 2979235783Skib i915_gem_retire_requests_ring(&dev_priv->rings[i]); 2980235783Skib} 2981235783Skib 2982235783Skibstatic int 2983235783Skibsandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2984235783Skib struct intel_ring_buffer *pipelined) 2985235783Skib{ 2986235783Skib struct drm_device *dev = obj->base.dev; 2987235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2988235783Skib u32 size = obj->gtt_space->size; 2989235783Skib int regnum = obj->fence_reg; 2990235783Skib uint64_t val; 2991235783Skib 2992235783Skib val = (uint64_t)((obj->gtt_offset + size - 4096) & 2993235783Skib 0xfffff000) << 32; 2994235783Skib val |= obj->gtt_offset & 0xfffff000; 2995235783Skib val |= (uint64_t)((obj->stride / 128) - 1) << 2996235783Skib SANDYBRIDGE_FENCE_PITCH_SHIFT; 2997235783Skib 2998235783Skib if (obj->tiling_mode == I915_TILING_Y) 2999235783Skib val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3000235783Skib val |= I965_FENCE_REG_VALID; 3001235783Skib 3002235783Skib if (pipelined) { 3003235783Skib int ret = intel_ring_begin(pipelined, 6); 3004235783Skib if (ret) 3005235783Skib return ret; 3006235783Skib 3007235783Skib intel_ring_emit(pipelined, MI_NOOP); 3008235783Skib intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 3009235783Skib intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 3010235783Skib intel_ring_emit(pipelined, (u32)val); 3011235783Skib intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 3012235783Skib intel_ring_emit(pipelined, (u32)(val >> 32)); 3013235783Skib intel_ring_advance(pipelined); 3014235783Skib } else 3015235783Skib I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 3016235783Skib 3017235783Skib return 0; 3018235783Skib} 3019235783Skib 3020235783Skibstatic int 3021235783Skibi965_write_fence_reg(struct drm_i915_gem_object *obj, 3022235783Skib struct intel_ring_buffer *pipelined) 3023235783Skib{ 3024235783Skib struct drm_device *dev = obj->base.dev; 3025235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 3026235783Skib u32 size = obj->gtt_space->size; 3027235783Skib int regnum = obj->fence_reg; 3028235783Skib uint64_t val; 3029235783Skib 3030235783Skib val = (uint64_t)((obj->gtt_offset + size - 4096) & 3031235783Skib 0xfffff000) << 32; 3032235783Skib val |= obj->gtt_offset & 0xfffff000; 3033235783Skib val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 3034235783Skib if (obj->tiling_mode == I915_TILING_Y) 3035235783Skib val |= 1 << I965_FENCE_TILING_Y_SHIFT; 3036235783Skib val |= I965_FENCE_REG_VALID; 3037235783Skib 3038235783Skib if (pipelined) { 3039235783Skib int ret = intel_ring_begin(pipelined, 6); 3040235783Skib if (ret) 3041235783Skib return ret; 3042235783Skib 3043235783Skib intel_ring_emit(pipelined, MI_NOOP); 3044235783Skib intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 3045235783Skib intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 3046235783Skib intel_ring_emit(pipelined, (u32)val); 3047235783Skib intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 3048235783Skib intel_ring_emit(pipelined, (u32)(val >> 32)); 3049235783Skib intel_ring_advance(pipelined); 3050235783Skib } else 3051235783Skib I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 3052235783Skib 3053235783Skib return 0; 3054235783Skib} 3055235783Skib 3056235783Skibstatic int 3057235783Skibi915_write_fence_reg(struct drm_i915_gem_object *obj, 3058235783Skib struct intel_ring_buffer *pipelined) 3059235783Skib{ 3060235783Skib struct drm_device *dev = obj->base.dev; 3061235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 3062235783Skib u32 size = obj->gtt_space->size; 3063235783Skib u32 fence_reg, val, pitch_val; 3064235783Skib int tile_width; 3065235783Skib 3066235783Skib if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3067235783Skib (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3068235783Skib printf( 3069235783Skib"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3070235783Skib obj->gtt_offset, obj->map_and_fenceable, size); 3071235783Skib return -EINVAL; 3072235783Skib } 3073235783Skib 3074235783Skib if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3075235783Skib tile_width = 128; 3076235783Skib else 3077235783Skib tile_width = 512; 3078235783Skib 3079235783Skib /* Note: pitch better be a power of two tile widths */ 3080235783Skib pitch_val = obj->stride / tile_width; 3081235783Skib pitch_val = ffs(pitch_val) - 1; 3082235783Skib 3083235783Skib val = obj->gtt_offset; 3084235783Skib if (obj->tiling_mode == I915_TILING_Y) 3085235783Skib val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3086235783Skib val |= I915_FENCE_SIZE_BITS(size); 3087235783Skib val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3088235783Skib val |= I830_FENCE_REG_VALID; 3089235783Skib 3090235783Skib fence_reg = obj->fence_reg; 3091235783Skib if (fence_reg < 8) 3092235783Skib fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3093235783Skib else 3094235783Skib fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3095235783Skib 3096235783Skib if (pipelined) { 3097235783Skib int ret = intel_ring_begin(pipelined, 4); 3098235783Skib if (ret) 3099235783Skib return ret; 3100235783Skib 3101235783Skib intel_ring_emit(pipelined, MI_NOOP); 3102235783Skib intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3103235783Skib intel_ring_emit(pipelined, fence_reg); 3104235783Skib intel_ring_emit(pipelined, val); 3105235783Skib intel_ring_advance(pipelined); 3106235783Skib } else 3107235783Skib I915_WRITE(fence_reg, val); 3108235783Skib 3109235783Skib return 0; 3110235783Skib} 3111235783Skib 3112235783Skibstatic int 3113235783Skibi830_write_fence_reg(struct drm_i915_gem_object *obj, 3114235783Skib struct intel_ring_buffer *pipelined) 3115235783Skib{ 3116235783Skib struct drm_device *dev = obj->base.dev; 3117235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 3118235783Skib u32 size = obj->gtt_space->size; 3119235783Skib int regnum = obj->fence_reg; 3120235783Skib uint32_t val; 3121235783Skib uint32_t pitch_val; 3122235783Skib 3123235783Skib if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3124235783Skib (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3125235783Skib printf( 3126235783Skib"object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3127235783Skib obj->gtt_offset, size); 3128235783Skib return -EINVAL; 3129235783Skib } 3130235783Skib 3131235783Skib pitch_val = obj->stride / 128; 3132235783Skib pitch_val = ffs(pitch_val) - 1; 3133235783Skib 3134235783Skib val = obj->gtt_offset; 3135235783Skib if (obj->tiling_mode == I915_TILING_Y) 3136235783Skib val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3137235783Skib val |= I830_FENCE_SIZE_BITS(size); 3138235783Skib val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3139235783Skib val |= I830_FENCE_REG_VALID; 3140235783Skib 3141235783Skib if (pipelined) { 3142235783Skib int ret = intel_ring_begin(pipelined, 4); 3143235783Skib if (ret) 3144235783Skib return ret; 3145235783Skib 3146235783Skib intel_ring_emit(pipelined, MI_NOOP); 3147235783Skib intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3148235783Skib intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 3149235783Skib intel_ring_emit(pipelined, val); 3150235783Skib intel_ring_advance(pipelined); 3151235783Skib } else 3152235783Skib I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 3153235783Skib 3154235783Skib return 0; 3155235783Skib} 3156235783Skib 3157235783Skibstatic bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 3158235783Skib{ 3159235783Skib return i915_seqno_passed(ring->get_seqno(ring), seqno); 3160235783Skib} 3161235783Skib 3162235783Skibstatic int 3163235783Skibi915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 3164235783Skib struct intel_ring_buffer *pipelined) 3165235783Skib{ 3166235783Skib int ret; 3167235783Skib 3168235783Skib if (obj->fenced_gpu_access) { 3169235783Skib if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3170235783Skib ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3171235783Skib obj->base.write_domain); 3172235783Skib if (ret) 3173235783Skib return ret; 3174235783Skib } 3175235783Skib 3176235783Skib obj->fenced_gpu_access = false; 3177235783Skib } 3178235783Skib 3179235783Skib if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3180235783Skib if (!ring_passed_seqno(obj->last_fenced_ring, 3181235783Skib obj->last_fenced_seqno)) { 3182235783Skib ret = i915_wait_request(obj->last_fenced_ring, 3183235783Skib obj->last_fenced_seqno, 3184235783Skib true); 3185235783Skib if (ret) 3186235783Skib return ret; 3187235783Skib } 3188235783Skib 3189235783Skib obj->last_fenced_seqno = 0; 3190235783Skib obj->last_fenced_ring = NULL; 3191235783Skib } 3192235783Skib 3193235783Skib /* Ensure that all CPU reads are completed before installing a fence 3194235783Skib * and all writes before removing the fence. 3195235783Skib */ 3196235783Skib if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3197235783Skib mb(); 3198235783Skib 3199235783Skib return 0; 3200235783Skib} 3201235783Skib 3202235783Skibint 3203235783Skibi915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3204235783Skib{ 3205235783Skib int ret; 3206235783Skib 3207235783Skib if (obj->tiling_mode) 3208235783Skib i915_gem_release_mmap(obj); 3209235783Skib 3210235783Skib ret = i915_gem_object_flush_fence(obj, NULL); 3211235783Skib if (ret) 3212235783Skib return ret; 3213235783Skib 3214235783Skib if (obj->fence_reg != I915_FENCE_REG_NONE) { 3215235783Skib struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3216235783Skib 3217235783Skib if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3218235783Skib printf("%s: pin_count %d\n", __func__, 3219235783Skib dev_priv->fence_regs[obj->fence_reg].pin_count); 3220235783Skib i915_gem_clear_fence_reg(obj->base.dev, 3221235783Skib &dev_priv->fence_regs[obj->fence_reg]); 3222235783Skib 3223235783Skib obj->fence_reg = I915_FENCE_REG_NONE; 3224235783Skib } 3225235783Skib 3226235783Skib return 0; 3227235783Skib} 3228235783Skib 3229235783Skibstatic struct drm_i915_fence_reg * 3230235783Skibi915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3231235783Skib{ 3232235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 3233235783Skib struct drm_i915_fence_reg *reg, *first, *avail; 3234235783Skib int i; 3235235783Skib 3236235783Skib /* First try to find a free reg */ 3237235783Skib avail = NULL; 3238235783Skib for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3239235783Skib reg = &dev_priv->fence_regs[i]; 3240235783Skib if (!reg->obj) 3241235783Skib return reg; 3242235783Skib 3243235783Skib if (!reg->pin_count) 3244235783Skib avail = reg; 3245235783Skib } 3246235783Skib 3247235783Skib if (avail == NULL) 3248235783Skib return NULL; 3249235783Skib 3250235783Skib /* None available, try to steal one or wait for a user to finish */ 3251235783Skib avail = first = NULL; 3252235783Skib list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3253235783Skib if (reg->pin_count) 3254235783Skib continue; 3255235783Skib 3256235783Skib if (first == NULL) 3257235783Skib first = reg; 3258235783Skib 3259235783Skib if (!pipelined || 3260235783Skib !reg->obj->last_fenced_ring || 3261235783Skib reg->obj->last_fenced_ring == pipelined) { 3262235783Skib avail = reg; 3263235783Skib break; 3264235783Skib } 3265235783Skib } 3266235783Skib 3267235783Skib if (avail == NULL) 3268235783Skib avail = first; 3269235783Skib 3270235783Skib return avail; 3271235783Skib} 3272235783Skib 3273235783Skibint 3274235783Skibi915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3275235783Skib struct intel_ring_buffer *pipelined) 3276235783Skib{ 3277235783Skib struct drm_device *dev = obj->base.dev; 3278235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 3279235783Skib struct drm_i915_fence_reg *reg; 3280235783Skib int ret; 3281235783Skib 3282235783Skib pipelined = NULL; 3283235783Skib ret = 0; 3284235783Skib 3285235783Skib if (obj->fence_reg != I915_FENCE_REG_NONE) { 3286235783Skib reg = &dev_priv->fence_regs[obj->fence_reg]; 3287235783Skib list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3288235783Skib 3289235783Skib if (obj->tiling_changed) { 3290235783Skib ret = i915_gem_object_flush_fence(obj, pipelined); 3291235783Skib if (ret) 3292235783Skib return ret; 3293235783Skib 3294235783Skib if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3295235783Skib pipelined = NULL; 3296235783Skib 3297235783Skib if (pipelined) { 3298235783Skib reg->setup_seqno = 3299235783Skib i915_gem_next_request_seqno(pipelined); 3300235783Skib obj->last_fenced_seqno = reg->setup_seqno; 3301235783Skib obj->last_fenced_ring = pipelined; 3302235783Skib } 3303235783Skib 3304235783Skib goto update; 3305235783Skib } 3306235783Skib 3307235783Skib if (!pipelined) { 3308235783Skib if (reg->setup_seqno) { 3309235783Skib if (!ring_passed_seqno(obj->last_fenced_ring, 3310235783Skib reg->setup_seqno)) { 3311235783Skib ret = i915_wait_request( 3312235783Skib obj->last_fenced_ring, 3313235783Skib reg->setup_seqno, 3314235783Skib true); 3315235783Skib if (ret) 3316235783Skib return ret; 3317235783Skib } 3318235783Skib 3319235783Skib reg->setup_seqno = 0; 3320235783Skib } 3321235783Skib } else if (obj->last_fenced_ring && 3322235783Skib obj->last_fenced_ring != pipelined) { 3323235783Skib ret = i915_gem_object_flush_fence(obj, pipelined); 3324235783Skib if (ret) 3325235783Skib return ret; 3326235783Skib } 3327235783Skib 3328235783Skib if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3329235783Skib pipelined = NULL; 3330235783Skib KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3331235783Skib 3332235783Skib if (obj->tiling_changed) { 3333235783Skib if (pipelined) { 3334235783Skib reg->setup_seqno = 3335235783Skib i915_gem_next_request_seqno(pipelined); 3336235783Skib obj->last_fenced_seqno = reg->setup_seqno; 3337235783Skib obj->last_fenced_ring = pipelined; 3338235783Skib } 3339235783Skib goto update; 3340235783Skib } 3341235783Skib 3342235783Skib return 0; 3343235783Skib } 3344235783Skib 3345235783Skib reg = i915_find_fence_reg(dev, pipelined); 3346235783Skib if (reg == NULL) 3347235783Skib return -EDEADLK; 3348235783Skib 3349235783Skib ret = i915_gem_object_flush_fence(obj, pipelined); 3350235783Skib if (ret) 3351235783Skib return ret; 3352235783Skib 3353235783Skib if (reg->obj) { 3354235783Skib struct drm_i915_gem_object *old = reg->obj; 3355235783Skib 3356235783Skib drm_gem_object_reference(&old->base); 3357235783Skib 3358235783Skib if (old->tiling_mode) 3359235783Skib i915_gem_release_mmap(old); 3360235783Skib 3361235783Skib ret = i915_gem_object_flush_fence(old, pipelined); 3362235783Skib if (ret) { 3363235783Skib drm_gem_object_unreference(&old->base); 3364235783Skib return ret; 3365235783Skib } 3366235783Skib 3367235783Skib if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3368235783Skib pipelined = NULL; 3369235783Skib 3370235783Skib old->fence_reg = I915_FENCE_REG_NONE; 3371235783Skib old->last_fenced_ring = pipelined; 3372235783Skib old->last_fenced_seqno = 3373235783Skib pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3374235783Skib 3375235783Skib drm_gem_object_unreference(&old->base); 3376235783Skib } else if (obj->last_fenced_seqno == 0) 3377235783Skib pipelined = NULL; 3378235783Skib 3379235783Skib reg->obj = obj; 3380235783Skib list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3381235783Skib obj->fence_reg = reg - dev_priv->fence_regs; 3382235783Skib obj->last_fenced_ring = pipelined; 3383235783Skib 3384235783Skib reg->setup_seqno = 3385235783Skib pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3386235783Skib obj->last_fenced_seqno = reg->setup_seqno; 3387235783Skib 3388235783Skibupdate: 3389235783Skib obj->tiling_changed = false; 3390235783Skib switch (INTEL_INFO(dev)->gen) { 3391235783Skib case 7: 3392235783Skib case 6: 3393235783Skib ret = sandybridge_write_fence_reg(obj, pipelined); 3394235783Skib break; 3395235783Skib case 5: 3396235783Skib case 4: 3397235783Skib ret = i965_write_fence_reg(obj, pipelined); 3398235783Skib break; 3399235783Skib case 3: 3400235783Skib ret = i915_write_fence_reg(obj, pipelined); 3401235783Skib break; 3402235783Skib case 2: 3403235783Skib ret = i830_write_fence_reg(obj, pipelined); 3404235783Skib break; 3405235783Skib } 3406235783Skib 3407235783Skib return ret; 3408235783Skib} 3409235783Skib 3410235783Skibstatic void 3411235783Skibi915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3412235783Skib{ 3413235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 3414235783Skib uint32_t fence_reg = reg - dev_priv->fence_regs; 3415235783Skib 3416235783Skib switch (INTEL_INFO(dev)->gen) { 3417235783Skib case 7: 3418235783Skib case 6: 3419235783Skib I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3420235783Skib break; 3421235783Skib case 5: 3422235783Skib case 4: 3423235783Skib I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3424235783Skib break; 3425235783Skib case 3: 3426235783Skib if (fence_reg >= 8) 3427235783Skib fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3428235783Skib else 3429235783Skib case 2: 3430235783Skib fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3431235783Skib 3432235783Skib I915_WRITE(fence_reg, 0); 3433235783Skib break; 3434235783Skib } 3435235783Skib 3436235783Skib list_del_init(®->lru_list); 3437235783Skib reg->obj = NULL; 3438235783Skib reg->setup_seqno = 0; 3439235783Skib reg->pin_count = 0; 3440235783Skib} 3441235783Skib 3442235783Skibint 3443235783Skibi915_gem_init_object(struct drm_gem_object *obj) 3444235783Skib{ 3445235783Skib 3446235783Skib printf("i915_gem_init_object called\n"); 3447235783Skib return (0); 3448235783Skib} 3449235783Skib 3450235783Skibstatic bool 3451235783Skibi915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3452235783Skib{ 3453235783Skib 3454235783Skib return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3455235783Skib} 3456235783Skib 3457235783Skibstatic void 3458235783Skibi915_gem_retire_task_handler(void *arg, int pending) 3459235783Skib{ 3460235783Skib drm_i915_private_t *dev_priv; 3461235783Skib struct drm_device *dev; 3462235783Skib bool idle; 3463235783Skib int i; 3464235783Skib 3465235783Skib dev_priv = arg; 3466235783Skib dev = dev_priv->dev; 3467235783Skib 3468235783Skib /* Come back later if the device is busy... */ 3469235783Skib if (!sx_try_xlock(&dev->dev_struct_lock)) { 3470235783Skib taskqueue_enqueue_timeout(dev_priv->tq, 3471235783Skib &dev_priv->mm.retire_task, hz); 3472235783Skib return; 3473235783Skib } 3474235783Skib 3475235783Skib CTR0(KTR_DRM, "retire_task"); 3476235783Skib 3477235783Skib i915_gem_retire_requests(dev); 3478235783Skib 3479235783Skib /* Send a periodic flush down the ring so we don't hold onto GEM 3480235783Skib * objects indefinitely. 3481235783Skib */ 3482235783Skib idle = true; 3483235783Skib for (i = 0; i < I915_NUM_RINGS; i++) { 3484235783Skib struct intel_ring_buffer *ring = &dev_priv->rings[i]; 3485235783Skib 3486235783Skib if (!list_empty(&ring->gpu_write_list)) { 3487235783Skib struct drm_i915_gem_request *request; 3488235783Skib int ret; 3489235783Skib 3490235783Skib ret = i915_gem_flush_ring(ring, 3491235783Skib 0, I915_GEM_GPU_DOMAINS); 3492235783Skib request = malloc(sizeof(*request), DRM_I915_GEM, 3493235783Skib M_WAITOK | M_ZERO); 3494235783Skib if (ret || request == NULL || 3495235783Skib i915_add_request(ring, NULL, request)) 3496235783Skib free(request, DRM_I915_GEM); 3497235783Skib } 3498235783Skib 3499235783Skib idle &= list_empty(&ring->request_list); 3500235783Skib } 3501235783Skib 3502235783Skib if (!dev_priv->mm.suspended && !idle) 3503235783Skib taskqueue_enqueue_timeout(dev_priv->tq, 3504235783Skib &dev_priv->mm.retire_task, hz); 3505235783Skib 3506235783Skib DRM_UNLOCK(dev); 3507235783Skib} 3508235783Skib 3509235783Skibvoid 3510235783Skibi915_gem_lastclose(struct drm_device *dev) 3511235783Skib{ 3512235783Skib int ret; 3513235783Skib 3514235783Skib if (drm_core_check_feature(dev, DRIVER_MODESET)) 3515235783Skib return; 3516235783Skib 3517235783Skib ret = i915_gem_idle(dev); 3518235783Skib if (ret != 0) 3519235783Skib DRM_ERROR("failed to idle hardware: %d\n", ret); 3520235783Skib} 3521235783Skib 3522235783Skibstatic int 3523235783Skibi915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3524235783Skib{ 3525235783Skib drm_i915_private_t *dev_priv; 3526235783Skib struct drm_i915_gem_phys_object *phys_obj; 3527235783Skib int ret; 3528235783Skib 3529235783Skib dev_priv = dev->dev_private; 3530235783Skib if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3531235783Skib return (0); 3532235783Skib 3533235783Skib phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3534235783Skib M_WAITOK | M_ZERO); 3535235783Skib 3536235783Skib phys_obj->id = id; 3537235783Skib 3538235783Skib phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3539235783Skib if (phys_obj->handle == NULL) { 3540235783Skib ret = -ENOMEM; 3541235783Skib goto free_obj; 3542235783Skib } 3543235783Skib pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3544235783Skib size / PAGE_SIZE, PAT_WRITE_COMBINING); 3545235783Skib 3546235783Skib dev_priv->mm.phys_objs[id - 1] = phys_obj; 3547235783Skib 3548235783Skib return (0); 3549235783Skib 3550235783Skibfree_obj: 3551235783Skib free(phys_obj, DRM_I915_GEM); 3552235783Skib return (ret); 3553235783Skib} 3554235783Skib 3555235783Skibstatic void 3556235783Skibi915_gem_free_phys_object(struct drm_device *dev, int id) 3557235783Skib{ 3558235783Skib drm_i915_private_t *dev_priv; 3559235783Skib struct drm_i915_gem_phys_object *phys_obj; 3560235783Skib 3561235783Skib dev_priv = dev->dev_private; 3562235783Skib if (dev_priv->mm.phys_objs[id - 1] == NULL) 3563235783Skib return; 3564235783Skib 3565235783Skib phys_obj = dev_priv->mm.phys_objs[id - 1]; 3566235783Skib if (phys_obj->cur_obj != NULL) 3567235783Skib i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3568235783Skib 3569235783Skib drm_pci_free(dev, phys_obj->handle); 3570235783Skib free(phys_obj, DRM_I915_GEM); 3571235783Skib dev_priv->mm.phys_objs[id - 1] = NULL; 3572235783Skib} 3573235783Skib 3574235783Skibvoid 3575235783Skibi915_gem_free_all_phys_object(struct drm_device *dev) 3576235783Skib{ 3577235783Skib int i; 3578235783Skib 3579235783Skib for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3580235783Skib i915_gem_free_phys_object(dev, i); 3581235783Skib} 3582235783Skib 3583235783Skibvoid 3584235783Skibi915_gem_detach_phys_object(struct drm_device *dev, 3585235783Skib struct drm_i915_gem_object *obj) 3586235783Skib{ 3587235783Skib vm_page_t m; 3588235783Skib struct sf_buf *sf; 3589235783Skib char *vaddr, *dst; 3590235783Skib int i, page_count; 3591235783Skib 3592235783Skib if (obj->phys_obj == NULL) 3593235783Skib return; 3594235783Skib vaddr = obj->phys_obj->handle->vaddr; 3595235783Skib 3596235783Skib page_count = obj->base.size / PAGE_SIZE; 3597248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 3598235783Skib for (i = 0; i < page_count; i++) { 3599235783Skib m = i915_gem_wire_page(obj->base.vm_obj, i); 3600235783Skib if (m == NULL) 3601235783Skib continue; /* XXX */ 3602235783Skib 3603248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3604235783Skib sf = sf_buf_alloc(m, 0); 3605235783Skib if (sf != NULL) { 3606235783Skib dst = (char *)sf_buf_kva(sf); 3607235783Skib memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3608235783Skib sf_buf_free(sf); 3609235783Skib } 3610235783Skib drm_clflush_pages(&m, 1); 3611235783Skib 3612248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 3613235783Skib vm_page_reference(m); 3614235783Skib vm_page_lock(m); 3615235783Skib vm_page_dirty(m); 3616267548Sattilio vm_page_unwire(m, PQ_INACTIVE); 3617235783Skib vm_page_unlock(m); 3618235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3619235783Skib } 3620248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3621235783Skib intel_gtt_chipset_flush(); 3622235783Skib 3623235783Skib obj->phys_obj->cur_obj = NULL; 3624235783Skib obj->phys_obj = NULL; 3625235783Skib} 3626235783Skib 3627235783Skibint 3628235783Skibi915_gem_attach_phys_object(struct drm_device *dev, 3629235783Skib struct drm_i915_gem_object *obj, int id, int align) 3630235783Skib{ 3631235783Skib drm_i915_private_t *dev_priv; 3632235783Skib vm_page_t m; 3633235783Skib struct sf_buf *sf; 3634235783Skib char *dst, *src; 3635235783Skib int i, page_count, ret; 3636235783Skib 3637235783Skib if (id > I915_MAX_PHYS_OBJECT) 3638235783Skib return (-EINVAL); 3639235783Skib 3640235783Skib if (obj->phys_obj != NULL) { 3641235783Skib if (obj->phys_obj->id == id) 3642235783Skib return (0); 3643235783Skib i915_gem_detach_phys_object(dev, obj); 3644235783Skib } 3645235783Skib 3646235783Skib dev_priv = dev->dev_private; 3647235783Skib if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3648235783Skib ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3649235783Skib if (ret != 0) { 3650235783Skib DRM_ERROR("failed to init phys object %d size: %zu\n", 3651235783Skib id, obj->base.size); 3652235783Skib return (ret); 3653235783Skib } 3654235783Skib } 3655235783Skib 3656235783Skib /* bind to the object */ 3657235783Skib obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3658235783Skib obj->phys_obj->cur_obj = obj; 3659235783Skib 3660235783Skib page_count = obj->base.size / PAGE_SIZE; 3661235783Skib 3662248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 3663235783Skib ret = 0; 3664235783Skib for (i = 0; i < page_count; i++) { 3665235783Skib m = i915_gem_wire_page(obj->base.vm_obj, i); 3666235783Skib if (m == NULL) { 3667235783Skib ret = -EIO; 3668235783Skib break; 3669235783Skib } 3670248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3671235783Skib sf = sf_buf_alloc(m, 0); 3672235783Skib src = (char *)sf_buf_kva(sf); 3673235783Skib dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3674235783Skib memcpy(dst, src, PAGE_SIZE); 3675235783Skib sf_buf_free(sf); 3676235783Skib 3677248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 3678235783Skib 3679235783Skib vm_page_reference(m); 3680235783Skib vm_page_lock(m); 3681267548Sattilio vm_page_unwire(m, PQ_INACTIVE); 3682235783Skib vm_page_unlock(m); 3683235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3684235783Skib } 3685248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 3686235783Skib 3687235783Skib return (0); 3688235783Skib} 3689235783Skib 3690235783Skibstatic int 3691235783Skibi915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3692235783Skib uint64_t data_ptr, uint64_t offset, uint64_t size, 3693235783Skib struct drm_file *file_priv) 3694235783Skib{ 3695235783Skib char *user_data, *vaddr; 3696235783Skib int ret; 3697235783Skib 3698235783Skib vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3699235783Skib user_data = (char *)(uintptr_t)data_ptr; 3700235783Skib 3701235783Skib if (copyin_nofault(user_data, vaddr, size) != 0) { 3702235783Skib /* The physical object once assigned is fixed for the lifetime 3703235783Skib * of the obj, so we can safely drop the lock and continue 3704235783Skib * to access vaddr. 3705235783Skib */ 3706235783Skib DRM_UNLOCK(dev); 3707235783Skib ret = -copyin(user_data, vaddr, size); 3708235783Skib DRM_LOCK(dev); 3709235783Skib if (ret != 0) 3710235783Skib return (ret); 3711235783Skib } 3712235783Skib 3713235783Skib intel_gtt_chipset_flush(); 3714235783Skib return (0); 3715235783Skib} 3716235783Skib 3717235783Skibstatic int 3718235783Skibi915_gpu_is_active(struct drm_device *dev) 3719235783Skib{ 3720235783Skib drm_i915_private_t *dev_priv; 3721235783Skib 3722235783Skib dev_priv = dev->dev_private; 3723235783Skib return (!list_empty(&dev_priv->mm.flushing_list) || 3724235783Skib !list_empty(&dev_priv->mm.active_list)); 3725235783Skib} 3726235783Skib 3727235783Skibstatic void 3728235783Skibi915_gem_lowmem(void *arg) 3729235783Skib{ 3730235783Skib struct drm_device *dev; 3731235783Skib struct drm_i915_private *dev_priv; 3732235783Skib struct drm_i915_gem_object *obj, *next; 3733235783Skib int cnt, cnt_fail, cnt_total; 3734235783Skib 3735235783Skib dev = arg; 3736235783Skib dev_priv = dev->dev_private; 3737235783Skib 3738235783Skib if (!sx_try_xlock(&dev->dev_struct_lock)) 3739235783Skib return; 3740235783Skib 3741235783Skib CTR0(KTR_DRM, "gem_lowmem"); 3742235783Skib 3743235783Skibrescan: 3744235783Skib /* first scan for clean buffers */ 3745235783Skib i915_gem_retire_requests(dev); 3746235783Skib 3747235783Skib cnt_total = cnt_fail = cnt = 0; 3748235783Skib 3749235783Skib list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3750235783Skib mm_list) { 3751235783Skib if (i915_gem_object_is_purgeable(obj)) { 3752235783Skib if (i915_gem_object_unbind(obj) != 0) 3753235783Skib cnt_total++; 3754235783Skib } else 3755235783Skib cnt_total++; 3756235783Skib } 3757235783Skib 3758235783Skib /* second pass, evict/count anything still on the inactive list */ 3759235783Skib list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3760235783Skib mm_list) { 3761235783Skib if (i915_gem_object_unbind(obj) == 0) 3762235783Skib cnt++; 3763235783Skib else 3764235783Skib cnt_fail++; 3765235783Skib } 3766235783Skib 3767235783Skib if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3768235783Skib /* 3769235783Skib * We are desperate for pages, so as a last resort, wait 3770235783Skib * for the GPU to finish and discard whatever we can. 3771235783Skib * This has a dramatic impact to reduce the number of 3772235783Skib * OOM-killer events whilst running the GPU aggressively. 3773235783Skib */ 3774235783Skib if (i915_gpu_idle(dev, true) == 0) 3775235783Skib goto rescan; 3776235783Skib } 3777235783Skib DRM_UNLOCK(dev); 3778235783Skib} 3779235783Skib 3780235783Skibvoid 3781235783Skibi915_gem_unload(struct drm_device *dev) 3782235783Skib{ 3783235783Skib struct drm_i915_private *dev_priv; 3784235783Skib 3785235783Skib dev_priv = dev->dev_private; 3786235783Skib EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3787235783Skib} 3788