1287174Sbapt/* 2235783Skib * Copyright �� 2008 Intel Corporation 3235783Skib * 4235783Skib * Permission is hereby granted, free of charge, to any person obtaining a 5235783Skib * copy of this software and associated documentation files (the "Software"), 6235783Skib * to deal in the Software without restriction, including without limitation 7235783Skib * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8235783Skib * and/or sell copies of the Software, and to permit persons to whom the 9235783Skib * Software is furnished to do so, subject to the following conditions: 10235783Skib * 11235783Skib * The above copyright notice and this permission notice (including the next 12235783Skib * paragraph) shall be included in all copies or substantial portions of the 13235783Skib * Software. 14235783Skib * 15235783Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16235783Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17235783Skib * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18235783Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19235783Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20235783Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21235783Skib * IN THE SOFTWARE. 22235783Skib * 23235783Skib * Authors: 24235783Skib * Eric Anholt <eric@anholt.net> 25235783Skib * 26235783Skib * Copyright (c) 2011 The FreeBSD Foundation 27235783Skib * All rights reserved. 28235783Skib * 29235783Skib * This software was developed by Konstantin Belousov under sponsorship from 30235783Skib * the FreeBSD Foundation. 31235783Skib * 32235783Skib * Redistribution and use in source and binary forms, with or without 33235783Skib * modification, are permitted provided that the following conditions 34235783Skib * are met: 35235783Skib * 1. Redistributions of source code must retain the above copyright 36235783Skib * notice, this list of conditions and the following disclaimer. 37235783Skib * 2. Redistributions in binary form must reproduce the above copyright 38235783Skib * notice, this list of conditions and the following disclaimer in the 39235783Skib * documentation and/or other materials provided with the distribution. 40235783Skib * 41235783Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42235783Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43235783Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44235783Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45235783Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46235783Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47235783Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48235783Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49235783Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50235783Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51235783Skib * SUCH DAMAGE. 52235783Skib */ 53235783Skib 54235783Skib#include <sys/cdefs.h> 55235783Skib__FBSDID("$FreeBSD: stable/11/sys/dev/drm2/i915/i915_gem.c 324129 2017-09-30 18:52:59Z alc $"); 56235783Skib 57235783Skib#include <dev/drm2/drmP.h> 58235783Skib#include <dev/drm2/i915/i915_drm.h> 59235783Skib#include <dev/drm2/i915/i915_drv.h> 60235783Skib#include <dev/drm2/i915/intel_drv.h> 61285988Sdumbbell 62235783Skib#include <sys/resourcevar.h> 63235783Skib#include <sys/sched.h> 64235783Skib#include <sys/sf_buf.h> 65235783Skib 66254141Sattilio#include <vm/vm.h> 67254141Sattilio#include <vm/vm_pageout.h> 68254141Sattilio 69277487Skib#include <machine/md_var.h> 70277487Skib 71285988Sdumbbellstatic void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); 72285988Sdumbbellstatic void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); 73285988Sdumbbellstatic __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 74285988Sdumbbell unsigned alignment, 75296548Sdumbbell bool map_and_fenceable, 76296548Sdumbbell bool nonblocking); 77285988Sdumbbellstatic int i915_gem_phys_pwrite(struct drm_device *dev, 78285988Sdumbbell struct drm_i915_gem_object *obj, 79285988Sdumbbell struct drm_i915_gem_pwrite *args, 80285988Sdumbbell struct drm_file *file); 81285988Sdumbbell 82285988Sdumbbellstatic void i915_gem_write_fence(struct drm_device *dev, int reg, 83285988Sdumbbell struct drm_i915_gem_object *obj); 84285988Sdumbbellstatic void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 85285988Sdumbbell struct drm_i915_fence_reg *fence, 86285988Sdumbbell bool enable); 87285988Sdumbbell 88296548Sdumbbellstatic void i915_gem_inactive_shrink(void *); 89296548Sdumbbellstatic long i915_gem_purge(struct drm_i915_private *dev_priv, long target); 90296548Sdumbbellstatic void i915_gem_shrink_all(struct drm_i915_private *dev_priv); 91285988Sdumbbellstatic void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 92285988Sdumbbell 93285988Sdumbbellstatic int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 94285988Sdumbbell off_t start, off_t end); 95285988Sdumbbell 96277487Skibstatic vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, 97277487Skib bool *fresh); 98235783Skib 99235783SkibMALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 100235783Skiblong i915_gem_wired_pages_cnt; 101235783Skib 102277487Skibstatic inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) 103277487Skib{ 104277487Skib if (obj->tiling_mode) 105277487Skib i915_gem_release_mmap(obj); 106277487Skib 107277487Skib /* As we do not have an associated fence register, we will force 108277487Skib * a tiling change if we ever need to acquire one. 109277487Skib */ 110277487Skib obj->fence_dirty = false; 111277487Skib obj->fence_reg = I915_FENCE_REG_NONE; 112277487Skib} 113277487Skib 114285988Sdumbbell/* some bookkeeping */ 115285988Sdumbbellstatic void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 116285988Sdumbbell size_t size) 117235783Skib{ 118235783Skib dev_priv->mm.object_count++; 119235783Skib dev_priv->mm.object_memory += size; 120235783Skib} 121235783Skib 122285988Sdumbbellstatic void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 123285988Sdumbbell size_t size) 124235783Skib{ 125235783Skib dev_priv->mm.object_count--; 126235783Skib dev_priv->mm.object_memory -= size; 127235783Skib} 128235783Skib 129235783Skibstatic int 130235783Skibi915_gem_wait_for_error(struct drm_device *dev) 131235783Skib{ 132285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 133296548Sdumbbell struct completion *x = &dev_priv->error_completion; 134235783Skib int ret; 135235783Skib 136296548Sdumbbell if (!atomic_read(&dev_priv->mm.wedged)) 137293837Sdumbbell return 0; 138235783Skib 139296548Sdumbbell /* 140296548Sdumbbell * Only wait 10 seconds for the gpu reset to complete to avoid hanging 141296548Sdumbbell * userspace. If it takes that long something really bad is going on and 142296548Sdumbbell * we should simply try to bail out and fail as gracefully as possible. 143296548Sdumbbell */ 144296548Sdumbbell ret = wait_for_completion_interruptible_timeout(x, 10*HZ); 145296548Sdumbbell if (ret == 0) { 146296548Sdumbbell DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); 147296548Sdumbbell return -EIO; 148296548Sdumbbell } else if (ret < 0) { 149296548Sdumbbell return ret; 150235783Skib } 151235783Skib 152296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) { 153285988Sdumbbell /* GPU is hung, bump the completion count to account for 154285988Sdumbbell * the token we just consumed so that we never hit zero and 155285988Sdumbbell * end up waiting upon a subsequent completion event that 156285988Sdumbbell * will never happen. 157285988Sdumbbell */ 158296548Sdumbbell mtx_lock(&x->lock); 159296548Sdumbbell x->done++; 160296548Sdumbbell mtx_unlock(&x->lock); 161235783Skib } 162285988Sdumbbell return 0; 163235783Skib} 164235783Skib 165285988Sdumbbellint i915_mutex_lock_interruptible(struct drm_device *dev) 166235783Skib{ 167235783Skib int ret; 168235783Skib 169235783Skib ret = i915_gem_wait_for_error(dev); 170285988Sdumbbell if (ret) 171285988Sdumbbell return ret; 172235783Skib 173235783Skib /* 174235783Skib * interruptible shall it be. might indeed be if dev_lock is 175235783Skib * changed to sx 176235783Skib */ 177296548Sdumbbell ret = sx_xlock_sig(&dev->dev_struct_lock); 178285988Sdumbbell if (ret) 179296548Sdumbbell return -EINTR; 180235783Skib 181296548Sdumbbell WARN_ON(i915_verify_lists(dev)); 182285988Sdumbbell return 0; 183235783Skib} 184235783Skib 185287174Sbaptstatic inline bool 186285988Sdumbbelli915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 187235783Skib{ 188296548Sdumbbell return obj->gtt_space && !obj->active; 189235783Skib} 190235783Skib 191235783Skibint 192235783Skibi915_gem_init_ioctl(struct drm_device *dev, void *data, 193285988Sdumbbell struct drm_file *file) 194235783Skib{ 195285988Sdumbbell struct drm_i915_gem_init *args = data; 196235783Skib 197277487Skib if (drm_core_check_feature(dev, DRIVER_MODESET)) 198277487Skib return -ENODEV; 199277487Skib 200235783Skib if (args->gtt_start >= args->gtt_end || 201235783Skib (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 202285988Sdumbbell return -EINVAL; 203235783Skib 204277487Skib /* GEM with user mode setting was never supported on ilk and later. */ 205277487Skib if (INTEL_INFO(dev)->gen >= 5) 206277487Skib return -ENODEV; 207277487Skib 208235783Skib /* 209235783Skib * XXXKIB. The second-time initialization should be guarded 210235783Skib * against. 211235783Skib */ 212280183Sdumbbell DRM_LOCK(dev); 213296548Sdumbbell i915_gem_init_global_gtt(dev, args->gtt_start, 214285988Sdumbbell args->gtt_end, args->gtt_end); 215280183Sdumbbell DRM_UNLOCK(dev); 216235783Skib 217296548Sdumbbell return 0; 218235783Skib} 219235783Skib 220235783Skibint 221285988Sdumbbelli915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 222285988Sdumbbell struct drm_file *file) 223235783Skib{ 224277487Skib struct drm_i915_private *dev_priv = dev->dev_private; 225285988Sdumbbell struct drm_i915_gem_get_aperture *args = data; 226235783Skib struct drm_i915_gem_object *obj; 227235783Skib size_t pinned; 228235783Skib 229235783Skib pinned = 0; 230235783Skib DRM_LOCK(dev); 231296548Sdumbbell list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) 232277487Skib if (obj->pin_count) 233277487Skib pinned += obj->gtt_space->size; 234235783Skib DRM_UNLOCK(dev); 235235783Skib 236235783Skib args->aper_size = dev_priv->mm.gtt_total; 237235783Skib args->aper_available_size = args->aper_size - pinned; 238235783Skib 239277487Skib return 0; 240235783Skib} 241235783Skib 242235783Skibstatic int 243285988Sdumbbelli915_gem_create(struct drm_file *file, 244285988Sdumbbell struct drm_device *dev, 245285988Sdumbbell uint64_t size, 246285988Sdumbbell uint32_t *handle_p) 247235783Skib{ 248235783Skib struct drm_i915_gem_object *obj; 249235783Skib int ret; 250285988Sdumbbell u32 handle; 251235783Skib 252235783Skib size = roundup(size, PAGE_SIZE); 253235783Skib if (size == 0) 254285988Sdumbbell return -EINVAL; 255235783Skib 256285988Sdumbbell /* Allocate the new object */ 257235783Skib obj = i915_gem_alloc_object(dev, size); 258235783Skib if (obj == NULL) 259285988Sdumbbell return -ENOMEM; 260235783Skib 261235783Skib ret = drm_gem_handle_create(file, &obj->base, &handle); 262285988Sdumbbell if (ret) { 263235783Skib drm_gem_object_release(&obj->base); 264235783Skib i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 265235783Skib free(obj, DRM_I915_GEM); 266285988Sdumbbell return ret; 267235783Skib } 268235783Skib 269235783Skib /* drop reference from allocate - handle holds it now */ 270235783Skib drm_gem_object_unreference(&obj->base); 271235783Skib CTR2(KTR_DRM, "object_create %p %x", obj, size); 272285988Sdumbbell 273235783Skib *handle_p = handle; 274285988Sdumbbell return 0; 275235783Skib} 276235783Skib 277235783Skibint 278285988Sdumbbelli915_gem_dumb_create(struct drm_file *file, 279285988Sdumbbell struct drm_device *dev, 280285988Sdumbbell struct drm_mode_create_dumb *args) 281235783Skib{ 282235783Skib /* have to work out size/pitch and return them */ 283235783Skib args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 284235783Skib args->size = args->pitch * args->height; 285285988Sdumbbell return i915_gem_create(file, dev, 286285988Sdumbbell args->size, &args->handle); 287235783Skib} 288235783Skib 289285988Sdumbbellint i915_gem_dumb_destroy(struct drm_file *file, 290285988Sdumbbell struct drm_device *dev, 291285988Sdumbbell uint32_t handle) 292235783Skib{ 293285988Sdumbbell return drm_gem_handle_delete(file, handle); 294235783Skib} 295235783Skib 296285988Sdumbbell/** 297285988Sdumbbell * Creates a new mm object and returns a handle to it. 298285988Sdumbbell */ 299235783Skibint 300235783Skibi915_gem_create_ioctl(struct drm_device *dev, void *data, 301285988Sdumbbell struct drm_file *file) 302235783Skib{ 303235783Skib struct drm_i915_gem_create *args = data; 304235783Skib 305285988Sdumbbell return i915_gem_create(file, dev, 306285988Sdumbbell args->size, &args->handle); 307235783Skib} 308235783Skib 309285988Sdumbbellstatic int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 310285988Sdumbbell{ 311285988Sdumbbell drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 312285988Sdumbbell 313285988Sdumbbell return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 314285988Sdumbbell obj->tiling_mode != I915_TILING_NONE; 315285988Sdumbbell} 316285988Sdumbbell 317277487Skibstatic inline int 318277487Skib__copy_to_user_swizzled(char __user *cpu_vaddr, 319277487Skib const char *gpu_vaddr, int gpu_offset, 320277487Skib int length) 321277487Skib{ 322277487Skib int ret, cpu_offset = 0; 323277487Skib 324277487Skib while (length > 0) { 325277487Skib int cacheline_end = roundup2(gpu_offset + 1, 64); 326277487Skib int this_length = min(cacheline_end - gpu_offset, length); 327277487Skib int swizzled_gpu_offset = gpu_offset ^ 64; 328277487Skib 329277487Skib ret = __copy_to_user(cpu_vaddr + cpu_offset, 330277487Skib gpu_vaddr + swizzled_gpu_offset, 331277487Skib this_length); 332277487Skib if (ret) 333277487Skib return ret + length; 334277487Skib 335277487Skib cpu_offset += this_length; 336277487Skib gpu_offset += this_length; 337277487Skib length -= this_length; 338277487Skib } 339277487Skib 340277487Skib return 0; 341277487Skib} 342277487Skib 343277487Skibstatic inline int 344277487Skib__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 345277487Skib const char __user *cpu_vaddr, 346277487Skib int length) 347277487Skib{ 348277487Skib int ret, cpu_offset = 0; 349277487Skib 350277487Skib while (length > 0) { 351277487Skib int cacheline_end = roundup2(gpu_offset + 1, 64); 352277487Skib int this_length = min(cacheline_end - gpu_offset, length); 353277487Skib int swizzled_gpu_offset = gpu_offset ^ 64; 354277487Skib 355277487Skib ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 356277487Skib cpu_vaddr + cpu_offset, 357277487Skib this_length); 358277487Skib if (ret) 359277487Skib return ret + length; 360277487Skib 361277487Skib cpu_offset += this_length; 362277487Skib gpu_offset += this_length; 363277487Skib length -= this_length; 364277487Skib } 365277487Skib 366277487Skib return 0; 367277487Skib} 368277487Skib 369277487Skib/* Per-page copy function for the shmem pread fastpath. 370277487Skib * Flushes invalid cachelines before reading the target if 371277487Skib * needs_clflush is set. */ 372277487Skibstatic int 373277487Skibshmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length, 374277487Skib char __user *user_data, 375277487Skib bool page_do_bit17_swizzling, bool needs_clflush) 376277487Skib{ 377277487Skib char *vaddr; 378277487Skib struct sf_buf *sf; 379277487Skib int ret; 380235783Skib 381277487Skib if (unlikely(page_do_bit17_swizzling)) 382277487Skib return -EINVAL; 383277487Skib 384277487Skib sched_pin(); 385277487Skib sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 386277487Skib if (sf == NULL) { 387235783Skib sched_unpin(); 388277487Skib return (-EFAULT); 389277487Skib } 390277487Skib vaddr = (char *)sf_buf_kva(sf); 391277487Skib if (needs_clflush) 392277487Skib drm_clflush_virt_range(vaddr + shmem_page_offset, 393277487Skib page_length); 394277487Skib ret = __copy_to_user_inatomic(user_data, 395277487Skib vaddr + shmem_page_offset, 396277487Skib page_length); 397277487Skib sf_buf_free(sf); 398277487Skib sched_unpin(); 399235783Skib 400277487Skib return ret ? -EFAULT : 0; 401277487Skib} 402277487Skib 403277487Skibstatic void 404277487Skibshmem_clflush_swizzled_range(char *addr, unsigned long length, 405277487Skib bool swizzled) 406277487Skib{ 407277487Skib if (unlikely(swizzled)) { 408277487Skib unsigned long start = (unsigned long) addr; 409277487Skib unsigned long end = (unsigned long) addr + length; 410277487Skib 411277487Skib /* For swizzling simply ensure that we always flush both 412277487Skib * channels. Lame, but simple and it works. Swizzled 413277487Skib * pwrite/pread is far from a hotpath - current userspace 414277487Skib * doesn't use it at all. */ 415296548Sdumbbell start = round_down(start, 128); 416296548Sdumbbell end = round_up(end, 128); 417277487Skib 418277487Skib drm_clflush_virt_range((void *)start, end - start); 419277487Skib } else { 420277487Skib drm_clflush_virt_range(addr, length); 421235783Skib } 422235783Skib 423235783Skib} 424235783Skib 425277487Skib/* Only difference to the fast-path function is that this can handle bit17 426277487Skib * and uses non-atomic copy and kmap functions. */ 427235783Skibstatic int 428277487Skibshmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length, 429277487Skib char __user *user_data, 430277487Skib bool page_do_bit17_swizzling, bool needs_clflush) 431235783Skib{ 432277487Skib char *vaddr; 433277487Skib struct sf_buf *sf; 434277487Skib int ret; 435235783Skib 436277487Skib sf = sf_buf_alloc(page, 0); 437277487Skib vaddr = (char *)sf_buf_kva(sf); 438277487Skib if (needs_clflush) 439277487Skib shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 440277487Skib page_length, 441277487Skib page_do_bit17_swizzling); 442235783Skib 443277487Skib if (page_do_bit17_swizzling) 444277487Skib ret = __copy_to_user_swizzled(user_data, 445277487Skib vaddr, shmem_page_offset, 446277487Skib page_length); 447277487Skib else 448277487Skib ret = __copy_to_user(user_data, 449277487Skib vaddr + shmem_page_offset, 450277487Skib page_length); 451277487Skib sf_buf_free(sf); 452277487Skib 453277487Skib return ret ? - EFAULT : 0; 454235783Skib} 455235783Skib 456235783Skibstatic int 457277487Skibi915_gem_shmem_pread(struct drm_device *dev, 458277487Skib struct drm_i915_gem_object *obj, 459277487Skib struct drm_i915_gem_pread *args, 460277487Skib struct drm_file *file) 461235783Skib{ 462277487Skib char __user *user_data; 463296548Sdumbbell ssize_t remain; 464296548Sdumbbell off_t offset; 465277487Skib int shmem_page_offset, page_length, ret = 0; 466277487Skib int obj_do_bit17_swizzling, page_do_bit17_swizzling; 467296548Sdumbbell int hit_slowpath = 0; 468277487Skib int prefaulted = 0; 469277487Skib int needs_clflush = 0; 470235783Skib 471277487Skib user_data = to_user_ptr(args->data_ptr); 472296548Sdumbbell remain = args->size; 473277487Skib 474277487Skib obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 475277487Skib 476277487Skib if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 477277487Skib /* If we're not in the cpu read domain, set ourself into the gtt 478277487Skib * read domain and manually flush cachelines (if required). This 479277487Skib * optimizes for the case when the gpu will dirty the data 480277487Skib * anyway again before the next pread happens. */ 481296548Sdumbbell if (obj->cache_level == I915_CACHE_NONE) 482296548Sdumbbell needs_clflush = 1; 483296548Sdumbbell if (obj->gtt_space) { 484296548Sdumbbell ret = i915_gem_object_set_to_gtt_domain(obj, false); 485296548Sdumbbell if (ret) 486296548Sdumbbell return ret; 487296548Sdumbbell } 488235783Skib } 489235783Skib 490296548Sdumbbell ret = i915_gem_object_get_pages(obj); 491277487Skib if (ret) 492277487Skib return ret; 493277487Skib 494277487Skib i915_gem_object_pin_pages(obj); 495277487Skib 496296548Sdumbbell offset = args->offset; 497296548Sdumbbell 498277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 499277487Skib for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 500277487Skib OFF_TO_IDX(offset));; page = vm_page_next(page)) { 501277487Skib VM_OBJECT_WUNLOCK(obj->base.vm_obj); 502277487Skib 503277487Skib if (remain <= 0) 504277487Skib break; 505277487Skib 506277487Skib /* Operation in this page 507277487Skib * 508277487Skib * shmem_page_offset = offset within page in shmem file 509277487Skib * page_length = bytes to copy for this page 510277487Skib */ 511277487Skib shmem_page_offset = offset_in_page(offset); 512277487Skib page_length = remain; 513277487Skib if ((shmem_page_offset + page_length) > PAGE_SIZE) 514277487Skib page_length = PAGE_SIZE - shmem_page_offset; 515277487Skib 516277487Skib page_do_bit17_swizzling = obj_do_bit17_swizzling && 517277487Skib (page_to_phys(page) & (1 << 17)) != 0; 518277487Skib 519277487Skib ret = shmem_pread_fast(page, shmem_page_offset, page_length, 520277487Skib user_data, page_do_bit17_swizzling, 521277487Skib needs_clflush); 522277487Skib if (ret == 0) 523277487Skib goto next_page; 524277487Skib 525296548Sdumbbell hit_slowpath = 1; 526277487Skib DRM_UNLOCK(dev); 527277487Skib 528296548Sdumbbell if (!prefaulted) { 529277487Skib ret = fault_in_multipages_writeable(user_data, remain); 530277487Skib /* Userspace is tricking us, but we've already clobbered 531277487Skib * its pages with the prefault and promised to write the 532277487Skib * data up to the first fault. Hence ignore any errors 533277487Skib * and just continue. */ 534277487Skib (void)ret; 535277487Skib prefaulted = 1; 536277487Skib } 537277487Skib 538277487Skib ret = shmem_pread_slow(page, shmem_page_offset, page_length, 539277487Skib user_data, page_do_bit17_swizzling, 540277487Skib needs_clflush); 541277487Skib 542277487Skib DRM_LOCK(dev); 543277487Skib 544277487Skibnext_page: 545277487Skib vm_page_reference(page); 546277487Skib 547277487Skib if (ret) 548277487Skib goto out; 549277487Skib 550277487Skib remain -= page_length; 551277487Skib user_data += page_length; 552277487Skib offset += page_length; 553277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 554277487Skib } 555277487Skib 556277487Skibout: 557277487Skib i915_gem_object_unpin_pages(obj); 558277487Skib 559296548Sdumbbell if (hit_slowpath) { 560296548Sdumbbell /* Fixup: Kill any reinstated backing storage pages */ 561296548Sdumbbell if (obj->madv == __I915_MADV_PURGED) 562296548Sdumbbell i915_gem_object_truncate(obj); 563296548Sdumbbell } 564296548Sdumbbell 565277487Skib return ret; 566277487Skib} 567277487Skib 568277487Skib/** 569277487Skib * Reads data from the object referenced by handle. 570277487Skib * 571277487Skib * On error, the contents of *data are undefined. 572277487Skib */ 573277487Skibint 574277487Skibi915_gem_pread_ioctl(struct drm_device *dev, void *data, 575277487Skib struct drm_file *file) 576277487Skib{ 577277487Skib struct drm_i915_gem_pread *args = data; 578277487Skib struct drm_i915_gem_object *obj; 579277487Skib int ret = 0; 580277487Skib 581277487Skib if (args->size == 0) 582277487Skib return 0; 583277487Skib 584277487Skib if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE)) 585277487Skib return -EFAULT; 586277487Skib 587235783Skib ret = i915_mutex_lock_interruptible(dev); 588277487Skib if (ret) 589277487Skib return ret; 590235783Skib 591277487Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 592235783Skib if (&obj->base == NULL) { 593235783Skib ret = -ENOENT; 594235783Skib goto unlock; 595235783Skib } 596277487Skib 597277487Skib /* Bounds check source. */ 598277487Skib if (args->offset > obj->base.size || 599277487Skib args->size > obj->base.size - args->offset) { 600235783Skib ret = -EINVAL; 601235783Skib goto out; 602235783Skib } 603235783Skib 604296548Sdumbbell#ifdef FREEBSD_WIP 605277487Skib /* prime objects have no backing filp to GEM pread/pwrite 606277487Skib * pages from. 607277487Skib */ 608277487Skib if (!obj->base.filp) { 609277487Skib ret = -EINVAL; 610277487Skib goto out; 611235783Skib } 612296548Sdumbbell#endif /* FREEBSD_WIP */ 613277487Skib 614277487Skib CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size); 615277487Skib 616277487Skib ret = i915_gem_shmem_pread(dev, obj, args, file); 617277487Skib 618235783Skibout: 619235783Skib drm_gem_object_unreference(&obj->base); 620235783Skibunlock: 621235783Skib DRM_UNLOCK(dev); 622277487Skib return ret; 623235783Skib} 624235783Skib 625277487Skib/* This is the fast write path which cannot handle 626277487Skib * page faults in the source data 627277487Skib */ 628277487Skib 629277487Skibstatic inline int 630296548Sdumbbellfast_user_write(vm_paddr_t mapping_addr, 631277487Skib off_t page_base, int page_offset, 632277487Skib char __user *user_data, 633277487Skib int length) 634235783Skib{ 635277487Skib void __iomem *vaddr_atomic; 636277487Skib void *vaddr; 637277487Skib unsigned long unwritten; 638235783Skib 639296548Sdumbbell vaddr_atomic = pmap_mapdev_attr(mapping_addr + page_base, 640277487Skib length, PAT_WRITE_COMBINING); 641277487Skib /* We can use the cpu mem copy function because this is X86. */ 642285988Sdumbbell vaddr = (char __force*)vaddr_atomic + page_offset; 643277487Skib unwritten = __copy_from_user_inatomic_nocache(vaddr, 644277487Skib user_data, length); 645277487Skib pmap_unmapdev((vm_offset_t)vaddr_atomic, length); 646277487Skib return unwritten; 647235783Skib} 648235783Skib 649277487Skib/** 650277487Skib * This is the fast pwrite path, where we copy the data directly from the 651277487Skib * user into the GTT, uncached. 652277487Skib */ 653277487Skibstatic int 654277487Skibi915_gem_gtt_pwrite_fast(struct drm_device *dev, 655277487Skib struct drm_i915_gem_object *obj, 656277487Skib struct drm_i915_gem_pwrite *args, 657277487Skib struct drm_file *file) 658277487Skib{ 659296548Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 660277487Skib ssize_t remain; 661277487Skib off_t offset, page_base; 662277487Skib char __user *user_data; 663277487Skib int page_offset, page_length, ret; 664277487Skib 665296548Sdumbbell ret = i915_gem_object_pin(obj, 0, true, true); 666285988Sdumbbell if (ret) 667277487Skib goto out; 668277487Skib 669277487Skib ret = i915_gem_object_set_to_gtt_domain(obj, true); 670277487Skib if (ret) 671277487Skib goto out_unpin; 672277487Skib 673277487Skib ret = i915_gem_object_put_fence(obj); 674277487Skib if (ret) 675277487Skib goto out_unpin; 676277487Skib 677277487Skib user_data = to_user_ptr(args->data_ptr); 678277487Skib remain = args->size; 679277487Skib 680277487Skib offset = obj->gtt_offset + args->offset; 681277487Skib 682277487Skib while (remain > 0) { 683277487Skib /* Operation in this page 684277487Skib * 685277487Skib * page_base = page offset within aperture 686277487Skib * page_offset = offset within page 687277487Skib * page_length = bytes to copy for this page 688277487Skib */ 689277487Skib page_base = offset & ~PAGE_MASK; 690277487Skib page_offset = offset_in_page(offset); 691277487Skib page_length = remain; 692277487Skib if ((page_offset + remain) > PAGE_SIZE) 693277487Skib page_length = PAGE_SIZE - page_offset; 694277487Skib 695277487Skib /* If we get a fault while copying data, then (presumably) our 696277487Skib * source page isn't available. Return the error and we'll 697277487Skib * retry in the slow path. 698277487Skib */ 699296548Sdumbbell if (fast_user_write(dev_priv->mm.gtt_base_addr, page_base, 700277487Skib page_offset, user_data, page_length)) { 701277487Skib ret = -EFAULT; 702277487Skib goto out_unpin; 703277487Skib } 704277487Skib 705277487Skib remain -= page_length; 706277487Skib user_data += page_length; 707277487Skib offset += page_length; 708277487Skib } 709277487Skib 710277487Skibout_unpin: 711277487Skib i915_gem_object_unpin(obj); 712277487Skibout: 713277487Skib return ret; 714277487Skib} 715277487Skib 716277487Skib/* Per-page copy function for the shmem pwrite fastpath. 717277487Skib * Flushes invalid cachelines before writing to the target if 718277487Skib * needs_clflush_before is set and flushes out any written cachelines after 719277487Skib * writing if needs_clflush is set. */ 720277487Skibstatic int 721277487Skibshmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length, 722277487Skib char __user *user_data, 723277487Skib bool page_do_bit17_swizzling, 724277487Skib bool needs_clflush_before, 725277487Skib bool needs_clflush_after) 726277487Skib{ 727277487Skib char *vaddr; 728277487Skib struct sf_buf *sf; 729277487Skib int ret; 730277487Skib 731277487Skib if (unlikely(page_do_bit17_swizzling)) 732277487Skib return -EINVAL; 733277487Skib 734277487Skib sched_pin(); 735277487Skib sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 736277487Skib if (sf == NULL) { 737277487Skib sched_unpin(); 738277487Skib return (-EFAULT); 739277487Skib } 740277487Skib vaddr = (char *)sf_buf_kva(sf); 741277487Skib if (needs_clflush_before) 742277487Skib drm_clflush_virt_range(vaddr + shmem_page_offset, 743277487Skib page_length); 744277487Skib ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, 745277487Skib user_data, 746277487Skib page_length); 747277487Skib if (needs_clflush_after) 748277487Skib drm_clflush_virt_range(vaddr + shmem_page_offset, 749277487Skib page_length); 750277487Skib sf_buf_free(sf); 751277487Skib sched_unpin(); 752277487Skib 753277487Skib return ret ? -EFAULT : 0; 754277487Skib} 755277487Skib 756277487Skib/* Only difference to the fast-path function is that this can handle bit17 757277487Skib * and uses non-atomic copy and kmap functions. */ 758277487Skibstatic int 759277487Skibshmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length, 760277487Skib char __user *user_data, 761277487Skib bool page_do_bit17_swizzling, 762277487Skib bool needs_clflush_before, 763277487Skib bool needs_clflush_after) 764277487Skib{ 765277487Skib char *vaddr; 766277487Skib struct sf_buf *sf; 767277487Skib int ret; 768277487Skib 769277487Skib sf = sf_buf_alloc(page, 0); 770277487Skib vaddr = (char *)sf_buf_kva(sf); 771277487Skib if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 772277487Skib shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 773277487Skib page_length, 774277487Skib page_do_bit17_swizzling); 775277487Skib if (page_do_bit17_swizzling) 776277487Skib ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 777277487Skib user_data, 778277487Skib page_length); 779277487Skib else 780277487Skib ret = __copy_from_user(vaddr + shmem_page_offset, 781277487Skib user_data, 782277487Skib page_length); 783277487Skib if (needs_clflush_after) 784277487Skib shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 785277487Skib page_length, 786277487Skib page_do_bit17_swizzling); 787277487Skib sf_buf_free(sf); 788277487Skib 789277487Skib return ret ? -EFAULT : 0; 790277487Skib} 791277487Skib 792277487Skibstatic int 793277487Skibi915_gem_shmem_pwrite(struct drm_device *dev, 794277487Skib struct drm_i915_gem_object *obj, 795277487Skib struct drm_i915_gem_pwrite *args, 796277487Skib struct drm_file *file) 797277487Skib{ 798296548Sdumbbell ssize_t remain; 799296548Sdumbbell off_t offset; 800277487Skib char __user *user_data; 801277487Skib int shmem_page_offset, page_length, ret = 0; 802277487Skib int obj_do_bit17_swizzling, page_do_bit17_swizzling; 803277487Skib int hit_slowpath = 0; 804277487Skib int needs_clflush_after = 0; 805277487Skib int needs_clflush_before = 0; 806277487Skib 807277487Skib user_data = to_user_ptr(args->data_ptr); 808296548Sdumbbell remain = args->size; 809277487Skib 810277487Skib obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 811277487Skib 812277487Skib if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 813277487Skib /* If we're not in the cpu write domain, set ourself into the gtt 814277487Skib * write domain and manually flush cachelines (if required). This 815277487Skib * optimizes for the case when the gpu will use the data 816277487Skib * right away and we therefore have to clflush anyway. */ 817296548Sdumbbell if (obj->cache_level == I915_CACHE_NONE) 818296548Sdumbbell needs_clflush_after = 1; 819296548Sdumbbell if (obj->gtt_space) { 820296548Sdumbbell ret = i915_gem_object_set_to_gtt_domain(obj, true); 821296548Sdumbbell if (ret) 822296548Sdumbbell return ret; 823296548Sdumbbell } 824277487Skib } 825296548Sdumbbell /* Same trick applies for invalidate partially written cachelines before 826296548Sdumbbell * writing. */ 827296548Sdumbbell if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) 828296548Sdumbbell && obj->cache_level == I915_CACHE_NONE) 829296548Sdumbbell needs_clflush_before = 1; 830277487Skib 831296548Sdumbbell ret = i915_gem_object_get_pages(obj); 832277487Skib if (ret) 833277487Skib return ret; 834277487Skib 835277487Skib i915_gem_object_pin_pages(obj); 836277487Skib 837296548Sdumbbell offset = args->offset; 838277487Skib obj->dirty = 1; 839277487Skib 840277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 841277487Skib for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 842277487Skib OFF_TO_IDX(offset));; page = vm_page_next(page)) { 843277487Skib VM_OBJECT_WUNLOCK(obj->base.vm_obj); 844277487Skib int partial_cacheline_write; 845277487Skib 846277487Skib if (remain <= 0) 847277487Skib break; 848277487Skib 849277487Skib /* Operation in this page 850277487Skib * 851277487Skib * shmem_page_offset = offset within page in shmem file 852277487Skib * page_length = bytes to copy for this page 853277487Skib */ 854277487Skib shmem_page_offset = offset_in_page(offset); 855277487Skib 856277487Skib page_length = remain; 857277487Skib if ((shmem_page_offset + page_length) > PAGE_SIZE) 858277487Skib page_length = PAGE_SIZE - shmem_page_offset; 859277487Skib 860277487Skib /* If we don't overwrite a cacheline completely we need to be 861277487Skib * careful to have up-to-date data by first clflushing. Don't 862277487Skib * overcomplicate things and flush the entire patch. */ 863277487Skib partial_cacheline_write = needs_clflush_before && 864277487Skib ((shmem_page_offset | page_length) 865277487Skib & (cpu_clflush_line_size - 1)); 866277487Skib 867277487Skib page_do_bit17_swizzling = obj_do_bit17_swizzling && 868277487Skib (page_to_phys(page) & (1 << 17)) != 0; 869277487Skib 870277487Skib ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 871277487Skib user_data, page_do_bit17_swizzling, 872277487Skib partial_cacheline_write, 873277487Skib needs_clflush_after); 874277487Skib if (ret == 0) 875277487Skib goto next_page; 876277487Skib 877277487Skib hit_slowpath = 1; 878277487Skib DRM_UNLOCK(dev); 879277487Skib ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 880277487Skib user_data, page_do_bit17_swizzling, 881277487Skib partial_cacheline_write, 882277487Skib needs_clflush_after); 883277487Skib 884277487Skib DRM_LOCK(dev); 885277487Skib 886277487Skibnext_page: 887277487Skib vm_page_dirty(page); 888277487Skib vm_page_reference(page); 889277487Skib 890277487Skib if (ret) 891277487Skib goto out; 892277487Skib 893277487Skib remain -= page_length; 894277487Skib user_data += page_length; 895277487Skib offset += page_length; 896277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 897277487Skib } 898277487Skib 899277487Skibout: 900277487Skib i915_gem_object_unpin_pages(obj); 901277487Skib 902277487Skib if (hit_slowpath) { 903296548Sdumbbell /* Fixup: Kill any reinstated backing storage pages */ 904296548Sdumbbell if (obj->madv == __I915_MADV_PURGED) 905296548Sdumbbell i915_gem_object_truncate(obj); 906296548Sdumbbell /* and flush dirty cachelines in case the object isn't in the cpu write 907296548Sdumbbell * domain anymore. */ 908296548Sdumbbell if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 909277487Skib i915_gem_clflush_object(obj); 910285988Sdumbbell i915_gem_chipset_flush(dev); 911277487Skib } 912277487Skib } 913277487Skib 914277487Skib if (needs_clflush_after) 915277487Skib i915_gem_chipset_flush(dev); 916277487Skib 917277487Skib return ret; 918277487Skib} 919277487Skib 920277487Skib/** 921277487Skib * Writes data to the object referenced by handle. 922277487Skib * 923277487Skib * On error, the contents of the buffer that were to be modified are undefined. 924277487Skib */ 925235783Skibint 926277487Skibi915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 927277487Skib struct drm_file *file) 928235783Skib{ 929277487Skib struct drm_i915_gem_pwrite *args = data; 930277487Skib struct drm_i915_gem_object *obj; 931277487Skib int ret; 932235783Skib 933277487Skib if (args->size == 0) 934277487Skib return 0; 935277487Skib 936277487Skib if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ)) 937277487Skib return -EFAULT; 938277487Skib 939296548Sdumbbell ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), 940296548Sdumbbell args->size); 941296548Sdumbbell if (ret) 942296548Sdumbbell return -EFAULT; 943277487Skib 944277487Skib ret = i915_mutex_lock_interruptible(dev); 945277487Skib if (ret) 946277487Skib return ret; 947277487Skib 948277487Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 949277487Skib if (&obj->base == NULL) { 950277487Skib ret = -ENOENT; 951277487Skib goto unlock; 952277487Skib } 953277487Skib 954277487Skib /* Bounds check destination. */ 955277487Skib if (args->offset > obj->base.size || 956277487Skib args->size > obj->base.size - args->offset) { 957277487Skib ret = -EINVAL; 958277487Skib goto out; 959277487Skib } 960277487Skib 961296548Sdumbbell#ifdef FREEBSD_WIP 962277487Skib /* prime objects have no backing filp to GEM pread/pwrite 963277487Skib * pages from. 964277487Skib */ 965277487Skib if (!obj->base.filp) { 966277487Skib ret = -EINVAL; 967277487Skib goto out; 968277487Skib } 969296548Sdumbbell#endif /* FREEBSD_WIP */ 970277487Skib 971277487Skib CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size); 972277487Skib 973277487Skib ret = -EFAULT; 974277487Skib /* We can only do the GTT pwrite on untiled buffers, as otherwise 975277487Skib * it would end up going through the fenced access, and we'll get 976277487Skib * different detiling behavior between reading and writing. 977277487Skib * pread/pwrite currently are reading and writing from the CPU 978277487Skib * perspective, requiring manual detiling by the client. 979277487Skib */ 980277487Skib if (obj->phys_obj) { 981277487Skib ret = i915_gem_phys_pwrite(dev, obj, args, file); 982277487Skib goto out; 983277487Skib } 984277487Skib 985296548Sdumbbell if (obj->cache_level == I915_CACHE_NONE && 986296548Sdumbbell obj->tiling_mode == I915_TILING_NONE && 987296548Sdumbbell obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 988277487Skib ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); 989277487Skib /* Note that the gtt paths might fail with non-page-backed user 990277487Skib * pointers (e.g. gtt mappings when moving data between 991277487Skib * textures). Fallback to the shmem path in that case. */ 992277487Skib } 993277487Skib 994277487Skib if (ret == -EFAULT || ret == -ENOSPC) 995277487Skib ret = i915_gem_shmem_pwrite(dev, obj, args, file); 996277487Skib 997277487Skibout: 998277487Skib drm_gem_object_unreference(&obj->base); 999277487Skibunlock: 1000277487Skib DRM_UNLOCK(dev); 1001277487Skib return ret; 1002235783Skib} 1003235783Skib 1004296548Sdumbbellint 1005296548Sdumbbelli915_gem_check_wedge(struct drm_i915_private *dev_priv, 1006296548Sdumbbell bool interruptible) 1007285988Sdumbbell{ 1008296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) { 1009296548Sdumbbell struct completion *x = &dev_priv->error_completion; 1010285988Sdumbbell bool recovery_complete; 1011285988Sdumbbell 1012285988Sdumbbell /* Give the error handler a chance to run. */ 1013296548Sdumbbell mtx_lock(&x->lock); 1014296548Sdumbbell recovery_complete = x->done > 0; 1015296548Sdumbbell mtx_unlock(&x->lock); 1016285988Sdumbbell 1017296548Sdumbbell /* Non-interruptible callers can't handle -EAGAIN, hence return 1018296548Sdumbbell * -EIO unconditionally for these. */ 1019296548Sdumbbell if (!interruptible) 1020296548Sdumbbell return -EIO; 1021296548Sdumbbell 1022296548Sdumbbell /* Recovery complete, but still wedged means reset failure. */ 1023296548Sdumbbell if (recovery_complete) 1024296548Sdumbbell return -EIO; 1025296548Sdumbbell 1026296548Sdumbbell return -EAGAIN; 1027285988Sdumbbell } 1028285988Sdumbbell 1029285988Sdumbbell return 0; 1030285988Sdumbbell} 1031285988Sdumbbell 1032285988Sdumbbell/* 1033285988Sdumbbell * Compare seqno against outstanding lazy request. Emit a request if they are 1034285988Sdumbbell * equal. 1035285988Sdumbbell */ 1036285988Sdumbbellstatic int 1037285988Sdumbbelli915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) 1038285988Sdumbbell{ 1039285988Sdumbbell int ret; 1040285988Sdumbbell 1041285988Sdumbbell DRM_LOCK_ASSERT(ring->dev); 1042285988Sdumbbell 1043285988Sdumbbell ret = 0; 1044296548Sdumbbell if (seqno == ring->outstanding_lazy_request) 1045296548Sdumbbell ret = i915_add_request(ring, NULL, NULL); 1046285988Sdumbbell 1047285988Sdumbbell return ret; 1048285988Sdumbbell} 1049285988Sdumbbell 1050296548Sdumbbell/** 1051296548Sdumbbell * __wait_seqno - wait until execution of seqno has finished 1052296548Sdumbbell * @ring: the ring expected to report seqno 1053296548Sdumbbell * @seqno: duh! 1054296548Sdumbbell * @interruptible: do an interruptible wait (normally yes) 1055296548Sdumbbell * @timeout: in - how long to wait (NULL forever); out - how much time remaining 1056296548Sdumbbell * 1057296548Sdumbbell * Returns 0 if the seqno was found within the alloted time. Else returns the 1058296548Sdumbbell * errno with remaining time filled in timeout argument. 1059296548Sdumbbell */ 1060285988Sdumbbellstatic int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 1061296548Sdumbbell bool interruptible, struct timespec *timeout) 1062285988Sdumbbell{ 1063285988Sdumbbell drm_i915_private_t *dev_priv = ring->dev->dev_private; 1064296548Sdumbbell struct timespec before, now, wait_time={1,0}; 1065296548Sdumbbell sbintime_t timeout_sbt; 1066296548Sdumbbell long end; 1067296548Sdumbbell bool wait_forever = true; 1068296548Sdumbbell int ret, flags; 1069285988Sdumbbell 1070296548Sdumbbell if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) 1071285988Sdumbbell return 0; 1072285988Sdumbbell 1073285988Sdumbbell CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 1074285988Sdumbbell 1075296548Sdumbbell if (timeout != NULL) { 1076296548Sdumbbell wait_time = *timeout; 1077296548Sdumbbell wait_forever = false; 1078285988Sdumbbell } 1079285988Sdumbbell 1080296548Sdumbbell timeout_sbt = tstosbt(wait_time); 1081296548Sdumbbell 1082296548Sdumbbell if (WARN_ON(!ring->irq_get(ring))) 1083296548Sdumbbell return -ENODEV; 1084296548Sdumbbell 1085296548Sdumbbell /* Record current time in case interrupted by signal, or wedged * */ 1086296548Sdumbbell getrawmonotonic(&before); 1087296548Sdumbbell 1088296548Sdumbbell#define EXIT_COND \ 1089296548Sdumbbell (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ 1090296548Sdumbbell atomic_read(&dev_priv->mm.wedged)) 1091285988Sdumbbell flags = interruptible ? PCATCH : 0; 1092296548Sdumbbell mtx_lock(&dev_priv->irq_lock); 1093296548Sdumbbell do { 1094296548Sdumbbell if (EXIT_COND) { 1095296548Sdumbbell end = 1; 1096296548Sdumbbell } else { 1097296548Sdumbbell ret = -msleep_sbt(&ring->irq_queue, &dev_priv->irq_lock, flags, 1098296548Sdumbbell "915gwr", timeout_sbt, 0, 0); 1099296548Sdumbbell 1100296548Sdumbbell /* 1101296548Sdumbbell * NOTE Linux<->FreeBSD: Convert msleep_sbt() return 1102296548Sdumbbell * value to something close to wait_event*_timeout() 1103296548Sdumbbell * functions used on Linux. 1104296548Sdumbbell * 1105296548Sdumbbell * >0 -> condition is true (end = time remaining) 1106296548Sdumbbell * =0 -> sleep timed out 1107296548Sdumbbell * <0 -> error (interrupted) 1108296548Sdumbbell * 1109296548Sdumbbell * We fake the remaining time by returning 1. We 1110296548Sdumbbell * compute a proper value later. 1111296548Sdumbbell */ 1112296548Sdumbbell if (EXIT_COND) 1113296548Sdumbbell /* We fake a remaining time of 1 tick. */ 1114296548Sdumbbell end = 1; 1115296548Sdumbbell else if (ret == -EINTR || ret == -ERESTART) 1116296548Sdumbbell /* Interrupted. */ 1117296548Sdumbbell end = -ERESTARTSYS; 1118296548Sdumbbell else 1119296548Sdumbbell /* Timeout. */ 1120296548Sdumbbell end = 0; 1121296548Sdumbbell } 1122296548Sdumbbell 1123296548Sdumbbell ret = i915_gem_check_wedge(dev_priv, interruptible); 1124296548Sdumbbell if (ret) 1125296548Sdumbbell end = ret; 1126296548Sdumbbell } while (end == 0 && wait_forever); 1127285988Sdumbbell mtx_unlock(&dev_priv->irq_lock); 1128285988Sdumbbell 1129296548Sdumbbell getrawmonotonic(&now); 1130285988Sdumbbell 1131296548Sdumbbell ring->irq_put(ring); 1132296548Sdumbbell CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, end); 1133296548Sdumbbell#undef EXIT_COND 1134296548Sdumbbell 1135296548Sdumbbell if (timeout) { 1136296548Sdumbbell timespecsub(&now, &before); 1137296548Sdumbbell timespecsub(timeout, &now); 1138296548Sdumbbell } 1139296548Sdumbbell 1140296548Sdumbbell switch (end) { 1141296548Sdumbbell case -EIO: 1142296548Sdumbbell case -EAGAIN: /* Wedged */ 1143296548Sdumbbell case -ERESTARTSYS: /* Signal */ 1144296548Sdumbbell case -ETIMEDOUT: /* Timeout */ 1145296548Sdumbbell return (int)end; 1146296548Sdumbbell case 0: /* Timeout */ 1147296548Sdumbbell return -ETIMEDOUT; 1148296548Sdumbbell default: /* Completed */ 1149296548Sdumbbell WARN_ON(end < 0); /* We're not aware of other errors */ 1150296548Sdumbbell return 0; 1151296548Sdumbbell } 1152285988Sdumbbell} 1153285988Sdumbbell 1154285988Sdumbbell/** 1155285988Sdumbbell * Waits for a sequence number to be signaled, and cleans up the 1156285988Sdumbbell * request and object lists appropriately for that event. 1157285988Sdumbbell */ 1158235783Skibint 1159296548Sdumbbelli915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) 1160285988Sdumbbell{ 1161285988Sdumbbell struct drm_device *dev = ring->dev; 1162285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 1163296548Sdumbbell bool interruptible = dev_priv->mm.interruptible; 1164285988Sdumbbell int ret; 1165285988Sdumbbell 1166296548Sdumbbell DRM_LOCK_ASSERT(dev); 1167296548Sdumbbell BUG_ON(seqno == 0); 1168285988Sdumbbell 1169296548Sdumbbell ret = i915_gem_check_wedge(dev_priv, interruptible); 1170285988Sdumbbell if (ret) 1171285988Sdumbbell return ret; 1172285988Sdumbbell 1173285988Sdumbbell ret = i915_gem_check_olr(ring, seqno); 1174285988Sdumbbell if (ret) 1175285988Sdumbbell return ret; 1176285988Sdumbbell 1177296548Sdumbbell return __wait_seqno(ring, seqno, interruptible, NULL); 1178285988Sdumbbell} 1179285988Sdumbbell 1180285988Sdumbbell/** 1181285988Sdumbbell * Ensures that all rendering to the object has completed and the object is 1182285988Sdumbbell * safe to unbind from the GTT or access from the CPU. 1183285988Sdumbbell */ 1184285988Sdumbbellstatic __must_check int 1185296548Sdumbbelli915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, 1186296548Sdumbbell bool readonly) 1187285988Sdumbbell{ 1188296548Sdumbbell struct intel_ring_buffer *ring = obj->ring; 1189296548Sdumbbell u32 seqno; 1190285988Sdumbbell int ret; 1191285988Sdumbbell 1192296548Sdumbbell seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; 1193296548Sdumbbell if (seqno == 0) 1194296548Sdumbbell return 0; 1195285988Sdumbbell 1196296548Sdumbbell ret = i915_wait_seqno(ring, seqno); 1197296548Sdumbbell if (ret) 1198296548Sdumbbell return ret; 1199296548Sdumbbell 1200296548Sdumbbell i915_gem_retire_requests_ring(ring); 1201296548Sdumbbell 1202296548Sdumbbell /* Manually manage the write flush as we may have not yet 1203296548Sdumbbell * retired the buffer. 1204296548Sdumbbell */ 1205296548Sdumbbell if (obj->last_write_seqno && 1206296548Sdumbbell i915_seqno_passed(seqno, obj->last_write_seqno)) { 1207296548Sdumbbell obj->last_write_seqno = 0; 1208296548Sdumbbell obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; 1209285988Sdumbbell } 1210285988Sdumbbell 1211285988Sdumbbell return 0; 1212285988Sdumbbell} 1213285988Sdumbbell 1214296548Sdumbbell/* A nonblocking variant of the above wait. This is a highly dangerous routine 1215296548Sdumbbell * as the object state may change during this call. 1216296548Sdumbbell */ 1217296548Sdumbbellstatic __must_check int 1218296548Sdumbbelli915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, 1219296548Sdumbbell bool readonly) 1220296548Sdumbbell{ 1221296548Sdumbbell struct drm_device *dev = obj->base.dev; 1222296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 1223296548Sdumbbell struct intel_ring_buffer *ring = obj->ring; 1224296548Sdumbbell u32 seqno; 1225296548Sdumbbell int ret; 1226296548Sdumbbell 1227296548Sdumbbell DRM_LOCK_ASSERT(dev); 1228296548Sdumbbell BUG_ON(!dev_priv->mm.interruptible); 1229296548Sdumbbell 1230296548Sdumbbell seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; 1231296548Sdumbbell if (seqno == 0) 1232296548Sdumbbell return 0; 1233296548Sdumbbell 1234296548Sdumbbell ret = i915_gem_check_wedge(dev_priv, true); 1235296548Sdumbbell if (ret) 1236296548Sdumbbell return ret; 1237296548Sdumbbell 1238296548Sdumbbell ret = i915_gem_check_olr(ring, seqno); 1239296548Sdumbbell if (ret) 1240296548Sdumbbell return ret; 1241296548Sdumbbell 1242296548Sdumbbell DRM_UNLOCK(dev); 1243296548Sdumbbell ret = __wait_seqno(ring, seqno, true, NULL); 1244296548Sdumbbell DRM_LOCK(dev); 1245296548Sdumbbell 1246296548Sdumbbell i915_gem_retire_requests_ring(ring); 1247296548Sdumbbell 1248296548Sdumbbell /* Manually manage the write flush as we may have not yet 1249296548Sdumbbell * retired the buffer. 1250296548Sdumbbell */ 1251296548Sdumbbell if (ret == 0 && 1252296548Sdumbbell obj->last_write_seqno && 1253296548Sdumbbell i915_seqno_passed(seqno, obj->last_write_seqno)) { 1254296548Sdumbbell obj->last_write_seqno = 0; 1255296548Sdumbbell obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; 1256296548Sdumbbell } 1257296548Sdumbbell 1258296548Sdumbbell return ret; 1259296548Sdumbbell} 1260296548Sdumbbell 1261296548Sdumbbell/** 1262296548Sdumbbell * Called when user space prepares to use an object with the CPU, either 1263296548Sdumbbell * through the mmap ioctl's mapping or a GTT mapping. 1264296548Sdumbbell */ 1265285988Sdumbbellint 1266235783Skibi915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1267285988Sdumbbell struct drm_file *file) 1268235783Skib{ 1269285988Sdumbbell struct drm_i915_gem_set_domain *args = data; 1270235783Skib struct drm_i915_gem_object *obj; 1271285988Sdumbbell uint32_t read_domains = args->read_domains; 1272285988Sdumbbell uint32_t write_domain = args->write_domain; 1273235783Skib int ret; 1274235783Skib 1275287174Sbapt /* Only handle setting domains to types used by the CPU. */ 1276287174Sbapt if (write_domain & I915_GEM_GPU_DOMAINS) 1277285988Sdumbbell return -EINVAL; 1278235783Skib 1279287174Sbapt if (read_domains & I915_GEM_GPU_DOMAINS) 1280287174Sbapt return -EINVAL; 1281287174Sbapt 1282287174Sbapt /* Having something in the write domain implies it's in the read 1283287174Sbapt * domain, and only that read domain. Enforce that in the request. 1284287174Sbapt */ 1285287174Sbapt if (write_domain != 0 && read_domains != write_domain) 1286287174Sbapt return -EINVAL; 1287287174Sbapt 1288235783Skib ret = i915_mutex_lock_interruptible(dev); 1289285988Sdumbbell if (ret) 1290285988Sdumbbell return ret; 1291235783Skib 1292235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1293235783Skib if (&obj->base == NULL) { 1294235783Skib ret = -ENOENT; 1295235783Skib goto unlock; 1296235783Skib } 1297235783Skib 1298296548Sdumbbell /* Try to flush the object off the GPU without holding the lock. 1299296548Sdumbbell * We will repeat the flush holding the lock in the normal manner 1300296548Sdumbbell * to catch cases where we are gazumped. 1301296548Sdumbbell */ 1302296548Sdumbbell ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain); 1303296548Sdumbbell if (ret) 1304296548Sdumbbell goto unref; 1305296548Sdumbbell 1306285988Sdumbbell if (read_domains & I915_GEM_DOMAIN_GTT) { 1307235783Skib ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1308285988Sdumbbell 1309285988Sdumbbell /* Silently promote "you're not bound, there was nothing to do" 1310285988Sdumbbell * to success, since the client was just asking us to 1311285988Sdumbbell * make sure everything was done. 1312285988Sdumbbell */ 1313235783Skib if (ret == -EINVAL) 1314235783Skib ret = 0; 1315285988Sdumbbell } else { 1316235783Skib ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1317285988Sdumbbell } 1318235783Skib 1319296548Sdumbbellunref: 1320235783Skib drm_gem_object_unreference(&obj->base); 1321235783Skibunlock: 1322235783Skib DRM_UNLOCK(dev); 1323285988Sdumbbell return ret; 1324235783Skib} 1325235783Skib 1326285988Sdumbbell/** 1327285988Sdumbbell * Called when user space has done writes to this buffer 1328285988Sdumbbell */ 1329235783Skibint 1330235783Skibi915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1331285988Sdumbbell struct drm_file *file) 1332235783Skib{ 1333285988Sdumbbell struct drm_i915_gem_sw_finish *args = data; 1334235783Skib struct drm_i915_gem_object *obj; 1335285988Sdumbbell int ret = 0; 1336235783Skib 1337285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 1338285988Sdumbbell if (ret) 1339285988Sdumbbell return ret; 1340277487Skib 1341235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1342235783Skib if (&obj->base == NULL) { 1343235783Skib ret = -ENOENT; 1344235783Skib goto unlock; 1345235783Skib } 1346285988Sdumbbell 1347285988Sdumbbell /* Pinned buffers may be scanout, so flush the cache */ 1348285988Sdumbbell if (obj->pin_count) 1349235783Skib i915_gem_object_flush_cpu_write_domain(obj); 1350285988Sdumbbell 1351235783Skib drm_gem_object_unreference(&obj->base); 1352235783Skibunlock: 1353235783Skib DRM_UNLOCK(dev); 1354285988Sdumbbell return ret; 1355235783Skib} 1356235783Skib 1357285988Sdumbbell/** 1358285988Sdumbbell * Maps the contents of an object, returning the address it is mapped 1359285988Sdumbbell * into. 1360285988Sdumbbell * 1361285988Sdumbbell * While the mapping holds a reference on the contents of the object, it doesn't 1362285988Sdumbbell * imply a ref on the object itself. 1363285988Sdumbbell */ 1364235783Skibint 1365235783Skibi915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1366285988Sdumbbell struct drm_file *file) 1367235783Skib{ 1368285988Sdumbbell struct drm_i915_gem_mmap *args = data; 1369235783Skib struct drm_gem_object *obj; 1370235783Skib struct proc *p; 1371235783Skib vm_map_t map; 1372235783Skib vm_offset_t addr; 1373235783Skib vm_size_t size; 1374235783Skib int error, rv; 1375235783Skib 1376235783Skib obj = drm_gem_object_lookup(dev, file, args->handle); 1377235783Skib if (obj == NULL) 1378285988Sdumbbell return -ENOENT; 1379285988Sdumbbell 1380296548Sdumbbell#ifdef FREEBSD_WIP 1381296548Sdumbbell /* prime objects have no backing filp to GEM mmap 1382296548Sdumbbell * pages from. 1383296548Sdumbbell */ 1384296548Sdumbbell if (!obj->filp) { 1385296548Sdumbbell drm_gem_object_unreference_unlocked(obj); 1386296548Sdumbbell return -EINVAL; 1387296548Sdumbbell } 1388296548Sdumbbell#endif /* FREEBSD_WIP */ 1389296548Sdumbbell 1390235783Skib error = 0; 1391235783Skib if (args->size == 0) 1392235783Skib goto out; 1393235783Skib p = curproc; 1394235783Skib map = &p->p_vmspace->vm_map; 1395235783Skib size = round_page(args->size); 1396235783Skib PROC_LOCK(p); 1397284215Smjg if (map->size + size > lim_cur_proc(p, RLIMIT_VMEM)) { 1398235783Skib PROC_UNLOCK(p); 1399280183Sdumbbell error = -ENOMEM; 1400235783Skib goto out; 1401235783Skib } 1402235783Skib PROC_UNLOCK(p); 1403235783Skib 1404235783Skib addr = 0; 1405235783Skib vm_object_reference(obj->vm_obj); 1406255426Sjhb rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1407253471Sjhb VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1408253497Skib VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1409235783Skib if (rv != KERN_SUCCESS) { 1410235783Skib vm_object_deallocate(obj->vm_obj); 1411235783Skib error = -vm_mmap_to_errno(rv); 1412235783Skib } else { 1413235783Skib args->addr_ptr = (uint64_t)addr; 1414235783Skib } 1415235783Skibout: 1416296548Sdumbbell drm_gem_object_unreference_unlocked(obj); 1417235783Skib return (error); 1418235783Skib} 1419235783Skib 1420235783Skibstatic int 1421235783Skibi915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1422235783Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 1423235783Skib{ 1424235783Skib 1425296548Sdumbbell /* 1426296548Sdumbbell * NOTE Linux<->FreeBSD: drm_gem_mmap_single() takes care of 1427296548Sdumbbell * calling drm_gem_object_reference(). That's why we don't 1428296548Sdumbbell * do this here. i915_gem_pager_dtor(), below, will call 1429296548Sdumbbell * drm_gem_object_unreference(). 1430296548Sdumbbell * 1431296548Sdumbbell * On Linux, drm_gem_vm_open() references the object because 1432296548Sdumbbell * it's called the mapping is copied. drm_gem_vm_open() is not 1433296548Sdumbbell * called when the mapping is created. So the possible sequences 1434296548Sdumbbell * are: 1435296548Sdumbbell * 1. drm_gem_mmap(): ref++ 1436296548Sdumbbell * 2. drm_gem_vm_close(): ref-- 1437296548Sdumbbell * 1438296548Sdumbbell * 1. drm_gem_mmap(): ref++ 1439296548Sdumbbell * 2. drm_gem_vm_open(): ref++ (for the copied vma) 1440296548Sdumbbell * 3. drm_gem_vm_close(): ref-- (for the copied vma) 1441296548Sdumbbell * 4. drm_gem_vm_close(): ref-- (for the initial vma) 1442296548Sdumbbell * 1443296548Sdumbbell * On FreeBSD, i915_gem_pager_ctor() is called once during the 1444296548Sdumbbell * creation of the mapping. No callback is called when the 1445296548Sdumbbell * mapping is shared during a fork(). i915_gem_pager_dtor() is 1446296548Sdumbbell * called when the last reference to the mapping is dropped. So 1447296548Sdumbbell * the only sequence is: 1448296548Sdumbbell * 1. drm_gem_mmap_single(): ref++ 1449296548Sdumbbell * 2. i915_gem_pager_ctor(): <noop> 1450296548Sdumbbell * 3. i915_gem_pager_dtor(): ref-- 1451296548Sdumbbell */ 1452296548Sdumbbell 1453235783Skib *color = 0; /* XXXKIB */ 1454235783Skib return (0); 1455235783Skib} 1456235783Skib 1457285988Sdumbbell/** 1458285988Sdumbbell * i915_gem_fault - fault a page into the GTT 1459285988Sdumbbell * vma: VMA in question 1460285988Sdumbbell * vmf: fault info 1461285988Sdumbbell * 1462285988Sdumbbell * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 1463285988Sdumbbell * from userspace. The fault handler takes care of binding the object to 1464285988Sdumbbell * the GTT (if needed), allocating and programming a fence register (again, 1465285988Sdumbbell * only if needed based on whether the old reg is still valid or the object 1466285988Sdumbbell * is tiled) and inserting a new PTE into the faulting process. 1467285988Sdumbbell * 1468285988Sdumbbell * Note that the faulting process may involve evicting existing objects 1469285988Sdumbbell * from the GTT and/or fence registers to make room. So performance may 1470285988Sdumbbell * suffer if the GTT working set is large or there are few fence registers 1471285988Sdumbbell * left. 1472285988Sdumbbell */ 1473285988Sdumbbell 1474235783Skibint i915_intr_pf; 1475235783Skib 1476235783Skibstatic int 1477312075Skibi915_gem_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type, 1478312075Skib vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) 1479235783Skib{ 1480296548Sdumbbell struct drm_gem_object *gem_obj = vm_obj->handle; 1481296548Sdumbbell struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 1482296548Sdumbbell struct drm_device *dev = obj->base.dev; 1483296548Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 1484297231Skib vm_page_t page; 1485296548Sdumbbell int ret = 0; 1486312075Skib bool write = (max_prot & VM_PROT_WRITE) != 0; 1487296548Sdumbbell bool pinned; 1488235783Skib 1489254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1490235783Skibretry: 1491296548Sdumbbell ret = 0; 1492296548Sdumbbell pinned = 0; 1493285988Sdumbbell page = NULL; 1494235783Skib 1495235783Skib if (i915_intr_pf) { 1496235783Skib ret = i915_mutex_lock_interruptible(dev); 1497296548Sdumbbell if (ret != 0) 1498235783Skib goto out; 1499235783Skib } else 1500235783Skib DRM_LOCK(dev); 1501235783Skib 1502251960Skib /* 1503251960Skib * Since the object lock was dropped, other thread might have 1504251960Skib * faulted on the same GTT address and instantiated the 1505251960Skib * mapping for the page. Recheck. 1506251960Skib */ 1507251960Skib VM_OBJECT_WLOCK(vm_obj); 1508312075Skib page = vm_page_lookup(vm_obj, pidx); 1509285988Sdumbbell if (page != NULL) { 1510285988Sdumbbell if (vm_page_busied(page)) { 1511251960Skib DRM_UNLOCK(dev); 1512285988Sdumbbell vm_page_lock(page); 1513254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1514307671Skib vm_page_busy_sleep(page, "915pee", false); 1515251960Skib goto retry; 1516251960Skib } 1517251960Skib goto have_page; 1518251960Skib } else 1519251960Skib VM_OBJECT_WUNLOCK(vm_obj); 1520251960Skib 1521235783Skib /* Now bind it into the GTT if needed */ 1522296548Sdumbbell ret = i915_gem_object_pin(obj, 0, true, false); 1523296548Sdumbbell if (ret) 1524296548Sdumbbell goto unlock; 1525296548Sdumbbell pinned = 1; 1526235783Skib 1527296548Sdumbbell ret = i915_gem_object_set_to_gtt_domain(obj, write); 1528296548Sdumbbell if (ret) 1529296548Sdumbbell goto unpin; 1530235783Skib 1531277487Skib ret = i915_gem_object_get_fence(obj); 1532296548Sdumbbell if (ret) 1533296548Sdumbbell goto unpin; 1534235783Skib 1535296548Sdumbbell obj->fault_mappable = true; 1536235783Skib 1537312075Skib page = PHYS_TO_VM_PAGE(dev_priv->mm.gtt_base_addr + obj->gtt_offset + 1538312075Skib IDX_TO_OFF(pidx)); 1539285988Sdumbbell if (page == NULL) { 1540235783Skib ret = -EFAULT; 1541296548Sdumbbell goto unpin; 1542235783Skib } 1543285988Sdumbbell KASSERT((page->flags & PG_FICTITIOUS) != 0, 1544312075Skib ("physical address %#jx not fictitious, page %p", 1545312075Skib (uintmax_t)(dev_priv->mm.gtt_base_addr + obj->gtt_offset + 1546312075Skib IDX_TO_OFF(pidx)), page)); 1547285988Sdumbbell KASSERT(page->wire_count == 1, ("wire_count not 1 %p", page)); 1548235783Skib 1549312075Skib VM_OBJECT_WLOCK(vm_obj); 1550285988Sdumbbell if (vm_page_busied(page)) { 1551296548Sdumbbell i915_gem_object_unpin(obj); 1552235783Skib DRM_UNLOCK(dev); 1553285988Sdumbbell vm_page_lock(page); 1554254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1555307671Skib vm_page_busy_sleep(page, "915pbs", false); 1556235783Skib goto retry; 1557235783Skib } 1558312075Skib if (vm_page_insert(page, vm_obj, pidx)) { 1559296548Sdumbbell i915_gem_object_unpin(obj); 1560254141Sattilio DRM_UNLOCK(dev); 1561254141Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1562254141Sattilio VM_WAIT; 1563254141Sattilio goto retry; 1564254141Sattilio } 1565285988Sdumbbell page->valid = VM_PAGE_BITS_ALL; 1566251960Skibhave_page: 1567285988Sdumbbell vm_page_xbusy(page); 1568235783Skib 1569312075Skib CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, pidx, fault_type, 1570285988Sdumbbell page->phys_addr); 1571296548Sdumbbell if (pinned) { 1572296548Sdumbbell /* 1573296548Sdumbbell * We may have not pinned the object if the page was 1574312075Skib * found by the call to vm_page_lookup(). 1575296548Sdumbbell */ 1576296548Sdumbbell i915_gem_object_unpin(obj); 1577296548Sdumbbell } 1578235783Skib DRM_UNLOCK(dev); 1579312075Skib *first = *last = pidx; 1580235783Skib return (VM_PAGER_OK); 1581235783Skib 1582296548Sdumbbellunpin: 1583296548Sdumbbell i915_gem_object_unpin(obj); 1584235783Skibunlock: 1585235783Skib DRM_UNLOCK(dev); 1586235783Skibout: 1587235783Skib KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1588312075Skib CTR4(KTR_DRM, "fault_fail %p %jx %x err %d", gem_obj, pidx, fault_type, 1589296548Sdumbbell -ret); 1590296720Sdumbbell if (ret == -ERESTARTSYS) { 1591296720Sdumbbell /* 1592296720Sdumbbell * NOTE Linux<->FreeBSD: Convert Linux' -ERESTARTSYS to 1593296720Sdumbbell * the more common -EINTR, so the page fault is retried. 1594296720Sdumbbell */ 1595296720Sdumbbell ret = -EINTR; 1596296720Sdumbbell } 1597235783Skib if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1598235783Skib kern_yield(PRI_USER); 1599254138Sattilio goto retry; 1600235783Skib } 1601248084Sattilio VM_OBJECT_WLOCK(vm_obj); 1602235783Skib return (VM_PAGER_ERROR); 1603235783Skib} 1604235783Skib 1605235783Skibstatic void 1606235783Skibi915_gem_pager_dtor(void *handle) 1607235783Skib{ 1608296548Sdumbbell struct drm_gem_object *obj = handle; 1609296548Sdumbbell struct drm_device *dev = obj->dev; 1610235783Skib 1611235783Skib DRM_LOCK(dev); 1612235783Skib drm_gem_object_unreference(obj); 1613235783Skib DRM_UNLOCK(dev); 1614235783Skib} 1615235783Skib 1616235783Skibstruct cdev_pager_ops i915_gem_pager_ops = { 1617312075Skib .cdev_pg_populate = i915_gem_pager_populate, 1618312075Skib .cdev_pg_ctor = i915_gem_pager_ctor, 1619312075Skib .cdev_pg_dtor = i915_gem_pager_dtor, 1620235783Skib}; 1621235783Skib 1622285988Sdumbbell/** 1623285988Sdumbbell * i915_gem_release_mmap - remove physical page mappings 1624285988Sdumbbell * @obj: obj in question 1625285988Sdumbbell * 1626285988Sdumbbell * Preserve the reservation of the mmapping with the DRM core code, but 1627285988Sdumbbell * relinquish ownership of the pages back to the system. 1628285988Sdumbbell * 1629285988Sdumbbell * It is vital that we remove the page mapping if we have mapped a tiled 1630285988Sdumbbell * object through the GTT and then lose the fence register due to 1631285988Sdumbbell * resource pressure. Similarly if the object has been moved out of the 1632285988Sdumbbell * aperture, than pages mapped into userspace must be revoked. Removing the 1633285988Sdumbbell * mapping will then trigger a page fault on the next user access, allowing 1634285988Sdumbbell * fixup by i915_gem_fault(). 1635285988Sdumbbell */ 1636235783Skibvoid 1637285988Sdumbbelli915_gem_release_mmap(struct drm_i915_gem_object *obj) 1638235783Skib{ 1639285988Sdumbbell vm_object_t devobj; 1640285988Sdumbbell vm_page_t page; 1641285988Sdumbbell int i, page_count; 1642235783Skib 1643285988Sdumbbell if (!obj->fault_mappable) 1644235783Skib return; 1645235783Skib 1646285988Sdumbbell CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 1647285988Sdumbbell OFF_TO_IDX(obj->base.size)); 1648285988Sdumbbell devobj = cdev_pager_lookup(obj); 1649285988Sdumbbell if (devobj != NULL) { 1650285988Sdumbbell page_count = OFF_TO_IDX(obj->base.size); 1651235783Skib 1652285988Sdumbbell VM_OBJECT_WLOCK(devobj); 1653285988Sdumbbellretry: 1654285988Sdumbbell for (i = 0; i < page_count; i++) { 1655285988Sdumbbell page = vm_page_lookup(devobj, i); 1656285988Sdumbbell if (page == NULL) 1657285988Sdumbbell continue; 1658285988Sdumbbell if (vm_page_sleep_if_busy(page, "915unm")) 1659285988Sdumbbell goto retry; 1660285988Sdumbbell cdev_pager_free_page(devobj, page); 1661235783Skib } 1662285988Sdumbbell VM_OBJECT_WUNLOCK(devobj); 1663285988Sdumbbell vm_object_deallocate(devobj); 1664235783Skib } 1665235783Skib 1666285988Sdumbbell obj->fault_mappable = false; 1667235783Skib} 1668235783Skib 1669235783Skibstatic uint32_t 1670235783Skibi915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1671235783Skib{ 1672235783Skib uint32_t gtt_size; 1673235783Skib 1674235783Skib if (INTEL_INFO(dev)->gen >= 4 || 1675235783Skib tiling_mode == I915_TILING_NONE) 1676285988Sdumbbell return size; 1677235783Skib 1678235783Skib /* Previous chips need a power-of-two fence region when tiling */ 1679235783Skib if (INTEL_INFO(dev)->gen == 3) 1680235783Skib gtt_size = 1024*1024; 1681235783Skib else 1682235783Skib gtt_size = 512*1024; 1683235783Skib 1684235783Skib while (gtt_size < size) 1685235783Skib gtt_size <<= 1; 1686235783Skib 1687285988Sdumbbell return gtt_size; 1688235783Skib} 1689235783Skib 1690235783Skib/** 1691235783Skib * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1692235783Skib * @obj: object to check 1693235783Skib * 1694235783Skib * Return the required GTT alignment for an object, taking into account 1695235783Skib * potential fence register mapping. 1696235783Skib */ 1697235783Skibstatic uint32_t 1698285988Sdumbbelli915_gem_get_gtt_alignment(struct drm_device *dev, 1699285988Sdumbbell uint32_t size, 1700285988Sdumbbell int tiling_mode) 1701235783Skib{ 1702235783Skib /* 1703235783Skib * Minimum alignment is 4k (GTT page size), but might be greater 1704235783Skib * if a fence register is needed for the object. 1705235783Skib */ 1706235783Skib if (INTEL_INFO(dev)->gen >= 4 || 1707235783Skib tiling_mode == I915_TILING_NONE) 1708285988Sdumbbell return 4096; 1709235783Skib 1710235783Skib /* 1711235783Skib * Previous chips need to be aligned to the size of the smallest 1712235783Skib * fence register that can contain the object. 1713235783Skib */ 1714285988Sdumbbell return i915_gem_get_gtt_size(dev, size, tiling_mode); 1715235783Skib} 1716235783Skib 1717285988Sdumbbell/** 1718285988Sdumbbell * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an 1719285988Sdumbbell * unfenced object 1720285988Sdumbbell * @dev: the device 1721285988Sdumbbell * @size: size of the object 1722285988Sdumbbell * @tiling_mode: tiling mode of the object 1723285988Sdumbbell * 1724285988Sdumbbell * Return the required GTT alignment for an object, only taking into account 1725285988Sdumbbell * unfenced tiled surface requirements. 1726285988Sdumbbell */ 1727235783Skibuint32_t 1728285988Sdumbbelli915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, 1729285988Sdumbbell uint32_t size, 1730285988Sdumbbell int tiling_mode) 1731235783Skib{ 1732235783Skib /* 1733235783Skib * Minimum alignment is 4k (GTT page size) for sane hw. 1734235783Skib */ 1735287174Sbapt if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || 1736287174Sbapt tiling_mode == I915_TILING_NONE) 1737285988Sdumbbell return 4096; 1738235783Skib 1739285988Sdumbbell /* Previous hardware however needs to be aligned to a power-of-two 1740235783Skib * tile height. The simplest method for determining this is to reuse 1741235783Skib * the power-of-tile object size. 1742285988Sdumbbell */ 1743285988Sdumbbell return i915_gem_get_gtt_size(dev, size, tiling_mode); 1744235783Skib} 1745235783Skib 1746296548Sdumbbellstatic int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) 1747296548Sdumbbell{ 1748296548Sdumbbell struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 1749296548Sdumbbell int ret; 1750296548Sdumbbell 1751296548Sdumbbell if (obj->base.on_map) 1752296548Sdumbbell return 0; 1753296548Sdumbbell 1754296548Sdumbbell dev_priv->mm.shrinker_no_lock_stealing = true; 1755296548Sdumbbell 1756296548Sdumbbell ret = drm_gem_create_mmap_offset(&obj->base); 1757296548Sdumbbell if (ret != -ENOSPC) 1758296548Sdumbbell goto out; 1759296548Sdumbbell 1760296548Sdumbbell /* Badly fragmented mmap space? The only way we can recover 1761296548Sdumbbell * space is by destroying unwanted objects. We can't randomly release 1762296548Sdumbbell * mmap_offsets as userspace expects them to be persistent for the 1763296548Sdumbbell * lifetime of the objects. The closest we can is to release the 1764296548Sdumbbell * offsets on purgeable objects by truncating it and marking it purged, 1765296548Sdumbbell * which prevents userspace from ever using that object again. 1766296548Sdumbbell */ 1767296548Sdumbbell i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT); 1768296548Sdumbbell ret = drm_gem_create_mmap_offset(&obj->base); 1769296548Sdumbbell if (ret != -ENOSPC) 1770296548Sdumbbell goto out; 1771296548Sdumbbell 1772296548Sdumbbell i915_gem_shrink_all(dev_priv); 1773296548Sdumbbell ret = drm_gem_create_mmap_offset(&obj->base); 1774296548Sdumbbellout: 1775296548Sdumbbell dev_priv->mm.shrinker_no_lock_stealing = false; 1776296548Sdumbbell 1777296548Sdumbbell return ret; 1778296548Sdumbbell} 1779296548Sdumbbell 1780296548Sdumbbellstatic void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) 1781296548Sdumbbell{ 1782296548Sdumbbell if (!obj->base.on_map) 1783296548Sdumbbell return; 1784296548Sdumbbell 1785296548Sdumbbell drm_gem_free_mmap_offset(&obj->base); 1786296548Sdumbbell} 1787296548Sdumbbell 1788285988Sdumbbellint 1789285988Sdumbbelli915_gem_mmap_gtt(struct drm_file *file, 1790285988Sdumbbell struct drm_device *dev, 1791285988Sdumbbell uint32_t handle, 1792285988Sdumbbell uint64_t *offset) 1793235783Skib{ 1794285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 1795285988Sdumbbell struct drm_i915_gem_object *obj; 1796235783Skib int ret; 1797235783Skib 1798285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 1799285988Sdumbbell if (ret) 1800285988Sdumbbell return ret; 1801235783Skib 1802285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1803285988Sdumbbell if (&obj->base == NULL) { 1804285988Sdumbbell ret = -ENOENT; 1805285988Sdumbbell goto unlock; 1806235783Skib } 1807235783Skib 1808285988Sdumbbell if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1809285988Sdumbbell ret = -E2BIG; 1810285988Sdumbbell goto out; 1811235783Skib } 1812235783Skib 1813285988Sdumbbell if (obj->madv != I915_MADV_WILLNEED) { 1814285988Sdumbbell DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1815285988Sdumbbell ret = -EINVAL; 1816285988Sdumbbell goto out; 1817235783Skib } 1818235783Skib 1819296548Sdumbbell ret = i915_gem_object_create_mmap_offset(obj); 1820285988Sdumbbell if (ret) 1821285988Sdumbbell goto out; 1822235783Skib 1823285988Sdumbbell *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1824285988Sdumbbell DRM_GEM_MAPPING_KEY; 1825235783Skib 1826285988Sdumbbellout: 1827285988Sdumbbell drm_gem_object_unreference(&obj->base); 1828285988Sdumbbellunlock: 1829285988Sdumbbell DRM_UNLOCK(dev); 1830285988Sdumbbell return ret; 1831235783Skib} 1832235783Skib 1833285988Sdumbbell/** 1834285988Sdumbbell * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1835285988Sdumbbell * @dev: DRM device 1836285988Sdumbbell * @data: GTT mapping ioctl data 1837285988Sdumbbell * @file: GEM object info 1838285988Sdumbbell * 1839285988Sdumbbell * Simply returns the fake offset to userspace so it can mmap it. 1840285988Sdumbbell * The mmap call will end up in drm_gem_mmap(), which will set things 1841285988Sdumbbell * up so we can get faults in the handler above. 1842285988Sdumbbell * 1843285988Sdumbbell * The fault handler will take care of binding the object into the GTT 1844285988Sdumbbell * (since it may have been evicted to make room for something), allocating 1845285988Sdumbbell * a fence register, and mapping the appropriate aperture address into 1846285988Sdumbbell * userspace. 1847285988Sdumbbell */ 1848277487Skibint 1849285988Sdumbbelli915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1850285988Sdumbbell struct drm_file *file) 1851235783Skib{ 1852285988Sdumbbell struct drm_i915_gem_mmap_gtt *args = data; 1853277487Skib 1854285988Sdumbbell return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); 1855277487Skib} 1856277487Skib 1857285988Sdumbbell/* Immediately discard the backing storage */ 1858285988Sdumbbellstatic void 1859285988Sdumbbelli915_gem_object_truncate(struct drm_i915_gem_object *obj) 1860277487Skib{ 1861285988Sdumbbell vm_object_t vm_obj; 1862235783Skib 1863285988Sdumbbell vm_obj = obj->base.vm_obj; 1864285988Sdumbbell VM_OBJECT_WLOCK(vm_obj); 1865285988Sdumbbell vm_object_page_remove(vm_obj, 0, 0, false); 1866285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 1867296548Sdumbbell i915_gem_object_free_mmap_offset(obj); 1868296548Sdumbbell 1869296548Sdumbbell obj->madv = __I915_MADV_PURGED; 1870235783Skib} 1871235783Skib 1872285988Sdumbbellstatic inline int 1873285988Sdumbbelli915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 1874235783Skib{ 1875285988Sdumbbell return obj->madv == I915_MADV_DONTNEED; 1876235783Skib} 1877235783Skib 1878277487Skibstatic void 1879277487Skibi915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj, 1880277487Skib vm_pindex_t si, vm_pindex_t ei) 1881277487Skib{ 1882277487Skib vm_object_t vm_obj; 1883285988Sdumbbell vm_page_t page; 1884277487Skib vm_pindex_t i; 1885277487Skib 1886277487Skib vm_obj = obj->base.vm_obj; 1887277487Skib VM_OBJECT_ASSERT_LOCKED(vm_obj); 1888285988Sdumbbell for (i = si, page = vm_page_lookup(vm_obj, i); i < ei; 1889285988Sdumbbell page = vm_page_next(page), i++) { 1890285988Sdumbbell KASSERT(page->pindex == i, ("pindex %jx %jx", 1891285988Sdumbbell (uintmax_t)page->pindex, (uintmax_t)i)); 1892285988Sdumbbell vm_page_lock(page); 1893324129Salc if (vm_page_unwire(page, PQ_INACTIVE)) 1894277487Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1895285988Sdumbbell vm_page_unlock(page); 1896277487Skib } 1897277487Skib} 1898277487Skib 1899235783Skib#define GEM_PARANOID_CHECK_GTT 0 1900235783Skib#if GEM_PARANOID_CHECK_GTT 1901235783Skibstatic void 1902235783Skibi915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 1903235783Skib int page_count) 1904235783Skib{ 1905235783Skib struct drm_i915_private *dev_priv; 1906235783Skib vm_paddr_t pa; 1907235783Skib unsigned long start, end; 1908235783Skib u_int i; 1909235783Skib int j; 1910235783Skib 1911235783Skib dev_priv = dev->dev_private; 1912235783Skib start = OFF_TO_IDX(dev_priv->mm.gtt_start); 1913235783Skib end = OFF_TO_IDX(dev_priv->mm.gtt_end); 1914235783Skib for (i = start; i < end; i++) { 1915235783Skib pa = intel_gtt_read_pte_paddr(i); 1916235783Skib for (j = 0; j < page_count; j++) { 1917235783Skib if (pa == VM_PAGE_TO_PHYS(ma[j])) { 1918235783Skib panic("Page %p in GTT pte index %d pte %x", 1919235783Skib ma[i], i, intel_gtt_read_pte(i)); 1920235783Skib } 1921235783Skib } 1922235783Skib } 1923235783Skib} 1924235783Skib#endif 1925235783Skib 1926235783Skibstatic void 1927235783Skibi915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 1928235783Skib{ 1929293837Sdumbbell int page_count = obj->base.size / PAGE_SIZE; 1930296548Sdumbbell int ret, i; 1931235783Skib 1932296548Sdumbbell BUG_ON(obj->madv == __I915_MADV_PURGED); 1933235783Skib 1934296548Sdumbbell ret = i915_gem_object_set_to_cpu_domain(obj, true); 1935296548Sdumbbell if (ret) { 1936296548Sdumbbell /* In the event of a disaster, abandon all caches and 1937296548Sdumbbell * hope for the best. 1938296548Sdumbbell */ 1939296548Sdumbbell WARN_ON(ret != -EIO); 1940296548Sdumbbell i915_gem_clflush_object(obj); 1941296548Sdumbbell obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1942296548Sdumbbell } 1943296548Sdumbbell 1944296548Sdumbbell if (i915_gem_object_needs_bit17_swizzle(obj)) 1945235783Skib i915_gem_object_save_bit_17_swizzle(obj); 1946293837Sdumbbell 1947235783Skib if (obj->madv == I915_MADV_DONTNEED) 1948235783Skib obj->dirty = 0; 1949293837Sdumbbell 1950248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 1951235783Skib#if GEM_PARANOID_CHECK_GTT 1952235783Skib i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 1953235783Skib#endif 1954235783Skib for (i = 0; i < page_count; i++) { 1955293837Sdumbbell vm_page_t page = obj->pages[i]; 1956293837Sdumbbell 1957235783Skib if (obj->dirty) 1958285988Sdumbbell vm_page_dirty(page); 1959293837Sdumbbell 1960235783Skib if (obj->madv == I915_MADV_WILLNEED) 1961285988Sdumbbell vm_page_reference(page); 1962293837Sdumbbell 1963285988Sdumbbell vm_page_lock(page); 1964267548Sattilio vm_page_unwire(obj->pages[i], PQ_ACTIVE); 1965285988Sdumbbell vm_page_unlock(page); 1966235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1967235783Skib } 1968248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 1969235783Skib obj->dirty = 0; 1970293837Sdumbbell 1971235783Skib free(obj->pages, DRM_I915_GEM); 1972235783Skib obj->pages = NULL; 1973235783Skib} 1974235783Skib 1975285988Sdumbbellstatic int 1976296548Sdumbbelli915_gem_object_put_pages(struct drm_i915_gem_object *obj) 1977235783Skib{ 1978296548Sdumbbell const struct drm_i915_gem_object_ops *ops = obj->ops; 1979235783Skib 1980296548Sdumbbell if (obj->pages == NULL) 1981296548Sdumbbell return 0; 1982285988Sdumbbell 1983296548Sdumbbell BUG_ON(obj->gtt_space); 1984285988Sdumbbell 1985296548Sdumbbell if (obj->pages_pin_count) 1986296548Sdumbbell return -EBUSY; 1987285988Sdumbbell 1988296548Sdumbbell /* ->put_pages might need to allocate memory for the bit17 swizzle 1989296548Sdumbbell * array, hence protect them from being reaped by removing them from gtt 1990296548Sdumbbell * lists early. */ 1991296548Sdumbbell list_del(&obj->gtt_list); 1992235783Skib 1993296548Sdumbbell ops->put_pages(obj); 1994296548Sdumbbell obj->pages = NULL; 1995235783Skib 1996296548Sdumbbell if (i915_gem_object_is_purgeable(obj)) 1997296548Sdumbbell i915_gem_object_truncate(obj); 1998285988Sdumbbell 1999296548Sdumbbell return 0; 2000296548Sdumbbell} 2001285988Sdumbbell 2002296548Sdumbbellstatic long 2003296548Sdumbbell__i915_gem_shrink(struct drm_i915_private *dev_priv, long target, 2004296548Sdumbbell bool purgeable_only) 2005296548Sdumbbell{ 2006296548Sdumbbell struct drm_i915_gem_object *obj, *next; 2007296548Sdumbbell long count = 0; 2008296548Sdumbbell 2009296548Sdumbbell list_for_each_entry_safe(obj, next, 2010296548Sdumbbell &dev_priv->mm.unbound_list, 2011296548Sdumbbell gtt_list) { 2012296548Sdumbbell if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && 2013296548Sdumbbell i915_gem_object_put_pages(obj) == 0) { 2014296548Sdumbbell count += obj->base.size >> PAGE_SHIFT; 2015296548Sdumbbell if (target != -1 && count >= target) 2016296548Sdumbbell return count; 2017296548Sdumbbell } 2018235783Skib } 2019235783Skib 2020296548Sdumbbell list_for_each_entry_safe(obj, next, 2021296548Sdumbbell &dev_priv->mm.inactive_list, 2022296548Sdumbbell mm_list) { 2023296548Sdumbbell if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && 2024296548Sdumbbell i915_gem_object_unbind(obj) == 0 && 2025296548Sdumbbell i915_gem_object_put_pages(obj) == 0) { 2026296548Sdumbbell count += obj->base.size >> PAGE_SHIFT; 2027296548Sdumbbell if (target != -1 && count >= target) 2028296548Sdumbbell return count; 2029296548Sdumbbell } 2030285988Sdumbbell } 2031285988Sdumbbell 2032296548Sdumbbell return count; 2033235783Skib} 2034235783Skib 2035296548Sdumbbellstatic long 2036296548Sdumbbelli915_gem_purge(struct drm_i915_private *dev_priv, long target) 2037296548Sdumbbell{ 2038296548Sdumbbell return __i915_gem_shrink(dev_priv, target, true); 2039296548Sdumbbell} 2040296548Sdumbbell 2041296548Sdumbbellstatic void 2042296548Sdumbbelli915_gem_shrink_all(struct drm_i915_private *dev_priv) 2043296548Sdumbbell{ 2044296548Sdumbbell struct drm_i915_gem_object *obj, *next; 2045296548Sdumbbell 2046296548Sdumbbell i915_gem_evict_everything(dev_priv->dev); 2047296548Sdumbbell 2048296548Sdumbbell list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list) 2049296548Sdumbbell i915_gem_object_put_pages(obj); 2050296548Sdumbbell} 2051296548Sdumbbell 2052285988Sdumbbellstatic int 2053285988Sdumbbelli915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 2054285988Sdumbbell off_t start, off_t end) 2055235783Skib{ 2056285988Sdumbbell vm_object_t vm_obj; 2057285988Sdumbbell vm_page_t page; 2058285988Sdumbbell vm_pindex_t si, ei, i; 2059285988Sdumbbell bool need_swizzle, fresh; 2060235783Skib 2061285988Sdumbbell need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0; 2062285988Sdumbbell vm_obj = obj->base.vm_obj; 2063285988Sdumbbell si = OFF_TO_IDX(trunc_page(start)); 2064285988Sdumbbell ei = OFF_TO_IDX(round_page(end)); 2065285988Sdumbbell VM_OBJECT_WLOCK(vm_obj); 2066285988Sdumbbell for (i = si; i < ei; i++) { 2067285988Sdumbbell page = i915_gem_wire_page(vm_obj, i, &fresh); 2068285988Sdumbbell if (page == NULL) 2069285988Sdumbbell goto failed; 2070285988Sdumbbell if (need_swizzle && fresh) 2071285988Sdumbbell i915_gem_object_do_bit_17_swizzle_page(obj, page); 2072285988Sdumbbell } 2073285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 2074285988Sdumbbell return (0); 2075285988Sdumbbellfailed: 2076285988Sdumbbell i915_gem_object_put_pages_range_locked(obj, si, i); 2077285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 2078285988Sdumbbell return (-EIO); 2079285988Sdumbbell} 2080235783Skib 2081285988Sdumbbellstatic int 2082296548Sdumbbelli915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 2083285988Sdumbbell{ 2084285988Sdumbbell vm_object_t vm_obj; 2085285988Sdumbbell vm_page_t page; 2086285988Sdumbbell vm_pindex_t i, page_count; 2087285988Sdumbbell int res; 2088285988Sdumbbell 2089296548Sdumbbell /* Assert that the object is not currently in any GPU domain. As it 2090296548Sdumbbell * wasn't in the GTT, there shouldn't be any way it could have been in 2091296548Sdumbbell * a GPU cache 2092296548Sdumbbell */ 2093296548Sdumbbell BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 2094296548Sdumbbell BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 2095285988Sdumbbell KASSERT(obj->pages == NULL, ("Obj already has pages")); 2096285988Sdumbbell 2097285988Sdumbbell page_count = OFF_TO_IDX(obj->base.size); 2098285988Sdumbbell obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2099285988Sdumbbell M_WAITOK); 2100285988Sdumbbell res = i915_gem_object_get_pages_range(obj, 0, obj->base.size); 2101285988Sdumbbell if (res != 0) { 2102285988Sdumbbell free(obj->pages, DRM_I915_GEM); 2103285988Sdumbbell obj->pages = NULL; 2104285988Sdumbbell return (res); 2105235783Skib } 2106285988Sdumbbell vm_obj = obj->base.vm_obj; 2107285988Sdumbbell VM_OBJECT_WLOCK(vm_obj); 2108285988Sdumbbell for (i = 0, page = vm_page_lookup(vm_obj, 0); i < page_count; 2109285988Sdumbbell i++, page = vm_page_next(page)) { 2110285988Sdumbbell KASSERT(page->pindex == i, ("pindex %jx %jx", 2111285988Sdumbbell (uintmax_t)page->pindex, (uintmax_t)i)); 2112285988Sdumbbell obj->pages[i] = page; 2113285988Sdumbbell } 2114285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 2115235783Skib return (0); 2116235783Skib} 2117235783Skib 2118296548Sdumbbell/* Ensure that the associated pages are gathered from the backing storage 2119296548Sdumbbell * and pinned into our object. i915_gem_object_get_pages() may be called 2120296548Sdumbbell * multiple times before they are released by a single call to 2121296548Sdumbbell * i915_gem_object_put_pages() - once the pages are no longer referenced 2122296548Sdumbbell * either as a result of memory pressure (reaping pages under the shrinker) 2123296548Sdumbbell * or as the object is itself released. 2124296548Sdumbbell */ 2125296548Sdumbbellint 2126296548Sdumbbelli915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2127296548Sdumbbell{ 2128296548Sdumbbell struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2129296548Sdumbbell const struct drm_i915_gem_object_ops *ops = obj->ops; 2130296548Sdumbbell int ret; 2131296548Sdumbbell 2132296548Sdumbbell if (obj->pages) 2133296548Sdumbbell return 0; 2134296548Sdumbbell 2135296548Sdumbbell BUG_ON(obj->pages_pin_count); 2136296548Sdumbbell 2137296548Sdumbbell ret = ops->get_pages(obj); 2138296548Sdumbbell if (ret) 2139296548Sdumbbell return ret; 2140296548Sdumbbell 2141296548Sdumbbell list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); 2142296548Sdumbbell return 0; 2143296548Sdumbbell} 2144296548Sdumbbell 2145235783Skibvoid 2146235783Skibi915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2147296548Sdumbbell struct intel_ring_buffer *ring) 2148235783Skib{ 2149235783Skib struct drm_device *dev = obj->base.dev; 2150235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2151296548Sdumbbell u32 seqno = intel_ring_get_seqno(ring); 2152235783Skib 2153296548Sdumbbell BUG_ON(ring == NULL); 2154235783Skib obj->ring = ring; 2155235783Skib 2156235783Skib /* Add a reference if we're newly entering the active list. */ 2157235783Skib if (!obj->active) { 2158235783Skib drm_gem_object_reference(&obj->base); 2159235783Skib obj->active = 1; 2160235783Skib } 2161235783Skib 2162235783Skib /* Move from whatever list we were on to the tail of execution. */ 2163235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2164235783Skib list_move_tail(&obj->ring_list, &ring->active_list); 2165235783Skib 2166296548Sdumbbell obj->last_read_seqno = seqno; 2167296548Sdumbbell 2168235783Skib if (obj->fenced_gpu_access) { 2169235783Skib obj->last_fenced_seqno = seqno; 2170235783Skib 2171235783Skib /* Bump MRU to take account of the delayed flush */ 2172235783Skib if (obj->fence_reg != I915_FENCE_REG_NONE) { 2173296548Sdumbbell struct drm_i915_fence_reg *reg; 2174296548Sdumbbell 2175235783Skib reg = &dev_priv->fence_regs[obj->fence_reg]; 2176235783Skib list_move_tail(®->lru_list, 2177235783Skib &dev_priv->mm.fence_list); 2178235783Skib } 2179235783Skib } 2180235783Skib} 2181235783Skib 2182235783Skibstatic void 2183235783Skibi915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2184235783Skib{ 2185235783Skib struct drm_device *dev = obj->base.dev; 2186235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2187235783Skib 2188296548Sdumbbell BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); 2189296548Sdumbbell BUG_ON(!obj->active); 2190296548Sdumbbell 2191277487Skib list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2192235783Skib 2193296548Sdumbbell list_del_init(&obj->ring_list); 2194235783Skib obj->ring = NULL; 2195235783Skib 2196296548Sdumbbell obj->last_read_seqno = 0; 2197296548Sdumbbell obj->last_write_seqno = 0; 2198296548Sdumbbell obj->base.write_domain = 0; 2199296548Sdumbbell 2200296548Sdumbbell obj->last_fenced_seqno = 0; 2201235783Skib obj->fenced_gpu_access = false; 2202235783Skib 2203235783Skib obj->active = 0; 2204235783Skib drm_gem_object_unreference(&obj->base); 2205235783Skib 2206235783Skib WARN_ON(i915_verify_lists(dev)); 2207235783Skib} 2208235783Skib 2209296548Sdumbbellstatic int 2210296548Sdumbbelli915_gem_handle_seqno_wrap(struct drm_device *dev) 2211235783Skib{ 2212296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 2213296548Sdumbbell struct intel_ring_buffer *ring; 2214296548Sdumbbell int ret, i, j; 2215235783Skib 2216296548Sdumbbell /* The hardware uses various monotonic 32-bit counters, if we 2217296548Sdumbbell * detect that they will wraparound we need to idle the GPU 2218296548Sdumbbell * and reset those counters. 2219296548Sdumbbell */ 2220296548Sdumbbell ret = 0; 2221296548Sdumbbell for_each_ring(ring, dev_priv, i) { 2222296548Sdumbbell for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) 2223296548Sdumbbell ret |= ring->sync_seqno[j] != 0; 2224296548Sdumbbell } 2225296548Sdumbbell if (ret == 0) 2226296548Sdumbbell return ret; 2227235783Skib 2228296548Sdumbbell ret = i915_gpu_idle(dev); 2229296548Sdumbbell if (ret) 2230296548Sdumbbell return ret; 2231296548Sdumbbell 2232296548Sdumbbell i915_gem_retire_requests(dev); 2233296548Sdumbbell for_each_ring(ring, dev_priv, i) { 2234296548Sdumbbell for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) 2235296548Sdumbbell ring->sync_seqno[j] = 0; 2236296548Sdumbbell } 2237296548Sdumbbell 2238296548Sdumbbell return 0; 2239235783Skib} 2240235783Skib 2241296548Sdumbbellint 2242296548Sdumbbelli915_gem_get_seqno(struct drm_device *dev, u32 *seqno) 2243235783Skib{ 2244296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 2245235783Skib 2246296548Sdumbbell /* reserve 0 for non-seqno */ 2247296548Sdumbbell if (dev_priv->next_seqno == 0) { 2248296548Sdumbbell int ret = i915_gem_handle_seqno_wrap(dev); 2249296548Sdumbbell if (ret) 2250296548Sdumbbell return ret; 2251296548Sdumbbell 2252296548Sdumbbell dev_priv->next_seqno = 1; 2253296548Sdumbbell } 2254296548Sdumbbell 2255296548Sdumbbell *seqno = dev_priv->next_seqno++; 2256296548Sdumbbell return 0; 2257235783Skib} 2258235783Skib 2259235783Skibint 2260285988Sdumbbelli915_add_request(struct intel_ring_buffer *ring, 2261285988Sdumbbell struct drm_file *file, 2262296548Sdumbbell u32 *out_seqno) 2263235783Skib{ 2264285988Sdumbbell drm_i915_private_t *dev_priv = ring->dev->dev_private; 2265296548Sdumbbell struct drm_i915_gem_request *request; 2266235783Skib u32 request_ring_position; 2267235783Skib int was_empty; 2268235783Skib int ret; 2269235783Skib 2270296548Sdumbbell /* 2271296548Sdumbbell * Emit any outstanding flushes - execbuf can fail to emit the flush 2272296548Sdumbbell * after having emitted the batchbuffer command. Hence we need to fix 2273296548Sdumbbell * things up similar to emitting the lazy request. The difference here 2274296548Sdumbbell * is that the flush _must_ happen before the next request, no matter 2275296548Sdumbbell * what. 2276296548Sdumbbell */ 2277296548Sdumbbell ret = intel_ring_flush_all_caches(ring); 2278296548Sdumbbell if (ret) 2279296548Sdumbbell return ret; 2280235783Skib 2281296548Sdumbbell request = malloc(sizeof(*request), DRM_I915_GEM, M_NOWAIT); 2282296548Sdumbbell if (request == NULL) 2283296548Sdumbbell return -ENOMEM; 2284296548Sdumbbell 2285296548Sdumbbell 2286296548Sdumbbell /* Record the position of the start of the request so that 2287296548Sdumbbell * should we detect the updated seqno part-way through the 2288296548Sdumbbell * GPU processing the request, we never over-estimate the 2289296548Sdumbbell * position of the head. 2290296548Sdumbbell */ 2291235783Skib request_ring_position = intel_ring_get_tail(ring); 2292235783Skib 2293296548Sdumbbell ret = ring->add_request(ring); 2294296548Sdumbbell if (ret) { 2295296548Sdumbbell free(request, DRM_I915_GEM); 2296296548Sdumbbell return ret; 2297296548Sdumbbell } 2298235783Skib 2299296548Sdumbbell request->seqno = intel_ring_get_seqno(ring); 2300235783Skib request->ring = ring; 2301235783Skib request->tail = request_ring_position; 2302296548Sdumbbell request->emitted_jiffies = jiffies; 2303235783Skib was_empty = list_empty(&ring->request_list); 2304235783Skib list_add_tail(&request->list, &ring->request_list); 2305296548Sdumbbell request->file_priv = NULL; 2306235783Skib 2307285988Sdumbbell if (file) { 2308296548Sdumbbell struct drm_i915_file_private *file_priv = file->driver_priv; 2309235783Skib 2310296548Sdumbbell mtx_lock(&file_priv->mm.lock); 2311235783Skib request->file_priv = file_priv; 2312235783Skib list_add_tail(&request->client_list, 2313285988Sdumbbell &file_priv->mm.request_list); 2314296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 2315235783Skib } 2316235783Skib 2317296548Sdumbbell CTR2(KTR_DRM, "request_add %s %d", ring->name, request->seqno); 2318235783Skib ring->outstanding_lazy_request = 0; 2319235783Skib 2320235783Skib if (!dev_priv->mm.suspended) { 2321235783Skib if (i915_enable_hangcheck) { 2322235783Skib callout_schedule(&dev_priv->hangcheck_timer, 2323235783Skib DRM_I915_HANGCHECK_PERIOD); 2324235783Skib } 2325296548Sdumbbell if (was_empty) { 2326296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 2327296548Sdumbbell &dev_priv->mm.retire_work, hz); 2328296548Sdumbbell intel_mark_busy(dev_priv->dev); 2329296548Sdumbbell } 2330235783Skib } 2331285988Sdumbbell 2332296548Sdumbbell if (out_seqno) 2333296548Sdumbbell *out_seqno = request->seqno; 2334285988Sdumbbell return 0; 2335235783Skib} 2336235783Skib 2337235783Skibstatic inline void 2338235783Skibi915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2339235783Skib{ 2340235783Skib struct drm_i915_file_private *file_priv = request->file_priv; 2341235783Skib 2342235783Skib if (!file_priv) 2343235783Skib return; 2344235783Skib 2345296548Sdumbbell mtx_lock(&file_priv->mm.lock); 2346285988Sdumbbell if (request->file_priv) { 2347235783Skib list_del(&request->client_list); 2348235783Skib request->file_priv = NULL; 2349235783Skib } 2350296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 2351235783Skib} 2352235783Skib 2353285988Sdumbbellstatic void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2354285988Sdumbbell struct intel_ring_buffer *ring) 2355235783Skib{ 2356235783Skib if (ring->dev != NULL) 2357235783Skib DRM_LOCK_ASSERT(ring->dev); 2358235783Skib 2359235783Skib while (!list_empty(&ring->request_list)) { 2360235783Skib struct drm_i915_gem_request *request; 2361235783Skib 2362235783Skib request = list_first_entry(&ring->request_list, 2363285988Sdumbbell struct drm_i915_gem_request, 2364285988Sdumbbell list); 2365235783Skib 2366235783Skib list_del(&request->list); 2367235783Skib i915_gem_request_remove_from_client(request); 2368235783Skib free(request, DRM_I915_GEM); 2369235783Skib } 2370235783Skib 2371235783Skib while (!list_empty(&ring->active_list)) { 2372235783Skib struct drm_i915_gem_object *obj; 2373235783Skib 2374235783Skib obj = list_first_entry(&ring->active_list, 2375285988Sdumbbell struct drm_i915_gem_object, 2376285988Sdumbbell ring_list); 2377235783Skib 2378235783Skib i915_gem_object_move_to_inactive(obj); 2379235783Skib } 2380235783Skib} 2381235783Skib 2382285988Sdumbbellstatic void i915_gem_reset_fences(struct drm_device *dev) 2383235783Skib{ 2384235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2385235783Skib int i; 2386235783Skib 2387235783Skib for (i = 0; i < dev_priv->num_fence_regs; i++) { 2388235783Skib struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2389235783Skib 2390277487Skib i915_gem_write_fence(dev, i, NULL); 2391235783Skib 2392277487Skib if (reg->obj) 2393277487Skib i915_gem_object_fence_lost(reg->obj); 2394235783Skib 2395277487Skib reg->pin_count = 0; 2396277487Skib reg->obj = NULL; 2397277487Skib INIT_LIST_HEAD(®->lru_list); 2398235783Skib } 2399277487Skib 2400277487Skib INIT_LIST_HEAD(&dev_priv->mm.fence_list); 2401235783Skib} 2402235783Skib 2403285988Sdumbbellvoid i915_gem_reset(struct drm_device *dev) 2404235783Skib{ 2405235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2406235783Skib struct drm_i915_gem_object *obj; 2407277487Skib struct intel_ring_buffer *ring; 2408235783Skib int i; 2409235783Skib 2410277487Skib for_each_ring(ring, dev_priv, i) 2411277487Skib i915_gem_reset_ring_lists(dev_priv, ring); 2412235783Skib 2413235783Skib /* Move everything out of the GPU domains to ensure we do any 2414235783Skib * necessary invalidation upon reuse. 2415235783Skib */ 2416296548Sdumbbell list_for_each_entry(obj, 2417296548Sdumbbell &dev_priv->mm.inactive_list, 2418296548Sdumbbell mm_list) 2419296548Sdumbbell { 2420235783Skib obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2421235783Skib } 2422235783Skib 2423235783Skib /* The fence registers are invalidated so clear them out */ 2424235783Skib i915_gem_reset_fences(dev); 2425235783Skib} 2426235783Skib 2427235783Skib/** 2428235783Skib * This function clears the request list as sequence numbers are passed. 2429235783Skib */ 2430235783Skibvoid 2431235783Skibi915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2432235783Skib{ 2433235783Skib uint32_t seqno; 2434235783Skib 2435235783Skib if (list_empty(&ring->request_list)) 2436235783Skib return; 2437235783Skib 2438296548Sdumbbell WARN_ON(i915_verify_lists(ring->dev)); 2439296548Sdumbbell 2440296548Sdumbbell seqno = ring->get_seqno(ring, true); 2441235783Skib CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 2442235783Skib 2443235783Skib while (!list_empty(&ring->request_list)) { 2444235783Skib struct drm_i915_gem_request *request; 2445235783Skib 2446235783Skib request = list_first_entry(&ring->request_list, 2447235783Skib struct drm_i915_gem_request, 2448235783Skib list); 2449235783Skib 2450235783Skib if (!i915_seqno_passed(seqno, request->seqno)) 2451235783Skib break; 2452235783Skib 2453235783Skib CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 2454235783Skib ring->name, seqno); 2455296548Sdumbbell /* We know the GPU must have read the request to have 2456296548Sdumbbell * sent us the seqno + interrupt, so use the position 2457296548Sdumbbell * of tail of the request to update the last known position 2458296548Sdumbbell * of the GPU head. 2459296548Sdumbbell */ 2460235783Skib ring->last_retired_head = request->tail; 2461235783Skib 2462235783Skib list_del(&request->list); 2463235783Skib i915_gem_request_remove_from_client(request); 2464235783Skib free(request, DRM_I915_GEM); 2465235783Skib } 2466235783Skib 2467235783Skib /* Move any buffers on the active list that are no longer referenced 2468235783Skib * by the ringbuffer to the flushing/inactive lists as appropriate. 2469235783Skib */ 2470235783Skib while (!list_empty(&ring->active_list)) { 2471235783Skib struct drm_i915_gem_object *obj; 2472235783Skib 2473235783Skib obj = list_first_entry(&ring->active_list, 2474235783Skib struct drm_i915_gem_object, 2475235783Skib ring_list); 2476235783Skib 2477296548Sdumbbell if (!i915_seqno_passed(seqno, obj->last_read_seqno)) 2478235783Skib break; 2479235783Skib 2480296548Sdumbbell i915_gem_object_move_to_inactive(obj); 2481235783Skib } 2482235783Skib 2483296548Sdumbbell if (unlikely(ring->trace_irq_seqno && 2484296548Sdumbbell i915_seqno_passed(seqno, ring->trace_irq_seqno))) { 2485235783Skib ring->irq_put(ring); 2486235783Skib ring->trace_irq_seqno = 0; 2487235783Skib } 2488296548Sdumbbell 2489296548Sdumbbell WARN_ON(i915_verify_lists(ring->dev)); 2490235783Skib} 2491235783Skib 2492235783Skibvoid 2493235783Skibi915_gem_retire_requests(struct drm_device *dev) 2494235783Skib{ 2495235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2496277487Skib struct intel_ring_buffer *ring; 2497235783Skib int i; 2498235783Skib 2499277487Skib for_each_ring(ring, dev_priv, i) 2500277487Skib i915_gem_retire_requests_ring(ring); 2501235783Skib} 2502235783Skib 2503285988Sdumbbellstatic void 2504296548Sdumbbelli915_gem_retire_work_handler(void *arg, int pending) 2505285988Sdumbbell{ 2506285988Sdumbbell drm_i915_private_t *dev_priv; 2507285988Sdumbbell struct drm_device *dev; 2508285988Sdumbbell struct intel_ring_buffer *ring; 2509285988Sdumbbell bool idle; 2510285988Sdumbbell int i; 2511285988Sdumbbell 2512285988Sdumbbell dev_priv = arg; 2513285988Sdumbbell dev = dev_priv->dev; 2514285988Sdumbbell 2515285988Sdumbbell /* Come back later if the device is busy... */ 2516285988Sdumbbell if (!sx_try_xlock(&dev->dev_struct_lock)) { 2517296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 2518296548Sdumbbell &dev_priv->mm.retire_work, hz); 2519285988Sdumbbell return; 2520285988Sdumbbell } 2521285988Sdumbbell 2522285988Sdumbbell CTR0(KTR_DRM, "retire_task"); 2523285988Sdumbbell 2524285988Sdumbbell i915_gem_retire_requests(dev); 2525285988Sdumbbell 2526285988Sdumbbell /* Send a periodic flush down the ring so we don't hold onto GEM 2527285988Sdumbbell * objects indefinitely. 2528285988Sdumbbell */ 2529285988Sdumbbell idle = true; 2530285988Sdumbbell for_each_ring(ring, dev_priv, i) { 2531296548Sdumbbell if (ring->gpu_caches_dirty) 2532296548Sdumbbell i915_add_request(ring, NULL, NULL); 2533285988Sdumbbell 2534285988Sdumbbell idle &= list_empty(&ring->request_list); 2535285988Sdumbbell } 2536285988Sdumbbell 2537285988Sdumbbell if (!dev_priv->mm.suspended && !idle) 2538296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 2539296548Sdumbbell &dev_priv->mm.retire_work, hz); 2540296548Sdumbbell if (idle) 2541296548Sdumbbell intel_mark_idle(dev); 2542285988Sdumbbell 2543285988Sdumbbell DRM_UNLOCK(dev); 2544285988Sdumbbell} 2545285988Sdumbbell 2546296548Sdumbbell/** 2547296548Sdumbbell * Ensures that an object will eventually get non-busy by flushing any required 2548296548Sdumbbell * write domains, emitting any outstanding lazy request and retiring and 2549296548Sdumbbell * completed requests. 2550296548Sdumbbell */ 2551296548Sdumbbellstatic int 2552296548Sdumbbelli915_gem_object_flush_active(struct drm_i915_gem_object *obj) 2553296548Sdumbbell{ 2554296548Sdumbbell int ret; 2555296548Sdumbbell 2556296548Sdumbbell if (obj->active) { 2557296548Sdumbbell ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); 2558296548Sdumbbell if (ret) 2559296548Sdumbbell return ret; 2560296548Sdumbbell 2561296548Sdumbbell i915_gem_retire_requests_ring(obj->ring); 2562296548Sdumbbell } 2563296548Sdumbbell 2564296548Sdumbbell return 0; 2565296548Sdumbbell} 2566296548Sdumbbell 2567296548Sdumbbell/** 2568296548Sdumbbell * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 2569296548Sdumbbell * @DRM_IOCTL_ARGS: standard ioctl arguments 2570296548Sdumbbell * 2571296548Sdumbbell * Returns 0 if successful, else an error is returned with the remaining time in 2572296548Sdumbbell * the timeout parameter. 2573296548Sdumbbell * -ETIME: object is still busy after timeout 2574296548Sdumbbell * -ERESTARTSYS: signal interrupted the wait 2575296548Sdumbbell * -ENONENT: object doesn't exist 2576296548Sdumbbell * Also possible, but rare: 2577296548Sdumbbell * -EAGAIN: GPU wedged 2578296548Sdumbbell * -ENOMEM: damn 2579296548Sdumbbell * -ENODEV: Internal IRQ fail 2580296548Sdumbbell * -E?: The add request failed 2581296548Sdumbbell * 2582296548Sdumbbell * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 2583296548Sdumbbell * non-zero timeout parameter the wait ioctl will wait for the given number of 2584296548Sdumbbell * nanoseconds on an object becoming unbusy. Since the wait itself does so 2585296548Sdumbbell * without holding struct_mutex the object may become re-busied before this 2586296548Sdumbbell * function completes. A similar but shorter * race condition exists in the busy 2587296548Sdumbbell * ioctl 2588296548Sdumbbell */ 2589285988Sdumbbellint 2590296548Sdumbbelli915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 2591296548Sdumbbell{ 2592296548Sdumbbell struct drm_i915_gem_wait *args = data; 2593296548Sdumbbell struct drm_i915_gem_object *obj; 2594296548Sdumbbell struct intel_ring_buffer *ring = NULL; 2595296548Sdumbbell struct timespec timeout_stack, *timeout = NULL; 2596296548Sdumbbell u32 seqno = 0; 2597296548Sdumbbell int ret = 0; 2598296548Sdumbbell 2599296548Sdumbbell if (args->timeout_ns >= 0) { 2600296548Sdumbbell timeout_stack.tv_sec = args->timeout_ns / 1000000; 2601296548Sdumbbell timeout_stack.tv_nsec = args->timeout_ns % 1000000; 2602296548Sdumbbell timeout = &timeout_stack; 2603296548Sdumbbell } 2604296548Sdumbbell 2605296548Sdumbbell ret = i915_mutex_lock_interruptible(dev); 2606296548Sdumbbell if (ret) 2607296548Sdumbbell return ret; 2608296548Sdumbbell 2609296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle)); 2610296548Sdumbbell if (&obj->base == NULL) { 2611296548Sdumbbell DRM_UNLOCK(dev); 2612296548Sdumbbell return -ENOENT; 2613296548Sdumbbell } 2614296548Sdumbbell 2615296548Sdumbbell /* Need to make sure the object gets inactive eventually. */ 2616296548Sdumbbell ret = i915_gem_object_flush_active(obj); 2617296548Sdumbbell if (ret) 2618296548Sdumbbell goto out; 2619296548Sdumbbell 2620296548Sdumbbell if (obj->active) { 2621296548Sdumbbell seqno = obj->last_read_seqno; 2622296548Sdumbbell ring = obj->ring; 2623296548Sdumbbell } 2624296548Sdumbbell 2625296548Sdumbbell if (seqno == 0) 2626296548Sdumbbell goto out; 2627296548Sdumbbell 2628296548Sdumbbell /* Do this after OLR check to make sure we make forward progress polling 2629296548Sdumbbell * on this IOCTL with a 0 timeout (like busy ioctl) 2630296548Sdumbbell */ 2631296548Sdumbbell if (!args->timeout_ns) { 2632296548Sdumbbell ret = -ETIMEDOUT; 2633296548Sdumbbell goto out; 2634296548Sdumbbell } 2635296548Sdumbbell 2636296548Sdumbbell drm_gem_object_unreference(&obj->base); 2637296548Sdumbbell DRM_UNLOCK(dev); 2638296548Sdumbbell 2639296548Sdumbbell ret = __wait_seqno(ring, seqno, true, timeout); 2640296548Sdumbbell if (timeout) { 2641296548Sdumbbell args->timeout_ns = timeout->tv_sec * 1000000 + timeout->tv_nsec; 2642296548Sdumbbell } 2643296548Sdumbbell return ret; 2644296548Sdumbbell 2645296548Sdumbbellout: 2646296548Sdumbbell drm_gem_object_unreference(&obj->base); 2647296548Sdumbbell DRM_UNLOCK(dev); 2648296548Sdumbbell return ret; 2649296548Sdumbbell} 2650296548Sdumbbell 2651296548Sdumbbell/** 2652296548Sdumbbell * i915_gem_object_sync - sync an object to a ring. 2653296548Sdumbbell * 2654296548Sdumbbell * @obj: object which may be in use on another ring. 2655296548Sdumbbell * @to: ring we wish to use the object on. May be NULL. 2656296548Sdumbbell * 2657296548Sdumbbell * This code is meant to abstract object synchronization with the GPU. 2658296548Sdumbbell * Calling with NULL implies synchronizing the object with the CPU 2659296548Sdumbbell * rather than a particular GPU ring. 2660296548Sdumbbell * 2661296548Sdumbbell * Returns 0 if successful, else propagates up the lower layer error. 2662296548Sdumbbell */ 2663296548Sdumbbellint 2664285988Sdumbbelli915_gem_object_sync(struct drm_i915_gem_object *obj, 2665285988Sdumbbell struct intel_ring_buffer *to) 2666285988Sdumbbell{ 2667285988Sdumbbell struct intel_ring_buffer *from = obj->ring; 2668285988Sdumbbell u32 seqno; 2669285988Sdumbbell int ret, idx; 2670285988Sdumbbell 2671285988Sdumbbell if (from == NULL || to == from) 2672285988Sdumbbell return 0; 2673285988Sdumbbell 2674285988Sdumbbell if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) 2675296548Sdumbbell return i915_gem_object_wait_rendering(obj, false); 2676285988Sdumbbell 2677285988Sdumbbell idx = intel_ring_sync_index(from, to); 2678285988Sdumbbell 2679296548Sdumbbell seqno = obj->last_read_seqno; 2680285988Sdumbbell if (seqno <= from->sync_seqno[idx]) 2681285988Sdumbbell return 0; 2682285988Sdumbbell 2683296548Sdumbbell ret = i915_gem_check_olr(obj->ring, seqno); 2684296548Sdumbbell if (ret) 2685296548Sdumbbell return ret; 2686285988Sdumbbell 2687285988Sdumbbell ret = to->sync_to(to, from, seqno); 2688285988Sdumbbell if (!ret) 2689296548Sdumbbell /* We use last_read_seqno because sync_to() 2690296548Sdumbbell * might have just caused seqno wrap under 2691296548Sdumbbell * the radar. 2692296548Sdumbbell */ 2693296548Sdumbbell from->sync_seqno[idx] = obj->last_read_seqno; 2694285988Sdumbbell 2695285988Sdumbbell return ret; 2696285988Sdumbbell} 2697285988Sdumbbell 2698285988Sdumbbellstatic void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2699285988Sdumbbell{ 2700285988Sdumbbell u32 old_write_domain, old_read_domains; 2701285988Sdumbbell 2702285988Sdumbbell /* Act a barrier for all accesses through the GTT */ 2703285988Sdumbbell mb(); 2704285988Sdumbbell 2705285988Sdumbbell /* Force a pagefault for domain tracking on next user access */ 2706285988Sdumbbell i915_gem_release_mmap(obj); 2707285988Sdumbbell 2708285988Sdumbbell if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2709285988Sdumbbell return; 2710285988Sdumbbell 2711285988Sdumbbell old_read_domains = obj->base.read_domains; 2712285988Sdumbbell old_write_domain = obj->base.write_domain; 2713285988Sdumbbell 2714285988Sdumbbell obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2715285988Sdumbbell obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2716285988Sdumbbell 2717285988Sdumbbell CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2718285988Sdumbbell obj, old_read_domains, old_write_domain); 2719285988Sdumbbell} 2720285988Sdumbbell 2721285988Sdumbbell/** 2722285988Sdumbbell * Unbinds an object from the GTT aperture. 2723285988Sdumbbell */ 2724285988Sdumbbellint 2725285988Sdumbbelli915_gem_object_unbind(struct drm_i915_gem_object *obj) 2726285988Sdumbbell{ 2727285988Sdumbbell drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 2728285988Sdumbbell int ret = 0; 2729285988Sdumbbell 2730285988Sdumbbell if (obj->gtt_space == NULL) 2731285988Sdumbbell return 0; 2732285988Sdumbbell 2733289109Sdumbbell if (obj->pin_count) 2734296548Sdumbbell return -EBUSY; 2735285988Sdumbbell 2736296548Sdumbbell BUG_ON(obj->pages == NULL); 2737296548Sdumbbell 2738285988Sdumbbell ret = i915_gem_object_finish_gpu(obj); 2739296548Sdumbbell if (ret) 2740285988Sdumbbell return ret; 2741296548Sdumbbell /* Continue on if we fail due to EIO, the GPU is hung so we 2742296548Sdumbbell * should be safe and we need to cleanup or else we might 2743296548Sdumbbell * cause memory corruption through use-after-free. 2744296548Sdumbbell */ 2745285988Sdumbbell 2746285988Sdumbbell i915_gem_object_finish_gtt(obj); 2747285988Sdumbbell 2748285988Sdumbbell /* release the fence reg _after_ flushing */ 2749285988Sdumbbell ret = i915_gem_object_put_fence(obj); 2750285988Sdumbbell if (ret) 2751285988Sdumbbell return ret; 2752285988Sdumbbell 2753285988Sdumbbell if (obj->has_global_gtt_mapping) 2754285988Sdumbbell i915_gem_gtt_unbind_object(obj); 2755285988Sdumbbell if (obj->has_aliasing_ppgtt_mapping) { 2756285988Sdumbbell i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2757285988Sdumbbell obj->has_aliasing_ppgtt_mapping = 0; 2758285988Sdumbbell } 2759285988Sdumbbell i915_gem_gtt_finish_object(obj); 2760285988Sdumbbell 2761296548Sdumbbell list_del(&obj->mm_list); 2762296548Sdumbbell list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); 2763296548Sdumbbell /* Avoid an unnecessary call to unbind on rebind. */ 2764285988Sdumbbell obj->map_and_fenceable = true; 2765285988Sdumbbell 2766285988Sdumbbell drm_mm_put_block(obj->gtt_space); 2767285988Sdumbbell obj->gtt_space = NULL; 2768285988Sdumbbell obj->gtt_offset = 0; 2769285988Sdumbbell 2770296548Sdumbbell return 0; 2771285988Sdumbbell} 2772285988Sdumbbell 2773285988Sdumbbellint i915_gpu_idle(struct drm_device *dev) 2774285988Sdumbbell{ 2775285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 2776285988Sdumbbell struct intel_ring_buffer *ring; 2777285988Sdumbbell int ret, i; 2778285988Sdumbbell 2779285988Sdumbbell /* Flush everything onto the inactive list. */ 2780285988Sdumbbell for_each_ring(ring, dev_priv, i) { 2781285988Sdumbbell ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); 2782285988Sdumbbell if (ret) 2783285988Sdumbbell return ret; 2784285988Sdumbbell 2785296548Sdumbbell ret = intel_ring_idle(ring); 2786285988Sdumbbell if (ret) 2787285988Sdumbbell return ret; 2788285988Sdumbbell } 2789285988Sdumbbell 2790285988Sdumbbell return 0; 2791285988Sdumbbell} 2792285988Sdumbbell 2793277487Skibstatic void sandybridge_write_fence_reg(struct drm_device *dev, int reg, 2794277487Skib struct drm_i915_gem_object *obj) 2795235783Skib{ 2796235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2797235783Skib uint64_t val; 2798235783Skib 2799277487Skib if (obj) { 2800277487Skib u32 size = obj->gtt_space->size; 2801235783Skib 2802277487Skib val = (uint64_t)((obj->gtt_offset + size - 4096) & 2803277487Skib 0xfffff000) << 32; 2804277487Skib val |= obj->gtt_offset & 0xfffff000; 2805277487Skib val |= (uint64_t)((obj->stride / 128) - 1) << 2806277487Skib SANDYBRIDGE_FENCE_PITCH_SHIFT; 2807235783Skib 2808277487Skib if (obj->tiling_mode == I915_TILING_Y) 2809277487Skib val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2810277487Skib val |= I965_FENCE_REG_VALID; 2811235783Skib } else 2812277487Skib val = 0; 2813235783Skib 2814277487Skib I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); 2815277487Skib POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); 2816235783Skib} 2817235783Skib 2818277487Skibstatic void i965_write_fence_reg(struct drm_device *dev, int reg, 2819277487Skib struct drm_i915_gem_object *obj) 2820235783Skib{ 2821235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2822235783Skib uint64_t val; 2823235783Skib 2824277487Skib if (obj) { 2825277487Skib u32 size = obj->gtt_space->size; 2826235783Skib 2827277487Skib val = (uint64_t)((obj->gtt_offset + size - 4096) & 2828277487Skib 0xfffff000) << 32; 2829277487Skib val |= obj->gtt_offset & 0xfffff000; 2830277487Skib val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2831277487Skib if (obj->tiling_mode == I915_TILING_Y) 2832277487Skib val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2833277487Skib val |= I965_FENCE_REG_VALID; 2834235783Skib } else 2835277487Skib val = 0; 2836235783Skib 2837277487Skib I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); 2838277487Skib POSTING_READ(FENCE_REG_965_0 + reg * 8); 2839235783Skib} 2840235783Skib 2841277487Skibstatic void i915_write_fence_reg(struct drm_device *dev, int reg, 2842277487Skib struct drm_i915_gem_object *obj) 2843235783Skib{ 2844235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2845277487Skib u32 val; 2846235783Skib 2847277487Skib if (obj) { 2848277487Skib u32 size = obj->gtt_space->size; 2849277487Skib int pitch_val; 2850277487Skib int tile_width; 2851235783Skib 2852296548Sdumbbell WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || 2853277487Skib (size & -size) != size || 2854296548Sdumbbell (obj->gtt_offset & (size - 1)), 2855277487Skib "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 2856277487Skib obj->gtt_offset, obj->map_and_fenceable, size); 2857235783Skib 2858277487Skib if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 2859277487Skib tile_width = 128; 2860277487Skib else 2861277487Skib tile_width = 512; 2862235783Skib 2863277487Skib /* Note: pitch better be a power of two tile widths */ 2864277487Skib pitch_val = obj->stride / tile_width; 2865277487Skib pitch_val = ffs(pitch_val) - 1; 2866235783Skib 2867277487Skib val = obj->gtt_offset; 2868277487Skib if (obj->tiling_mode == I915_TILING_Y) 2869277487Skib val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2870277487Skib val |= I915_FENCE_SIZE_BITS(size); 2871277487Skib val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2872277487Skib val |= I830_FENCE_REG_VALID; 2873277487Skib } else 2874277487Skib val = 0; 2875277487Skib 2876277487Skib if (reg < 8) 2877277487Skib reg = FENCE_REG_830_0 + reg * 4; 2878235783Skib else 2879277487Skib reg = FENCE_REG_945_8 + (reg - 8) * 4; 2880235783Skib 2881277487Skib I915_WRITE(reg, val); 2882277487Skib POSTING_READ(reg); 2883235783Skib} 2884235783Skib 2885277487Skibstatic void i830_write_fence_reg(struct drm_device *dev, int reg, 2886277487Skib struct drm_i915_gem_object *obj) 2887235783Skib{ 2888235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2889235783Skib uint32_t val; 2890235783Skib 2891277487Skib if (obj) { 2892277487Skib u32 size = obj->gtt_space->size; 2893277487Skib uint32_t pitch_val; 2894235783Skib 2895296548Sdumbbell WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || 2896277487Skib (size & -size) != size || 2897296548Sdumbbell (obj->gtt_offset & (size - 1)), 2898277487Skib "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 2899277487Skib obj->gtt_offset, size); 2900235783Skib 2901277487Skib pitch_val = obj->stride / 128; 2902277487Skib pitch_val = ffs(pitch_val) - 1; 2903235783Skib 2904277487Skib val = obj->gtt_offset; 2905277487Skib if (obj->tiling_mode == I915_TILING_Y) 2906277487Skib val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2907277487Skib val |= I830_FENCE_SIZE_BITS(size); 2908277487Skib val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2909277487Skib val |= I830_FENCE_REG_VALID; 2910235783Skib } else 2911277487Skib val = 0; 2912235783Skib 2913277487Skib I915_WRITE(FENCE_REG_830_0 + reg * 4, val); 2914277487Skib POSTING_READ(FENCE_REG_830_0 + reg * 4); 2915235783Skib} 2916235783Skib 2917277487Skibstatic void i915_gem_write_fence(struct drm_device *dev, int reg, 2918277487Skib struct drm_i915_gem_object *obj) 2919235783Skib{ 2920277487Skib switch (INTEL_INFO(dev)->gen) { 2921277487Skib case 7: 2922277487Skib case 6: sandybridge_write_fence_reg(dev, reg, obj); break; 2923277487Skib case 5: 2924277487Skib case 4: i965_write_fence_reg(dev, reg, obj); break; 2925277487Skib case 3: i915_write_fence_reg(dev, reg, obj); break; 2926277487Skib case 2: i830_write_fence_reg(dev, reg, obj); break; 2927277487Skib default: break; 2928277487Skib } 2929235783Skib} 2930235783Skib 2931277487Skibstatic inline int fence_number(struct drm_i915_private *dev_priv, 2932277487Skib struct drm_i915_fence_reg *fence) 2933277487Skib{ 2934277487Skib return fence - dev_priv->fence_regs; 2935277487Skib} 2936277487Skib 2937296548Sdumbbellstatic void i915_gem_write_fence__ipi(void *data) 2938296548Sdumbbell{ 2939296548Sdumbbell wbinvd(); 2940296548Sdumbbell} 2941296548Sdumbbell 2942277487Skibstatic void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 2943277487Skib struct drm_i915_fence_reg *fence, 2944277487Skib bool enable) 2945277487Skib{ 2946285988Sdumbbell struct drm_device *dev = obj->base.dev; 2947285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 2948285988Sdumbbell int fence_reg = fence_number(dev_priv, fence); 2949277487Skib 2950296548Sdumbbell /* In order to fully serialize access to the fenced region and 2951296548Sdumbbell * the update to the fence register we need to take extreme 2952296548Sdumbbell * measures on SNB+. In theory, the write to the fence register 2953296548Sdumbbell * flushes all memory transactions before, and coupled with the 2954296548Sdumbbell * mb() placed around the register write we serialise all memory 2955296548Sdumbbell * operations with respect to the changes in the tiler. Yet, on 2956296548Sdumbbell * SNB+ we need to take a step further and emit an explicit wbinvd() 2957296548Sdumbbell * on each processor in order to manually flush all memory 2958296548Sdumbbell * transactions before updating the fence register. 2959296548Sdumbbell */ 2960296548Sdumbbell if (HAS_LLC(obj->base.dev)) 2961296548Sdumbbell on_each_cpu(i915_gem_write_fence__ipi, NULL, 1); 2962285988Sdumbbell i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL); 2963277487Skib 2964277487Skib if (enable) { 2965285988Sdumbbell obj->fence_reg = fence_reg; 2966277487Skib fence->obj = obj; 2967277487Skib list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); 2968277487Skib } else { 2969277487Skib obj->fence_reg = I915_FENCE_REG_NONE; 2970277487Skib fence->obj = NULL; 2971277487Skib list_del_init(&fence->lru_list); 2972277487Skib } 2973277487Skib} 2974277487Skib 2975235783Skibstatic int 2976277487Skibi915_gem_object_flush_fence(struct drm_i915_gem_object *obj) 2977235783Skib{ 2978277487Skib if (obj->last_fenced_seqno) { 2979296548Sdumbbell int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); 2980277487Skib if (ret) 2981277487Skib return ret; 2982235783Skib 2983235783Skib obj->last_fenced_seqno = 0; 2984235783Skib } 2985235783Skib 2986235783Skib /* Ensure that all CPU reads are completed before installing a fence 2987235783Skib * and all writes before removing the fence. 2988235783Skib */ 2989235783Skib if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 2990235783Skib mb(); 2991235783Skib 2992296548Sdumbbell obj->fenced_gpu_access = false; 2993235783Skib return 0; 2994235783Skib} 2995235783Skib 2996235783Skibint 2997235783Skibi915_gem_object_put_fence(struct drm_i915_gem_object *obj) 2998235783Skib{ 2999277487Skib struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3000235783Skib int ret; 3001235783Skib 3002277487Skib ret = i915_gem_object_flush_fence(obj); 3003235783Skib if (ret) 3004235783Skib return ret; 3005235783Skib 3006277487Skib if (obj->fence_reg == I915_FENCE_REG_NONE) 3007277487Skib return 0; 3008235783Skib 3009277487Skib i915_gem_object_update_fence(obj, 3010277487Skib &dev_priv->fence_regs[obj->fence_reg], 3011277487Skib false); 3012277487Skib i915_gem_object_fence_lost(obj); 3013235783Skib 3014235783Skib return 0; 3015235783Skib} 3016235783Skib 3017235783Skibstatic struct drm_i915_fence_reg * 3018277487Skibi915_find_fence_reg(struct drm_device *dev) 3019235783Skib{ 3020235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 3021277487Skib struct drm_i915_fence_reg *reg, *avail; 3022235783Skib int i; 3023235783Skib 3024235783Skib /* First try to find a free reg */ 3025235783Skib avail = NULL; 3026235783Skib for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3027235783Skib reg = &dev_priv->fence_regs[i]; 3028235783Skib if (!reg->obj) 3029235783Skib return reg; 3030235783Skib 3031235783Skib if (!reg->pin_count) 3032235783Skib avail = reg; 3033235783Skib } 3034235783Skib 3035235783Skib if (avail == NULL) 3036235783Skib return NULL; 3037235783Skib 3038235783Skib /* None available, try to steal one or wait for a user to finish */ 3039235783Skib list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3040235783Skib if (reg->pin_count) 3041235783Skib continue; 3042235783Skib 3043277487Skib return reg; 3044235783Skib } 3045235783Skib 3046277487Skib return NULL; 3047235783Skib} 3048235783Skib 3049285988Sdumbbell/** 3050285988Sdumbbell * i915_gem_object_get_fence - set up fencing for an object 3051285988Sdumbbell * @obj: object to map through a fence reg 3052285988Sdumbbell * 3053285988Sdumbbell * When mapping objects through the GTT, userspace wants to be able to write 3054285988Sdumbbell * to them without having to worry about swizzling if the object is tiled. 3055285988Sdumbbell * This function walks the fence regs looking for a free one for @obj, 3056285988Sdumbbell * stealing one if it can't find any. 3057285988Sdumbbell * 3058285988Sdumbbell * It then sets up the reg based on the object's properties: address, pitch 3059285988Sdumbbell * and tiling format. 3060285988Sdumbbell * 3061285988Sdumbbell * For an untiled surface, this removes any existing fence. 3062285988Sdumbbell */ 3063235783Skibint 3064277487Skibi915_gem_object_get_fence(struct drm_i915_gem_object *obj) 3065235783Skib{ 3066235783Skib struct drm_device *dev = obj->base.dev; 3067235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 3068277487Skib bool enable = obj->tiling_mode != I915_TILING_NONE; 3069235783Skib struct drm_i915_fence_reg *reg; 3070235783Skib int ret; 3071235783Skib 3072277487Skib /* Have we updated the tiling parameters upon the object and so 3073277487Skib * will need to serialise the write to the associated fence register? 3074277487Skib */ 3075277487Skib if (obj->fence_dirty) { 3076277487Skib ret = i915_gem_object_flush_fence(obj); 3077277487Skib if (ret) 3078277487Skib return ret; 3079277487Skib } 3080277487Skib 3081285988Sdumbbell /* Just update our place in the LRU if our fence is getting reused. */ 3082235783Skib if (obj->fence_reg != I915_FENCE_REG_NONE) { 3083235783Skib reg = &dev_priv->fence_regs[obj->fence_reg]; 3084277487Skib if (!obj->fence_dirty) { 3085277487Skib list_move_tail(®->lru_list, 3086277487Skib &dev_priv->mm.fence_list); 3087277487Skib return 0; 3088235783Skib } 3089277487Skib } else if (enable) { 3090277487Skib reg = i915_find_fence_reg(dev); 3091277487Skib if (reg == NULL) 3092277487Skib return -EDEADLK; 3093235783Skib 3094277487Skib if (reg->obj) { 3095277487Skib struct drm_i915_gem_object *old = reg->obj; 3096235783Skib 3097277487Skib ret = i915_gem_object_flush_fence(old); 3098235783Skib if (ret) 3099235783Skib return ret; 3100235783Skib 3101277487Skib i915_gem_object_fence_lost(old); 3102235783Skib } 3103277487Skib } else 3104235783Skib return 0; 3105235783Skib 3106277487Skib i915_gem_object_update_fence(obj, reg, enable); 3107277487Skib obj->fence_dirty = false; 3108235783Skib 3109277487Skib return 0; 3110235783Skib} 3111235783Skib 3112296548Sdumbbellstatic bool i915_gem_valid_gtt_space(struct drm_device *dev, 3113296548Sdumbbell struct drm_mm_node *gtt_space, 3114296548Sdumbbell unsigned long cache_level) 3115296548Sdumbbell{ 3116296548Sdumbbell struct drm_mm_node *other; 3117296548Sdumbbell 3118296548Sdumbbell /* On non-LLC machines we have to be careful when putting differing 3119296548Sdumbbell * types of snoopable memory together to avoid the prefetcher 3120298931Spfg * crossing memory domains and dying. 3121296548Sdumbbell */ 3122296548Sdumbbell if (HAS_LLC(dev)) 3123296548Sdumbbell return true; 3124296548Sdumbbell 3125296548Sdumbbell if (gtt_space == NULL) 3126296548Sdumbbell return true; 3127296548Sdumbbell 3128296548Sdumbbell if (list_empty(>t_space->node_list)) 3129296548Sdumbbell return true; 3130296548Sdumbbell 3131296548Sdumbbell other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); 3132296548Sdumbbell if (other->allocated && !other->hole_follows && other->color != cache_level) 3133296548Sdumbbell return false; 3134296548Sdumbbell 3135296548Sdumbbell other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); 3136296548Sdumbbell if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) 3137296548Sdumbbell return false; 3138296548Sdumbbell 3139296548Sdumbbell return true; 3140296548Sdumbbell} 3141296548Sdumbbell 3142296548Sdumbbellstatic void i915_gem_verify_gtt(struct drm_device *dev) 3143296548Sdumbbell{ 3144296548Sdumbbell#if WATCH_GTT 3145296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 3146296548Sdumbbell struct drm_i915_gem_object *obj; 3147296548Sdumbbell int err = 0; 3148296548Sdumbbell 3149296548Sdumbbell list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { 3150296548Sdumbbell if (obj->gtt_space == NULL) { 3151296548Sdumbbell DRM_ERROR("object found on GTT list with no space reserved\n"); 3152296548Sdumbbell err++; 3153296548Sdumbbell continue; 3154296548Sdumbbell } 3155296548Sdumbbell 3156296548Sdumbbell if (obj->cache_level != obj->gtt_space->color) { 3157296548Sdumbbell DRM_ERROR("object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", 3158296548Sdumbbell obj->gtt_space->start, 3159296548Sdumbbell obj->gtt_space->start + obj->gtt_space->size, 3160296548Sdumbbell obj->cache_level, 3161296548Sdumbbell obj->gtt_space->color); 3162296548Sdumbbell err++; 3163296548Sdumbbell continue; 3164296548Sdumbbell } 3165296548Sdumbbell 3166296548Sdumbbell if (!i915_gem_valid_gtt_space(dev, 3167296548Sdumbbell obj->gtt_space, 3168296548Sdumbbell obj->cache_level)) { 3169296548Sdumbbell DRM_ERROR("invalid GTT space found at [%08lx, %08lx] - color=%x\n", 3170296548Sdumbbell obj->gtt_space->start, 3171296548Sdumbbell obj->gtt_space->start + obj->gtt_space->size, 3172296548Sdumbbell obj->cache_level); 3173296548Sdumbbell err++; 3174296548Sdumbbell continue; 3175296548Sdumbbell } 3176296548Sdumbbell } 3177296548Sdumbbell 3178296548Sdumbbell WARN_ON(err); 3179296548Sdumbbell#endif 3180296548Sdumbbell} 3181296548Sdumbbell 3182285988Sdumbbell/** 3183285988Sdumbbell * Finds free space in the GTT aperture and binds the object there. 3184285988Sdumbbell */ 3185285988Sdumbbellstatic int 3186285988Sdumbbelli915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 3187285988Sdumbbell unsigned alignment, 3188296548Sdumbbell bool map_and_fenceable, 3189296548Sdumbbell bool nonblocking) 3190235783Skib{ 3191285988Sdumbbell struct drm_device *dev = obj->base.dev; 3192285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 3193296548Sdumbbell struct drm_mm_node *node; 3194285988Sdumbbell u32 size, fence_size, fence_alignment, unfenced_alignment; 3195285988Sdumbbell bool mappable, fenceable; 3196285988Sdumbbell int ret; 3197235783Skib 3198285988Sdumbbell if (obj->madv != I915_MADV_WILLNEED) { 3199285988Sdumbbell DRM_ERROR("Attempting to bind a purgeable object\n"); 3200285988Sdumbbell return -EINVAL; 3201285988Sdumbbell } 3202285988Sdumbbell 3203285988Sdumbbell fence_size = i915_gem_get_gtt_size(dev, 3204285988Sdumbbell obj->base.size, 3205285988Sdumbbell obj->tiling_mode); 3206285988Sdumbbell fence_alignment = i915_gem_get_gtt_alignment(dev, 3207285988Sdumbbell obj->base.size, 3208285988Sdumbbell obj->tiling_mode); 3209285988Sdumbbell unfenced_alignment = 3210285988Sdumbbell i915_gem_get_unfenced_gtt_alignment(dev, 3211285988Sdumbbell obj->base.size, 3212285988Sdumbbell obj->tiling_mode); 3213285988Sdumbbell 3214285988Sdumbbell if (alignment == 0) 3215285988Sdumbbell alignment = map_and_fenceable ? fence_alignment : 3216285988Sdumbbell unfenced_alignment; 3217285988Sdumbbell if (map_and_fenceable && alignment & (fence_alignment - 1)) { 3218285988Sdumbbell DRM_ERROR("Invalid object alignment requested %u\n", alignment); 3219285988Sdumbbell return -EINVAL; 3220285988Sdumbbell } 3221285988Sdumbbell 3222285988Sdumbbell size = map_and_fenceable ? fence_size : obj->base.size; 3223285988Sdumbbell 3224285988Sdumbbell /* If the object is bigger than the entire aperture, reject it early 3225285988Sdumbbell * before evicting everything in a vain attempt to find space. 3226285988Sdumbbell */ 3227285988Sdumbbell if (obj->base.size > 3228285988Sdumbbell (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 3229285988Sdumbbell DRM_ERROR("Attempting to bind an object larger than the aperture\n"); 3230285988Sdumbbell return -E2BIG; 3231285988Sdumbbell } 3232285988Sdumbbell 3233296548Sdumbbell ret = i915_gem_object_get_pages(obj); 3234296548Sdumbbell if (ret) 3235296548Sdumbbell return ret; 3236296548Sdumbbell 3237296548Sdumbbell i915_gem_object_pin_pages(obj); 3238296548Sdumbbell 3239296681Sdumbbell node = malloc(sizeof(*node), DRM_MEM_MM, M_NOWAIT | M_ZERO); 3240296548Sdumbbell if (node == NULL) { 3241296548Sdumbbell i915_gem_object_unpin_pages(obj); 3242296548Sdumbbell return -ENOMEM; 3243296548Sdumbbell } 3244296548Sdumbbell 3245285988Sdumbbell search_free: 3246285988Sdumbbell if (map_and_fenceable) 3247296548Sdumbbell ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node, 3248296548Sdumbbell size, alignment, obj->cache_level, 3249296548Sdumbbell 0, dev_priv->mm.gtt_mappable_end); 3250285988Sdumbbell else 3251296548Sdumbbell ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node, 3252296548Sdumbbell size, alignment, obj->cache_level); 3253296548Sdumbbell if (ret) { 3254285988Sdumbbell ret = i915_gem_evict_something(dev, size, alignment, 3255296548Sdumbbell obj->cache_level, 3256296548Sdumbbell map_and_fenceable, 3257296548Sdumbbell nonblocking); 3258296548Sdumbbell if (ret == 0) 3259296548Sdumbbell goto search_free; 3260296548Sdumbbell 3261296548Sdumbbell i915_gem_object_unpin_pages(obj); 3262296681Sdumbbell free(node, DRM_MEM_MM); 3263285988Sdumbbell return ret; 3264285988Sdumbbell } 3265296548Sdumbbell if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) { 3266296548Sdumbbell i915_gem_object_unpin_pages(obj); 3267296548Sdumbbell drm_mm_put_block(node); 3268296548Sdumbbell return -EINVAL; 3269296548Sdumbbell } 3270285988Sdumbbell 3271285988Sdumbbell ret = i915_gem_gtt_prepare_object(obj); 3272285988Sdumbbell if (ret) { 3273296548Sdumbbell i915_gem_object_unpin_pages(obj); 3274296548Sdumbbell drm_mm_put_block(node); 3275296548Sdumbbell return ret; 3276285988Sdumbbell } 3277285988Sdumbbell 3278296548Sdumbbell list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list); 3279285988Sdumbbell list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3280285988Sdumbbell 3281296548Sdumbbell obj->gtt_space = node; 3282296548Sdumbbell obj->gtt_offset = node->start; 3283285988Sdumbbell 3284285988Sdumbbell fenceable = 3285296548Sdumbbell node->size == fence_size && 3286296548Sdumbbell (node->start & (fence_alignment - 1)) == 0; 3287285988Sdumbbell 3288285988Sdumbbell mappable = 3289285988Sdumbbell obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 3290285988Sdumbbell 3291285988Sdumbbell obj->map_and_fenceable = mappable && fenceable; 3292285988Sdumbbell 3293296548Sdumbbell i915_gem_object_unpin_pages(obj); 3294285988Sdumbbell CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 3295285988Sdumbbell obj->base.size, map_and_fenceable); 3296296548Sdumbbell i915_gem_verify_gtt(dev); 3297285988Sdumbbell return 0; 3298235783Skib} 3299235783Skib 3300285988Sdumbbellvoid 3301285988Sdumbbelli915_gem_clflush_object(struct drm_i915_gem_object *obj) 3302235783Skib{ 3303285988Sdumbbell /* If we don't have a page list set up, then we're not pinned 3304285988Sdumbbell * to GPU, and we can ignore the cache flush because it'll happen 3305285988Sdumbbell * again at bind time. 3306285988Sdumbbell */ 3307285988Sdumbbell if (obj->pages == NULL) 3308285988Sdumbbell return; 3309235783Skib 3310285988Sdumbbell /* If the GPU is snooping the contents of the CPU cache, 3311285988Sdumbbell * we do not need to manually clear the CPU cache lines. However, 3312285988Sdumbbell * the caches are only snooped when the render cache is 3313285988Sdumbbell * flushed/invalidated. As we always have to emit invalidations 3314285988Sdumbbell * and flushes when moving into and out of the RENDER domain, correct 3315285988Sdumbbell * snooping behaviour occurs naturally as the result of our domain 3316285988Sdumbbell * tracking. 3317285988Sdumbbell */ 3318285988Sdumbbell if (obj->cache_level != I915_CACHE_NONE) 3319285988Sdumbbell return; 3320285988Sdumbbell 3321285988Sdumbbell CTR1(KTR_DRM, "object_clflush %p", obj); 3322285988Sdumbbell 3323285988Sdumbbell drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 3324235783Skib} 3325235783Skib 3326285988Sdumbbell/** Flushes the GTT write domain for the object if it's dirty. */ 3327235783Skibstatic void 3328285988Sdumbbelli915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 3329235783Skib{ 3330285988Sdumbbell uint32_t old_write_domain; 3331235783Skib 3332285988Sdumbbell if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 3333285988Sdumbbell return; 3334235783Skib 3335285988Sdumbbell /* No actual flushing is required for the GTT write domain. Writes 3336285988Sdumbbell * to it immediately go to main memory as far as we know, so there's 3337285988Sdumbbell * no chipset flush. It also doesn't land in render cache. 3338285988Sdumbbell * 3339285988Sdumbbell * However, we do have to enforce the order so that all writes through 3340285988Sdumbbell * the GTT land before any writes to the device, such as updates to 3341285988Sdumbbell * the GATT itself. 3342285988Sdumbbell */ 3343285988Sdumbbell wmb(); 3344285988Sdumbbell 3345285988Sdumbbell old_write_domain = obj->base.write_domain; 3346285988Sdumbbell obj->base.write_domain = 0; 3347285988Sdumbbell 3348285988Sdumbbell CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 3349285988Sdumbbell obj->base.read_domains, old_write_domain); 3350285988Sdumbbell} 3351285988Sdumbbell 3352285988Sdumbbell/** Flushes the CPU write domain for the object if it's dirty. */ 3353285988Sdumbbellstatic void 3354285988Sdumbbelli915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 3355285988Sdumbbell{ 3356285988Sdumbbell uint32_t old_write_domain; 3357285988Sdumbbell 3358285988Sdumbbell if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 3359235783Skib return; 3360285988Sdumbbell 3361285988Sdumbbell i915_gem_clflush_object(obj); 3362296548Sdumbbell i915_gem_chipset_flush(obj->base.dev); 3363285988Sdumbbell old_write_domain = obj->base.write_domain; 3364285988Sdumbbell obj->base.write_domain = 0; 3365285988Sdumbbell 3366285988Sdumbbell CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 3367285988Sdumbbell obj->base.read_domains, old_write_domain); 3368285988Sdumbbell} 3369285988Sdumbbell 3370285988Sdumbbell/** 3371285988Sdumbbell * Moves a single object to the GTT read, and possibly write domain. 3372285988Sdumbbell * 3373285988Sdumbbell * This function returns when the move is complete, including waiting on 3374285988Sdumbbell * flushes to occur. 3375285988Sdumbbell */ 3376285988Sdumbbellint 3377285988Sdumbbelli915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 3378285988Sdumbbell{ 3379285988Sdumbbell drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 3380285988Sdumbbell uint32_t old_write_domain, old_read_domains; 3381285988Sdumbbell int ret; 3382285988Sdumbbell 3383285988Sdumbbell /* Not valid to be called on unbound objects. */ 3384285988Sdumbbell if (obj->gtt_space == NULL) 3385285988Sdumbbell return -EINVAL; 3386285988Sdumbbell 3387285988Sdumbbell if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 3388285988Sdumbbell return 0; 3389285988Sdumbbell 3390296548Sdumbbell ret = i915_gem_object_wait_rendering(obj, !write); 3391285988Sdumbbell if (ret) 3392287174Sbapt return ret; 3393285988Sdumbbell 3394285988Sdumbbell i915_gem_object_flush_cpu_write_domain(obj); 3395235783Skib 3396285988Sdumbbell old_write_domain = obj->base.write_domain; 3397285988Sdumbbell old_read_domains = obj->base.read_domains; 3398235783Skib 3399285988Sdumbbell /* It should now be out of any other write domains, and we can update 3400285988Sdumbbell * the domain values for our changes. 3401235783Skib */ 3402296548Sdumbbell BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 3403285988Sdumbbell obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3404285988Sdumbbell if (write) { 3405285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_GTT; 3406285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_GTT; 3407285988Sdumbbell obj->dirty = 1; 3408285988Sdumbbell } 3409235783Skib 3410285988Sdumbbell CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 3411285988Sdumbbell old_read_domains, old_write_domain); 3412235783Skib 3413285988Sdumbbell /* And bump the LRU for this access */ 3414285988Sdumbbell if (i915_gem_object_is_inactive(obj)) 3415285988Sdumbbell list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3416285988Sdumbbell 3417285988Sdumbbell return 0; 3418285988Sdumbbell} 3419285988Sdumbbell 3420285988Sdumbbellint i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3421285988Sdumbbell enum i915_cache_level cache_level) 3422285988Sdumbbell{ 3423285988Sdumbbell struct drm_device *dev = obj->base.dev; 3424285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 3425285988Sdumbbell int ret; 3426285988Sdumbbell 3427285988Sdumbbell if (obj->cache_level == cache_level) 3428285988Sdumbbell return 0; 3429285988Sdumbbell 3430285988Sdumbbell if (obj->pin_count) { 3431285988Sdumbbell DRM_DEBUG("can not change the cache level of pinned objects\n"); 3432285988Sdumbbell return -EBUSY; 3433285988Sdumbbell } 3434285988Sdumbbell 3435296548Sdumbbell if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) { 3436296548Sdumbbell ret = i915_gem_object_unbind(obj); 3437296548Sdumbbell if (ret) 3438296548Sdumbbell return ret; 3439296548Sdumbbell } 3440296548Sdumbbell 3441285988Sdumbbell if (obj->gtt_space) { 3442285988Sdumbbell ret = i915_gem_object_finish_gpu(obj); 3443285988Sdumbbell if (ret) 3444285988Sdumbbell return ret; 3445285988Sdumbbell 3446285988Sdumbbell i915_gem_object_finish_gtt(obj); 3447285988Sdumbbell 3448285988Sdumbbell /* Before SandyBridge, you could not use tiling or fence 3449285988Sdumbbell * registers with snooped memory, so relinquish any fences 3450285988Sdumbbell * currently pointing to our region in the aperture. 3451285988Sdumbbell */ 3452296548Sdumbbell if (INTEL_INFO(dev)->gen < 6) { 3453285988Sdumbbell ret = i915_gem_object_put_fence(obj); 3454285988Sdumbbell if (ret) 3455285988Sdumbbell return ret; 3456235783Skib } 3457235783Skib 3458285988Sdumbbell if (obj->has_global_gtt_mapping) 3459285988Sdumbbell i915_gem_gtt_bind_object(obj, cache_level); 3460285988Sdumbbell if (obj->has_aliasing_ppgtt_mapping) 3461285988Sdumbbell i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 3462285988Sdumbbell obj, cache_level); 3463296548Sdumbbell 3464296548Sdumbbell obj->gtt_space->color = cache_level; 3465235783Skib } 3466235783Skib 3467285988Sdumbbell if (cache_level == I915_CACHE_NONE) { 3468285988Sdumbbell u32 old_read_domains, old_write_domain; 3469285988Sdumbbell 3470285988Sdumbbell /* If we're coming from LLC cached, then we haven't 3471285988Sdumbbell * actually been tracking whether the data is in the 3472285988Sdumbbell * CPU cache or not, since we only allow one bit set 3473285988Sdumbbell * in obj->write_domain and have been skipping the clflushes. 3474285988Sdumbbell * Just set it to the CPU cache for now. 3475285988Sdumbbell */ 3476296548Sdumbbell WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); 3477296548Sdumbbell WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU); 3478285988Sdumbbell 3479285988Sdumbbell old_read_domains = obj->base.read_domains; 3480285988Sdumbbell old_write_domain = obj->base.write_domain; 3481285988Sdumbbell 3482285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3483285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3484285988Sdumbbell 3485285988Sdumbbell CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 3486285988Sdumbbell obj, old_read_domains, old_write_domain); 3487285988Sdumbbell } 3488285988Sdumbbell 3489285988Sdumbbell obj->cache_level = cache_level; 3490296548Sdumbbell i915_gem_verify_gtt(dev); 3491285988Sdumbbell return 0; 3492285988Sdumbbell} 3493285988Sdumbbell 3494296548Sdumbbellint i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 3495296548Sdumbbell struct drm_file *file) 3496296548Sdumbbell{ 3497296548Sdumbbell struct drm_i915_gem_caching *args = data; 3498296548Sdumbbell struct drm_i915_gem_object *obj; 3499296548Sdumbbell int ret; 3500296548Sdumbbell 3501296548Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3502296548Sdumbbell if (ret) 3503296548Sdumbbell return ret; 3504296548Sdumbbell 3505296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3506296548Sdumbbell if (&obj->base == NULL) { 3507296548Sdumbbell ret = -ENOENT; 3508296548Sdumbbell goto unlock; 3509296548Sdumbbell } 3510296548Sdumbbell 3511296548Sdumbbell args->caching = obj->cache_level != I915_CACHE_NONE; 3512296548Sdumbbell 3513296548Sdumbbell drm_gem_object_unreference(&obj->base); 3514296548Sdumbbellunlock: 3515296548Sdumbbell DRM_UNLOCK(dev); 3516296548Sdumbbell return ret; 3517296548Sdumbbell} 3518296548Sdumbbell 3519296548Sdumbbellint i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 3520296548Sdumbbell struct drm_file *file) 3521296548Sdumbbell{ 3522296548Sdumbbell struct drm_i915_gem_caching *args = data; 3523296548Sdumbbell struct drm_i915_gem_object *obj; 3524296548Sdumbbell enum i915_cache_level level; 3525296548Sdumbbell int ret; 3526296548Sdumbbell 3527296548Sdumbbell switch (args->caching) { 3528296548Sdumbbell case I915_CACHING_NONE: 3529296548Sdumbbell level = I915_CACHE_NONE; 3530296548Sdumbbell break; 3531296548Sdumbbell case I915_CACHING_CACHED: 3532296548Sdumbbell level = I915_CACHE_LLC; 3533296548Sdumbbell break; 3534296548Sdumbbell default: 3535296548Sdumbbell return -EINVAL; 3536296548Sdumbbell } 3537296548Sdumbbell 3538296548Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3539296548Sdumbbell if (ret) 3540296548Sdumbbell return ret; 3541296548Sdumbbell 3542296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3543296548Sdumbbell if (&obj->base == NULL) { 3544296548Sdumbbell ret = -ENOENT; 3545296548Sdumbbell goto unlock; 3546296548Sdumbbell } 3547296548Sdumbbell 3548296548Sdumbbell ret = i915_gem_object_set_cache_level(obj, level); 3549296548Sdumbbell 3550296548Sdumbbell drm_gem_object_unreference(&obj->base); 3551296548Sdumbbellunlock: 3552296548Sdumbbell DRM_UNLOCK(dev); 3553296548Sdumbbell return ret; 3554296548Sdumbbell} 3555296548Sdumbbell 3556285988Sdumbbellstatic bool is_pin_display(struct drm_i915_gem_object *obj) 3557285988Sdumbbell{ 3558285988Sdumbbell /* There are 3 sources that pin objects: 3559285988Sdumbbell * 1. The display engine (scanouts, sprites, cursors); 3560285988Sdumbbell * 2. Reservations for execbuffer; 3561285988Sdumbbell * 3. The user. 3562285988Sdumbbell * 3563285988Sdumbbell * We can ignore reservations as we hold the struct_mutex and 3564285988Sdumbbell * are only called outside of the reservation path. The user 3565285988Sdumbbell * can only increment pin_count once, and so if after 3566285988Sdumbbell * subtracting the potential reference by the user, any pin_count 3567285988Sdumbbell * remains, it must be due to another use by the display engine. 3568285988Sdumbbell */ 3569285988Sdumbbell return obj->pin_count - !!obj->user_pin_count; 3570285988Sdumbbell} 3571285988Sdumbbell 3572296548Sdumbbell/* 3573296548Sdumbbell * Prepare buffer for display plane (scanout, cursors, etc). 3574296548Sdumbbell * Can be called from an uninterruptible phase (modesetting) and allows 3575296548Sdumbbell * any flushes to be pipelined (for pageflips). 3576296548Sdumbbell */ 3577285988Sdumbbellint 3578285988Sdumbbelli915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 3579285988Sdumbbell u32 alignment, 3580285988Sdumbbell struct intel_ring_buffer *pipelined) 3581285988Sdumbbell{ 3582285988Sdumbbell u32 old_read_domains, old_write_domain; 3583285988Sdumbbell int ret; 3584285988Sdumbbell 3585285988Sdumbbell if (pipelined != obj->ring) { 3586285988Sdumbbell ret = i915_gem_object_sync(obj, pipelined); 3587285988Sdumbbell if (ret) 3588285988Sdumbbell return ret; 3589285988Sdumbbell } 3590285988Sdumbbell 3591285988Sdumbbell /* Mark the pin_display early so that we account for the 3592285988Sdumbbell * display coherency whilst setting up the cache domains. 3593285988Sdumbbell */ 3594285988Sdumbbell obj->pin_display = true; 3595285988Sdumbbell 3596285988Sdumbbell /* The display engine is not coherent with the LLC cache on gen6. As 3597285988Sdumbbell * a result, we make sure that the pinning that is about to occur is 3598285988Sdumbbell * done with uncached PTEs. This is lowest common denominator for all 3599285988Sdumbbell * chipsets. 3600285988Sdumbbell * 3601285988Sdumbbell * However for gen6+, we could do better by using the GFDT bit instead 3602285988Sdumbbell * of uncaching, which would allow us to flush all the LLC-cached data 3603285988Sdumbbell * with that bit in the PTE to main memory with just one PIPE_CONTROL. 3604285988Sdumbbell */ 3605285988Sdumbbell ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 3606285988Sdumbbell if (ret) 3607285988Sdumbbell goto err_unpin_display; 3608285988Sdumbbell 3609285988Sdumbbell /* As the user may map the buffer once pinned in the display plane 3610285988Sdumbbell * (e.g. libkms for the bootup splash), we have to ensure that we 3611285988Sdumbbell * always use map_and_fenceable for all scanout buffers. 3612285988Sdumbbell */ 3613296548Sdumbbell ret = i915_gem_object_pin(obj, alignment, true, false); 3614285988Sdumbbell if (ret) 3615285988Sdumbbell goto err_unpin_display; 3616285988Sdumbbell 3617285988Sdumbbell i915_gem_object_flush_cpu_write_domain(obj); 3618285988Sdumbbell 3619285988Sdumbbell old_write_domain = obj->base.write_domain; 3620285988Sdumbbell old_read_domains = obj->base.read_domains; 3621285988Sdumbbell 3622296548Sdumbbell /* It should now be out of any other write domains, and we can update 3623296548Sdumbbell * the domain values for our changes. 3624296548Sdumbbell */ 3625296548Sdumbbell obj->base.write_domain = 0; 3626285988Sdumbbell obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3627285988Sdumbbell 3628285988Sdumbbell CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 3629296548Sdumbbell obj, old_read_domains, old_write_domain); 3630285988Sdumbbell 3631285988Sdumbbell return 0; 3632285988Sdumbbell 3633285988Sdumbbellerr_unpin_display: 3634285988Sdumbbell obj->pin_display = is_pin_display(obj); 3635285988Sdumbbell return ret; 3636285988Sdumbbell} 3637285988Sdumbbell 3638285988Sdumbbellvoid 3639285988Sdumbbelli915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) 3640285988Sdumbbell{ 3641285988Sdumbbell i915_gem_object_unpin(obj); 3642285988Sdumbbell obj->pin_display = is_pin_display(obj); 3643285988Sdumbbell} 3644285988Sdumbbell 3645285988Sdumbbellint 3646285988Sdumbbelli915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 3647285988Sdumbbell{ 3648285988Sdumbbell int ret; 3649285988Sdumbbell 3650285988Sdumbbell if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 3651285988Sdumbbell return 0; 3652285988Sdumbbell 3653296548Sdumbbell ret = i915_gem_object_wait_rendering(obj, false); 3654285988Sdumbbell if (ret) 3655285988Sdumbbell return ret; 3656285988Sdumbbell 3657285988Sdumbbell /* Ensure that we invalidate the GPU's caches and TLBs. */ 3658285988Sdumbbell obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 3659285988Sdumbbell return 0; 3660285988Sdumbbell} 3661285988Sdumbbell 3662287174Sbapt/** 3663287174Sbapt * Moves a single object to the CPU read, and possibly write domain. 3664287174Sbapt * 3665287174Sbapt * This function returns when the move is complete, including waiting on 3666287174Sbapt * flushes to occur. 3667287174Sbapt */ 3668285988Sdumbbellint 3669285988Sdumbbelli915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 3670285988Sdumbbell{ 3671285988Sdumbbell uint32_t old_write_domain, old_read_domains; 3672285988Sdumbbell int ret; 3673285988Sdumbbell 3674285988Sdumbbell if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 3675285988Sdumbbell return 0; 3676285988Sdumbbell 3677296548Sdumbbell ret = i915_gem_object_wait_rendering(obj, !write); 3678285988Sdumbbell if (ret) 3679285988Sdumbbell return ret; 3680285988Sdumbbell 3681285988Sdumbbell i915_gem_object_flush_gtt_write_domain(obj); 3682285988Sdumbbell 3683285988Sdumbbell old_write_domain = obj->base.write_domain; 3684285988Sdumbbell old_read_domains = obj->base.read_domains; 3685285988Sdumbbell 3686285988Sdumbbell /* Flush the CPU cache if it's still invalid. */ 3687285988Sdumbbell if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 3688285988Sdumbbell i915_gem_clflush_object(obj); 3689285988Sdumbbell 3690285988Sdumbbell obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 3691285988Sdumbbell } 3692285988Sdumbbell 3693285988Sdumbbell /* It should now be out of any other write domains, and we can update 3694285988Sdumbbell * the domain values for our changes. 3695285988Sdumbbell */ 3696296548Sdumbbell BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3697285988Sdumbbell 3698285988Sdumbbell /* If we're writing through the CPU, then the GPU read domains will 3699285988Sdumbbell * need to be invalidated at next use. 3700285988Sdumbbell */ 3701285988Sdumbbell if (write) { 3702285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3703285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3704285988Sdumbbell } 3705285988Sdumbbell 3706285988Sdumbbell CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 3707285988Sdumbbell old_read_domains, old_write_domain); 3708285988Sdumbbell 3709285988Sdumbbell return 0; 3710285988Sdumbbell} 3711285988Sdumbbell 3712285988Sdumbbell/* Throttle our rendering by waiting until the ring has completed our requests 3713285988Sdumbbell * emitted over 20 msec ago. 3714285988Sdumbbell * 3715285988Sdumbbell * Note that if we were to use the current jiffies each time around the loop, 3716285988Sdumbbell * we wouldn't escape the function with any frames outstanding if the time to 3717285988Sdumbbell * render a frame was over 20ms. 3718285988Sdumbbell * 3719285988Sdumbbell * This should get us reasonable parallelism between CPU and GPU but also 3720285988Sdumbbell * relatively low latency when blocking on a particular request to finish. 3721285988Sdumbbell */ 3722285988Sdumbbellstatic int 3723285988Sdumbbelli915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 3724285988Sdumbbell{ 3725285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 3726285988Sdumbbell struct drm_i915_file_private *file_priv = file->driver_priv; 3727296548Sdumbbell unsigned long recent_enough = jiffies - msecs_to_jiffies(20); 3728285988Sdumbbell struct drm_i915_gem_request *request; 3729285988Sdumbbell struct intel_ring_buffer *ring = NULL; 3730285988Sdumbbell u32 seqno = 0; 3731285988Sdumbbell int ret; 3732285988Sdumbbell 3733296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) 3734285988Sdumbbell return -EIO; 3735285988Sdumbbell 3736296548Sdumbbell mtx_lock(&file_priv->mm.lock); 3737285988Sdumbbell list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 3738285988Sdumbbell if (time_after_eq(request->emitted_jiffies, recent_enough)) 3739285988Sdumbbell break; 3740296548Sdumbbell 3741285988Sdumbbell ring = request->ring; 3742285988Sdumbbell seqno = request->seqno; 3743285988Sdumbbell } 3744296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 3745296548Sdumbbell 3746285988Sdumbbell if (seqno == 0) 3747285988Sdumbbell return 0; 3748285988Sdumbbell 3749296548Sdumbbell ret = __wait_seqno(ring, seqno, true, NULL); 3750285988Sdumbbell if (ret == 0) 3751296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 3752296548Sdumbbell &dev_priv->mm.retire_work, 0); 3753235783Skib 3754285988Sdumbbell return ret; 3755285988Sdumbbell} 3756285988Sdumbbell 3757285988Sdumbbellint 3758285988Sdumbbelli915_gem_object_pin(struct drm_i915_gem_object *obj, 3759285988Sdumbbell uint32_t alignment, 3760296548Sdumbbell bool map_and_fenceable, 3761296548Sdumbbell bool nonblocking) 3762285988Sdumbbell{ 3763285988Sdumbbell int ret; 3764285988Sdumbbell 3765296548Sdumbbell if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) 3766285988Sdumbbell return -EBUSY; 3767285988Sdumbbell 3768285988Sdumbbell if (obj->gtt_space != NULL) { 3769285988Sdumbbell if ((alignment && obj->gtt_offset & (alignment - 1)) || 3770285988Sdumbbell (map_and_fenceable && !obj->map_and_fenceable)) { 3771296548Sdumbbell WARN(obj->pin_count, 3772296548Sdumbbell "bo is already pinned with incorrect alignment:" 3773285988Sdumbbell " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 3774285988Sdumbbell " obj->map_and_fenceable=%d\n", 3775285988Sdumbbell obj->gtt_offset, alignment, 3776285988Sdumbbell map_and_fenceable, 3777285988Sdumbbell obj->map_and_fenceable); 3778285988Sdumbbell ret = i915_gem_object_unbind(obj); 3779285988Sdumbbell if (ret) 3780285988Sdumbbell return ret; 3781285988Sdumbbell } 3782285988Sdumbbell } 3783285988Sdumbbell 3784285988Sdumbbell if (obj->gtt_space == NULL) { 3785296548Sdumbbell struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3786296548Sdumbbell 3787285988Sdumbbell ret = i915_gem_object_bind_to_gtt(obj, alignment, 3788296548Sdumbbell map_and_fenceable, 3789296548Sdumbbell nonblocking); 3790285988Sdumbbell if (ret) 3791285988Sdumbbell return ret; 3792296548Sdumbbell 3793296548Sdumbbell if (!dev_priv->mm.aliasing_ppgtt) 3794296548Sdumbbell i915_gem_gtt_bind_object(obj, obj->cache_level); 3795285988Sdumbbell } 3796285988Sdumbbell 3797285988Sdumbbell if (!obj->has_global_gtt_mapping && map_and_fenceable) 3798285988Sdumbbell i915_gem_gtt_bind_object(obj, obj->cache_level); 3799285988Sdumbbell 3800285988Sdumbbell obj->pin_count++; 3801285988Sdumbbell obj->pin_mappable |= map_and_fenceable; 3802285988Sdumbbell 3803285988Sdumbbell return 0; 3804285988Sdumbbell} 3805285988Sdumbbell 3806285988Sdumbbellvoid 3807285988Sdumbbelli915_gem_object_unpin(struct drm_i915_gem_object *obj) 3808285988Sdumbbell{ 3809296548Sdumbbell BUG_ON(obj->pin_count == 0); 3810296548Sdumbbell BUG_ON(obj->gtt_space == NULL); 3811285988Sdumbbell 3812285988Sdumbbell if (--obj->pin_count == 0) 3813285988Sdumbbell obj->pin_mappable = false; 3814285988Sdumbbell} 3815285988Sdumbbell 3816285988Sdumbbellint 3817285988Sdumbbelli915_gem_pin_ioctl(struct drm_device *dev, void *data, 3818285988Sdumbbell struct drm_file *file) 3819285988Sdumbbell{ 3820285988Sdumbbell struct drm_i915_gem_pin *args = data; 3821285988Sdumbbell struct drm_i915_gem_object *obj; 3822285988Sdumbbell int ret; 3823285988Sdumbbell 3824285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3825285988Sdumbbell if (ret) 3826285988Sdumbbell return ret; 3827285988Sdumbbell 3828296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3829296548Sdumbbell if (&obj->base == NULL) { 3830285988Sdumbbell ret = -ENOENT; 3831285988Sdumbbell goto unlock; 3832285988Sdumbbell } 3833285988Sdumbbell 3834285988Sdumbbell if (obj->madv != I915_MADV_WILLNEED) { 3835285988Sdumbbell DRM_ERROR("Attempting to pin a purgeable buffer\n"); 3836285988Sdumbbell ret = -EINVAL; 3837285988Sdumbbell goto out; 3838285988Sdumbbell } 3839285988Sdumbbell 3840285988Sdumbbell if (obj->pin_filp != NULL && obj->pin_filp != file) { 3841285988Sdumbbell DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3842285988Sdumbbell args->handle); 3843285988Sdumbbell ret = -EINVAL; 3844285988Sdumbbell goto out; 3845285988Sdumbbell } 3846285988Sdumbbell 3847296548Sdumbbell if (obj->user_pin_count == 0) { 3848296548Sdumbbell ret = i915_gem_object_pin(obj, args->alignment, true, false); 3849285988Sdumbbell if (ret) 3850285988Sdumbbell goto out; 3851285988Sdumbbell } 3852285988Sdumbbell 3853296548Sdumbbell obj->user_pin_count++; 3854296548Sdumbbell obj->pin_filp = file; 3855296548Sdumbbell 3856285988Sdumbbell /* XXX - flush the CPU caches for pinned objects 3857285988Sdumbbell * as the X server doesn't manage domains yet 3858285988Sdumbbell */ 3859285988Sdumbbell i915_gem_object_flush_cpu_write_domain(obj); 3860285988Sdumbbell args->offset = obj->gtt_offset; 3861285988Sdumbbellout: 3862285988Sdumbbell drm_gem_object_unreference(&obj->base); 3863285988Sdumbbellunlock: 3864235783Skib DRM_UNLOCK(dev); 3865285988Sdumbbell return ret; 3866235783Skib} 3867235783Skib 3868285988Sdumbbellint 3869285988Sdumbbelli915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3870285988Sdumbbell struct drm_file *file) 3871285988Sdumbbell{ 3872285988Sdumbbell struct drm_i915_gem_pin *args = data; 3873285988Sdumbbell struct drm_i915_gem_object *obj; 3874285988Sdumbbell int ret; 3875285988Sdumbbell 3876285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3877285988Sdumbbell if (ret) 3878285988Sdumbbell return ret; 3879285988Sdumbbell 3880285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3881285988Sdumbbell if (&obj->base == NULL) { 3882285988Sdumbbell ret = -ENOENT; 3883285988Sdumbbell goto unlock; 3884285988Sdumbbell } 3885285988Sdumbbell 3886285988Sdumbbell if (obj->pin_filp != file) { 3887285988Sdumbbell DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3888285988Sdumbbell args->handle); 3889285988Sdumbbell ret = -EINVAL; 3890285988Sdumbbell goto out; 3891285988Sdumbbell } 3892285988Sdumbbell obj->user_pin_count--; 3893285988Sdumbbell if (obj->user_pin_count == 0) { 3894285988Sdumbbell obj->pin_filp = NULL; 3895285988Sdumbbell i915_gem_object_unpin(obj); 3896285988Sdumbbell } 3897285988Sdumbbell 3898285988Sdumbbellout: 3899285988Sdumbbell drm_gem_object_unreference(&obj->base); 3900285988Sdumbbellunlock: 3901285988Sdumbbell DRM_UNLOCK(dev); 3902285988Sdumbbell return ret; 3903285988Sdumbbell} 3904285988Sdumbbell 3905285988Sdumbbellint 3906285988Sdumbbelli915_gem_busy_ioctl(struct drm_device *dev, void *data, 3907285988Sdumbbell struct drm_file *file) 3908285988Sdumbbell{ 3909285988Sdumbbell struct drm_i915_gem_busy *args = data; 3910285988Sdumbbell struct drm_i915_gem_object *obj; 3911285988Sdumbbell int ret; 3912285988Sdumbbell 3913285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3914285988Sdumbbell if (ret) 3915285988Sdumbbell return ret; 3916285988Sdumbbell 3917285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3918285988Sdumbbell if (&obj->base == NULL) { 3919285988Sdumbbell ret = -ENOENT; 3920285988Sdumbbell goto unlock; 3921285988Sdumbbell } 3922285988Sdumbbell 3923296548Sdumbbell /* Count all active objects as busy, even if they are currently not used 3924296548Sdumbbell * by the gpu. Users of this interface expect objects to eventually 3925296548Sdumbbell * become non-busy without any further actions, therefore emit any 3926296548Sdumbbell * necessary flushes here. 3927296548Sdumbbell */ 3928296548Sdumbbell ret = i915_gem_object_flush_active(obj); 3929296548Sdumbbell 3930285988Sdumbbell args->busy = obj->active; 3931296548Sdumbbell if (obj->ring) { 3932296548Sdumbbell BUILD_BUG_ON(I915_NUM_RINGS > 16); 3933296548Sdumbbell args->busy |= intel_ring_flag(obj->ring) << 16; 3934285988Sdumbbell } 3935285988Sdumbbell 3936285988Sdumbbell drm_gem_object_unreference(&obj->base); 3937285988Sdumbbellunlock: 3938285988Sdumbbell DRM_UNLOCK(dev); 3939285988Sdumbbell return ret; 3940285988Sdumbbell} 3941285988Sdumbbell 3942285988Sdumbbellint 3943285988Sdumbbelli915_gem_throttle_ioctl(struct drm_device *dev, void *data, 3944285988Sdumbbell struct drm_file *file_priv) 3945285988Sdumbbell{ 3946285988Sdumbbell return i915_gem_ring_throttle(dev, file_priv); 3947285988Sdumbbell} 3948285988Sdumbbell 3949285988Sdumbbellint 3950285988Sdumbbelli915_gem_madvise_ioctl(struct drm_device *dev, void *data, 3951285988Sdumbbell struct drm_file *file_priv) 3952285988Sdumbbell{ 3953285988Sdumbbell struct drm_i915_gem_madvise *args = data; 3954285988Sdumbbell struct drm_i915_gem_object *obj; 3955285988Sdumbbell int ret; 3956285988Sdumbbell 3957285988Sdumbbell switch (args->madv) { 3958285988Sdumbbell case I915_MADV_DONTNEED: 3959285988Sdumbbell case I915_MADV_WILLNEED: 3960285988Sdumbbell break; 3961285988Sdumbbell default: 3962285988Sdumbbell return -EINVAL; 3963285988Sdumbbell } 3964285988Sdumbbell 3965285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3966285988Sdumbbell if (ret) 3967285988Sdumbbell return ret; 3968285988Sdumbbell 3969285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 3970285988Sdumbbell if (&obj->base == NULL) { 3971285988Sdumbbell ret = -ENOENT; 3972285988Sdumbbell goto unlock; 3973285988Sdumbbell } 3974285988Sdumbbell 3975285988Sdumbbell if (obj->pin_count) { 3976285988Sdumbbell ret = -EINVAL; 3977285988Sdumbbell goto out; 3978285988Sdumbbell } 3979285988Sdumbbell 3980296548Sdumbbell if (obj->madv != __I915_MADV_PURGED) 3981285988Sdumbbell obj->madv = args->madv; 3982285988Sdumbbell 3983285988Sdumbbell /* if the object is no longer attached, discard its backing storage */ 3984296548Sdumbbell if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) 3985285988Sdumbbell i915_gem_object_truncate(obj); 3986285988Sdumbbell 3987296548Sdumbbell args->retained = obj->madv != __I915_MADV_PURGED; 3988285988Sdumbbell 3989285988Sdumbbellout: 3990285988Sdumbbell drm_gem_object_unreference(&obj->base); 3991285988Sdumbbellunlock: 3992285988Sdumbbell DRM_UNLOCK(dev); 3993285988Sdumbbell return ret; 3994285988Sdumbbell} 3995285988Sdumbbell 3996296548Sdumbbellvoid i915_gem_object_init(struct drm_i915_gem_object *obj, 3997296548Sdumbbell const struct drm_i915_gem_object_ops *ops) 3998296548Sdumbbell{ 3999296548Sdumbbell INIT_LIST_HEAD(&obj->mm_list); 4000296548Sdumbbell INIT_LIST_HEAD(&obj->gtt_list); 4001296548Sdumbbell INIT_LIST_HEAD(&obj->ring_list); 4002296548Sdumbbell INIT_LIST_HEAD(&obj->exec_list); 4003296548Sdumbbell 4004296548Sdumbbell obj->ops = ops; 4005296548Sdumbbell 4006296548Sdumbbell obj->fence_reg = I915_FENCE_REG_NONE; 4007296548Sdumbbell obj->madv = I915_MADV_WILLNEED; 4008296548Sdumbbell /* Avoid an unnecessary call to unbind on the first bind. */ 4009296548Sdumbbell obj->map_and_fenceable = true; 4010296548Sdumbbell 4011296548Sdumbbell i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); 4012296548Sdumbbell} 4013296548Sdumbbell 4014296548Sdumbbellstatic const struct drm_i915_gem_object_ops i915_gem_object_ops = { 4015296548Sdumbbell .get_pages = i915_gem_object_get_pages_gtt, 4016296548Sdumbbell .put_pages = i915_gem_object_put_pages_gtt, 4017296548Sdumbbell}; 4018296548Sdumbbell 4019285988Sdumbbellstruct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, 4020285988Sdumbbell size_t size) 4021285988Sdumbbell{ 4022285988Sdumbbell struct drm_i915_gem_object *obj; 4023285988Sdumbbell 4024285988Sdumbbell obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 4025296548Sdumbbell if (obj == NULL) 4026296548Sdumbbell return NULL; 4027285988Sdumbbell 4028285988Sdumbbell if (drm_gem_object_init(dev, &obj->base, size) != 0) { 4029285988Sdumbbell free(obj, DRM_I915_GEM); 4030285988Sdumbbell return NULL; 4031285988Sdumbbell } 4032285988Sdumbbell 4033296548Sdumbbell#ifdef FREEBSD_WIP 4034296548Sdumbbell mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 4035296548Sdumbbell if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { 4036296548Sdumbbell /* 965gm cannot relocate objects above 4GiB. */ 4037296548Sdumbbell mask &= ~__GFP_HIGHMEM; 4038296548Sdumbbell mask |= __GFP_DMA32; 4039296548Sdumbbell } 4040296548Sdumbbell 4041296548Sdumbbell mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; 4042296548Sdumbbell mapping_set_gfp_mask(mapping, mask); 4043296548Sdumbbell#endif /* FREEBSD_WIP */ 4044296548Sdumbbell 4045296548Sdumbbell i915_gem_object_init(obj, &i915_gem_object_ops); 4046296548Sdumbbell 4047285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_CPU; 4048285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_CPU; 4049285988Sdumbbell 4050285988Sdumbbell if (HAS_LLC(dev)) { 4051285988Sdumbbell /* On some devices, we can have the GPU use the LLC (the CPU 4052285988Sdumbbell * cache) for about a 10% performance improvement 4053285988Sdumbbell * compared to uncached. Graphics requests other than 4054285988Sdumbbell * display scanout are coherent with the CPU in 4055285988Sdumbbell * accessing this cache. This means in this mode we 4056285988Sdumbbell * don't need to clflush on the CPU side, and on the 4057285988Sdumbbell * GPU side we only need to flush internal caches to 4058285988Sdumbbell * get data visible to the CPU. 4059285988Sdumbbell * 4060285988Sdumbbell * However, we maintain the display planes as UC, and so 4061285988Sdumbbell * need to rebind when first used as such. 4062285988Sdumbbell */ 4063285988Sdumbbell obj->cache_level = I915_CACHE_LLC; 4064285988Sdumbbell } else 4065285988Sdumbbell obj->cache_level = I915_CACHE_NONE; 4066285988Sdumbbell 4067285988Sdumbbell return obj; 4068285988Sdumbbell} 4069285988Sdumbbell 4070285988Sdumbbellint i915_gem_init_object(struct drm_gem_object *obj) 4071285988Sdumbbell{ 4072285988Sdumbbell printf("i915_gem_init_object called\n"); 4073285988Sdumbbell 4074285988Sdumbbell return 0; 4075285988Sdumbbell} 4076285988Sdumbbell 4077285988Sdumbbellvoid i915_gem_free_object(struct drm_gem_object *gem_obj) 4078285988Sdumbbell{ 4079285988Sdumbbell struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 4080285988Sdumbbell struct drm_device *dev = obj->base.dev; 4081285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4082285988Sdumbbell 4083285988Sdumbbell CTR1(KTR_DRM, "object_destroy_tail %p", obj); 4084285988Sdumbbell 4085285988Sdumbbell if (obj->phys_obj) 4086285988Sdumbbell i915_gem_detach_phys_object(dev, obj); 4087285988Sdumbbell 4088285988Sdumbbell obj->pin_count = 0; 4089296548Sdumbbell if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { 4090285988Sdumbbell bool was_interruptible; 4091285988Sdumbbell 4092285988Sdumbbell was_interruptible = dev_priv->mm.interruptible; 4093285988Sdumbbell dev_priv->mm.interruptible = false; 4094285988Sdumbbell 4095296548Sdumbbell WARN_ON(i915_gem_object_unbind(obj)); 4096285988Sdumbbell 4097285988Sdumbbell dev_priv->mm.interruptible = was_interruptible; 4098285988Sdumbbell } 4099285988Sdumbbell 4100296548Sdumbbell obj->pages_pin_count = 0; 4101296548Sdumbbell i915_gem_object_put_pages(obj); 4102296548Sdumbbell i915_gem_object_free_mmap_offset(obj); 4103296548Sdumbbell 4104296548Sdumbbell BUG_ON(obj->pages); 4105296548Sdumbbell 4106296548Sdumbbell#ifdef FREEBSD_WIP 4107296548Sdumbbell if (obj->base.import_attach) 4108296548Sdumbbell drm_prime_gem_destroy(&obj->base, NULL); 4109296548Sdumbbell#endif /* FREEBSD_WIP */ 4110296548Sdumbbell 4111285988Sdumbbell drm_gem_object_release(&obj->base); 4112285988Sdumbbell i915_gem_info_remove_obj(dev_priv, obj->base.size); 4113285988Sdumbbell 4114285988Sdumbbell free(obj->bit_17, DRM_I915_GEM); 4115285988Sdumbbell free(obj, DRM_I915_GEM); 4116285988Sdumbbell} 4117285988Sdumbbell 4118285988Sdumbbellint 4119285988Sdumbbelli915_gem_idle(struct drm_device *dev) 4120285988Sdumbbell{ 4121285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4122285988Sdumbbell int ret; 4123285988Sdumbbell 4124285988Sdumbbell DRM_LOCK(dev); 4125285988Sdumbbell 4126285988Sdumbbell if (dev_priv->mm.suspended) { 4127285988Sdumbbell DRM_UNLOCK(dev); 4128285988Sdumbbell return 0; 4129285988Sdumbbell } 4130285988Sdumbbell 4131285988Sdumbbell ret = i915_gpu_idle(dev); 4132285988Sdumbbell if (ret) { 4133285988Sdumbbell DRM_UNLOCK(dev); 4134285988Sdumbbell return ret; 4135285988Sdumbbell } 4136285988Sdumbbell i915_gem_retire_requests(dev); 4137285988Sdumbbell 4138285988Sdumbbell /* Under UMS, be paranoid and evict. */ 4139296548Sdumbbell if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4140296548Sdumbbell i915_gem_evict_everything(dev); 4141285988Sdumbbell 4142285988Sdumbbell i915_gem_reset_fences(dev); 4143285988Sdumbbell 4144285988Sdumbbell /* Hack! Don't let anybody do execbuf while we don't control the chip. 4145285988Sdumbbell * We need to replace this with a semaphore, or something. 4146285988Sdumbbell * And not confound mm.suspended! 4147285988Sdumbbell */ 4148285988Sdumbbell dev_priv->mm.suspended = 1; 4149285988Sdumbbell callout_stop(&dev_priv->hangcheck_timer); 4150285988Sdumbbell 4151285988Sdumbbell i915_kernel_lost_context(dev); 4152285988Sdumbbell i915_gem_cleanup_ringbuffer(dev); 4153285988Sdumbbell 4154285988Sdumbbell DRM_UNLOCK(dev); 4155285988Sdumbbell 4156285988Sdumbbell /* Cancel the retire work handler, which should be idle now. */ 4157296548Sdumbbell taskqueue_cancel_timeout(dev_priv->wq, &dev_priv->mm.retire_work, NULL); 4158285988Sdumbbell 4159296548Sdumbbell return 0; 4160285988Sdumbbell} 4161285988Sdumbbell 4162296548Sdumbbellvoid i915_gem_l3_remap(struct drm_device *dev) 4163296548Sdumbbell{ 4164296548Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4165296548Sdumbbell u32 misccpctl; 4166296548Sdumbbell int i; 4167296548Sdumbbell 4168296548Sdumbbell if (!HAS_L3_GPU_CACHE(dev)) 4169296548Sdumbbell return; 4170296548Sdumbbell 4171296548Sdumbbell if (!dev_priv->l3_parity.remap_info) 4172296548Sdumbbell return; 4173296548Sdumbbell 4174296548Sdumbbell misccpctl = I915_READ(GEN7_MISCCPCTL); 4175296548Sdumbbell I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); 4176296548Sdumbbell POSTING_READ(GEN7_MISCCPCTL); 4177296548Sdumbbell 4178296548Sdumbbell for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { 4179296548Sdumbbell u32 remap = I915_READ(GEN7_L3LOG_BASE + i); 4180296548Sdumbbell if (remap && remap != dev_priv->l3_parity.remap_info[i/4]) 4181296548Sdumbbell DRM_DEBUG("0x%x was already programmed to %x\n", 4182296548Sdumbbell GEN7_L3LOG_BASE + i, remap); 4183296548Sdumbbell if (remap && !dev_priv->l3_parity.remap_info[i/4]) 4184296548Sdumbbell DRM_DEBUG_DRIVER("Clearing remapped register\n"); 4185296548Sdumbbell I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]); 4186296548Sdumbbell } 4187296548Sdumbbell 4188296548Sdumbbell /* Make sure all the writes land before disabling dop clock gating */ 4189296548Sdumbbell POSTING_READ(GEN7_L3LOG_BASE); 4190296548Sdumbbell 4191296548Sdumbbell I915_WRITE(GEN7_MISCCPCTL, misccpctl); 4192296548Sdumbbell} 4193296548Sdumbbell 4194285988Sdumbbellvoid i915_gem_init_swizzling(struct drm_device *dev) 4195285988Sdumbbell{ 4196285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4197285988Sdumbbell 4198285988Sdumbbell if (INTEL_INFO(dev)->gen < 5 || 4199285988Sdumbbell dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 4200285988Sdumbbell return; 4201285988Sdumbbell 4202285988Sdumbbell I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 4203285988Sdumbbell DISP_TILE_SURFACE_SWIZZLING); 4204285988Sdumbbell 4205285988Sdumbbell if (IS_GEN5(dev)) 4206285988Sdumbbell return; 4207285988Sdumbbell 4208285988Sdumbbell I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 4209285988Sdumbbell if (IS_GEN6(dev)) 4210285988Sdumbbell I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 4211285988Sdumbbell else 4212285988Sdumbbell I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 4213285988Sdumbbell} 4214285988Sdumbbell 4215296548Sdumbbellstatic bool 4216296548Sdumbbellintel_enable_blt(struct drm_device *dev) 4217296548Sdumbbell{ 4218296548Sdumbbell if (!HAS_BLT(dev)) 4219296548Sdumbbell return false; 4220296548Sdumbbell 4221296548Sdumbbell /* The blitter was dysfunctional on early prototypes */ 4222296548Sdumbbell if (IS_GEN6(dev) && pci_get_revid(dev->dev) < 8) { 4223296548Sdumbbell DRM_INFO("BLT not supported on this pre-production hardware;" 4224296548Sdumbbell " graphics performance will be degraded.\n"); 4225296548Sdumbbell return false; 4226296548Sdumbbell } 4227296548Sdumbbell 4228296548Sdumbbell return true; 4229296548Sdumbbell} 4230296548Sdumbbell 4231285988Sdumbbellint 4232285988Sdumbbelli915_gem_init_hw(struct drm_device *dev) 4233285988Sdumbbell{ 4234285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4235285988Sdumbbell int ret; 4236285988Sdumbbell 4237296548Sdumbbell#ifdef FREEBSD_WIP 4238296548Sdumbbell if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) 4239296548Sdumbbell return -EIO; 4240296548Sdumbbell#endif /* FREEBSD_WIP */ 4241296548Sdumbbell 4242296548Sdumbbell if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) 4243296548Sdumbbell I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); 4244296548Sdumbbell 4245296548Sdumbbell i915_gem_l3_remap(dev); 4246296548Sdumbbell 4247285988Sdumbbell i915_gem_init_swizzling(dev); 4248285988Sdumbbell 4249285988Sdumbbell ret = intel_init_render_ring_buffer(dev); 4250285988Sdumbbell if (ret) 4251285988Sdumbbell return ret; 4252285988Sdumbbell 4253285988Sdumbbell if (HAS_BSD(dev)) { 4254285988Sdumbbell ret = intel_init_bsd_ring_buffer(dev); 4255285988Sdumbbell if (ret) 4256285988Sdumbbell goto cleanup_render_ring; 4257285988Sdumbbell } 4258285988Sdumbbell 4259296548Sdumbbell if (intel_enable_blt(dev)) { 4260285988Sdumbbell ret = intel_init_blt_ring_buffer(dev); 4261285988Sdumbbell if (ret) 4262285988Sdumbbell goto cleanup_bsd_ring; 4263285988Sdumbbell } 4264285988Sdumbbell 4265285988Sdumbbell dev_priv->next_seqno = 1; 4266285988Sdumbbell 4267285988Sdumbbell /* 4268285988Sdumbbell * XXX: There was some w/a described somewhere suggesting loading 4269285988Sdumbbell * contexts before PPGTT. 4270285988Sdumbbell */ 4271285988Sdumbbell i915_gem_context_init(dev); 4272285988Sdumbbell i915_gem_init_ppgtt(dev); 4273285988Sdumbbell 4274285988Sdumbbell return 0; 4275285988Sdumbbell 4276285988Sdumbbellcleanup_bsd_ring: 4277296548Sdumbbell intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); 4278285988Sdumbbellcleanup_render_ring: 4279296548Sdumbbell intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); 4280285988Sdumbbell return ret; 4281285988Sdumbbell} 4282285988Sdumbbell 4283285988Sdumbbellstatic bool 4284285988Sdumbbellintel_enable_ppgtt(struct drm_device *dev) 4285285988Sdumbbell{ 4286285988Sdumbbell if (i915_enable_ppgtt >= 0) 4287285988Sdumbbell return i915_enable_ppgtt; 4288285988Sdumbbell 4289296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU 4290285988Sdumbbell /* Disable ppgtt on SNB if VT-d is on. */ 4291296548Sdumbbell if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) 4292285988Sdumbbell return false; 4293296548Sdumbbell#endif 4294285988Sdumbbell 4295285988Sdumbbell return true; 4296285988Sdumbbell} 4297285988Sdumbbell 4298285988Sdumbbellint i915_gem_init(struct drm_device *dev) 4299285988Sdumbbell{ 4300285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 4301285988Sdumbbell unsigned long gtt_size, mappable_size; 4302285988Sdumbbell int ret; 4303285988Sdumbbell 4304296548Sdumbbell gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; 4305296548Sdumbbell mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; 4306285988Sdumbbell 4307285988Sdumbbell DRM_LOCK(dev); 4308285988Sdumbbell if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { 4309285988Sdumbbell /* PPGTT pdes are stolen from global gtt ptes, so shrink the 4310285988Sdumbbell * aperture accordingly when using aliasing ppgtt. */ 4311285988Sdumbbell gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; 4312285988Sdumbbell 4313285988Sdumbbell i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); 4314285988Sdumbbell 4315285988Sdumbbell ret = i915_gem_init_aliasing_ppgtt(dev); 4316285988Sdumbbell if (ret) { 4317285988Sdumbbell DRM_UNLOCK(dev); 4318285988Sdumbbell return ret; 4319285988Sdumbbell } 4320285988Sdumbbell } else { 4321285988Sdumbbell /* Let GEM Manage all of the aperture. 4322285988Sdumbbell * 4323285988Sdumbbell * However, leave one page at the end still bound to the scratch 4324285988Sdumbbell * page. There are a number of places where the hardware 4325285988Sdumbbell * apparently prefetches past the end of the object, and we've 4326285988Sdumbbell * seen multiple hangs with the GPU head pointer stuck in a 4327285988Sdumbbell * batchbuffer bound at the last page of the aperture. One page 4328285988Sdumbbell * should be enough to keep any prefetching inside of the 4329285988Sdumbbell * aperture. 4330285988Sdumbbell */ 4331285988Sdumbbell i915_gem_init_global_gtt(dev, 0, mappable_size, 4332285988Sdumbbell gtt_size); 4333285988Sdumbbell } 4334285988Sdumbbell 4335285988Sdumbbell ret = i915_gem_init_hw(dev); 4336285988Sdumbbell DRM_UNLOCK(dev); 4337285988Sdumbbell if (ret) { 4338285988Sdumbbell i915_gem_cleanup_aliasing_ppgtt(dev); 4339285988Sdumbbell return ret; 4340285988Sdumbbell } 4341285988Sdumbbell 4342285988Sdumbbell /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ 4343285988Sdumbbell if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4344285988Sdumbbell dev_priv->dri1.allow_batchbuffer = 1; 4345285988Sdumbbell return 0; 4346285988Sdumbbell} 4347285988Sdumbbell 4348235783Skibvoid 4349285988Sdumbbelli915_gem_cleanup_ringbuffer(struct drm_device *dev) 4350285988Sdumbbell{ 4351285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4352285988Sdumbbell struct intel_ring_buffer *ring; 4353285988Sdumbbell int i; 4354285988Sdumbbell 4355285988Sdumbbell for_each_ring(ring, dev_priv, i) 4356285988Sdumbbell intel_cleanup_ring_buffer(ring); 4357285988Sdumbbell} 4358285988Sdumbbell 4359285988Sdumbbellint 4360285988Sdumbbelli915_gem_entervt_ioctl(struct drm_device *dev, void *data, 4361285988Sdumbbell struct drm_file *file_priv) 4362285988Sdumbbell{ 4363285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4364285988Sdumbbell int ret; 4365285988Sdumbbell 4366285988Sdumbbell if (drm_core_check_feature(dev, DRIVER_MODESET)) 4367285988Sdumbbell return 0; 4368285988Sdumbbell 4369296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) { 4370285988Sdumbbell DRM_ERROR("Reenabling wedged hardware, good luck\n"); 4371296548Sdumbbell atomic_set(&dev_priv->mm.wedged, 0); 4372285988Sdumbbell } 4373285988Sdumbbell 4374285988Sdumbbell DRM_LOCK(dev); 4375285988Sdumbbell dev_priv->mm.suspended = 0; 4376285988Sdumbbell 4377285988Sdumbbell ret = i915_gem_init_hw(dev); 4378285988Sdumbbell if (ret != 0) { 4379285988Sdumbbell DRM_UNLOCK(dev); 4380285988Sdumbbell return ret; 4381285988Sdumbbell } 4382285988Sdumbbell 4383296548Sdumbbell BUG_ON(!list_empty(&dev_priv->mm.active_list)); 4384285988Sdumbbell DRM_UNLOCK(dev); 4385285988Sdumbbell 4386285988Sdumbbell ret = drm_irq_install(dev); 4387285988Sdumbbell if (ret) 4388285988Sdumbbell goto cleanup_ringbuffer; 4389285988Sdumbbell 4390285988Sdumbbell return 0; 4391285988Sdumbbell 4392285988Sdumbbellcleanup_ringbuffer: 4393285988Sdumbbell DRM_LOCK(dev); 4394285988Sdumbbell i915_gem_cleanup_ringbuffer(dev); 4395285988Sdumbbell dev_priv->mm.suspended = 1; 4396285988Sdumbbell DRM_UNLOCK(dev); 4397285988Sdumbbell 4398285988Sdumbbell return ret; 4399285988Sdumbbell} 4400285988Sdumbbell 4401285988Sdumbbellint 4402285988Sdumbbelli915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 4403285988Sdumbbell struct drm_file *file_priv) 4404285988Sdumbbell{ 4405285988Sdumbbell if (drm_core_check_feature(dev, DRIVER_MODESET)) 4406285988Sdumbbell return 0; 4407285988Sdumbbell 4408285988Sdumbbell drm_irq_uninstall(dev); 4409285988Sdumbbell return i915_gem_idle(dev); 4410285988Sdumbbell} 4411285988Sdumbbell 4412285988Sdumbbellvoid 4413235783Skibi915_gem_lastclose(struct drm_device *dev) 4414235783Skib{ 4415235783Skib int ret; 4416235783Skib 4417235783Skib if (drm_core_check_feature(dev, DRIVER_MODESET)) 4418235783Skib return; 4419235783Skib 4420235783Skib ret = i915_gem_idle(dev); 4421285988Sdumbbell if (ret) 4422235783Skib DRM_ERROR("failed to idle hardware: %d\n", ret); 4423235783Skib} 4424235783Skib 4425285988Sdumbbellstatic void 4426285988Sdumbbellinit_ring_lists(struct intel_ring_buffer *ring) 4427235783Skib{ 4428285988Sdumbbell INIT_LIST_HEAD(&ring->active_list); 4429285988Sdumbbell INIT_LIST_HEAD(&ring->request_list); 4430285988Sdumbbell} 4431285988Sdumbbell 4432285988Sdumbbellvoid 4433285988Sdumbbelli915_gem_load(struct drm_device *dev) 4434285988Sdumbbell{ 4435285988Sdumbbell int i; 4436285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4437285988Sdumbbell 4438285988Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.active_list); 4439285988Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 4440296548Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.unbound_list); 4441296548Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.bound_list); 4442285988Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.fence_list); 4443285988Sdumbbell for (i = 0; i < I915_NUM_RINGS; i++) 4444296548Sdumbbell init_ring_lists(&dev_priv->ring[i]); 4445285988Sdumbbell for (i = 0; i < I915_MAX_NUM_FENCES; i++) 4446285988Sdumbbell INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 4447296548Sdumbbell TIMEOUT_TASK_INIT(dev_priv->wq, &dev_priv->mm.retire_work, 0, 4448296548Sdumbbell i915_gem_retire_work_handler, dev_priv); 4449296548Sdumbbell init_completion(&dev_priv->error_completion); 4450285988Sdumbbell 4451285988Sdumbbell /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 4452285988Sdumbbell if (IS_GEN3(dev)) { 4453285988Sdumbbell I915_WRITE(MI_ARB_STATE, 4454285988Sdumbbell _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 4455285988Sdumbbell } 4456285988Sdumbbell 4457285988Sdumbbell dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 4458285988Sdumbbell 4459285988Sdumbbell /* Old X drivers will take 0-2 for front, back, depth buffers */ 4460285988Sdumbbell if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4461285988Sdumbbell dev_priv->fence_reg_start = 3; 4462285988Sdumbbell 4463285988Sdumbbell if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 4464285988Sdumbbell dev_priv->num_fence_regs = 16; 4465285988Sdumbbell else 4466285988Sdumbbell dev_priv->num_fence_regs = 8; 4467285988Sdumbbell 4468285988Sdumbbell /* Initialize fence registers to zero */ 4469285988Sdumbbell i915_gem_reset_fences(dev); 4470285988Sdumbbell 4471285988Sdumbbell i915_gem_detect_bit_6_swizzle(dev); 4472296548Sdumbbell DRM_INIT_WAITQUEUE(&dev_priv->pending_flip_queue); 4473296548Sdumbbell 4474285988Sdumbbell dev_priv->mm.interruptible = true; 4475285988Sdumbbell 4476296548Sdumbbell dev_priv->mm.inactive_shrinker = EVENTHANDLER_REGISTER(vm_lowmem, 4477296548Sdumbbell i915_gem_inactive_shrink, dev, EVENTHANDLER_PRI_ANY); 4478285988Sdumbbell} 4479285988Sdumbbell 4480287174Sbapt/* 4481287174Sbapt * Create a physically contiguous memory object for this object 4482287174Sbapt * e.g. for cursor + overlay regs 4483287174Sbapt */ 4484285988Sdumbbellstatic int i915_gem_init_phys_object(struct drm_device *dev, 4485285988Sdumbbell int id, int size, int align) 4486285988Sdumbbell{ 4487285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4488235783Skib struct drm_i915_gem_phys_object *phys_obj; 4489235783Skib int ret; 4490235783Skib 4491285988Sdumbbell if (dev_priv->mm.phys_objs[id - 1] || !size) 4492285988Sdumbbell return 0; 4493235783Skib 4494285988Sdumbbell phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), 4495285988Sdumbbell DRM_I915_GEM, M_WAITOK | M_ZERO); 4496296548Sdumbbell if (!phys_obj) 4497296548Sdumbbell return -ENOMEM; 4498235783Skib 4499235783Skib phys_obj->id = id; 4500235783Skib 4501280183Sdumbbell phys_obj->handle = drm_pci_alloc(dev, size, align, BUS_SPACE_MAXADDR); 4502285988Sdumbbell if (!phys_obj->handle) { 4503235783Skib ret = -ENOMEM; 4504285988Sdumbbell goto kfree_obj; 4505235783Skib } 4506296548Sdumbbell#ifdef CONFIG_X86 4507235783Skib pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 4508235783Skib size / PAGE_SIZE, PAT_WRITE_COMBINING); 4509296548Sdumbbell#endif 4510235783Skib 4511235783Skib dev_priv->mm.phys_objs[id - 1] = phys_obj; 4512235783Skib 4513285988Sdumbbell return 0; 4514285988Sdumbbellkfree_obj: 4515235783Skib free(phys_obj, DRM_I915_GEM); 4516285988Sdumbbell return ret; 4517235783Skib} 4518235783Skib 4519285988Sdumbbellstatic void i915_gem_free_phys_object(struct drm_device *dev, int id) 4520235783Skib{ 4521285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4522235783Skib struct drm_i915_gem_phys_object *phys_obj; 4523235783Skib 4524285988Sdumbbell if (!dev_priv->mm.phys_objs[id - 1]) 4525235783Skib return; 4526235783Skib 4527235783Skib phys_obj = dev_priv->mm.phys_objs[id - 1]; 4528285988Sdumbbell if (phys_obj->cur_obj) { 4529235783Skib i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 4530285988Sdumbbell } 4531235783Skib 4532296548Sdumbbell#ifdef FREEBSD_WIP 4533296548Sdumbbell#ifdef CONFIG_X86 4534296548Sdumbbell set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); 4535296548Sdumbbell#endif 4536296548Sdumbbell#endif /* FREEBSD_WIP */ 4537296548Sdumbbell 4538235783Skib drm_pci_free(dev, phys_obj->handle); 4539235783Skib free(phys_obj, DRM_I915_GEM); 4540235783Skib dev_priv->mm.phys_objs[id - 1] = NULL; 4541235783Skib} 4542235783Skib 4543285988Sdumbbellvoid i915_gem_free_all_phys_object(struct drm_device *dev) 4544235783Skib{ 4545235783Skib int i; 4546235783Skib 4547235783Skib for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 4548235783Skib i915_gem_free_phys_object(dev, i); 4549235783Skib} 4550235783Skib 4551285988Sdumbbellvoid i915_gem_detach_phys_object(struct drm_device *dev, 4552285988Sdumbbell struct drm_i915_gem_object *obj) 4553235783Skib{ 4554235783Skib struct sf_buf *sf; 4555296548Sdumbbell char *vaddr; 4556296548Sdumbbell char *dst; 4557296548Sdumbbell int i; 4558296548Sdumbbell int page_count; 4559235783Skib 4560285988Sdumbbell if (!obj->phys_obj) 4561235783Skib return; 4562235783Skib vaddr = obj->phys_obj->handle->vaddr; 4563235783Skib 4564235783Skib page_count = obj->base.size / PAGE_SIZE; 4565248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4566235783Skib for (i = 0; i < page_count; i++) { 4567296548Sdumbbell vm_page_t page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4568285988Sdumbbell if (page == NULL) 4569235783Skib continue; /* XXX */ 4570235783Skib 4571248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4572285988Sdumbbell sf = sf_buf_alloc(page, 0); 4573235783Skib if (sf != NULL) { 4574235783Skib dst = (char *)sf_buf_kva(sf); 4575235783Skib memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 4576235783Skib sf_buf_free(sf); 4577235783Skib } 4578285988Sdumbbell drm_clflush_pages(&page, 1); 4579235783Skib 4580248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4581285988Sdumbbell vm_page_reference(page); 4582285988Sdumbbell vm_page_lock(page); 4583285988Sdumbbell vm_page_dirty(page); 4584285988Sdumbbell vm_page_unwire(page, PQ_INACTIVE); 4585285988Sdumbbell vm_page_unlock(page); 4586235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4587235783Skib } 4588248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4589296548Sdumbbell i915_gem_chipset_flush(dev); 4590235783Skib 4591235783Skib obj->phys_obj->cur_obj = NULL; 4592235783Skib obj->phys_obj = NULL; 4593235783Skib} 4594235783Skib 4595235783Skibint 4596235783Skibi915_gem_attach_phys_object(struct drm_device *dev, 4597285988Sdumbbell struct drm_i915_gem_object *obj, 4598285988Sdumbbell int id, 4599285988Sdumbbell int align) 4600235783Skib{ 4601285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4602235783Skib struct sf_buf *sf; 4603235783Skib char *dst, *src; 4604285988Sdumbbell int ret = 0; 4605285988Sdumbbell int page_count; 4606285988Sdumbbell int i; 4607235783Skib 4608235783Skib if (id > I915_MAX_PHYS_OBJECT) 4609285988Sdumbbell return -EINVAL; 4610235783Skib 4611285988Sdumbbell if (obj->phys_obj) { 4612235783Skib if (obj->phys_obj->id == id) 4613285988Sdumbbell return 0; 4614235783Skib i915_gem_detach_phys_object(dev, obj); 4615235783Skib } 4616235783Skib 4617285988Sdumbbell /* create a new object */ 4618285988Sdumbbell if (!dev_priv->mm.phys_objs[id - 1]) { 4619285988Sdumbbell ret = i915_gem_init_phys_object(dev, id, 4620285988Sdumbbell obj->base.size, align); 4621285988Sdumbbell if (ret) { 4622235783Skib DRM_ERROR("failed to init phys object %d size: %zu\n", 4623235783Skib id, obj->base.size); 4624285988Sdumbbell return ret; 4625235783Skib } 4626235783Skib } 4627235783Skib 4628235783Skib /* bind to the object */ 4629235783Skib obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4630235783Skib obj->phys_obj->cur_obj = obj; 4631235783Skib 4632235783Skib page_count = obj->base.size / PAGE_SIZE; 4633235783Skib 4634248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4635235783Skib for (i = 0; i < page_count; i++) { 4636296548Sdumbbell vm_page_t page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4637285988Sdumbbell if (page == NULL) { 4638235783Skib ret = -EIO; 4639235783Skib break; 4640235783Skib } 4641248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4642285988Sdumbbell sf = sf_buf_alloc(page, 0); 4643235783Skib src = (char *)sf_buf_kva(sf); 4644235783Skib dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 4645235783Skib memcpy(dst, src, PAGE_SIZE); 4646235783Skib sf_buf_free(sf); 4647235783Skib 4648248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4649235783Skib 4650285988Sdumbbell vm_page_reference(page); 4651285988Sdumbbell vm_page_lock(page); 4652285988Sdumbbell vm_page_unwire(page, PQ_INACTIVE); 4653285988Sdumbbell vm_page_unlock(page); 4654235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4655235783Skib } 4656248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4657235783Skib 4658285988Sdumbbell return ret; 4659235783Skib} 4660235783Skib 4661235783Skibstatic int 4662285988Sdumbbelli915_gem_phys_pwrite(struct drm_device *dev, 4663285988Sdumbbell struct drm_i915_gem_object *obj, 4664285988Sdumbbell struct drm_i915_gem_pwrite *args, 4665285988Sdumbbell struct drm_file *file_priv) 4666235783Skib{ 4667285988Sdumbbell void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset; 4668285988Sdumbbell char __user *user_data = to_user_ptr(args->data_ptr); 4669235783Skib 4670285988Sdumbbell if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 4671285988Sdumbbell unsigned long unwritten; 4672285988Sdumbbell 4673285988Sdumbbell /* The physical object once assigned is fixed for the lifetime 4674285988Sdumbbell * of the obj, so we can safely drop the lock and continue 4675285988Sdumbbell * to access vaddr. 4676285988Sdumbbell */ 4677285988Sdumbbell DRM_UNLOCK(dev); 4678285988Sdumbbell unwritten = copy_from_user(vaddr, user_data, args->size); 4679285988Sdumbbell DRM_LOCK(dev); 4680285988Sdumbbell if (unwritten) 4681285988Sdumbbell return -EFAULT; 4682285988Sdumbbell } 4683285988Sdumbbell 4684285988Sdumbbell i915_gem_chipset_flush(dev); 4685285988Sdumbbell return 0; 4686235783Skib} 4687235783Skib 4688285988Sdumbbellvoid i915_gem_release(struct drm_device *dev, struct drm_file *file) 4689235783Skib{ 4690285988Sdumbbell struct drm_i915_file_private *file_priv = file->driver_priv; 4691235783Skib 4692285988Sdumbbell /* Clean up our request list when the client is going away, so that 4693285988Sdumbbell * later retire_requests won't dereference our soon-to-be-gone 4694285988Sdumbbell * file_priv. 4695285988Sdumbbell */ 4696296548Sdumbbell mtx_lock(&file_priv->mm.lock); 4697285988Sdumbbell while (!list_empty(&file_priv->mm.request_list)) { 4698285988Sdumbbell struct drm_i915_gem_request *request; 4699235783Skib 4700285988Sdumbbell request = list_first_entry(&file_priv->mm.request_list, 4701285988Sdumbbell struct drm_i915_gem_request, 4702285988Sdumbbell client_list); 4703285988Sdumbbell list_del(&request->client_list); 4704285988Sdumbbell request->file_priv = NULL; 4705235783Skib } 4706296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 4707285988Sdumbbell} 4708235783Skib 4709296548Sdumbbellstatic void 4710296548Sdumbbelli915_gem_inactive_shrink(void *arg) 4711296548Sdumbbell{ 4712296548Sdumbbell struct drm_device *dev = arg; 4713296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 4714296548Sdumbbell int pass1, pass2; 4715296548Sdumbbell 4716296548Sdumbbell if (!sx_try_xlock(&dev->dev_struct_lock)) { 4717296548Sdumbbell return; 4718296548Sdumbbell } 4719296548Sdumbbell 4720296548Sdumbbell CTR0(KTR_DRM, "gem_lowmem"); 4721296548Sdumbbell 4722296548Sdumbbell pass1 = i915_gem_purge(dev_priv, -1); 4723296548Sdumbbell pass2 = __i915_gem_shrink(dev_priv, -1, false); 4724296548Sdumbbell 4725296548Sdumbbell if (pass2 <= pass1 / 100) 4726296548Sdumbbell i915_gem_shrink_all(dev_priv); 4727296548Sdumbbell 4728296548Sdumbbell DRM_UNLOCK(dev); 4729296548Sdumbbell} 4730296548Sdumbbell 4731285988Sdumbbellstatic vm_page_t 4732285988Sdumbbelli915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh) 4733285988Sdumbbell{ 4734285988Sdumbbell vm_page_t page; 4735285988Sdumbbell int rv; 4736235783Skib 4737285988Sdumbbell VM_OBJECT_ASSERT_WLOCKED(object); 4738324083Smarkj page = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | 4739324083Smarkj VM_ALLOC_WIRED); 4740285988Sdumbbell if (page->valid != VM_PAGE_BITS_ALL) { 4741324083Smarkj vm_page_xbusy(page); 4742285988Sdumbbell if (vm_pager_has_page(object, pindex, NULL, NULL)) { 4743292373Sglebius rv = vm_pager_get_pages(object, &page, 1, NULL, NULL); 4744285988Sdumbbell if (rv != VM_PAGER_OK) { 4745285988Sdumbbell vm_page_lock(page); 4746324083Smarkj vm_page_unwire(page, PQ_NONE); 4747285988Sdumbbell vm_page_free(page); 4748285988Sdumbbell vm_page_unlock(page); 4749285988Sdumbbell return (NULL); 4750285988Sdumbbell } 4751285988Sdumbbell if (fresh != NULL) 4752285988Sdumbbell *fresh = true; 4753285988Sdumbbell } else { 4754285988Sdumbbell pmap_zero_page(page); 4755285988Sdumbbell page->valid = VM_PAGE_BITS_ALL; 4756285988Sdumbbell page->dirty = 0; 4757285988Sdumbbell if (fresh != NULL) 4758285988Sdumbbell *fresh = false; 4759285988Sdumbbell } 4760324083Smarkj vm_page_xunbusy(page); 4761324083Smarkj } else if (fresh != NULL) 4762285988Sdumbbell *fresh = false; 4763285988Sdumbbell atomic_add_long(&i915_gem_wired_pages_cnt, 1); 4764285988Sdumbbell return (page); 4765235783Skib} 4766