i915_gem.c revision 296681
1287174Sbapt/* 2235783Skib * Copyright �� 2008 Intel Corporation 3235783Skib * 4235783Skib * Permission is hereby granted, free of charge, to any person obtaining a 5235783Skib * copy of this software and associated documentation files (the "Software"), 6235783Skib * to deal in the Software without restriction, including without limitation 7235783Skib * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8235783Skib * and/or sell copies of the Software, and to permit persons to whom the 9235783Skib * Software is furnished to do so, subject to the following conditions: 10235783Skib * 11235783Skib * The above copyright notice and this permission notice (including the next 12235783Skib * paragraph) shall be included in all copies or substantial portions of the 13235783Skib * Software. 14235783Skib * 15235783Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16235783Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17235783Skib * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18235783Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19235783Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20235783Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21235783Skib * IN THE SOFTWARE. 22235783Skib * 23235783Skib * Authors: 24235783Skib * Eric Anholt <eric@anholt.net> 25235783Skib * 26235783Skib * Copyright (c) 2011 The FreeBSD Foundation 27235783Skib * All rights reserved. 28235783Skib * 29235783Skib * This software was developed by Konstantin Belousov under sponsorship from 30235783Skib * the FreeBSD Foundation. 31235783Skib * 32235783Skib * Redistribution and use in source and binary forms, with or without 33235783Skib * modification, are permitted provided that the following conditions 34235783Skib * are met: 35235783Skib * 1. Redistributions of source code must retain the above copyright 36235783Skib * notice, this list of conditions and the following disclaimer. 37235783Skib * 2. Redistributions in binary form must reproduce the above copyright 38235783Skib * notice, this list of conditions and the following disclaimer in the 39235783Skib * documentation and/or other materials provided with the distribution. 40235783Skib * 41235783Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42235783Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43235783Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44235783Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45235783Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46235783Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47235783Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48235783Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49235783Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50235783Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51235783Skib * SUCH DAMAGE. 52235783Skib */ 53235783Skib 54235783Skib#include <sys/cdefs.h> 55235783Skib__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 296681 2016-03-11 21:00:14Z dumbbell $"); 56235783Skib 57235783Skib#include <dev/drm2/drmP.h> 58235783Skib#include <dev/drm2/i915/i915_drm.h> 59235783Skib#include <dev/drm2/i915/i915_drv.h> 60235783Skib#include <dev/drm2/i915/intel_drv.h> 61285988Sdumbbell 62235783Skib#include <sys/resourcevar.h> 63235783Skib#include <sys/sched.h> 64235783Skib#include <sys/sf_buf.h> 65235783Skib 66254141Sattilio#include <vm/vm.h> 67254141Sattilio#include <vm/vm_pageout.h> 68254141Sattilio 69277487Skib#include <machine/md_var.h> 70277487Skib 71285988Sdumbbellstatic void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); 72285988Sdumbbellstatic void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); 73285988Sdumbbellstatic __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 74285988Sdumbbell unsigned alignment, 75296548Sdumbbell bool map_and_fenceable, 76296548Sdumbbell bool nonblocking); 77285988Sdumbbellstatic int i915_gem_phys_pwrite(struct drm_device *dev, 78285988Sdumbbell struct drm_i915_gem_object *obj, 79285988Sdumbbell struct drm_i915_gem_pwrite *args, 80285988Sdumbbell struct drm_file *file); 81285988Sdumbbell 82285988Sdumbbellstatic void i915_gem_write_fence(struct drm_device *dev, int reg, 83285988Sdumbbell struct drm_i915_gem_object *obj); 84285988Sdumbbellstatic void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 85285988Sdumbbell struct drm_i915_fence_reg *fence, 86285988Sdumbbell bool enable); 87285988Sdumbbell 88296548Sdumbbellstatic void i915_gem_inactive_shrink(void *); 89296548Sdumbbellstatic long i915_gem_purge(struct drm_i915_private *dev_priv, long target); 90296548Sdumbbellstatic void i915_gem_shrink_all(struct drm_i915_private *dev_priv); 91285988Sdumbbellstatic void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 92285988Sdumbbell 93285988Sdumbbellstatic int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 94285988Sdumbbell off_t start, off_t end); 95285988Sdumbbell 96277487Skibstatic vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, 97277487Skib bool *fresh); 98235783Skib 99235783SkibMALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 100235783Skiblong i915_gem_wired_pages_cnt; 101235783Skib 102277487Skibstatic inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) 103277487Skib{ 104277487Skib if (obj->tiling_mode) 105277487Skib i915_gem_release_mmap(obj); 106277487Skib 107277487Skib /* As we do not have an associated fence register, we will force 108277487Skib * a tiling change if we ever need to acquire one. 109277487Skib */ 110277487Skib obj->fence_dirty = false; 111277487Skib obj->fence_reg = I915_FENCE_REG_NONE; 112277487Skib} 113277487Skib 114285988Sdumbbell/* some bookkeeping */ 115285988Sdumbbellstatic void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 116285988Sdumbbell size_t size) 117235783Skib{ 118235783Skib dev_priv->mm.object_count++; 119235783Skib dev_priv->mm.object_memory += size; 120235783Skib} 121235783Skib 122285988Sdumbbellstatic void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 123285988Sdumbbell size_t size) 124235783Skib{ 125235783Skib dev_priv->mm.object_count--; 126235783Skib dev_priv->mm.object_memory -= size; 127235783Skib} 128235783Skib 129235783Skibstatic int 130235783Skibi915_gem_wait_for_error(struct drm_device *dev) 131235783Skib{ 132285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 133296548Sdumbbell struct completion *x = &dev_priv->error_completion; 134235783Skib int ret; 135235783Skib 136296548Sdumbbell if (!atomic_read(&dev_priv->mm.wedged)) 137293837Sdumbbell return 0; 138235783Skib 139296548Sdumbbell /* 140296548Sdumbbell * Only wait 10 seconds for the gpu reset to complete to avoid hanging 141296548Sdumbbell * userspace. If it takes that long something really bad is going on and 142296548Sdumbbell * we should simply try to bail out and fail as gracefully as possible. 143296548Sdumbbell */ 144296548Sdumbbell ret = wait_for_completion_interruptible_timeout(x, 10*HZ); 145296548Sdumbbell if (ret == 0) { 146296548Sdumbbell DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); 147296548Sdumbbell return -EIO; 148296548Sdumbbell } else if (ret < 0) { 149296548Sdumbbell return ret; 150235783Skib } 151235783Skib 152296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) { 153285988Sdumbbell /* GPU is hung, bump the completion count to account for 154285988Sdumbbell * the token we just consumed so that we never hit zero and 155285988Sdumbbell * end up waiting upon a subsequent completion event that 156285988Sdumbbell * will never happen. 157285988Sdumbbell */ 158296548Sdumbbell mtx_lock(&x->lock); 159296548Sdumbbell x->done++; 160296548Sdumbbell mtx_unlock(&x->lock); 161235783Skib } 162285988Sdumbbell return 0; 163235783Skib} 164235783Skib 165285988Sdumbbellint i915_mutex_lock_interruptible(struct drm_device *dev) 166235783Skib{ 167235783Skib int ret; 168235783Skib 169235783Skib ret = i915_gem_wait_for_error(dev); 170285988Sdumbbell if (ret) 171285988Sdumbbell return ret; 172235783Skib 173235783Skib /* 174235783Skib * interruptible shall it be. might indeed be if dev_lock is 175235783Skib * changed to sx 176235783Skib */ 177296548Sdumbbell ret = sx_xlock_sig(&dev->dev_struct_lock); 178285988Sdumbbell if (ret) 179296548Sdumbbell return -EINTR; 180235783Skib 181296548Sdumbbell WARN_ON(i915_verify_lists(dev)); 182285988Sdumbbell return 0; 183235783Skib} 184235783Skib 185287174Sbaptstatic inline bool 186285988Sdumbbelli915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 187235783Skib{ 188296548Sdumbbell return obj->gtt_space && !obj->active; 189235783Skib} 190235783Skib 191235783Skibint 192235783Skibi915_gem_init_ioctl(struct drm_device *dev, void *data, 193285988Sdumbbell struct drm_file *file) 194235783Skib{ 195285988Sdumbbell struct drm_i915_gem_init *args = data; 196235783Skib 197277487Skib if (drm_core_check_feature(dev, DRIVER_MODESET)) 198277487Skib return -ENODEV; 199277487Skib 200235783Skib if (args->gtt_start >= args->gtt_end || 201235783Skib (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 202285988Sdumbbell return -EINVAL; 203235783Skib 204277487Skib /* GEM with user mode setting was never supported on ilk and later. */ 205277487Skib if (INTEL_INFO(dev)->gen >= 5) 206277487Skib return -ENODEV; 207277487Skib 208235783Skib /* 209235783Skib * XXXKIB. The second-time initialization should be guarded 210235783Skib * against. 211235783Skib */ 212280183Sdumbbell DRM_LOCK(dev); 213296548Sdumbbell i915_gem_init_global_gtt(dev, args->gtt_start, 214285988Sdumbbell args->gtt_end, args->gtt_end); 215280183Sdumbbell DRM_UNLOCK(dev); 216235783Skib 217296548Sdumbbell return 0; 218235783Skib} 219235783Skib 220235783Skibint 221285988Sdumbbelli915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 222285988Sdumbbell struct drm_file *file) 223235783Skib{ 224277487Skib struct drm_i915_private *dev_priv = dev->dev_private; 225285988Sdumbbell struct drm_i915_gem_get_aperture *args = data; 226235783Skib struct drm_i915_gem_object *obj; 227235783Skib size_t pinned; 228235783Skib 229235783Skib pinned = 0; 230235783Skib DRM_LOCK(dev); 231296548Sdumbbell list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) 232277487Skib if (obj->pin_count) 233277487Skib pinned += obj->gtt_space->size; 234235783Skib DRM_UNLOCK(dev); 235235783Skib 236235783Skib args->aper_size = dev_priv->mm.gtt_total; 237235783Skib args->aper_available_size = args->aper_size - pinned; 238235783Skib 239277487Skib return 0; 240235783Skib} 241235783Skib 242235783Skibstatic int 243285988Sdumbbelli915_gem_create(struct drm_file *file, 244285988Sdumbbell struct drm_device *dev, 245285988Sdumbbell uint64_t size, 246285988Sdumbbell uint32_t *handle_p) 247235783Skib{ 248235783Skib struct drm_i915_gem_object *obj; 249235783Skib int ret; 250285988Sdumbbell u32 handle; 251235783Skib 252235783Skib size = roundup(size, PAGE_SIZE); 253235783Skib if (size == 0) 254285988Sdumbbell return -EINVAL; 255235783Skib 256285988Sdumbbell /* Allocate the new object */ 257235783Skib obj = i915_gem_alloc_object(dev, size); 258235783Skib if (obj == NULL) 259285988Sdumbbell return -ENOMEM; 260235783Skib 261235783Skib ret = drm_gem_handle_create(file, &obj->base, &handle); 262285988Sdumbbell if (ret) { 263235783Skib drm_gem_object_release(&obj->base); 264235783Skib i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 265235783Skib free(obj, DRM_I915_GEM); 266285988Sdumbbell return ret; 267235783Skib } 268235783Skib 269235783Skib /* drop reference from allocate - handle holds it now */ 270235783Skib drm_gem_object_unreference(&obj->base); 271235783Skib CTR2(KTR_DRM, "object_create %p %x", obj, size); 272285988Sdumbbell 273235783Skib *handle_p = handle; 274285988Sdumbbell return 0; 275235783Skib} 276235783Skib 277235783Skibint 278285988Sdumbbelli915_gem_dumb_create(struct drm_file *file, 279285988Sdumbbell struct drm_device *dev, 280285988Sdumbbell struct drm_mode_create_dumb *args) 281235783Skib{ 282235783Skib /* have to work out size/pitch and return them */ 283235783Skib args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 284235783Skib args->size = args->pitch * args->height; 285285988Sdumbbell return i915_gem_create(file, dev, 286285988Sdumbbell args->size, &args->handle); 287235783Skib} 288235783Skib 289285988Sdumbbellint i915_gem_dumb_destroy(struct drm_file *file, 290285988Sdumbbell struct drm_device *dev, 291285988Sdumbbell uint32_t handle) 292235783Skib{ 293285988Sdumbbell return drm_gem_handle_delete(file, handle); 294235783Skib} 295235783Skib 296285988Sdumbbell/** 297285988Sdumbbell * Creates a new mm object and returns a handle to it. 298285988Sdumbbell */ 299235783Skibint 300235783Skibi915_gem_create_ioctl(struct drm_device *dev, void *data, 301285988Sdumbbell struct drm_file *file) 302235783Skib{ 303235783Skib struct drm_i915_gem_create *args = data; 304235783Skib 305285988Sdumbbell return i915_gem_create(file, dev, 306285988Sdumbbell args->size, &args->handle); 307235783Skib} 308235783Skib 309285988Sdumbbellstatic int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 310285988Sdumbbell{ 311285988Sdumbbell drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 312285988Sdumbbell 313285988Sdumbbell return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 314285988Sdumbbell obj->tiling_mode != I915_TILING_NONE; 315285988Sdumbbell} 316285988Sdumbbell 317277487Skibstatic inline int 318277487Skib__copy_to_user_swizzled(char __user *cpu_vaddr, 319277487Skib const char *gpu_vaddr, int gpu_offset, 320277487Skib int length) 321277487Skib{ 322277487Skib int ret, cpu_offset = 0; 323277487Skib 324277487Skib while (length > 0) { 325277487Skib int cacheline_end = roundup2(gpu_offset + 1, 64); 326277487Skib int this_length = min(cacheline_end - gpu_offset, length); 327277487Skib int swizzled_gpu_offset = gpu_offset ^ 64; 328277487Skib 329277487Skib ret = __copy_to_user(cpu_vaddr + cpu_offset, 330277487Skib gpu_vaddr + swizzled_gpu_offset, 331277487Skib this_length); 332277487Skib if (ret) 333277487Skib return ret + length; 334277487Skib 335277487Skib cpu_offset += this_length; 336277487Skib gpu_offset += this_length; 337277487Skib length -= this_length; 338277487Skib } 339277487Skib 340277487Skib return 0; 341277487Skib} 342277487Skib 343277487Skibstatic inline int 344277487Skib__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 345277487Skib const char __user *cpu_vaddr, 346277487Skib int length) 347277487Skib{ 348277487Skib int ret, cpu_offset = 0; 349277487Skib 350277487Skib while (length > 0) { 351277487Skib int cacheline_end = roundup2(gpu_offset + 1, 64); 352277487Skib int this_length = min(cacheline_end - gpu_offset, length); 353277487Skib int swizzled_gpu_offset = gpu_offset ^ 64; 354277487Skib 355277487Skib ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 356277487Skib cpu_vaddr + cpu_offset, 357277487Skib this_length); 358277487Skib if (ret) 359277487Skib return ret + length; 360277487Skib 361277487Skib cpu_offset += this_length; 362277487Skib gpu_offset += this_length; 363277487Skib length -= this_length; 364277487Skib } 365277487Skib 366277487Skib return 0; 367277487Skib} 368277487Skib 369277487Skib/* Per-page copy function for the shmem pread fastpath. 370277487Skib * Flushes invalid cachelines before reading the target if 371277487Skib * needs_clflush is set. */ 372277487Skibstatic int 373277487Skibshmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length, 374277487Skib char __user *user_data, 375277487Skib bool page_do_bit17_swizzling, bool needs_clflush) 376277487Skib{ 377277487Skib char *vaddr; 378277487Skib struct sf_buf *sf; 379277487Skib int ret; 380235783Skib 381277487Skib if (unlikely(page_do_bit17_swizzling)) 382277487Skib return -EINVAL; 383277487Skib 384277487Skib sched_pin(); 385277487Skib sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 386277487Skib if (sf == NULL) { 387235783Skib sched_unpin(); 388277487Skib return (-EFAULT); 389277487Skib } 390277487Skib vaddr = (char *)sf_buf_kva(sf); 391277487Skib if (needs_clflush) 392277487Skib drm_clflush_virt_range(vaddr + shmem_page_offset, 393277487Skib page_length); 394277487Skib ret = __copy_to_user_inatomic(user_data, 395277487Skib vaddr + shmem_page_offset, 396277487Skib page_length); 397277487Skib sf_buf_free(sf); 398277487Skib sched_unpin(); 399235783Skib 400277487Skib return ret ? -EFAULT : 0; 401277487Skib} 402277487Skib 403277487Skibstatic void 404277487Skibshmem_clflush_swizzled_range(char *addr, unsigned long length, 405277487Skib bool swizzled) 406277487Skib{ 407277487Skib if (unlikely(swizzled)) { 408277487Skib unsigned long start = (unsigned long) addr; 409277487Skib unsigned long end = (unsigned long) addr + length; 410277487Skib 411277487Skib /* For swizzling simply ensure that we always flush both 412277487Skib * channels. Lame, but simple and it works. Swizzled 413277487Skib * pwrite/pread is far from a hotpath - current userspace 414277487Skib * doesn't use it at all. */ 415296548Sdumbbell start = round_down(start, 128); 416296548Sdumbbell end = round_up(end, 128); 417277487Skib 418277487Skib drm_clflush_virt_range((void *)start, end - start); 419277487Skib } else { 420277487Skib drm_clflush_virt_range(addr, length); 421235783Skib } 422235783Skib 423235783Skib} 424235783Skib 425277487Skib/* Only difference to the fast-path function is that this can handle bit17 426277487Skib * and uses non-atomic copy and kmap functions. */ 427235783Skibstatic int 428277487Skibshmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length, 429277487Skib char __user *user_data, 430277487Skib bool page_do_bit17_swizzling, bool needs_clflush) 431235783Skib{ 432277487Skib char *vaddr; 433277487Skib struct sf_buf *sf; 434277487Skib int ret; 435235783Skib 436277487Skib sf = sf_buf_alloc(page, 0); 437277487Skib vaddr = (char *)sf_buf_kva(sf); 438277487Skib if (needs_clflush) 439277487Skib shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 440277487Skib page_length, 441277487Skib page_do_bit17_swizzling); 442235783Skib 443277487Skib if (page_do_bit17_swizzling) 444277487Skib ret = __copy_to_user_swizzled(user_data, 445277487Skib vaddr, shmem_page_offset, 446277487Skib page_length); 447277487Skib else 448277487Skib ret = __copy_to_user(user_data, 449277487Skib vaddr + shmem_page_offset, 450277487Skib page_length); 451277487Skib sf_buf_free(sf); 452277487Skib 453277487Skib return ret ? - EFAULT : 0; 454235783Skib} 455235783Skib 456235783Skibstatic int 457277487Skibi915_gem_shmem_pread(struct drm_device *dev, 458277487Skib struct drm_i915_gem_object *obj, 459277487Skib struct drm_i915_gem_pread *args, 460277487Skib struct drm_file *file) 461235783Skib{ 462277487Skib char __user *user_data; 463296548Sdumbbell ssize_t remain; 464296548Sdumbbell off_t offset; 465277487Skib int shmem_page_offset, page_length, ret = 0; 466277487Skib int obj_do_bit17_swizzling, page_do_bit17_swizzling; 467296548Sdumbbell int hit_slowpath = 0; 468277487Skib int prefaulted = 0; 469277487Skib int needs_clflush = 0; 470235783Skib 471277487Skib user_data = to_user_ptr(args->data_ptr); 472296548Sdumbbell remain = args->size; 473277487Skib 474277487Skib obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 475277487Skib 476277487Skib if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 477277487Skib /* If we're not in the cpu read domain, set ourself into the gtt 478277487Skib * read domain and manually flush cachelines (if required). This 479277487Skib * optimizes for the case when the gpu will dirty the data 480277487Skib * anyway again before the next pread happens. */ 481296548Sdumbbell if (obj->cache_level == I915_CACHE_NONE) 482296548Sdumbbell needs_clflush = 1; 483296548Sdumbbell if (obj->gtt_space) { 484296548Sdumbbell ret = i915_gem_object_set_to_gtt_domain(obj, false); 485296548Sdumbbell if (ret) 486296548Sdumbbell return ret; 487296548Sdumbbell } 488235783Skib } 489235783Skib 490296548Sdumbbell ret = i915_gem_object_get_pages(obj); 491277487Skib if (ret) 492277487Skib return ret; 493277487Skib 494277487Skib i915_gem_object_pin_pages(obj); 495277487Skib 496296548Sdumbbell offset = args->offset; 497296548Sdumbbell 498277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 499277487Skib for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 500277487Skib OFF_TO_IDX(offset));; page = vm_page_next(page)) { 501277487Skib VM_OBJECT_WUNLOCK(obj->base.vm_obj); 502277487Skib 503277487Skib if (remain <= 0) 504277487Skib break; 505277487Skib 506277487Skib /* Operation in this page 507277487Skib * 508277487Skib * shmem_page_offset = offset within page in shmem file 509277487Skib * page_length = bytes to copy for this page 510277487Skib */ 511277487Skib shmem_page_offset = offset_in_page(offset); 512277487Skib page_length = remain; 513277487Skib if ((shmem_page_offset + page_length) > PAGE_SIZE) 514277487Skib page_length = PAGE_SIZE - shmem_page_offset; 515277487Skib 516277487Skib page_do_bit17_swizzling = obj_do_bit17_swizzling && 517277487Skib (page_to_phys(page) & (1 << 17)) != 0; 518277487Skib 519277487Skib ret = shmem_pread_fast(page, shmem_page_offset, page_length, 520277487Skib user_data, page_do_bit17_swizzling, 521277487Skib needs_clflush); 522277487Skib if (ret == 0) 523277487Skib goto next_page; 524277487Skib 525296548Sdumbbell hit_slowpath = 1; 526277487Skib DRM_UNLOCK(dev); 527277487Skib 528296548Sdumbbell if (!prefaulted) { 529277487Skib ret = fault_in_multipages_writeable(user_data, remain); 530277487Skib /* Userspace is tricking us, but we've already clobbered 531277487Skib * its pages with the prefault and promised to write the 532277487Skib * data up to the first fault. Hence ignore any errors 533277487Skib * and just continue. */ 534277487Skib (void)ret; 535277487Skib prefaulted = 1; 536277487Skib } 537277487Skib 538277487Skib ret = shmem_pread_slow(page, shmem_page_offset, page_length, 539277487Skib user_data, page_do_bit17_swizzling, 540277487Skib needs_clflush); 541277487Skib 542277487Skib DRM_LOCK(dev); 543277487Skib 544277487Skibnext_page: 545277487Skib vm_page_reference(page); 546277487Skib 547277487Skib if (ret) 548277487Skib goto out; 549277487Skib 550277487Skib remain -= page_length; 551277487Skib user_data += page_length; 552277487Skib offset += page_length; 553277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 554277487Skib } 555277487Skib 556277487Skibout: 557277487Skib i915_gem_object_unpin_pages(obj); 558277487Skib 559296548Sdumbbell if (hit_slowpath) { 560296548Sdumbbell /* Fixup: Kill any reinstated backing storage pages */ 561296548Sdumbbell if (obj->madv == __I915_MADV_PURGED) 562296548Sdumbbell i915_gem_object_truncate(obj); 563296548Sdumbbell } 564296548Sdumbbell 565277487Skib return ret; 566277487Skib} 567277487Skib 568277487Skib/** 569277487Skib * Reads data from the object referenced by handle. 570277487Skib * 571277487Skib * On error, the contents of *data are undefined. 572277487Skib */ 573277487Skibint 574277487Skibi915_gem_pread_ioctl(struct drm_device *dev, void *data, 575277487Skib struct drm_file *file) 576277487Skib{ 577277487Skib struct drm_i915_gem_pread *args = data; 578277487Skib struct drm_i915_gem_object *obj; 579277487Skib int ret = 0; 580277487Skib 581277487Skib if (args->size == 0) 582277487Skib return 0; 583277487Skib 584277487Skib if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE)) 585277487Skib return -EFAULT; 586277487Skib 587235783Skib ret = i915_mutex_lock_interruptible(dev); 588277487Skib if (ret) 589277487Skib return ret; 590235783Skib 591277487Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 592235783Skib if (&obj->base == NULL) { 593235783Skib ret = -ENOENT; 594235783Skib goto unlock; 595235783Skib } 596277487Skib 597277487Skib /* Bounds check source. */ 598277487Skib if (args->offset > obj->base.size || 599277487Skib args->size > obj->base.size - args->offset) { 600235783Skib ret = -EINVAL; 601235783Skib goto out; 602235783Skib } 603235783Skib 604296548Sdumbbell#ifdef FREEBSD_WIP 605277487Skib /* prime objects have no backing filp to GEM pread/pwrite 606277487Skib * pages from. 607277487Skib */ 608277487Skib if (!obj->base.filp) { 609277487Skib ret = -EINVAL; 610277487Skib goto out; 611235783Skib } 612296548Sdumbbell#endif /* FREEBSD_WIP */ 613277487Skib 614277487Skib CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size); 615277487Skib 616277487Skib ret = i915_gem_shmem_pread(dev, obj, args, file); 617277487Skib 618235783Skibout: 619235783Skib drm_gem_object_unreference(&obj->base); 620235783Skibunlock: 621235783Skib DRM_UNLOCK(dev); 622277487Skib return ret; 623235783Skib} 624235783Skib 625277487Skib/* This is the fast write path which cannot handle 626277487Skib * page faults in the source data 627277487Skib */ 628277487Skib 629277487Skibstatic inline int 630296548Sdumbbellfast_user_write(vm_paddr_t mapping_addr, 631277487Skib off_t page_base, int page_offset, 632277487Skib char __user *user_data, 633277487Skib int length) 634235783Skib{ 635277487Skib void __iomem *vaddr_atomic; 636277487Skib void *vaddr; 637277487Skib unsigned long unwritten; 638235783Skib 639296548Sdumbbell vaddr_atomic = pmap_mapdev_attr(mapping_addr + page_base, 640277487Skib length, PAT_WRITE_COMBINING); 641277487Skib /* We can use the cpu mem copy function because this is X86. */ 642285988Sdumbbell vaddr = (char __force*)vaddr_atomic + page_offset; 643277487Skib unwritten = __copy_from_user_inatomic_nocache(vaddr, 644277487Skib user_data, length); 645277487Skib pmap_unmapdev((vm_offset_t)vaddr_atomic, length); 646277487Skib return unwritten; 647235783Skib} 648235783Skib 649277487Skib/** 650277487Skib * This is the fast pwrite path, where we copy the data directly from the 651277487Skib * user into the GTT, uncached. 652277487Skib */ 653277487Skibstatic int 654277487Skibi915_gem_gtt_pwrite_fast(struct drm_device *dev, 655277487Skib struct drm_i915_gem_object *obj, 656277487Skib struct drm_i915_gem_pwrite *args, 657277487Skib struct drm_file *file) 658277487Skib{ 659296548Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 660277487Skib ssize_t remain; 661277487Skib off_t offset, page_base; 662277487Skib char __user *user_data; 663277487Skib int page_offset, page_length, ret; 664277487Skib 665296548Sdumbbell ret = i915_gem_object_pin(obj, 0, true, true); 666285988Sdumbbell if (ret) 667277487Skib goto out; 668277487Skib 669277487Skib ret = i915_gem_object_set_to_gtt_domain(obj, true); 670277487Skib if (ret) 671277487Skib goto out_unpin; 672277487Skib 673277487Skib ret = i915_gem_object_put_fence(obj); 674277487Skib if (ret) 675277487Skib goto out_unpin; 676277487Skib 677277487Skib user_data = to_user_ptr(args->data_ptr); 678277487Skib remain = args->size; 679277487Skib 680277487Skib offset = obj->gtt_offset + args->offset; 681277487Skib 682277487Skib while (remain > 0) { 683277487Skib /* Operation in this page 684277487Skib * 685277487Skib * page_base = page offset within aperture 686277487Skib * page_offset = offset within page 687277487Skib * page_length = bytes to copy for this page 688277487Skib */ 689277487Skib page_base = offset & ~PAGE_MASK; 690277487Skib page_offset = offset_in_page(offset); 691277487Skib page_length = remain; 692277487Skib if ((page_offset + remain) > PAGE_SIZE) 693277487Skib page_length = PAGE_SIZE - page_offset; 694277487Skib 695277487Skib /* If we get a fault while copying data, then (presumably) our 696277487Skib * source page isn't available. Return the error and we'll 697277487Skib * retry in the slow path. 698277487Skib */ 699296548Sdumbbell if (fast_user_write(dev_priv->mm.gtt_base_addr, page_base, 700277487Skib page_offset, user_data, page_length)) { 701277487Skib ret = -EFAULT; 702277487Skib goto out_unpin; 703277487Skib } 704277487Skib 705277487Skib remain -= page_length; 706277487Skib user_data += page_length; 707277487Skib offset += page_length; 708277487Skib } 709277487Skib 710277487Skibout_unpin: 711277487Skib i915_gem_object_unpin(obj); 712277487Skibout: 713277487Skib return ret; 714277487Skib} 715277487Skib 716277487Skib/* Per-page copy function for the shmem pwrite fastpath. 717277487Skib * Flushes invalid cachelines before writing to the target if 718277487Skib * needs_clflush_before is set and flushes out any written cachelines after 719277487Skib * writing if needs_clflush is set. */ 720277487Skibstatic int 721277487Skibshmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length, 722277487Skib char __user *user_data, 723277487Skib bool page_do_bit17_swizzling, 724277487Skib bool needs_clflush_before, 725277487Skib bool needs_clflush_after) 726277487Skib{ 727277487Skib char *vaddr; 728277487Skib struct sf_buf *sf; 729277487Skib int ret; 730277487Skib 731277487Skib if (unlikely(page_do_bit17_swizzling)) 732277487Skib return -EINVAL; 733277487Skib 734277487Skib sched_pin(); 735277487Skib sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); 736277487Skib if (sf == NULL) { 737277487Skib sched_unpin(); 738277487Skib return (-EFAULT); 739277487Skib } 740277487Skib vaddr = (char *)sf_buf_kva(sf); 741277487Skib if (needs_clflush_before) 742277487Skib drm_clflush_virt_range(vaddr + shmem_page_offset, 743277487Skib page_length); 744277487Skib ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, 745277487Skib user_data, 746277487Skib page_length); 747277487Skib if (needs_clflush_after) 748277487Skib drm_clflush_virt_range(vaddr + shmem_page_offset, 749277487Skib page_length); 750277487Skib sf_buf_free(sf); 751277487Skib sched_unpin(); 752277487Skib 753277487Skib return ret ? -EFAULT : 0; 754277487Skib} 755277487Skib 756277487Skib/* Only difference to the fast-path function is that this can handle bit17 757277487Skib * and uses non-atomic copy and kmap functions. */ 758277487Skibstatic int 759277487Skibshmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length, 760277487Skib char __user *user_data, 761277487Skib bool page_do_bit17_swizzling, 762277487Skib bool needs_clflush_before, 763277487Skib bool needs_clflush_after) 764277487Skib{ 765277487Skib char *vaddr; 766277487Skib struct sf_buf *sf; 767277487Skib int ret; 768277487Skib 769277487Skib sf = sf_buf_alloc(page, 0); 770277487Skib vaddr = (char *)sf_buf_kva(sf); 771277487Skib if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 772277487Skib shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 773277487Skib page_length, 774277487Skib page_do_bit17_swizzling); 775277487Skib if (page_do_bit17_swizzling) 776277487Skib ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 777277487Skib user_data, 778277487Skib page_length); 779277487Skib else 780277487Skib ret = __copy_from_user(vaddr + shmem_page_offset, 781277487Skib user_data, 782277487Skib page_length); 783277487Skib if (needs_clflush_after) 784277487Skib shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 785277487Skib page_length, 786277487Skib page_do_bit17_swizzling); 787277487Skib sf_buf_free(sf); 788277487Skib 789277487Skib return ret ? -EFAULT : 0; 790277487Skib} 791277487Skib 792277487Skibstatic int 793277487Skibi915_gem_shmem_pwrite(struct drm_device *dev, 794277487Skib struct drm_i915_gem_object *obj, 795277487Skib struct drm_i915_gem_pwrite *args, 796277487Skib struct drm_file *file) 797277487Skib{ 798296548Sdumbbell ssize_t remain; 799296548Sdumbbell off_t offset; 800277487Skib char __user *user_data; 801277487Skib int shmem_page_offset, page_length, ret = 0; 802277487Skib int obj_do_bit17_swizzling, page_do_bit17_swizzling; 803277487Skib int hit_slowpath = 0; 804277487Skib int needs_clflush_after = 0; 805277487Skib int needs_clflush_before = 0; 806277487Skib 807277487Skib user_data = to_user_ptr(args->data_ptr); 808296548Sdumbbell remain = args->size; 809277487Skib 810277487Skib obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 811277487Skib 812277487Skib if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 813277487Skib /* If we're not in the cpu write domain, set ourself into the gtt 814277487Skib * write domain and manually flush cachelines (if required). This 815277487Skib * optimizes for the case when the gpu will use the data 816277487Skib * right away and we therefore have to clflush anyway. */ 817296548Sdumbbell if (obj->cache_level == I915_CACHE_NONE) 818296548Sdumbbell needs_clflush_after = 1; 819296548Sdumbbell if (obj->gtt_space) { 820296548Sdumbbell ret = i915_gem_object_set_to_gtt_domain(obj, true); 821296548Sdumbbell if (ret) 822296548Sdumbbell return ret; 823296548Sdumbbell } 824277487Skib } 825296548Sdumbbell /* Same trick applies for invalidate partially written cachelines before 826296548Sdumbbell * writing. */ 827296548Sdumbbell if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) 828296548Sdumbbell && obj->cache_level == I915_CACHE_NONE) 829296548Sdumbbell needs_clflush_before = 1; 830277487Skib 831296548Sdumbbell ret = i915_gem_object_get_pages(obj); 832277487Skib if (ret) 833277487Skib return ret; 834277487Skib 835277487Skib i915_gem_object_pin_pages(obj); 836277487Skib 837296548Sdumbbell offset = args->offset; 838277487Skib obj->dirty = 1; 839277487Skib 840277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 841277487Skib for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, 842277487Skib OFF_TO_IDX(offset));; page = vm_page_next(page)) { 843277487Skib VM_OBJECT_WUNLOCK(obj->base.vm_obj); 844277487Skib int partial_cacheline_write; 845277487Skib 846277487Skib if (remain <= 0) 847277487Skib break; 848277487Skib 849277487Skib /* Operation in this page 850277487Skib * 851277487Skib * shmem_page_offset = offset within page in shmem file 852277487Skib * page_length = bytes to copy for this page 853277487Skib */ 854277487Skib shmem_page_offset = offset_in_page(offset); 855277487Skib 856277487Skib page_length = remain; 857277487Skib if ((shmem_page_offset + page_length) > PAGE_SIZE) 858277487Skib page_length = PAGE_SIZE - shmem_page_offset; 859277487Skib 860277487Skib /* If we don't overwrite a cacheline completely we need to be 861277487Skib * careful to have up-to-date data by first clflushing. Don't 862277487Skib * overcomplicate things and flush the entire patch. */ 863277487Skib partial_cacheline_write = needs_clflush_before && 864277487Skib ((shmem_page_offset | page_length) 865277487Skib & (cpu_clflush_line_size - 1)); 866277487Skib 867277487Skib page_do_bit17_swizzling = obj_do_bit17_swizzling && 868277487Skib (page_to_phys(page) & (1 << 17)) != 0; 869277487Skib 870277487Skib ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 871277487Skib user_data, page_do_bit17_swizzling, 872277487Skib partial_cacheline_write, 873277487Skib needs_clflush_after); 874277487Skib if (ret == 0) 875277487Skib goto next_page; 876277487Skib 877277487Skib hit_slowpath = 1; 878277487Skib DRM_UNLOCK(dev); 879277487Skib ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 880277487Skib user_data, page_do_bit17_swizzling, 881277487Skib partial_cacheline_write, 882277487Skib needs_clflush_after); 883277487Skib 884277487Skib DRM_LOCK(dev); 885277487Skib 886277487Skibnext_page: 887277487Skib vm_page_dirty(page); 888277487Skib vm_page_reference(page); 889277487Skib 890277487Skib if (ret) 891277487Skib goto out; 892277487Skib 893277487Skib remain -= page_length; 894277487Skib user_data += page_length; 895277487Skib offset += page_length; 896277487Skib VM_OBJECT_WLOCK(obj->base.vm_obj); 897277487Skib } 898277487Skib 899277487Skibout: 900277487Skib i915_gem_object_unpin_pages(obj); 901277487Skib 902277487Skib if (hit_slowpath) { 903296548Sdumbbell /* Fixup: Kill any reinstated backing storage pages */ 904296548Sdumbbell if (obj->madv == __I915_MADV_PURGED) 905296548Sdumbbell i915_gem_object_truncate(obj); 906296548Sdumbbell /* and flush dirty cachelines in case the object isn't in the cpu write 907296548Sdumbbell * domain anymore. */ 908296548Sdumbbell if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 909277487Skib i915_gem_clflush_object(obj); 910285988Sdumbbell i915_gem_chipset_flush(dev); 911277487Skib } 912277487Skib } 913277487Skib 914277487Skib if (needs_clflush_after) 915277487Skib i915_gem_chipset_flush(dev); 916277487Skib 917277487Skib return ret; 918277487Skib} 919277487Skib 920277487Skib/** 921277487Skib * Writes data to the object referenced by handle. 922277487Skib * 923277487Skib * On error, the contents of the buffer that were to be modified are undefined. 924277487Skib */ 925235783Skibint 926277487Skibi915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 927277487Skib struct drm_file *file) 928235783Skib{ 929277487Skib struct drm_i915_gem_pwrite *args = data; 930277487Skib struct drm_i915_gem_object *obj; 931277487Skib int ret; 932235783Skib 933277487Skib if (args->size == 0) 934277487Skib return 0; 935277487Skib 936277487Skib if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ)) 937277487Skib return -EFAULT; 938277487Skib 939296548Sdumbbell ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), 940296548Sdumbbell args->size); 941296548Sdumbbell if (ret) 942296548Sdumbbell return -EFAULT; 943277487Skib 944277487Skib ret = i915_mutex_lock_interruptible(dev); 945277487Skib if (ret) 946277487Skib return ret; 947277487Skib 948277487Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 949277487Skib if (&obj->base == NULL) { 950277487Skib ret = -ENOENT; 951277487Skib goto unlock; 952277487Skib } 953277487Skib 954277487Skib /* Bounds check destination. */ 955277487Skib if (args->offset > obj->base.size || 956277487Skib args->size > obj->base.size - args->offset) { 957277487Skib ret = -EINVAL; 958277487Skib goto out; 959277487Skib } 960277487Skib 961296548Sdumbbell#ifdef FREEBSD_WIP 962277487Skib /* prime objects have no backing filp to GEM pread/pwrite 963277487Skib * pages from. 964277487Skib */ 965277487Skib if (!obj->base.filp) { 966277487Skib ret = -EINVAL; 967277487Skib goto out; 968277487Skib } 969296548Sdumbbell#endif /* FREEBSD_WIP */ 970277487Skib 971277487Skib CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size); 972277487Skib 973277487Skib ret = -EFAULT; 974277487Skib /* We can only do the GTT pwrite on untiled buffers, as otherwise 975277487Skib * it would end up going through the fenced access, and we'll get 976277487Skib * different detiling behavior between reading and writing. 977277487Skib * pread/pwrite currently are reading and writing from the CPU 978277487Skib * perspective, requiring manual detiling by the client. 979277487Skib */ 980277487Skib if (obj->phys_obj) { 981277487Skib ret = i915_gem_phys_pwrite(dev, obj, args, file); 982277487Skib goto out; 983277487Skib } 984277487Skib 985296548Sdumbbell if (obj->cache_level == I915_CACHE_NONE && 986296548Sdumbbell obj->tiling_mode == I915_TILING_NONE && 987296548Sdumbbell obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 988277487Skib ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); 989277487Skib /* Note that the gtt paths might fail with non-page-backed user 990277487Skib * pointers (e.g. gtt mappings when moving data between 991277487Skib * textures). Fallback to the shmem path in that case. */ 992277487Skib } 993277487Skib 994277487Skib if (ret == -EFAULT || ret == -ENOSPC) 995277487Skib ret = i915_gem_shmem_pwrite(dev, obj, args, file); 996277487Skib 997277487Skibout: 998277487Skib drm_gem_object_unreference(&obj->base); 999277487Skibunlock: 1000277487Skib DRM_UNLOCK(dev); 1001277487Skib return ret; 1002235783Skib} 1003235783Skib 1004296548Sdumbbellint 1005296548Sdumbbelli915_gem_check_wedge(struct drm_i915_private *dev_priv, 1006296548Sdumbbell bool interruptible) 1007285988Sdumbbell{ 1008296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) { 1009296548Sdumbbell struct completion *x = &dev_priv->error_completion; 1010285988Sdumbbell bool recovery_complete; 1011285988Sdumbbell 1012285988Sdumbbell /* Give the error handler a chance to run. */ 1013296548Sdumbbell mtx_lock(&x->lock); 1014296548Sdumbbell recovery_complete = x->done > 0; 1015296548Sdumbbell mtx_unlock(&x->lock); 1016285988Sdumbbell 1017296548Sdumbbell /* Non-interruptible callers can't handle -EAGAIN, hence return 1018296548Sdumbbell * -EIO unconditionally for these. */ 1019296548Sdumbbell if (!interruptible) 1020296548Sdumbbell return -EIO; 1021296548Sdumbbell 1022296548Sdumbbell /* Recovery complete, but still wedged means reset failure. */ 1023296548Sdumbbell if (recovery_complete) 1024296548Sdumbbell return -EIO; 1025296548Sdumbbell 1026296548Sdumbbell return -EAGAIN; 1027285988Sdumbbell } 1028285988Sdumbbell 1029285988Sdumbbell return 0; 1030285988Sdumbbell} 1031285988Sdumbbell 1032285988Sdumbbell/* 1033285988Sdumbbell * Compare seqno against outstanding lazy request. Emit a request if they are 1034285988Sdumbbell * equal. 1035285988Sdumbbell */ 1036285988Sdumbbellstatic int 1037285988Sdumbbelli915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) 1038285988Sdumbbell{ 1039285988Sdumbbell int ret; 1040285988Sdumbbell 1041285988Sdumbbell DRM_LOCK_ASSERT(ring->dev); 1042285988Sdumbbell 1043285988Sdumbbell ret = 0; 1044296548Sdumbbell if (seqno == ring->outstanding_lazy_request) 1045296548Sdumbbell ret = i915_add_request(ring, NULL, NULL); 1046285988Sdumbbell 1047285988Sdumbbell return ret; 1048285988Sdumbbell} 1049285988Sdumbbell 1050296548Sdumbbell/** 1051296548Sdumbbell * __wait_seqno - wait until execution of seqno has finished 1052296548Sdumbbell * @ring: the ring expected to report seqno 1053296548Sdumbbell * @seqno: duh! 1054296548Sdumbbell * @interruptible: do an interruptible wait (normally yes) 1055296548Sdumbbell * @timeout: in - how long to wait (NULL forever); out - how much time remaining 1056296548Sdumbbell * 1057296548Sdumbbell * Returns 0 if the seqno was found within the alloted time. Else returns the 1058296548Sdumbbell * errno with remaining time filled in timeout argument. 1059296548Sdumbbell */ 1060285988Sdumbbellstatic int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, 1061296548Sdumbbell bool interruptible, struct timespec *timeout) 1062285988Sdumbbell{ 1063285988Sdumbbell drm_i915_private_t *dev_priv = ring->dev->dev_private; 1064296548Sdumbbell struct timespec before, now, wait_time={1,0}; 1065296548Sdumbbell sbintime_t timeout_sbt; 1066296548Sdumbbell long end; 1067296548Sdumbbell bool wait_forever = true; 1068296548Sdumbbell int ret, flags; 1069285988Sdumbbell 1070296548Sdumbbell if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) 1071285988Sdumbbell return 0; 1072285988Sdumbbell 1073285988Sdumbbell CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); 1074285988Sdumbbell 1075296548Sdumbbell if (timeout != NULL) { 1076296548Sdumbbell wait_time = *timeout; 1077296548Sdumbbell wait_forever = false; 1078285988Sdumbbell } 1079285988Sdumbbell 1080296548Sdumbbell timeout_sbt = tstosbt(wait_time); 1081296548Sdumbbell 1082296548Sdumbbell if (WARN_ON(!ring->irq_get(ring))) 1083296548Sdumbbell return -ENODEV; 1084296548Sdumbbell 1085296548Sdumbbell /* Record current time in case interrupted by signal, or wedged * */ 1086296548Sdumbbell getrawmonotonic(&before); 1087296548Sdumbbell 1088296548Sdumbbell#define EXIT_COND \ 1089296548Sdumbbell (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ 1090296548Sdumbbell atomic_read(&dev_priv->mm.wedged)) 1091285988Sdumbbell flags = interruptible ? PCATCH : 0; 1092296548Sdumbbell mtx_lock(&dev_priv->irq_lock); 1093296548Sdumbbell do { 1094296548Sdumbbell if (EXIT_COND) { 1095296548Sdumbbell end = 1; 1096296548Sdumbbell } else { 1097296548Sdumbbell ret = -msleep_sbt(&ring->irq_queue, &dev_priv->irq_lock, flags, 1098296548Sdumbbell "915gwr", timeout_sbt, 0, 0); 1099296548Sdumbbell 1100296548Sdumbbell /* 1101296548Sdumbbell * NOTE Linux<->FreeBSD: Convert msleep_sbt() return 1102296548Sdumbbell * value to something close to wait_event*_timeout() 1103296548Sdumbbell * functions used on Linux. 1104296548Sdumbbell * 1105296548Sdumbbell * >0 -> condition is true (end = time remaining) 1106296548Sdumbbell * =0 -> sleep timed out 1107296548Sdumbbell * <0 -> error (interrupted) 1108296548Sdumbbell * 1109296548Sdumbbell * We fake the remaining time by returning 1. We 1110296548Sdumbbell * compute a proper value later. 1111296548Sdumbbell */ 1112296548Sdumbbell if (EXIT_COND) 1113296548Sdumbbell /* We fake a remaining time of 1 tick. */ 1114296548Sdumbbell end = 1; 1115296548Sdumbbell else if (ret == -EINTR || ret == -ERESTART) 1116296548Sdumbbell /* Interrupted. */ 1117296548Sdumbbell end = -ERESTARTSYS; 1118296548Sdumbbell else 1119296548Sdumbbell /* Timeout. */ 1120296548Sdumbbell end = 0; 1121296548Sdumbbell } 1122296548Sdumbbell 1123296548Sdumbbell ret = i915_gem_check_wedge(dev_priv, interruptible); 1124296548Sdumbbell if (ret) 1125296548Sdumbbell end = ret; 1126296548Sdumbbell } while (end == 0 && wait_forever); 1127285988Sdumbbell mtx_unlock(&dev_priv->irq_lock); 1128285988Sdumbbell 1129296548Sdumbbell getrawmonotonic(&now); 1130285988Sdumbbell 1131296548Sdumbbell ring->irq_put(ring); 1132296548Sdumbbell CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, end); 1133296548Sdumbbell#undef EXIT_COND 1134296548Sdumbbell 1135296548Sdumbbell if (timeout) { 1136296548Sdumbbell timespecsub(&now, &before); 1137296548Sdumbbell timespecsub(timeout, &now); 1138296548Sdumbbell } 1139296548Sdumbbell 1140296548Sdumbbell switch (end) { 1141296548Sdumbbell case -EIO: 1142296548Sdumbbell case -EAGAIN: /* Wedged */ 1143296548Sdumbbell case -ERESTARTSYS: /* Signal */ 1144296548Sdumbbell case -ETIMEDOUT: /* Timeout */ 1145296548Sdumbbell return (int)end; 1146296548Sdumbbell case 0: /* Timeout */ 1147296548Sdumbbell return -ETIMEDOUT; 1148296548Sdumbbell default: /* Completed */ 1149296548Sdumbbell WARN_ON(end < 0); /* We're not aware of other errors */ 1150296548Sdumbbell return 0; 1151296548Sdumbbell } 1152285988Sdumbbell} 1153285988Sdumbbell 1154285988Sdumbbell/** 1155285988Sdumbbell * Waits for a sequence number to be signaled, and cleans up the 1156285988Sdumbbell * request and object lists appropriately for that event. 1157285988Sdumbbell */ 1158235783Skibint 1159296548Sdumbbelli915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) 1160285988Sdumbbell{ 1161285988Sdumbbell struct drm_device *dev = ring->dev; 1162285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 1163296548Sdumbbell bool interruptible = dev_priv->mm.interruptible; 1164285988Sdumbbell int ret; 1165285988Sdumbbell 1166296548Sdumbbell DRM_LOCK_ASSERT(dev); 1167296548Sdumbbell BUG_ON(seqno == 0); 1168285988Sdumbbell 1169296548Sdumbbell ret = i915_gem_check_wedge(dev_priv, interruptible); 1170285988Sdumbbell if (ret) 1171285988Sdumbbell return ret; 1172285988Sdumbbell 1173285988Sdumbbell ret = i915_gem_check_olr(ring, seqno); 1174285988Sdumbbell if (ret) 1175285988Sdumbbell return ret; 1176285988Sdumbbell 1177296548Sdumbbell return __wait_seqno(ring, seqno, interruptible, NULL); 1178285988Sdumbbell} 1179285988Sdumbbell 1180285988Sdumbbell/** 1181285988Sdumbbell * Ensures that all rendering to the object has completed and the object is 1182285988Sdumbbell * safe to unbind from the GTT or access from the CPU. 1183285988Sdumbbell */ 1184285988Sdumbbellstatic __must_check int 1185296548Sdumbbelli915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, 1186296548Sdumbbell bool readonly) 1187285988Sdumbbell{ 1188296548Sdumbbell struct intel_ring_buffer *ring = obj->ring; 1189296548Sdumbbell u32 seqno; 1190285988Sdumbbell int ret; 1191285988Sdumbbell 1192296548Sdumbbell seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; 1193296548Sdumbbell if (seqno == 0) 1194296548Sdumbbell return 0; 1195285988Sdumbbell 1196296548Sdumbbell ret = i915_wait_seqno(ring, seqno); 1197296548Sdumbbell if (ret) 1198296548Sdumbbell return ret; 1199296548Sdumbbell 1200296548Sdumbbell i915_gem_retire_requests_ring(ring); 1201296548Sdumbbell 1202296548Sdumbbell /* Manually manage the write flush as we may have not yet 1203296548Sdumbbell * retired the buffer. 1204296548Sdumbbell */ 1205296548Sdumbbell if (obj->last_write_seqno && 1206296548Sdumbbell i915_seqno_passed(seqno, obj->last_write_seqno)) { 1207296548Sdumbbell obj->last_write_seqno = 0; 1208296548Sdumbbell obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; 1209285988Sdumbbell } 1210285988Sdumbbell 1211285988Sdumbbell return 0; 1212285988Sdumbbell} 1213285988Sdumbbell 1214296548Sdumbbell/* A nonblocking variant of the above wait. This is a highly dangerous routine 1215296548Sdumbbell * as the object state may change during this call. 1216296548Sdumbbell */ 1217296548Sdumbbellstatic __must_check int 1218296548Sdumbbelli915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, 1219296548Sdumbbell bool readonly) 1220296548Sdumbbell{ 1221296548Sdumbbell struct drm_device *dev = obj->base.dev; 1222296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 1223296548Sdumbbell struct intel_ring_buffer *ring = obj->ring; 1224296548Sdumbbell u32 seqno; 1225296548Sdumbbell int ret; 1226296548Sdumbbell 1227296548Sdumbbell DRM_LOCK_ASSERT(dev); 1228296548Sdumbbell BUG_ON(!dev_priv->mm.interruptible); 1229296548Sdumbbell 1230296548Sdumbbell seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; 1231296548Sdumbbell if (seqno == 0) 1232296548Sdumbbell return 0; 1233296548Sdumbbell 1234296548Sdumbbell ret = i915_gem_check_wedge(dev_priv, true); 1235296548Sdumbbell if (ret) 1236296548Sdumbbell return ret; 1237296548Sdumbbell 1238296548Sdumbbell ret = i915_gem_check_olr(ring, seqno); 1239296548Sdumbbell if (ret) 1240296548Sdumbbell return ret; 1241296548Sdumbbell 1242296548Sdumbbell DRM_UNLOCK(dev); 1243296548Sdumbbell ret = __wait_seqno(ring, seqno, true, NULL); 1244296548Sdumbbell DRM_LOCK(dev); 1245296548Sdumbbell 1246296548Sdumbbell i915_gem_retire_requests_ring(ring); 1247296548Sdumbbell 1248296548Sdumbbell /* Manually manage the write flush as we may have not yet 1249296548Sdumbbell * retired the buffer. 1250296548Sdumbbell */ 1251296548Sdumbbell if (ret == 0 && 1252296548Sdumbbell obj->last_write_seqno && 1253296548Sdumbbell i915_seqno_passed(seqno, obj->last_write_seqno)) { 1254296548Sdumbbell obj->last_write_seqno = 0; 1255296548Sdumbbell obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; 1256296548Sdumbbell } 1257296548Sdumbbell 1258296548Sdumbbell return ret; 1259296548Sdumbbell} 1260296548Sdumbbell 1261296548Sdumbbell/** 1262296548Sdumbbell * Called when user space prepares to use an object with the CPU, either 1263296548Sdumbbell * through the mmap ioctl's mapping or a GTT mapping. 1264296548Sdumbbell */ 1265285988Sdumbbellint 1266235783Skibi915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1267285988Sdumbbell struct drm_file *file) 1268235783Skib{ 1269285988Sdumbbell struct drm_i915_gem_set_domain *args = data; 1270235783Skib struct drm_i915_gem_object *obj; 1271285988Sdumbbell uint32_t read_domains = args->read_domains; 1272285988Sdumbbell uint32_t write_domain = args->write_domain; 1273235783Skib int ret; 1274235783Skib 1275287174Sbapt /* Only handle setting domains to types used by the CPU. */ 1276287174Sbapt if (write_domain & I915_GEM_GPU_DOMAINS) 1277285988Sdumbbell return -EINVAL; 1278235783Skib 1279287174Sbapt if (read_domains & I915_GEM_GPU_DOMAINS) 1280287174Sbapt return -EINVAL; 1281287174Sbapt 1282287174Sbapt /* Having something in the write domain implies it's in the read 1283287174Sbapt * domain, and only that read domain. Enforce that in the request. 1284287174Sbapt */ 1285287174Sbapt if (write_domain != 0 && read_domains != write_domain) 1286287174Sbapt return -EINVAL; 1287287174Sbapt 1288235783Skib ret = i915_mutex_lock_interruptible(dev); 1289285988Sdumbbell if (ret) 1290285988Sdumbbell return ret; 1291235783Skib 1292235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1293235783Skib if (&obj->base == NULL) { 1294235783Skib ret = -ENOENT; 1295235783Skib goto unlock; 1296235783Skib } 1297235783Skib 1298296548Sdumbbell /* Try to flush the object off the GPU without holding the lock. 1299296548Sdumbbell * We will repeat the flush holding the lock in the normal manner 1300296548Sdumbbell * to catch cases where we are gazumped. 1301296548Sdumbbell */ 1302296548Sdumbbell ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain); 1303296548Sdumbbell if (ret) 1304296548Sdumbbell goto unref; 1305296548Sdumbbell 1306285988Sdumbbell if (read_domains & I915_GEM_DOMAIN_GTT) { 1307235783Skib ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1308285988Sdumbbell 1309285988Sdumbbell /* Silently promote "you're not bound, there was nothing to do" 1310285988Sdumbbell * to success, since the client was just asking us to 1311285988Sdumbbell * make sure everything was done. 1312285988Sdumbbell */ 1313235783Skib if (ret == -EINVAL) 1314235783Skib ret = 0; 1315285988Sdumbbell } else { 1316235783Skib ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1317285988Sdumbbell } 1318235783Skib 1319296548Sdumbbellunref: 1320235783Skib drm_gem_object_unreference(&obj->base); 1321235783Skibunlock: 1322235783Skib DRM_UNLOCK(dev); 1323285988Sdumbbell return ret; 1324235783Skib} 1325235783Skib 1326285988Sdumbbell/** 1327285988Sdumbbell * Called when user space has done writes to this buffer 1328285988Sdumbbell */ 1329235783Skibint 1330235783Skibi915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1331285988Sdumbbell struct drm_file *file) 1332235783Skib{ 1333285988Sdumbbell struct drm_i915_gem_sw_finish *args = data; 1334235783Skib struct drm_i915_gem_object *obj; 1335285988Sdumbbell int ret = 0; 1336235783Skib 1337285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 1338285988Sdumbbell if (ret) 1339285988Sdumbbell return ret; 1340277487Skib 1341235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1342235783Skib if (&obj->base == NULL) { 1343235783Skib ret = -ENOENT; 1344235783Skib goto unlock; 1345235783Skib } 1346285988Sdumbbell 1347285988Sdumbbell /* Pinned buffers may be scanout, so flush the cache */ 1348285988Sdumbbell if (obj->pin_count) 1349235783Skib i915_gem_object_flush_cpu_write_domain(obj); 1350285988Sdumbbell 1351235783Skib drm_gem_object_unreference(&obj->base); 1352235783Skibunlock: 1353235783Skib DRM_UNLOCK(dev); 1354285988Sdumbbell return ret; 1355235783Skib} 1356235783Skib 1357285988Sdumbbell/** 1358285988Sdumbbell * Maps the contents of an object, returning the address it is mapped 1359285988Sdumbbell * into. 1360285988Sdumbbell * 1361285988Sdumbbell * While the mapping holds a reference on the contents of the object, it doesn't 1362285988Sdumbbell * imply a ref on the object itself. 1363285988Sdumbbell */ 1364235783Skibint 1365235783Skibi915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1366285988Sdumbbell struct drm_file *file) 1367235783Skib{ 1368285988Sdumbbell struct drm_i915_gem_mmap *args = data; 1369235783Skib struct drm_gem_object *obj; 1370235783Skib struct proc *p; 1371235783Skib vm_map_t map; 1372235783Skib vm_offset_t addr; 1373235783Skib vm_size_t size; 1374235783Skib int error, rv; 1375235783Skib 1376235783Skib obj = drm_gem_object_lookup(dev, file, args->handle); 1377235783Skib if (obj == NULL) 1378285988Sdumbbell return -ENOENT; 1379285988Sdumbbell 1380296548Sdumbbell#ifdef FREEBSD_WIP 1381296548Sdumbbell /* prime objects have no backing filp to GEM mmap 1382296548Sdumbbell * pages from. 1383296548Sdumbbell */ 1384296548Sdumbbell if (!obj->filp) { 1385296548Sdumbbell drm_gem_object_unreference_unlocked(obj); 1386296548Sdumbbell return -EINVAL; 1387296548Sdumbbell } 1388296548Sdumbbell#endif /* FREEBSD_WIP */ 1389296548Sdumbbell 1390235783Skib error = 0; 1391235783Skib if (args->size == 0) 1392235783Skib goto out; 1393235783Skib p = curproc; 1394235783Skib map = &p->p_vmspace->vm_map; 1395235783Skib size = round_page(args->size); 1396235783Skib PROC_LOCK(p); 1397284215Smjg if (map->size + size > lim_cur_proc(p, RLIMIT_VMEM)) { 1398235783Skib PROC_UNLOCK(p); 1399280183Sdumbbell error = -ENOMEM; 1400235783Skib goto out; 1401235783Skib } 1402235783Skib PROC_UNLOCK(p); 1403235783Skib 1404235783Skib addr = 0; 1405235783Skib vm_object_reference(obj->vm_obj); 1406255426Sjhb rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, 1407253471Sjhb VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 1408253497Skib VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); 1409235783Skib if (rv != KERN_SUCCESS) { 1410235783Skib vm_object_deallocate(obj->vm_obj); 1411235783Skib error = -vm_mmap_to_errno(rv); 1412235783Skib } else { 1413235783Skib args->addr_ptr = (uint64_t)addr; 1414235783Skib } 1415235783Skibout: 1416296548Sdumbbell drm_gem_object_unreference_unlocked(obj); 1417235783Skib return (error); 1418235783Skib} 1419235783Skib 1420235783Skibstatic int 1421235783Skibi915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1422235783Skib vm_ooffset_t foff, struct ucred *cred, u_short *color) 1423235783Skib{ 1424235783Skib 1425296548Sdumbbell /* 1426296548Sdumbbell * NOTE Linux<->FreeBSD: drm_gem_mmap_single() takes care of 1427296548Sdumbbell * calling drm_gem_object_reference(). That's why we don't 1428296548Sdumbbell * do this here. i915_gem_pager_dtor(), below, will call 1429296548Sdumbbell * drm_gem_object_unreference(). 1430296548Sdumbbell * 1431296548Sdumbbell * On Linux, drm_gem_vm_open() references the object because 1432296548Sdumbbell * it's called the mapping is copied. drm_gem_vm_open() is not 1433296548Sdumbbell * called when the mapping is created. So the possible sequences 1434296548Sdumbbell * are: 1435296548Sdumbbell * 1. drm_gem_mmap(): ref++ 1436296548Sdumbbell * 2. drm_gem_vm_close(): ref-- 1437296548Sdumbbell * 1438296548Sdumbbell * 1. drm_gem_mmap(): ref++ 1439296548Sdumbbell * 2. drm_gem_vm_open(): ref++ (for the copied vma) 1440296548Sdumbbell * 3. drm_gem_vm_close(): ref-- (for the copied vma) 1441296548Sdumbbell * 4. drm_gem_vm_close(): ref-- (for the initial vma) 1442296548Sdumbbell * 1443296548Sdumbbell * On FreeBSD, i915_gem_pager_ctor() is called once during the 1444296548Sdumbbell * creation of the mapping. No callback is called when the 1445296548Sdumbbell * mapping is shared during a fork(). i915_gem_pager_dtor() is 1446296548Sdumbbell * called when the last reference to the mapping is dropped. So 1447296548Sdumbbell * the only sequence is: 1448296548Sdumbbell * 1. drm_gem_mmap_single(): ref++ 1449296548Sdumbbell * 2. i915_gem_pager_ctor(): <noop> 1450296548Sdumbbell * 3. i915_gem_pager_dtor(): ref-- 1451296548Sdumbbell */ 1452296548Sdumbbell 1453235783Skib *color = 0; /* XXXKIB */ 1454235783Skib return (0); 1455235783Skib} 1456235783Skib 1457285988Sdumbbell/** 1458285988Sdumbbell * i915_gem_fault - fault a page into the GTT 1459285988Sdumbbell * vma: VMA in question 1460285988Sdumbbell * vmf: fault info 1461285988Sdumbbell * 1462285988Sdumbbell * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 1463285988Sdumbbell * from userspace. The fault handler takes care of binding the object to 1464285988Sdumbbell * the GTT (if needed), allocating and programming a fence register (again, 1465285988Sdumbbell * only if needed based on whether the old reg is still valid or the object 1466285988Sdumbbell * is tiled) and inserting a new PTE into the faulting process. 1467285988Sdumbbell * 1468285988Sdumbbell * Note that the faulting process may involve evicting existing objects 1469285988Sdumbbell * from the GTT and/or fence registers to make room. So performance may 1470285988Sdumbbell * suffer if the GTT working set is large or there are few fence registers 1471285988Sdumbbell * left. 1472285988Sdumbbell */ 1473285988Sdumbbell 1474235783Skibint i915_intr_pf; 1475235783Skib 1476235783Skibstatic int 1477235783Skibi915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1478235783Skib vm_page_t *mres) 1479235783Skib{ 1480296548Sdumbbell struct drm_gem_object *gem_obj = vm_obj->handle; 1481296548Sdumbbell struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 1482296548Sdumbbell struct drm_device *dev = obj->base.dev; 1483296548Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 1484285988Sdumbbell vm_page_t page, oldpage; 1485296548Sdumbbell int ret = 0; 1486296548Sdumbbell#ifdef FREEBSD_WIP 1487296548Sdumbbell bool write = (prot & VM_PROT_WRITE) != 0; 1488296548Sdumbbell#else 1489296548Sdumbbell bool write = true; 1490296548Sdumbbell#endif /* FREEBSD_WIP */ 1491296548Sdumbbell bool pinned; 1492235783Skib 1493235783Skib vm_object_pip_add(vm_obj, 1); 1494235783Skib 1495235783Skib /* 1496235783Skib * Remove the placeholder page inserted by vm_fault() from the 1497235783Skib * object before dropping the object lock. If 1498235783Skib * i915_gem_release_mmap() is active in parallel on this gem 1499235783Skib * object, then it owns the drm device sx and might find the 1500235783Skib * placeholder already. Then, since the page is busy, 1501235783Skib * i915_gem_release_mmap() sleeps waiting for the busy state 1502296548Sdumbbell * of the page cleared. We will be unable to acquire drm 1503235783Skib * device lock until i915_gem_release_mmap() is able to make a 1504235783Skib * progress. 1505235783Skib */ 1506235783Skib if (*mres != NULL) { 1507285988Sdumbbell oldpage = *mres; 1508285988Sdumbbell vm_page_lock(oldpage); 1509285988Sdumbbell vm_page_remove(oldpage); 1510285988Sdumbbell vm_page_unlock(oldpage); 1511235783Skib *mres = NULL; 1512235783Skib } else 1513285988Sdumbbell oldpage = NULL; 1514254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1515235783Skibretry: 1516296548Sdumbbell ret = 0; 1517296548Sdumbbell pinned = 0; 1518285988Sdumbbell page = NULL; 1519235783Skib 1520235783Skib if (i915_intr_pf) { 1521235783Skib ret = i915_mutex_lock_interruptible(dev); 1522296548Sdumbbell if (ret != 0) 1523235783Skib goto out; 1524235783Skib } else 1525235783Skib DRM_LOCK(dev); 1526235783Skib 1527251960Skib /* 1528251960Skib * Since the object lock was dropped, other thread might have 1529251960Skib * faulted on the same GTT address and instantiated the 1530251960Skib * mapping for the page. Recheck. 1531251960Skib */ 1532251960Skib VM_OBJECT_WLOCK(vm_obj); 1533285988Sdumbbell page = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1534285988Sdumbbell if (page != NULL) { 1535285988Sdumbbell if (vm_page_busied(page)) { 1536251960Skib DRM_UNLOCK(dev); 1537285988Sdumbbell vm_page_lock(page); 1538254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1539285988Sdumbbell vm_page_busy_sleep(page, "915pee"); 1540251960Skib goto retry; 1541251960Skib } 1542251960Skib goto have_page; 1543251960Skib } else 1544251960Skib VM_OBJECT_WUNLOCK(vm_obj); 1545251960Skib 1546235783Skib /* Now bind it into the GTT if needed */ 1547296548Sdumbbell ret = i915_gem_object_pin(obj, 0, true, false); 1548296548Sdumbbell if (ret) 1549296548Sdumbbell goto unlock; 1550296548Sdumbbell pinned = 1; 1551235783Skib 1552296548Sdumbbell ret = i915_gem_object_set_to_gtt_domain(obj, write); 1553296548Sdumbbell if (ret) 1554296548Sdumbbell goto unpin; 1555235783Skib 1556277487Skib ret = i915_gem_object_get_fence(obj); 1557296548Sdumbbell if (ret) 1558296548Sdumbbell goto unpin; 1559235783Skib 1560296548Sdumbbell obj->fault_mappable = true; 1561235783Skib 1562248084Sattilio VM_OBJECT_WLOCK(vm_obj); 1563296548Sdumbbell page = PHYS_TO_VM_PAGE(dev_priv->mm.gtt_base_addr + obj->gtt_offset + offset); 1564285988Sdumbbell KASSERT((page->flags & PG_FICTITIOUS) != 0, 1565269634Sroyger ("physical address %#jx not fictitious", 1566296548Sdumbbell (uintmax_t)(dev_priv->mm.gtt_base_addr + obj->gtt_offset + offset))); 1567285988Sdumbbell if (page == NULL) { 1568265102Skib VM_OBJECT_WUNLOCK(vm_obj); 1569235783Skib ret = -EFAULT; 1570296548Sdumbbell goto unpin; 1571235783Skib } 1572285988Sdumbbell KASSERT((page->flags & PG_FICTITIOUS) != 0, 1573285988Sdumbbell ("not fictitious %p", page)); 1574285988Sdumbbell KASSERT(page->wire_count == 1, ("wire_count not 1 %p", page)); 1575235783Skib 1576285988Sdumbbell if (vm_page_busied(page)) { 1577296548Sdumbbell i915_gem_object_unpin(obj); 1578235783Skib DRM_UNLOCK(dev); 1579285988Sdumbbell vm_page_lock(page); 1580254138Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1581285988Sdumbbell vm_page_busy_sleep(page, "915pbs"); 1582235783Skib goto retry; 1583235783Skib } 1584285988Sdumbbell if (vm_page_insert(page, vm_obj, OFF_TO_IDX(offset))) { 1585296548Sdumbbell i915_gem_object_unpin(obj); 1586254141Sattilio DRM_UNLOCK(dev); 1587254141Sattilio VM_OBJECT_WUNLOCK(vm_obj); 1588254141Sattilio VM_WAIT; 1589254141Sattilio goto retry; 1590254141Sattilio } 1591285988Sdumbbell page->valid = VM_PAGE_BITS_ALL; 1592251960Skibhave_page: 1593285988Sdumbbell *mres = page; 1594285988Sdumbbell vm_page_xbusy(page); 1595235783Skib 1596235783Skib CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, 1597285988Sdumbbell page->phys_addr); 1598296548Sdumbbell if (pinned) { 1599296548Sdumbbell /* 1600296548Sdumbbell * We may have not pinned the object if the page was 1601296548Sdumbbell * found by the call to vm_page_lookup() 1602296548Sdumbbell */ 1603296548Sdumbbell i915_gem_object_unpin(obj); 1604296548Sdumbbell } 1605235783Skib DRM_UNLOCK(dev); 1606285988Sdumbbell if (oldpage != NULL) { 1607285988Sdumbbell vm_page_lock(oldpage); 1608285988Sdumbbell vm_page_free(oldpage); 1609285988Sdumbbell vm_page_unlock(oldpage); 1610235783Skib } 1611235783Skib vm_object_pip_wakeup(vm_obj); 1612235783Skib return (VM_PAGER_OK); 1613235783Skib 1614296548Sdumbbellunpin: 1615296548Sdumbbell i915_gem_object_unpin(obj); 1616235783Skibunlock: 1617235783Skib DRM_UNLOCK(dev); 1618235783Skibout: 1619235783Skib KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1620296548Sdumbbell CTR4(KTR_DRM, "fault_fail %p %jx %x err %d", gem_obj, offset, prot, 1621296548Sdumbbell -ret); 1622235783Skib if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1623235783Skib kern_yield(PRI_USER); 1624254138Sattilio goto retry; 1625235783Skib } 1626248084Sattilio VM_OBJECT_WLOCK(vm_obj); 1627235783Skib vm_object_pip_wakeup(vm_obj); 1628235783Skib return (VM_PAGER_ERROR); 1629235783Skib} 1630235783Skib 1631235783Skibstatic void 1632235783Skibi915_gem_pager_dtor(void *handle) 1633235783Skib{ 1634296548Sdumbbell struct drm_gem_object *obj = handle; 1635296548Sdumbbell struct drm_device *dev = obj->dev; 1636235783Skib 1637235783Skib DRM_LOCK(dev); 1638235783Skib drm_gem_object_unreference(obj); 1639235783Skib DRM_UNLOCK(dev); 1640235783Skib} 1641235783Skib 1642235783Skibstruct cdev_pager_ops i915_gem_pager_ops = { 1643235783Skib .cdev_pg_fault = i915_gem_pager_fault, 1644235783Skib .cdev_pg_ctor = i915_gem_pager_ctor, 1645235783Skib .cdev_pg_dtor = i915_gem_pager_dtor 1646235783Skib}; 1647235783Skib 1648285988Sdumbbell/** 1649285988Sdumbbell * i915_gem_release_mmap - remove physical page mappings 1650285988Sdumbbell * @obj: obj in question 1651285988Sdumbbell * 1652285988Sdumbbell * Preserve the reservation of the mmapping with the DRM core code, but 1653285988Sdumbbell * relinquish ownership of the pages back to the system. 1654285988Sdumbbell * 1655285988Sdumbbell * It is vital that we remove the page mapping if we have mapped a tiled 1656285988Sdumbbell * object through the GTT and then lose the fence register due to 1657285988Sdumbbell * resource pressure. Similarly if the object has been moved out of the 1658285988Sdumbbell * aperture, than pages mapped into userspace must be revoked. Removing the 1659285988Sdumbbell * mapping will then trigger a page fault on the next user access, allowing 1660285988Sdumbbell * fixup by i915_gem_fault(). 1661285988Sdumbbell */ 1662235783Skibvoid 1663285988Sdumbbelli915_gem_release_mmap(struct drm_i915_gem_object *obj) 1664235783Skib{ 1665285988Sdumbbell vm_object_t devobj; 1666285988Sdumbbell vm_page_t page; 1667285988Sdumbbell int i, page_count; 1668235783Skib 1669285988Sdumbbell if (!obj->fault_mappable) 1670235783Skib return; 1671235783Skib 1672285988Sdumbbell CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, 1673285988Sdumbbell OFF_TO_IDX(obj->base.size)); 1674285988Sdumbbell devobj = cdev_pager_lookup(obj); 1675285988Sdumbbell if (devobj != NULL) { 1676285988Sdumbbell page_count = OFF_TO_IDX(obj->base.size); 1677235783Skib 1678285988Sdumbbell VM_OBJECT_WLOCK(devobj); 1679285988Sdumbbellretry: 1680285988Sdumbbell for (i = 0; i < page_count; i++) { 1681285988Sdumbbell page = vm_page_lookup(devobj, i); 1682285988Sdumbbell if (page == NULL) 1683285988Sdumbbell continue; 1684285988Sdumbbell if (vm_page_sleep_if_busy(page, "915unm")) 1685285988Sdumbbell goto retry; 1686285988Sdumbbell cdev_pager_free_page(devobj, page); 1687235783Skib } 1688285988Sdumbbell VM_OBJECT_WUNLOCK(devobj); 1689285988Sdumbbell vm_object_deallocate(devobj); 1690235783Skib } 1691235783Skib 1692285988Sdumbbell obj->fault_mappable = false; 1693235783Skib} 1694235783Skib 1695235783Skibstatic uint32_t 1696235783Skibi915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1697235783Skib{ 1698235783Skib uint32_t gtt_size; 1699235783Skib 1700235783Skib if (INTEL_INFO(dev)->gen >= 4 || 1701235783Skib tiling_mode == I915_TILING_NONE) 1702285988Sdumbbell return size; 1703235783Skib 1704235783Skib /* Previous chips need a power-of-two fence region when tiling */ 1705235783Skib if (INTEL_INFO(dev)->gen == 3) 1706235783Skib gtt_size = 1024*1024; 1707235783Skib else 1708235783Skib gtt_size = 512*1024; 1709235783Skib 1710235783Skib while (gtt_size < size) 1711235783Skib gtt_size <<= 1; 1712235783Skib 1713285988Sdumbbell return gtt_size; 1714235783Skib} 1715235783Skib 1716235783Skib/** 1717235783Skib * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1718235783Skib * @obj: object to check 1719235783Skib * 1720235783Skib * Return the required GTT alignment for an object, taking into account 1721235783Skib * potential fence register mapping. 1722235783Skib */ 1723235783Skibstatic uint32_t 1724285988Sdumbbelli915_gem_get_gtt_alignment(struct drm_device *dev, 1725285988Sdumbbell uint32_t size, 1726285988Sdumbbell int tiling_mode) 1727235783Skib{ 1728235783Skib /* 1729235783Skib * Minimum alignment is 4k (GTT page size), but might be greater 1730235783Skib * if a fence register is needed for the object. 1731235783Skib */ 1732235783Skib if (INTEL_INFO(dev)->gen >= 4 || 1733235783Skib tiling_mode == I915_TILING_NONE) 1734285988Sdumbbell return 4096; 1735235783Skib 1736235783Skib /* 1737235783Skib * Previous chips need to be aligned to the size of the smallest 1738235783Skib * fence register that can contain the object. 1739235783Skib */ 1740285988Sdumbbell return i915_gem_get_gtt_size(dev, size, tiling_mode); 1741235783Skib} 1742235783Skib 1743285988Sdumbbell/** 1744285988Sdumbbell * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an 1745285988Sdumbbell * unfenced object 1746285988Sdumbbell * @dev: the device 1747285988Sdumbbell * @size: size of the object 1748285988Sdumbbell * @tiling_mode: tiling mode of the object 1749285988Sdumbbell * 1750285988Sdumbbell * Return the required GTT alignment for an object, only taking into account 1751285988Sdumbbell * unfenced tiled surface requirements. 1752285988Sdumbbell */ 1753235783Skibuint32_t 1754285988Sdumbbelli915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, 1755285988Sdumbbell uint32_t size, 1756285988Sdumbbell int tiling_mode) 1757235783Skib{ 1758235783Skib /* 1759235783Skib * Minimum alignment is 4k (GTT page size) for sane hw. 1760235783Skib */ 1761287174Sbapt if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || 1762287174Sbapt tiling_mode == I915_TILING_NONE) 1763285988Sdumbbell return 4096; 1764235783Skib 1765285988Sdumbbell /* Previous hardware however needs to be aligned to a power-of-two 1766235783Skib * tile height. The simplest method for determining this is to reuse 1767235783Skib * the power-of-tile object size. 1768285988Sdumbbell */ 1769285988Sdumbbell return i915_gem_get_gtt_size(dev, size, tiling_mode); 1770235783Skib} 1771235783Skib 1772296548Sdumbbellstatic int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) 1773296548Sdumbbell{ 1774296548Sdumbbell struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 1775296548Sdumbbell int ret; 1776296548Sdumbbell 1777296548Sdumbbell if (obj->base.on_map) 1778296548Sdumbbell return 0; 1779296548Sdumbbell 1780296548Sdumbbell dev_priv->mm.shrinker_no_lock_stealing = true; 1781296548Sdumbbell 1782296548Sdumbbell ret = drm_gem_create_mmap_offset(&obj->base); 1783296548Sdumbbell if (ret != -ENOSPC) 1784296548Sdumbbell goto out; 1785296548Sdumbbell 1786296548Sdumbbell /* Badly fragmented mmap space? The only way we can recover 1787296548Sdumbbell * space is by destroying unwanted objects. We can't randomly release 1788296548Sdumbbell * mmap_offsets as userspace expects them to be persistent for the 1789296548Sdumbbell * lifetime of the objects. The closest we can is to release the 1790296548Sdumbbell * offsets on purgeable objects by truncating it and marking it purged, 1791296548Sdumbbell * which prevents userspace from ever using that object again. 1792296548Sdumbbell */ 1793296548Sdumbbell i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT); 1794296548Sdumbbell ret = drm_gem_create_mmap_offset(&obj->base); 1795296548Sdumbbell if (ret != -ENOSPC) 1796296548Sdumbbell goto out; 1797296548Sdumbbell 1798296548Sdumbbell i915_gem_shrink_all(dev_priv); 1799296548Sdumbbell ret = drm_gem_create_mmap_offset(&obj->base); 1800296548Sdumbbellout: 1801296548Sdumbbell dev_priv->mm.shrinker_no_lock_stealing = false; 1802296548Sdumbbell 1803296548Sdumbbell return ret; 1804296548Sdumbbell} 1805296548Sdumbbell 1806296548Sdumbbellstatic void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) 1807296548Sdumbbell{ 1808296548Sdumbbell if (!obj->base.on_map) 1809296548Sdumbbell return; 1810296548Sdumbbell 1811296548Sdumbbell drm_gem_free_mmap_offset(&obj->base); 1812296548Sdumbbell} 1813296548Sdumbbell 1814285988Sdumbbellint 1815285988Sdumbbelli915_gem_mmap_gtt(struct drm_file *file, 1816285988Sdumbbell struct drm_device *dev, 1817285988Sdumbbell uint32_t handle, 1818285988Sdumbbell uint64_t *offset) 1819235783Skib{ 1820285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 1821285988Sdumbbell struct drm_i915_gem_object *obj; 1822235783Skib int ret; 1823235783Skib 1824285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 1825285988Sdumbbell if (ret) 1826285988Sdumbbell return ret; 1827235783Skib 1828285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1829285988Sdumbbell if (&obj->base == NULL) { 1830285988Sdumbbell ret = -ENOENT; 1831285988Sdumbbell goto unlock; 1832235783Skib } 1833235783Skib 1834285988Sdumbbell if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1835285988Sdumbbell ret = -E2BIG; 1836285988Sdumbbell goto out; 1837235783Skib } 1838235783Skib 1839285988Sdumbbell if (obj->madv != I915_MADV_WILLNEED) { 1840285988Sdumbbell DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1841285988Sdumbbell ret = -EINVAL; 1842285988Sdumbbell goto out; 1843235783Skib } 1844235783Skib 1845296548Sdumbbell ret = i915_gem_object_create_mmap_offset(obj); 1846285988Sdumbbell if (ret) 1847285988Sdumbbell goto out; 1848235783Skib 1849285988Sdumbbell *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1850285988Sdumbbell DRM_GEM_MAPPING_KEY; 1851235783Skib 1852285988Sdumbbellout: 1853285988Sdumbbell drm_gem_object_unreference(&obj->base); 1854285988Sdumbbellunlock: 1855285988Sdumbbell DRM_UNLOCK(dev); 1856285988Sdumbbell return ret; 1857235783Skib} 1858235783Skib 1859285988Sdumbbell/** 1860285988Sdumbbell * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1861285988Sdumbbell * @dev: DRM device 1862285988Sdumbbell * @data: GTT mapping ioctl data 1863285988Sdumbbell * @file: GEM object info 1864285988Sdumbbell * 1865285988Sdumbbell * Simply returns the fake offset to userspace so it can mmap it. 1866285988Sdumbbell * The mmap call will end up in drm_gem_mmap(), which will set things 1867285988Sdumbbell * up so we can get faults in the handler above. 1868285988Sdumbbell * 1869285988Sdumbbell * The fault handler will take care of binding the object into the GTT 1870285988Sdumbbell * (since it may have been evicted to make room for something), allocating 1871285988Sdumbbell * a fence register, and mapping the appropriate aperture address into 1872285988Sdumbbell * userspace. 1873285988Sdumbbell */ 1874277487Skibint 1875285988Sdumbbelli915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1876285988Sdumbbell struct drm_file *file) 1877235783Skib{ 1878285988Sdumbbell struct drm_i915_gem_mmap_gtt *args = data; 1879277487Skib 1880285988Sdumbbell return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); 1881277487Skib} 1882277487Skib 1883285988Sdumbbell/* Immediately discard the backing storage */ 1884285988Sdumbbellstatic void 1885285988Sdumbbelli915_gem_object_truncate(struct drm_i915_gem_object *obj) 1886277487Skib{ 1887285988Sdumbbell vm_object_t vm_obj; 1888235783Skib 1889285988Sdumbbell vm_obj = obj->base.vm_obj; 1890285988Sdumbbell VM_OBJECT_WLOCK(vm_obj); 1891285988Sdumbbell vm_object_page_remove(vm_obj, 0, 0, false); 1892285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 1893296548Sdumbbell i915_gem_object_free_mmap_offset(obj); 1894296548Sdumbbell 1895296548Sdumbbell obj->madv = __I915_MADV_PURGED; 1896235783Skib} 1897235783Skib 1898285988Sdumbbellstatic inline int 1899285988Sdumbbelli915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 1900235783Skib{ 1901285988Sdumbbell return obj->madv == I915_MADV_DONTNEED; 1902235783Skib} 1903235783Skib 1904277487Skibstatic void 1905277487Skibi915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj, 1906277487Skib vm_pindex_t si, vm_pindex_t ei) 1907277487Skib{ 1908277487Skib vm_object_t vm_obj; 1909285988Sdumbbell vm_page_t page; 1910277487Skib vm_pindex_t i; 1911277487Skib 1912277487Skib vm_obj = obj->base.vm_obj; 1913277487Skib VM_OBJECT_ASSERT_LOCKED(vm_obj); 1914285988Sdumbbell for (i = si, page = vm_page_lookup(vm_obj, i); i < ei; 1915285988Sdumbbell page = vm_page_next(page), i++) { 1916285988Sdumbbell KASSERT(page->pindex == i, ("pindex %jx %jx", 1917285988Sdumbbell (uintmax_t)page->pindex, (uintmax_t)i)); 1918285988Sdumbbell vm_page_lock(page); 1919285988Sdumbbell vm_page_unwire(page, PQ_INACTIVE); 1920285988Sdumbbell if (page->wire_count == 0) 1921277487Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1922285988Sdumbbell vm_page_unlock(page); 1923277487Skib } 1924277487Skib} 1925277487Skib 1926235783Skib#define GEM_PARANOID_CHECK_GTT 0 1927235783Skib#if GEM_PARANOID_CHECK_GTT 1928235783Skibstatic void 1929235783Skibi915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 1930235783Skib int page_count) 1931235783Skib{ 1932235783Skib struct drm_i915_private *dev_priv; 1933235783Skib vm_paddr_t pa; 1934235783Skib unsigned long start, end; 1935235783Skib u_int i; 1936235783Skib int j; 1937235783Skib 1938235783Skib dev_priv = dev->dev_private; 1939235783Skib start = OFF_TO_IDX(dev_priv->mm.gtt_start); 1940235783Skib end = OFF_TO_IDX(dev_priv->mm.gtt_end); 1941235783Skib for (i = start; i < end; i++) { 1942235783Skib pa = intel_gtt_read_pte_paddr(i); 1943235783Skib for (j = 0; j < page_count; j++) { 1944235783Skib if (pa == VM_PAGE_TO_PHYS(ma[j])) { 1945235783Skib panic("Page %p in GTT pte index %d pte %x", 1946235783Skib ma[i], i, intel_gtt_read_pte(i)); 1947235783Skib } 1948235783Skib } 1949235783Skib } 1950235783Skib} 1951235783Skib#endif 1952235783Skib 1953235783Skibstatic void 1954235783Skibi915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 1955235783Skib{ 1956293837Sdumbbell int page_count = obj->base.size / PAGE_SIZE; 1957296548Sdumbbell int ret, i; 1958235783Skib 1959296548Sdumbbell BUG_ON(obj->madv == __I915_MADV_PURGED); 1960235783Skib 1961296548Sdumbbell ret = i915_gem_object_set_to_cpu_domain(obj, true); 1962296548Sdumbbell if (ret) { 1963296548Sdumbbell /* In the event of a disaster, abandon all caches and 1964296548Sdumbbell * hope for the best. 1965296548Sdumbbell */ 1966296548Sdumbbell WARN_ON(ret != -EIO); 1967296548Sdumbbell i915_gem_clflush_object(obj); 1968296548Sdumbbell obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1969296548Sdumbbell } 1970296548Sdumbbell 1971296548Sdumbbell if (i915_gem_object_needs_bit17_swizzle(obj)) 1972235783Skib i915_gem_object_save_bit_17_swizzle(obj); 1973293837Sdumbbell 1974235783Skib if (obj->madv == I915_MADV_DONTNEED) 1975235783Skib obj->dirty = 0; 1976293837Sdumbbell 1977248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 1978235783Skib#if GEM_PARANOID_CHECK_GTT 1979235783Skib i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 1980235783Skib#endif 1981235783Skib for (i = 0; i < page_count; i++) { 1982293837Sdumbbell vm_page_t page = obj->pages[i]; 1983293837Sdumbbell 1984235783Skib if (obj->dirty) 1985285988Sdumbbell vm_page_dirty(page); 1986293837Sdumbbell 1987235783Skib if (obj->madv == I915_MADV_WILLNEED) 1988285988Sdumbbell vm_page_reference(page); 1989293837Sdumbbell 1990285988Sdumbbell vm_page_lock(page); 1991267548Sattilio vm_page_unwire(obj->pages[i], PQ_ACTIVE); 1992285988Sdumbbell vm_page_unlock(page); 1993235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1994235783Skib } 1995248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 1996235783Skib obj->dirty = 0; 1997293837Sdumbbell 1998235783Skib free(obj->pages, DRM_I915_GEM); 1999235783Skib obj->pages = NULL; 2000235783Skib} 2001235783Skib 2002285988Sdumbbellstatic int 2003296548Sdumbbelli915_gem_object_put_pages(struct drm_i915_gem_object *obj) 2004235783Skib{ 2005296548Sdumbbell const struct drm_i915_gem_object_ops *ops = obj->ops; 2006235783Skib 2007296548Sdumbbell if (obj->pages == NULL) 2008296548Sdumbbell return 0; 2009285988Sdumbbell 2010296548Sdumbbell BUG_ON(obj->gtt_space); 2011285988Sdumbbell 2012296548Sdumbbell if (obj->pages_pin_count) 2013296548Sdumbbell return -EBUSY; 2014285988Sdumbbell 2015296548Sdumbbell /* ->put_pages might need to allocate memory for the bit17 swizzle 2016296548Sdumbbell * array, hence protect them from being reaped by removing them from gtt 2017296548Sdumbbell * lists early. */ 2018296548Sdumbbell list_del(&obj->gtt_list); 2019235783Skib 2020296548Sdumbbell ops->put_pages(obj); 2021296548Sdumbbell obj->pages = NULL; 2022235783Skib 2023296548Sdumbbell if (i915_gem_object_is_purgeable(obj)) 2024296548Sdumbbell i915_gem_object_truncate(obj); 2025285988Sdumbbell 2026296548Sdumbbell return 0; 2027296548Sdumbbell} 2028285988Sdumbbell 2029296548Sdumbbellstatic long 2030296548Sdumbbell__i915_gem_shrink(struct drm_i915_private *dev_priv, long target, 2031296548Sdumbbell bool purgeable_only) 2032296548Sdumbbell{ 2033296548Sdumbbell struct drm_i915_gem_object *obj, *next; 2034296548Sdumbbell long count = 0; 2035296548Sdumbbell 2036296548Sdumbbell list_for_each_entry_safe(obj, next, 2037296548Sdumbbell &dev_priv->mm.unbound_list, 2038296548Sdumbbell gtt_list) { 2039296548Sdumbbell if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && 2040296548Sdumbbell i915_gem_object_put_pages(obj) == 0) { 2041296548Sdumbbell count += obj->base.size >> PAGE_SHIFT; 2042296548Sdumbbell if (target != -1 && count >= target) 2043296548Sdumbbell return count; 2044296548Sdumbbell } 2045235783Skib } 2046235783Skib 2047296548Sdumbbell list_for_each_entry_safe(obj, next, 2048296548Sdumbbell &dev_priv->mm.inactive_list, 2049296548Sdumbbell mm_list) { 2050296548Sdumbbell if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && 2051296548Sdumbbell i915_gem_object_unbind(obj) == 0 && 2052296548Sdumbbell i915_gem_object_put_pages(obj) == 0) { 2053296548Sdumbbell count += obj->base.size >> PAGE_SHIFT; 2054296548Sdumbbell if (target != -1 && count >= target) 2055296548Sdumbbell return count; 2056296548Sdumbbell } 2057285988Sdumbbell } 2058285988Sdumbbell 2059296548Sdumbbell return count; 2060235783Skib} 2061235783Skib 2062296548Sdumbbellstatic long 2063296548Sdumbbelli915_gem_purge(struct drm_i915_private *dev_priv, long target) 2064296548Sdumbbell{ 2065296548Sdumbbell return __i915_gem_shrink(dev_priv, target, true); 2066296548Sdumbbell} 2067296548Sdumbbell 2068296548Sdumbbellstatic void 2069296548Sdumbbelli915_gem_shrink_all(struct drm_i915_private *dev_priv) 2070296548Sdumbbell{ 2071296548Sdumbbell struct drm_i915_gem_object *obj, *next; 2072296548Sdumbbell 2073296548Sdumbbell i915_gem_evict_everything(dev_priv->dev); 2074296548Sdumbbell 2075296548Sdumbbell list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list) 2076296548Sdumbbell i915_gem_object_put_pages(obj); 2077296548Sdumbbell} 2078296548Sdumbbell 2079285988Sdumbbellstatic int 2080285988Sdumbbelli915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, 2081285988Sdumbbell off_t start, off_t end) 2082235783Skib{ 2083285988Sdumbbell vm_object_t vm_obj; 2084285988Sdumbbell vm_page_t page; 2085285988Sdumbbell vm_pindex_t si, ei, i; 2086285988Sdumbbell bool need_swizzle, fresh; 2087235783Skib 2088285988Sdumbbell need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0; 2089285988Sdumbbell vm_obj = obj->base.vm_obj; 2090285988Sdumbbell si = OFF_TO_IDX(trunc_page(start)); 2091285988Sdumbbell ei = OFF_TO_IDX(round_page(end)); 2092285988Sdumbbell VM_OBJECT_WLOCK(vm_obj); 2093285988Sdumbbell for (i = si; i < ei; i++) { 2094285988Sdumbbell page = i915_gem_wire_page(vm_obj, i, &fresh); 2095285988Sdumbbell if (page == NULL) 2096285988Sdumbbell goto failed; 2097285988Sdumbbell if (need_swizzle && fresh) 2098285988Sdumbbell i915_gem_object_do_bit_17_swizzle_page(obj, page); 2099285988Sdumbbell } 2100285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 2101285988Sdumbbell return (0); 2102285988Sdumbbellfailed: 2103285988Sdumbbell i915_gem_object_put_pages_range_locked(obj, si, i); 2104285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 2105285988Sdumbbell return (-EIO); 2106285988Sdumbbell} 2107235783Skib 2108285988Sdumbbellstatic int 2109296548Sdumbbelli915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 2110285988Sdumbbell{ 2111285988Sdumbbell vm_object_t vm_obj; 2112285988Sdumbbell vm_page_t page; 2113285988Sdumbbell vm_pindex_t i, page_count; 2114285988Sdumbbell int res; 2115285988Sdumbbell 2116296548Sdumbbell /* Assert that the object is not currently in any GPU domain. As it 2117296548Sdumbbell * wasn't in the GTT, there shouldn't be any way it could have been in 2118296548Sdumbbell * a GPU cache 2119296548Sdumbbell */ 2120296548Sdumbbell BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 2121296548Sdumbbell BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 2122285988Sdumbbell KASSERT(obj->pages == NULL, ("Obj already has pages")); 2123285988Sdumbbell 2124285988Sdumbbell page_count = OFF_TO_IDX(obj->base.size); 2125285988Sdumbbell obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2126285988Sdumbbell M_WAITOK); 2127285988Sdumbbell res = i915_gem_object_get_pages_range(obj, 0, obj->base.size); 2128285988Sdumbbell if (res != 0) { 2129285988Sdumbbell free(obj->pages, DRM_I915_GEM); 2130285988Sdumbbell obj->pages = NULL; 2131285988Sdumbbell return (res); 2132235783Skib } 2133285988Sdumbbell vm_obj = obj->base.vm_obj; 2134285988Sdumbbell VM_OBJECT_WLOCK(vm_obj); 2135285988Sdumbbell for (i = 0, page = vm_page_lookup(vm_obj, 0); i < page_count; 2136285988Sdumbbell i++, page = vm_page_next(page)) { 2137285988Sdumbbell KASSERT(page->pindex == i, ("pindex %jx %jx", 2138285988Sdumbbell (uintmax_t)page->pindex, (uintmax_t)i)); 2139285988Sdumbbell obj->pages[i] = page; 2140285988Sdumbbell } 2141285988Sdumbbell VM_OBJECT_WUNLOCK(vm_obj); 2142235783Skib return (0); 2143235783Skib} 2144235783Skib 2145296548Sdumbbell/* Ensure that the associated pages are gathered from the backing storage 2146296548Sdumbbell * and pinned into our object. i915_gem_object_get_pages() may be called 2147296548Sdumbbell * multiple times before they are released by a single call to 2148296548Sdumbbell * i915_gem_object_put_pages() - once the pages are no longer referenced 2149296548Sdumbbell * either as a result of memory pressure (reaping pages under the shrinker) 2150296548Sdumbbell * or as the object is itself released. 2151296548Sdumbbell */ 2152296548Sdumbbellint 2153296548Sdumbbelli915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2154296548Sdumbbell{ 2155296548Sdumbbell struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2156296548Sdumbbell const struct drm_i915_gem_object_ops *ops = obj->ops; 2157296548Sdumbbell int ret; 2158296548Sdumbbell 2159296548Sdumbbell if (obj->pages) 2160296548Sdumbbell return 0; 2161296548Sdumbbell 2162296548Sdumbbell BUG_ON(obj->pages_pin_count); 2163296548Sdumbbell 2164296548Sdumbbell ret = ops->get_pages(obj); 2165296548Sdumbbell if (ret) 2166296548Sdumbbell return ret; 2167296548Sdumbbell 2168296548Sdumbbell list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); 2169296548Sdumbbell return 0; 2170296548Sdumbbell} 2171296548Sdumbbell 2172235783Skibvoid 2173235783Skibi915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2174296548Sdumbbell struct intel_ring_buffer *ring) 2175235783Skib{ 2176235783Skib struct drm_device *dev = obj->base.dev; 2177235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2178296548Sdumbbell u32 seqno = intel_ring_get_seqno(ring); 2179235783Skib 2180296548Sdumbbell BUG_ON(ring == NULL); 2181235783Skib obj->ring = ring; 2182235783Skib 2183235783Skib /* Add a reference if we're newly entering the active list. */ 2184235783Skib if (!obj->active) { 2185235783Skib drm_gem_object_reference(&obj->base); 2186235783Skib obj->active = 1; 2187235783Skib } 2188235783Skib 2189235783Skib /* Move from whatever list we were on to the tail of execution. */ 2190235783Skib list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2191235783Skib list_move_tail(&obj->ring_list, &ring->active_list); 2192235783Skib 2193296548Sdumbbell obj->last_read_seqno = seqno; 2194296548Sdumbbell 2195235783Skib if (obj->fenced_gpu_access) { 2196235783Skib obj->last_fenced_seqno = seqno; 2197235783Skib 2198235783Skib /* Bump MRU to take account of the delayed flush */ 2199235783Skib if (obj->fence_reg != I915_FENCE_REG_NONE) { 2200296548Sdumbbell struct drm_i915_fence_reg *reg; 2201296548Sdumbbell 2202235783Skib reg = &dev_priv->fence_regs[obj->fence_reg]; 2203235783Skib list_move_tail(®->lru_list, 2204235783Skib &dev_priv->mm.fence_list); 2205235783Skib } 2206235783Skib } 2207235783Skib} 2208235783Skib 2209235783Skibstatic void 2210235783Skibi915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2211235783Skib{ 2212235783Skib struct drm_device *dev = obj->base.dev; 2213235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2214235783Skib 2215296548Sdumbbell BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); 2216296548Sdumbbell BUG_ON(!obj->active); 2217296548Sdumbbell 2218277487Skib list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2219235783Skib 2220296548Sdumbbell list_del_init(&obj->ring_list); 2221235783Skib obj->ring = NULL; 2222235783Skib 2223296548Sdumbbell obj->last_read_seqno = 0; 2224296548Sdumbbell obj->last_write_seqno = 0; 2225296548Sdumbbell obj->base.write_domain = 0; 2226296548Sdumbbell 2227296548Sdumbbell obj->last_fenced_seqno = 0; 2228235783Skib obj->fenced_gpu_access = false; 2229235783Skib 2230235783Skib obj->active = 0; 2231235783Skib drm_gem_object_unreference(&obj->base); 2232235783Skib 2233235783Skib WARN_ON(i915_verify_lists(dev)); 2234235783Skib} 2235235783Skib 2236296548Sdumbbellstatic int 2237296548Sdumbbelli915_gem_handle_seqno_wrap(struct drm_device *dev) 2238235783Skib{ 2239296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 2240296548Sdumbbell struct intel_ring_buffer *ring; 2241296548Sdumbbell int ret, i, j; 2242235783Skib 2243296548Sdumbbell /* The hardware uses various monotonic 32-bit counters, if we 2244296548Sdumbbell * detect that they will wraparound we need to idle the GPU 2245296548Sdumbbell * and reset those counters. 2246296548Sdumbbell */ 2247296548Sdumbbell ret = 0; 2248296548Sdumbbell for_each_ring(ring, dev_priv, i) { 2249296548Sdumbbell for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) 2250296548Sdumbbell ret |= ring->sync_seqno[j] != 0; 2251296548Sdumbbell } 2252296548Sdumbbell if (ret == 0) 2253296548Sdumbbell return ret; 2254235783Skib 2255296548Sdumbbell ret = i915_gpu_idle(dev); 2256296548Sdumbbell if (ret) 2257296548Sdumbbell return ret; 2258296548Sdumbbell 2259296548Sdumbbell i915_gem_retire_requests(dev); 2260296548Sdumbbell for_each_ring(ring, dev_priv, i) { 2261296548Sdumbbell for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) 2262296548Sdumbbell ring->sync_seqno[j] = 0; 2263296548Sdumbbell } 2264296548Sdumbbell 2265296548Sdumbbell return 0; 2266235783Skib} 2267235783Skib 2268296548Sdumbbellint 2269296548Sdumbbelli915_gem_get_seqno(struct drm_device *dev, u32 *seqno) 2270235783Skib{ 2271296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 2272235783Skib 2273296548Sdumbbell /* reserve 0 for non-seqno */ 2274296548Sdumbbell if (dev_priv->next_seqno == 0) { 2275296548Sdumbbell int ret = i915_gem_handle_seqno_wrap(dev); 2276296548Sdumbbell if (ret) 2277296548Sdumbbell return ret; 2278296548Sdumbbell 2279296548Sdumbbell dev_priv->next_seqno = 1; 2280296548Sdumbbell } 2281296548Sdumbbell 2282296548Sdumbbell *seqno = dev_priv->next_seqno++; 2283296548Sdumbbell return 0; 2284235783Skib} 2285235783Skib 2286235783Skibint 2287285988Sdumbbelli915_add_request(struct intel_ring_buffer *ring, 2288285988Sdumbbell struct drm_file *file, 2289296548Sdumbbell u32 *out_seqno) 2290235783Skib{ 2291285988Sdumbbell drm_i915_private_t *dev_priv = ring->dev->dev_private; 2292296548Sdumbbell struct drm_i915_gem_request *request; 2293235783Skib u32 request_ring_position; 2294235783Skib int was_empty; 2295235783Skib int ret; 2296235783Skib 2297296548Sdumbbell /* 2298296548Sdumbbell * Emit any outstanding flushes - execbuf can fail to emit the flush 2299296548Sdumbbell * after having emitted the batchbuffer command. Hence we need to fix 2300296548Sdumbbell * things up similar to emitting the lazy request. The difference here 2301296548Sdumbbell * is that the flush _must_ happen before the next request, no matter 2302296548Sdumbbell * what. 2303296548Sdumbbell */ 2304296548Sdumbbell ret = intel_ring_flush_all_caches(ring); 2305296548Sdumbbell if (ret) 2306296548Sdumbbell return ret; 2307235783Skib 2308296548Sdumbbell request = malloc(sizeof(*request), DRM_I915_GEM, M_NOWAIT); 2309296548Sdumbbell if (request == NULL) 2310296548Sdumbbell return -ENOMEM; 2311296548Sdumbbell 2312296548Sdumbbell 2313296548Sdumbbell /* Record the position of the start of the request so that 2314296548Sdumbbell * should we detect the updated seqno part-way through the 2315296548Sdumbbell * GPU processing the request, we never over-estimate the 2316296548Sdumbbell * position of the head. 2317296548Sdumbbell */ 2318235783Skib request_ring_position = intel_ring_get_tail(ring); 2319235783Skib 2320296548Sdumbbell ret = ring->add_request(ring); 2321296548Sdumbbell if (ret) { 2322296548Sdumbbell free(request, DRM_I915_GEM); 2323296548Sdumbbell return ret; 2324296548Sdumbbell } 2325235783Skib 2326296548Sdumbbell request->seqno = intel_ring_get_seqno(ring); 2327235783Skib request->ring = ring; 2328235783Skib request->tail = request_ring_position; 2329296548Sdumbbell request->emitted_jiffies = jiffies; 2330235783Skib was_empty = list_empty(&ring->request_list); 2331235783Skib list_add_tail(&request->list, &ring->request_list); 2332296548Sdumbbell request->file_priv = NULL; 2333235783Skib 2334285988Sdumbbell if (file) { 2335296548Sdumbbell struct drm_i915_file_private *file_priv = file->driver_priv; 2336235783Skib 2337296548Sdumbbell mtx_lock(&file_priv->mm.lock); 2338235783Skib request->file_priv = file_priv; 2339235783Skib list_add_tail(&request->client_list, 2340285988Sdumbbell &file_priv->mm.request_list); 2341296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 2342235783Skib } 2343235783Skib 2344296548Sdumbbell CTR2(KTR_DRM, "request_add %s %d", ring->name, request->seqno); 2345235783Skib ring->outstanding_lazy_request = 0; 2346235783Skib 2347235783Skib if (!dev_priv->mm.suspended) { 2348235783Skib if (i915_enable_hangcheck) { 2349235783Skib callout_schedule(&dev_priv->hangcheck_timer, 2350235783Skib DRM_I915_HANGCHECK_PERIOD); 2351235783Skib } 2352296548Sdumbbell if (was_empty) { 2353296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 2354296548Sdumbbell &dev_priv->mm.retire_work, hz); 2355296548Sdumbbell intel_mark_busy(dev_priv->dev); 2356296548Sdumbbell } 2357235783Skib } 2358285988Sdumbbell 2359296548Sdumbbell if (out_seqno) 2360296548Sdumbbell *out_seqno = request->seqno; 2361285988Sdumbbell return 0; 2362235783Skib} 2363235783Skib 2364235783Skibstatic inline void 2365235783Skibi915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2366235783Skib{ 2367235783Skib struct drm_i915_file_private *file_priv = request->file_priv; 2368235783Skib 2369235783Skib if (!file_priv) 2370235783Skib return; 2371235783Skib 2372296548Sdumbbell mtx_lock(&file_priv->mm.lock); 2373285988Sdumbbell if (request->file_priv) { 2374235783Skib list_del(&request->client_list); 2375235783Skib request->file_priv = NULL; 2376235783Skib } 2377296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 2378235783Skib} 2379235783Skib 2380285988Sdumbbellstatic void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2381285988Sdumbbell struct intel_ring_buffer *ring) 2382235783Skib{ 2383235783Skib if (ring->dev != NULL) 2384235783Skib DRM_LOCK_ASSERT(ring->dev); 2385235783Skib 2386235783Skib while (!list_empty(&ring->request_list)) { 2387235783Skib struct drm_i915_gem_request *request; 2388235783Skib 2389235783Skib request = list_first_entry(&ring->request_list, 2390285988Sdumbbell struct drm_i915_gem_request, 2391285988Sdumbbell list); 2392235783Skib 2393235783Skib list_del(&request->list); 2394235783Skib i915_gem_request_remove_from_client(request); 2395235783Skib free(request, DRM_I915_GEM); 2396235783Skib } 2397235783Skib 2398235783Skib while (!list_empty(&ring->active_list)) { 2399235783Skib struct drm_i915_gem_object *obj; 2400235783Skib 2401235783Skib obj = list_first_entry(&ring->active_list, 2402285988Sdumbbell struct drm_i915_gem_object, 2403285988Sdumbbell ring_list); 2404235783Skib 2405235783Skib i915_gem_object_move_to_inactive(obj); 2406235783Skib } 2407235783Skib} 2408235783Skib 2409285988Sdumbbellstatic void i915_gem_reset_fences(struct drm_device *dev) 2410235783Skib{ 2411235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2412235783Skib int i; 2413235783Skib 2414235783Skib for (i = 0; i < dev_priv->num_fence_regs; i++) { 2415235783Skib struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2416235783Skib 2417277487Skib i915_gem_write_fence(dev, i, NULL); 2418235783Skib 2419277487Skib if (reg->obj) 2420277487Skib i915_gem_object_fence_lost(reg->obj); 2421235783Skib 2422277487Skib reg->pin_count = 0; 2423277487Skib reg->obj = NULL; 2424277487Skib INIT_LIST_HEAD(®->lru_list); 2425235783Skib } 2426277487Skib 2427277487Skib INIT_LIST_HEAD(&dev_priv->mm.fence_list); 2428235783Skib} 2429235783Skib 2430285988Sdumbbellvoid i915_gem_reset(struct drm_device *dev) 2431235783Skib{ 2432235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 2433235783Skib struct drm_i915_gem_object *obj; 2434277487Skib struct intel_ring_buffer *ring; 2435235783Skib int i; 2436235783Skib 2437277487Skib for_each_ring(ring, dev_priv, i) 2438277487Skib i915_gem_reset_ring_lists(dev_priv, ring); 2439235783Skib 2440235783Skib /* Move everything out of the GPU domains to ensure we do any 2441235783Skib * necessary invalidation upon reuse. 2442235783Skib */ 2443296548Sdumbbell list_for_each_entry(obj, 2444296548Sdumbbell &dev_priv->mm.inactive_list, 2445296548Sdumbbell mm_list) 2446296548Sdumbbell { 2447235783Skib obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2448235783Skib } 2449235783Skib 2450235783Skib /* The fence registers are invalidated so clear them out */ 2451235783Skib i915_gem_reset_fences(dev); 2452235783Skib} 2453235783Skib 2454235783Skib/** 2455235783Skib * This function clears the request list as sequence numbers are passed. 2456235783Skib */ 2457235783Skibvoid 2458235783Skibi915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2459235783Skib{ 2460235783Skib uint32_t seqno; 2461235783Skib 2462235783Skib if (list_empty(&ring->request_list)) 2463235783Skib return; 2464235783Skib 2465296548Sdumbbell WARN_ON(i915_verify_lists(ring->dev)); 2466296548Sdumbbell 2467296548Sdumbbell seqno = ring->get_seqno(ring, true); 2468235783Skib CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); 2469235783Skib 2470235783Skib while (!list_empty(&ring->request_list)) { 2471235783Skib struct drm_i915_gem_request *request; 2472235783Skib 2473235783Skib request = list_first_entry(&ring->request_list, 2474235783Skib struct drm_i915_gem_request, 2475235783Skib list); 2476235783Skib 2477235783Skib if (!i915_seqno_passed(seqno, request->seqno)) 2478235783Skib break; 2479235783Skib 2480235783Skib CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", 2481235783Skib ring->name, seqno); 2482296548Sdumbbell /* We know the GPU must have read the request to have 2483296548Sdumbbell * sent us the seqno + interrupt, so use the position 2484296548Sdumbbell * of tail of the request to update the last known position 2485296548Sdumbbell * of the GPU head. 2486296548Sdumbbell */ 2487235783Skib ring->last_retired_head = request->tail; 2488235783Skib 2489235783Skib list_del(&request->list); 2490235783Skib i915_gem_request_remove_from_client(request); 2491235783Skib free(request, DRM_I915_GEM); 2492235783Skib } 2493235783Skib 2494235783Skib /* Move any buffers on the active list that are no longer referenced 2495235783Skib * by the ringbuffer to the flushing/inactive lists as appropriate. 2496235783Skib */ 2497235783Skib while (!list_empty(&ring->active_list)) { 2498235783Skib struct drm_i915_gem_object *obj; 2499235783Skib 2500235783Skib obj = list_first_entry(&ring->active_list, 2501235783Skib struct drm_i915_gem_object, 2502235783Skib ring_list); 2503235783Skib 2504296548Sdumbbell if (!i915_seqno_passed(seqno, obj->last_read_seqno)) 2505235783Skib break; 2506235783Skib 2507296548Sdumbbell i915_gem_object_move_to_inactive(obj); 2508235783Skib } 2509235783Skib 2510296548Sdumbbell if (unlikely(ring->trace_irq_seqno && 2511296548Sdumbbell i915_seqno_passed(seqno, ring->trace_irq_seqno))) { 2512235783Skib ring->irq_put(ring); 2513235783Skib ring->trace_irq_seqno = 0; 2514235783Skib } 2515296548Sdumbbell 2516296548Sdumbbell WARN_ON(i915_verify_lists(ring->dev)); 2517235783Skib} 2518235783Skib 2519235783Skibvoid 2520235783Skibi915_gem_retire_requests(struct drm_device *dev) 2521235783Skib{ 2522235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2523277487Skib struct intel_ring_buffer *ring; 2524235783Skib int i; 2525235783Skib 2526277487Skib for_each_ring(ring, dev_priv, i) 2527277487Skib i915_gem_retire_requests_ring(ring); 2528235783Skib} 2529235783Skib 2530285988Sdumbbellstatic void 2531296548Sdumbbelli915_gem_retire_work_handler(void *arg, int pending) 2532285988Sdumbbell{ 2533285988Sdumbbell drm_i915_private_t *dev_priv; 2534285988Sdumbbell struct drm_device *dev; 2535285988Sdumbbell struct intel_ring_buffer *ring; 2536285988Sdumbbell bool idle; 2537285988Sdumbbell int i; 2538285988Sdumbbell 2539285988Sdumbbell dev_priv = arg; 2540285988Sdumbbell dev = dev_priv->dev; 2541285988Sdumbbell 2542285988Sdumbbell /* Come back later if the device is busy... */ 2543285988Sdumbbell if (!sx_try_xlock(&dev->dev_struct_lock)) { 2544296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 2545296548Sdumbbell &dev_priv->mm.retire_work, hz); 2546285988Sdumbbell return; 2547285988Sdumbbell } 2548285988Sdumbbell 2549285988Sdumbbell CTR0(KTR_DRM, "retire_task"); 2550285988Sdumbbell 2551285988Sdumbbell i915_gem_retire_requests(dev); 2552285988Sdumbbell 2553285988Sdumbbell /* Send a periodic flush down the ring so we don't hold onto GEM 2554285988Sdumbbell * objects indefinitely. 2555285988Sdumbbell */ 2556285988Sdumbbell idle = true; 2557285988Sdumbbell for_each_ring(ring, dev_priv, i) { 2558296548Sdumbbell if (ring->gpu_caches_dirty) 2559296548Sdumbbell i915_add_request(ring, NULL, NULL); 2560285988Sdumbbell 2561285988Sdumbbell idle &= list_empty(&ring->request_list); 2562285988Sdumbbell } 2563285988Sdumbbell 2564285988Sdumbbell if (!dev_priv->mm.suspended && !idle) 2565296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 2566296548Sdumbbell &dev_priv->mm.retire_work, hz); 2567296548Sdumbbell if (idle) 2568296548Sdumbbell intel_mark_idle(dev); 2569285988Sdumbbell 2570285988Sdumbbell DRM_UNLOCK(dev); 2571285988Sdumbbell} 2572285988Sdumbbell 2573296548Sdumbbell/** 2574296548Sdumbbell * Ensures that an object will eventually get non-busy by flushing any required 2575296548Sdumbbell * write domains, emitting any outstanding lazy request and retiring and 2576296548Sdumbbell * completed requests. 2577296548Sdumbbell */ 2578296548Sdumbbellstatic int 2579296548Sdumbbelli915_gem_object_flush_active(struct drm_i915_gem_object *obj) 2580296548Sdumbbell{ 2581296548Sdumbbell int ret; 2582296548Sdumbbell 2583296548Sdumbbell if (obj->active) { 2584296548Sdumbbell ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); 2585296548Sdumbbell if (ret) 2586296548Sdumbbell return ret; 2587296548Sdumbbell 2588296548Sdumbbell i915_gem_retire_requests_ring(obj->ring); 2589296548Sdumbbell } 2590296548Sdumbbell 2591296548Sdumbbell return 0; 2592296548Sdumbbell} 2593296548Sdumbbell 2594296548Sdumbbell/** 2595296548Sdumbbell * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 2596296548Sdumbbell * @DRM_IOCTL_ARGS: standard ioctl arguments 2597296548Sdumbbell * 2598296548Sdumbbell * Returns 0 if successful, else an error is returned with the remaining time in 2599296548Sdumbbell * the timeout parameter. 2600296548Sdumbbell * -ETIME: object is still busy after timeout 2601296548Sdumbbell * -ERESTARTSYS: signal interrupted the wait 2602296548Sdumbbell * -ENONENT: object doesn't exist 2603296548Sdumbbell * Also possible, but rare: 2604296548Sdumbbell * -EAGAIN: GPU wedged 2605296548Sdumbbell * -ENOMEM: damn 2606296548Sdumbbell * -ENODEV: Internal IRQ fail 2607296548Sdumbbell * -E?: The add request failed 2608296548Sdumbbell * 2609296548Sdumbbell * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 2610296548Sdumbbell * non-zero timeout parameter the wait ioctl will wait for the given number of 2611296548Sdumbbell * nanoseconds on an object becoming unbusy. Since the wait itself does so 2612296548Sdumbbell * without holding struct_mutex the object may become re-busied before this 2613296548Sdumbbell * function completes. A similar but shorter * race condition exists in the busy 2614296548Sdumbbell * ioctl 2615296548Sdumbbell */ 2616285988Sdumbbellint 2617296548Sdumbbelli915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 2618296548Sdumbbell{ 2619296548Sdumbbell struct drm_i915_gem_wait *args = data; 2620296548Sdumbbell struct drm_i915_gem_object *obj; 2621296548Sdumbbell struct intel_ring_buffer *ring = NULL; 2622296548Sdumbbell struct timespec timeout_stack, *timeout = NULL; 2623296548Sdumbbell u32 seqno = 0; 2624296548Sdumbbell int ret = 0; 2625296548Sdumbbell 2626296548Sdumbbell if (args->timeout_ns >= 0) { 2627296548Sdumbbell timeout_stack.tv_sec = args->timeout_ns / 1000000; 2628296548Sdumbbell timeout_stack.tv_nsec = args->timeout_ns % 1000000; 2629296548Sdumbbell timeout = &timeout_stack; 2630296548Sdumbbell } 2631296548Sdumbbell 2632296548Sdumbbell ret = i915_mutex_lock_interruptible(dev); 2633296548Sdumbbell if (ret) 2634296548Sdumbbell return ret; 2635296548Sdumbbell 2636296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle)); 2637296548Sdumbbell if (&obj->base == NULL) { 2638296548Sdumbbell DRM_UNLOCK(dev); 2639296548Sdumbbell return -ENOENT; 2640296548Sdumbbell } 2641296548Sdumbbell 2642296548Sdumbbell /* Need to make sure the object gets inactive eventually. */ 2643296548Sdumbbell ret = i915_gem_object_flush_active(obj); 2644296548Sdumbbell if (ret) 2645296548Sdumbbell goto out; 2646296548Sdumbbell 2647296548Sdumbbell if (obj->active) { 2648296548Sdumbbell seqno = obj->last_read_seqno; 2649296548Sdumbbell ring = obj->ring; 2650296548Sdumbbell } 2651296548Sdumbbell 2652296548Sdumbbell if (seqno == 0) 2653296548Sdumbbell goto out; 2654296548Sdumbbell 2655296548Sdumbbell /* Do this after OLR check to make sure we make forward progress polling 2656296548Sdumbbell * on this IOCTL with a 0 timeout (like busy ioctl) 2657296548Sdumbbell */ 2658296548Sdumbbell if (!args->timeout_ns) { 2659296548Sdumbbell ret = -ETIMEDOUT; 2660296548Sdumbbell goto out; 2661296548Sdumbbell } 2662296548Sdumbbell 2663296548Sdumbbell drm_gem_object_unreference(&obj->base); 2664296548Sdumbbell DRM_UNLOCK(dev); 2665296548Sdumbbell 2666296548Sdumbbell ret = __wait_seqno(ring, seqno, true, timeout); 2667296548Sdumbbell if (timeout) { 2668296548Sdumbbell args->timeout_ns = timeout->tv_sec * 1000000 + timeout->tv_nsec; 2669296548Sdumbbell } 2670296548Sdumbbell return ret; 2671296548Sdumbbell 2672296548Sdumbbellout: 2673296548Sdumbbell drm_gem_object_unreference(&obj->base); 2674296548Sdumbbell DRM_UNLOCK(dev); 2675296548Sdumbbell return ret; 2676296548Sdumbbell} 2677296548Sdumbbell 2678296548Sdumbbell/** 2679296548Sdumbbell * i915_gem_object_sync - sync an object to a ring. 2680296548Sdumbbell * 2681296548Sdumbbell * @obj: object which may be in use on another ring. 2682296548Sdumbbell * @to: ring we wish to use the object on. May be NULL. 2683296548Sdumbbell * 2684296548Sdumbbell * This code is meant to abstract object synchronization with the GPU. 2685296548Sdumbbell * Calling with NULL implies synchronizing the object with the CPU 2686296548Sdumbbell * rather than a particular GPU ring. 2687296548Sdumbbell * 2688296548Sdumbbell * Returns 0 if successful, else propagates up the lower layer error. 2689296548Sdumbbell */ 2690296548Sdumbbellint 2691285988Sdumbbelli915_gem_object_sync(struct drm_i915_gem_object *obj, 2692285988Sdumbbell struct intel_ring_buffer *to) 2693285988Sdumbbell{ 2694285988Sdumbbell struct intel_ring_buffer *from = obj->ring; 2695285988Sdumbbell u32 seqno; 2696285988Sdumbbell int ret, idx; 2697285988Sdumbbell 2698285988Sdumbbell if (from == NULL || to == from) 2699285988Sdumbbell return 0; 2700285988Sdumbbell 2701285988Sdumbbell if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) 2702296548Sdumbbell return i915_gem_object_wait_rendering(obj, false); 2703285988Sdumbbell 2704285988Sdumbbell idx = intel_ring_sync_index(from, to); 2705285988Sdumbbell 2706296548Sdumbbell seqno = obj->last_read_seqno; 2707285988Sdumbbell if (seqno <= from->sync_seqno[idx]) 2708285988Sdumbbell return 0; 2709285988Sdumbbell 2710296548Sdumbbell ret = i915_gem_check_olr(obj->ring, seqno); 2711296548Sdumbbell if (ret) 2712296548Sdumbbell return ret; 2713285988Sdumbbell 2714285988Sdumbbell ret = to->sync_to(to, from, seqno); 2715285988Sdumbbell if (!ret) 2716296548Sdumbbell /* We use last_read_seqno because sync_to() 2717296548Sdumbbell * might have just caused seqno wrap under 2718296548Sdumbbell * the radar. 2719296548Sdumbbell */ 2720296548Sdumbbell from->sync_seqno[idx] = obj->last_read_seqno; 2721285988Sdumbbell 2722285988Sdumbbell return ret; 2723285988Sdumbbell} 2724285988Sdumbbell 2725285988Sdumbbellstatic void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2726285988Sdumbbell{ 2727285988Sdumbbell u32 old_write_domain, old_read_domains; 2728285988Sdumbbell 2729285988Sdumbbell /* Act a barrier for all accesses through the GTT */ 2730285988Sdumbbell mb(); 2731285988Sdumbbell 2732285988Sdumbbell /* Force a pagefault for domain tracking on next user access */ 2733285988Sdumbbell i915_gem_release_mmap(obj); 2734285988Sdumbbell 2735285988Sdumbbell if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2736285988Sdumbbell return; 2737285988Sdumbbell 2738285988Sdumbbell old_read_domains = obj->base.read_domains; 2739285988Sdumbbell old_write_domain = obj->base.write_domain; 2740285988Sdumbbell 2741285988Sdumbbell obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2742285988Sdumbbell obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2743285988Sdumbbell 2744285988Sdumbbell CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", 2745285988Sdumbbell obj, old_read_domains, old_write_domain); 2746285988Sdumbbell} 2747285988Sdumbbell 2748285988Sdumbbell/** 2749285988Sdumbbell * Unbinds an object from the GTT aperture. 2750285988Sdumbbell */ 2751285988Sdumbbellint 2752285988Sdumbbelli915_gem_object_unbind(struct drm_i915_gem_object *obj) 2753285988Sdumbbell{ 2754285988Sdumbbell drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 2755285988Sdumbbell int ret = 0; 2756285988Sdumbbell 2757285988Sdumbbell if (obj->gtt_space == NULL) 2758285988Sdumbbell return 0; 2759285988Sdumbbell 2760289109Sdumbbell if (obj->pin_count) 2761296548Sdumbbell return -EBUSY; 2762285988Sdumbbell 2763296548Sdumbbell BUG_ON(obj->pages == NULL); 2764296548Sdumbbell 2765285988Sdumbbell ret = i915_gem_object_finish_gpu(obj); 2766296548Sdumbbell if (ret) 2767285988Sdumbbell return ret; 2768296548Sdumbbell /* Continue on if we fail due to EIO, the GPU is hung so we 2769296548Sdumbbell * should be safe and we need to cleanup or else we might 2770296548Sdumbbell * cause memory corruption through use-after-free. 2771296548Sdumbbell */ 2772285988Sdumbbell 2773285988Sdumbbell i915_gem_object_finish_gtt(obj); 2774285988Sdumbbell 2775285988Sdumbbell /* release the fence reg _after_ flushing */ 2776285988Sdumbbell ret = i915_gem_object_put_fence(obj); 2777285988Sdumbbell if (ret) 2778285988Sdumbbell return ret; 2779285988Sdumbbell 2780285988Sdumbbell if (obj->has_global_gtt_mapping) 2781285988Sdumbbell i915_gem_gtt_unbind_object(obj); 2782285988Sdumbbell if (obj->has_aliasing_ppgtt_mapping) { 2783285988Sdumbbell i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2784285988Sdumbbell obj->has_aliasing_ppgtt_mapping = 0; 2785285988Sdumbbell } 2786285988Sdumbbell i915_gem_gtt_finish_object(obj); 2787285988Sdumbbell 2788296548Sdumbbell list_del(&obj->mm_list); 2789296548Sdumbbell list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); 2790296548Sdumbbell /* Avoid an unnecessary call to unbind on rebind. */ 2791285988Sdumbbell obj->map_and_fenceable = true; 2792285988Sdumbbell 2793285988Sdumbbell drm_mm_put_block(obj->gtt_space); 2794285988Sdumbbell obj->gtt_space = NULL; 2795285988Sdumbbell obj->gtt_offset = 0; 2796285988Sdumbbell 2797296548Sdumbbell return 0; 2798285988Sdumbbell} 2799285988Sdumbbell 2800285988Sdumbbellint i915_gpu_idle(struct drm_device *dev) 2801285988Sdumbbell{ 2802285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 2803285988Sdumbbell struct intel_ring_buffer *ring; 2804285988Sdumbbell int ret, i; 2805285988Sdumbbell 2806285988Sdumbbell /* Flush everything onto the inactive list. */ 2807285988Sdumbbell for_each_ring(ring, dev_priv, i) { 2808285988Sdumbbell ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); 2809285988Sdumbbell if (ret) 2810285988Sdumbbell return ret; 2811285988Sdumbbell 2812296548Sdumbbell ret = intel_ring_idle(ring); 2813285988Sdumbbell if (ret) 2814285988Sdumbbell return ret; 2815285988Sdumbbell } 2816285988Sdumbbell 2817285988Sdumbbell return 0; 2818285988Sdumbbell} 2819285988Sdumbbell 2820277487Skibstatic void sandybridge_write_fence_reg(struct drm_device *dev, int reg, 2821277487Skib struct drm_i915_gem_object *obj) 2822235783Skib{ 2823235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2824235783Skib uint64_t val; 2825235783Skib 2826277487Skib if (obj) { 2827277487Skib u32 size = obj->gtt_space->size; 2828235783Skib 2829277487Skib val = (uint64_t)((obj->gtt_offset + size - 4096) & 2830277487Skib 0xfffff000) << 32; 2831277487Skib val |= obj->gtt_offset & 0xfffff000; 2832277487Skib val |= (uint64_t)((obj->stride / 128) - 1) << 2833277487Skib SANDYBRIDGE_FENCE_PITCH_SHIFT; 2834235783Skib 2835277487Skib if (obj->tiling_mode == I915_TILING_Y) 2836277487Skib val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2837277487Skib val |= I965_FENCE_REG_VALID; 2838235783Skib } else 2839277487Skib val = 0; 2840235783Skib 2841277487Skib I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); 2842277487Skib POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); 2843235783Skib} 2844235783Skib 2845277487Skibstatic void i965_write_fence_reg(struct drm_device *dev, int reg, 2846277487Skib struct drm_i915_gem_object *obj) 2847235783Skib{ 2848235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2849235783Skib uint64_t val; 2850235783Skib 2851277487Skib if (obj) { 2852277487Skib u32 size = obj->gtt_space->size; 2853235783Skib 2854277487Skib val = (uint64_t)((obj->gtt_offset + size - 4096) & 2855277487Skib 0xfffff000) << 32; 2856277487Skib val |= obj->gtt_offset & 0xfffff000; 2857277487Skib val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2858277487Skib if (obj->tiling_mode == I915_TILING_Y) 2859277487Skib val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2860277487Skib val |= I965_FENCE_REG_VALID; 2861235783Skib } else 2862277487Skib val = 0; 2863235783Skib 2864277487Skib I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); 2865277487Skib POSTING_READ(FENCE_REG_965_0 + reg * 8); 2866235783Skib} 2867235783Skib 2868277487Skibstatic void i915_write_fence_reg(struct drm_device *dev, int reg, 2869277487Skib struct drm_i915_gem_object *obj) 2870235783Skib{ 2871235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2872277487Skib u32 val; 2873235783Skib 2874277487Skib if (obj) { 2875277487Skib u32 size = obj->gtt_space->size; 2876277487Skib int pitch_val; 2877277487Skib int tile_width; 2878235783Skib 2879296548Sdumbbell WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || 2880277487Skib (size & -size) != size || 2881296548Sdumbbell (obj->gtt_offset & (size - 1)), 2882277487Skib "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 2883277487Skib obj->gtt_offset, obj->map_and_fenceable, size); 2884235783Skib 2885277487Skib if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 2886277487Skib tile_width = 128; 2887277487Skib else 2888277487Skib tile_width = 512; 2889235783Skib 2890277487Skib /* Note: pitch better be a power of two tile widths */ 2891277487Skib pitch_val = obj->stride / tile_width; 2892277487Skib pitch_val = ffs(pitch_val) - 1; 2893235783Skib 2894277487Skib val = obj->gtt_offset; 2895277487Skib if (obj->tiling_mode == I915_TILING_Y) 2896277487Skib val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2897277487Skib val |= I915_FENCE_SIZE_BITS(size); 2898277487Skib val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2899277487Skib val |= I830_FENCE_REG_VALID; 2900277487Skib } else 2901277487Skib val = 0; 2902277487Skib 2903277487Skib if (reg < 8) 2904277487Skib reg = FENCE_REG_830_0 + reg * 4; 2905235783Skib else 2906277487Skib reg = FENCE_REG_945_8 + (reg - 8) * 4; 2907235783Skib 2908277487Skib I915_WRITE(reg, val); 2909277487Skib POSTING_READ(reg); 2910235783Skib} 2911235783Skib 2912277487Skibstatic void i830_write_fence_reg(struct drm_device *dev, int reg, 2913277487Skib struct drm_i915_gem_object *obj) 2914235783Skib{ 2915235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 2916235783Skib uint32_t val; 2917235783Skib 2918277487Skib if (obj) { 2919277487Skib u32 size = obj->gtt_space->size; 2920277487Skib uint32_t pitch_val; 2921235783Skib 2922296548Sdumbbell WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || 2923277487Skib (size & -size) != size || 2924296548Sdumbbell (obj->gtt_offset & (size - 1)), 2925277487Skib "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 2926277487Skib obj->gtt_offset, size); 2927235783Skib 2928277487Skib pitch_val = obj->stride / 128; 2929277487Skib pitch_val = ffs(pitch_val) - 1; 2930235783Skib 2931277487Skib val = obj->gtt_offset; 2932277487Skib if (obj->tiling_mode == I915_TILING_Y) 2933277487Skib val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2934277487Skib val |= I830_FENCE_SIZE_BITS(size); 2935277487Skib val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2936277487Skib val |= I830_FENCE_REG_VALID; 2937235783Skib } else 2938277487Skib val = 0; 2939235783Skib 2940277487Skib I915_WRITE(FENCE_REG_830_0 + reg * 4, val); 2941277487Skib POSTING_READ(FENCE_REG_830_0 + reg * 4); 2942235783Skib} 2943235783Skib 2944277487Skibstatic void i915_gem_write_fence(struct drm_device *dev, int reg, 2945277487Skib struct drm_i915_gem_object *obj) 2946235783Skib{ 2947277487Skib switch (INTEL_INFO(dev)->gen) { 2948277487Skib case 7: 2949277487Skib case 6: sandybridge_write_fence_reg(dev, reg, obj); break; 2950277487Skib case 5: 2951277487Skib case 4: i965_write_fence_reg(dev, reg, obj); break; 2952277487Skib case 3: i915_write_fence_reg(dev, reg, obj); break; 2953277487Skib case 2: i830_write_fence_reg(dev, reg, obj); break; 2954277487Skib default: break; 2955277487Skib } 2956235783Skib} 2957235783Skib 2958277487Skibstatic inline int fence_number(struct drm_i915_private *dev_priv, 2959277487Skib struct drm_i915_fence_reg *fence) 2960277487Skib{ 2961277487Skib return fence - dev_priv->fence_regs; 2962277487Skib} 2963277487Skib 2964296548Sdumbbellstatic void i915_gem_write_fence__ipi(void *data) 2965296548Sdumbbell{ 2966296548Sdumbbell wbinvd(); 2967296548Sdumbbell} 2968296548Sdumbbell 2969277487Skibstatic void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, 2970277487Skib struct drm_i915_fence_reg *fence, 2971277487Skib bool enable) 2972277487Skib{ 2973285988Sdumbbell struct drm_device *dev = obj->base.dev; 2974285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 2975285988Sdumbbell int fence_reg = fence_number(dev_priv, fence); 2976277487Skib 2977296548Sdumbbell /* In order to fully serialize access to the fenced region and 2978296548Sdumbbell * the update to the fence register we need to take extreme 2979296548Sdumbbell * measures on SNB+. In theory, the write to the fence register 2980296548Sdumbbell * flushes all memory transactions before, and coupled with the 2981296548Sdumbbell * mb() placed around the register write we serialise all memory 2982296548Sdumbbell * operations with respect to the changes in the tiler. Yet, on 2983296548Sdumbbell * SNB+ we need to take a step further and emit an explicit wbinvd() 2984296548Sdumbbell * on each processor in order to manually flush all memory 2985296548Sdumbbell * transactions before updating the fence register. 2986296548Sdumbbell */ 2987296548Sdumbbell if (HAS_LLC(obj->base.dev)) 2988296548Sdumbbell on_each_cpu(i915_gem_write_fence__ipi, NULL, 1); 2989285988Sdumbbell i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL); 2990277487Skib 2991277487Skib if (enable) { 2992285988Sdumbbell obj->fence_reg = fence_reg; 2993277487Skib fence->obj = obj; 2994277487Skib list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); 2995277487Skib } else { 2996277487Skib obj->fence_reg = I915_FENCE_REG_NONE; 2997277487Skib fence->obj = NULL; 2998277487Skib list_del_init(&fence->lru_list); 2999277487Skib } 3000277487Skib} 3001277487Skib 3002235783Skibstatic int 3003277487Skibi915_gem_object_flush_fence(struct drm_i915_gem_object *obj) 3004235783Skib{ 3005277487Skib if (obj->last_fenced_seqno) { 3006296548Sdumbbell int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); 3007277487Skib if (ret) 3008277487Skib return ret; 3009235783Skib 3010235783Skib obj->last_fenced_seqno = 0; 3011235783Skib } 3012235783Skib 3013235783Skib /* Ensure that all CPU reads are completed before installing a fence 3014235783Skib * and all writes before removing the fence. 3015235783Skib */ 3016235783Skib if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3017235783Skib mb(); 3018235783Skib 3019296548Sdumbbell obj->fenced_gpu_access = false; 3020235783Skib return 0; 3021235783Skib} 3022235783Skib 3023235783Skibint 3024235783Skibi915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3025235783Skib{ 3026277487Skib struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3027235783Skib int ret; 3028235783Skib 3029277487Skib ret = i915_gem_object_flush_fence(obj); 3030235783Skib if (ret) 3031235783Skib return ret; 3032235783Skib 3033277487Skib if (obj->fence_reg == I915_FENCE_REG_NONE) 3034277487Skib return 0; 3035235783Skib 3036277487Skib i915_gem_object_update_fence(obj, 3037277487Skib &dev_priv->fence_regs[obj->fence_reg], 3038277487Skib false); 3039277487Skib i915_gem_object_fence_lost(obj); 3040235783Skib 3041235783Skib return 0; 3042235783Skib} 3043235783Skib 3044235783Skibstatic struct drm_i915_fence_reg * 3045277487Skibi915_find_fence_reg(struct drm_device *dev) 3046235783Skib{ 3047235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 3048277487Skib struct drm_i915_fence_reg *reg, *avail; 3049235783Skib int i; 3050235783Skib 3051235783Skib /* First try to find a free reg */ 3052235783Skib avail = NULL; 3053235783Skib for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3054235783Skib reg = &dev_priv->fence_regs[i]; 3055235783Skib if (!reg->obj) 3056235783Skib return reg; 3057235783Skib 3058235783Skib if (!reg->pin_count) 3059235783Skib avail = reg; 3060235783Skib } 3061235783Skib 3062235783Skib if (avail == NULL) 3063235783Skib return NULL; 3064235783Skib 3065235783Skib /* None available, try to steal one or wait for a user to finish */ 3066235783Skib list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3067235783Skib if (reg->pin_count) 3068235783Skib continue; 3069235783Skib 3070277487Skib return reg; 3071235783Skib } 3072235783Skib 3073277487Skib return NULL; 3074235783Skib} 3075235783Skib 3076285988Sdumbbell/** 3077285988Sdumbbell * i915_gem_object_get_fence - set up fencing for an object 3078285988Sdumbbell * @obj: object to map through a fence reg 3079285988Sdumbbell * 3080285988Sdumbbell * When mapping objects through the GTT, userspace wants to be able to write 3081285988Sdumbbell * to them without having to worry about swizzling if the object is tiled. 3082285988Sdumbbell * This function walks the fence regs looking for a free one for @obj, 3083285988Sdumbbell * stealing one if it can't find any. 3084285988Sdumbbell * 3085285988Sdumbbell * It then sets up the reg based on the object's properties: address, pitch 3086285988Sdumbbell * and tiling format. 3087285988Sdumbbell * 3088285988Sdumbbell * For an untiled surface, this removes any existing fence. 3089285988Sdumbbell */ 3090235783Skibint 3091277487Skibi915_gem_object_get_fence(struct drm_i915_gem_object *obj) 3092235783Skib{ 3093235783Skib struct drm_device *dev = obj->base.dev; 3094235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 3095277487Skib bool enable = obj->tiling_mode != I915_TILING_NONE; 3096235783Skib struct drm_i915_fence_reg *reg; 3097235783Skib int ret; 3098235783Skib 3099277487Skib /* Have we updated the tiling parameters upon the object and so 3100277487Skib * will need to serialise the write to the associated fence register? 3101277487Skib */ 3102277487Skib if (obj->fence_dirty) { 3103277487Skib ret = i915_gem_object_flush_fence(obj); 3104277487Skib if (ret) 3105277487Skib return ret; 3106277487Skib } 3107277487Skib 3108285988Sdumbbell /* Just update our place in the LRU if our fence is getting reused. */ 3109235783Skib if (obj->fence_reg != I915_FENCE_REG_NONE) { 3110235783Skib reg = &dev_priv->fence_regs[obj->fence_reg]; 3111277487Skib if (!obj->fence_dirty) { 3112277487Skib list_move_tail(®->lru_list, 3113277487Skib &dev_priv->mm.fence_list); 3114277487Skib return 0; 3115235783Skib } 3116277487Skib } else if (enable) { 3117277487Skib reg = i915_find_fence_reg(dev); 3118277487Skib if (reg == NULL) 3119277487Skib return -EDEADLK; 3120235783Skib 3121277487Skib if (reg->obj) { 3122277487Skib struct drm_i915_gem_object *old = reg->obj; 3123235783Skib 3124277487Skib ret = i915_gem_object_flush_fence(old); 3125235783Skib if (ret) 3126235783Skib return ret; 3127235783Skib 3128277487Skib i915_gem_object_fence_lost(old); 3129235783Skib } 3130277487Skib } else 3131235783Skib return 0; 3132235783Skib 3133277487Skib i915_gem_object_update_fence(obj, reg, enable); 3134277487Skib obj->fence_dirty = false; 3135235783Skib 3136277487Skib return 0; 3137235783Skib} 3138235783Skib 3139296548Sdumbbellstatic bool i915_gem_valid_gtt_space(struct drm_device *dev, 3140296548Sdumbbell struct drm_mm_node *gtt_space, 3141296548Sdumbbell unsigned long cache_level) 3142296548Sdumbbell{ 3143296548Sdumbbell struct drm_mm_node *other; 3144296548Sdumbbell 3145296548Sdumbbell /* On non-LLC machines we have to be careful when putting differing 3146296548Sdumbbell * types of snoopable memory together to avoid the prefetcher 3147296548Sdumbbell * crossing memory domains and dieing. 3148296548Sdumbbell */ 3149296548Sdumbbell if (HAS_LLC(dev)) 3150296548Sdumbbell return true; 3151296548Sdumbbell 3152296548Sdumbbell if (gtt_space == NULL) 3153296548Sdumbbell return true; 3154296548Sdumbbell 3155296548Sdumbbell if (list_empty(>t_space->node_list)) 3156296548Sdumbbell return true; 3157296548Sdumbbell 3158296548Sdumbbell other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); 3159296548Sdumbbell if (other->allocated && !other->hole_follows && other->color != cache_level) 3160296548Sdumbbell return false; 3161296548Sdumbbell 3162296548Sdumbbell other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); 3163296548Sdumbbell if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) 3164296548Sdumbbell return false; 3165296548Sdumbbell 3166296548Sdumbbell return true; 3167296548Sdumbbell} 3168296548Sdumbbell 3169296548Sdumbbellstatic void i915_gem_verify_gtt(struct drm_device *dev) 3170296548Sdumbbell{ 3171296548Sdumbbell#if WATCH_GTT 3172296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 3173296548Sdumbbell struct drm_i915_gem_object *obj; 3174296548Sdumbbell int err = 0; 3175296548Sdumbbell 3176296548Sdumbbell list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { 3177296548Sdumbbell if (obj->gtt_space == NULL) { 3178296548Sdumbbell DRM_ERROR("object found on GTT list with no space reserved\n"); 3179296548Sdumbbell err++; 3180296548Sdumbbell continue; 3181296548Sdumbbell } 3182296548Sdumbbell 3183296548Sdumbbell if (obj->cache_level != obj->gtt_space->color) { 3184296548Sdumbbell DRM_ERROR("object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", 3185296548Sdumbbell obj->gtt_space->start, 3186296548Sdumbbell obj->gtt_space->start + obj->gtt_space->size, 3187296548Sdumbbell obj->cache_level, 3188296548Sdumbbell obj->gtt_space->color); 3189296548Sdumbbell err++; 3190296548Sdumbbell continue; 3191296548Sdumbbell } 3192296548Sdumbbell 3193296548Sdumbbell if (!i915_gem_valid_gtt_space(dev, 3194296548Sdumbbell obj->gtt_space, 3195296548Sdumbbell obj->cache_level)) { 3196296548Sdumbbell DRM_ERROR("invalid GTT space found at [%08lx, %08lx] - color=%x\n", 3197296548Sdumbbell obj->gtt_space->start, 3198296548Sdumbbell obj->gtt_space->start + obj->gtt_space->size, 3199296548Sdumbbell obj->cache_level); 3200296548Sdumbbell err++; 3201296548Sdumbbell continue; 3202296548Sdumbbell } 3203296548Sdumbbell } 3204296548Sdumbbell 3205296548Sdumbbell WARN_ON(err); 3206296548Sdumbbell#endif 3207296548Sdumbbell} 3208296548Sdumbbell 3209285988Sdumbbell/** 3210285988Sdumbbell * Finds free space in the GTT aperture and binds the object there. 3211285988Sdumbbell */ 3212285988Sdumbbellstatic int 3213285988Sdumbbelli915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 3214285988Sdumbbell unsigned alignment, 3215296548Sdumbbell bool map_and_fenceable, 3216296548Sdumbbell bool nonblocking) 3217235783Skib{ 3218285988Sdumbbell struct drm_device *dev = obj->base.dev; 3219285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 3220296548Sdumbbell struct drm_mm_node *node; 3221285988Sdumbbell u32 size, fence_size, fence_alignment, unfenced_alignment; 3222285988Sdumbbell bool mappable, fenceable; 3223285988Sdumbbell int ret; 3224235783Skib 3225285988Sdumbbell if (obj->madv != I915_MADV_WILLNEED) { 3226285988Sdumbbell DRM_ERROR("Attempting to bind a purgeable object\n"); 3227285988Sdumbbell return -EINVAL; 3228285988Sdumbbell } 3229285988Sdumbbell 3230285988Sdumbbell fence_size = i915_gem_get_gtt_size(dev, 3231285988Sdumbbell obj->base.size, 3232285988Sdumbbell obj->tiling_mode); 3233285988Sdumbbell fence_alignment = i915_gem_get_gtt_alignment(dev, 3234285988Sdumbbell obj->base.size, 3235285988Sdumbbell obj->tiling_mode); 3236285988Sdumbbell unfenced_alignment = 3237285988Sdumbbell i915_gem_get_unfenced_gtt_alignment(dev, 3238285988Sdumbbell obj->base.size, 3239285988Sdumbbell obj->tiling_mode); 3240285988Sdumbbell 3241285988Sdumbbell if (alignment == 0) 3242285988Sdumbbell alignment = map_and_fenceable ? fence_alignment : 3243285988Sdumbbell unfenced_alignment; 3244285988Sdumbbell if (map_and_fenceable && alignment & (fence_alignment - 1)) { 3245285988Sdumbbell DRM_ERROR("Invalid object alignment requested %u\n", alignment); 3246285988Sdumbbell return -EINVAL; 3247285988Sdumbbell } 3248285988Sdumbbell 3249285988Sdumbbell size = map_and_fenceable ? fence_size : obj->base.size; 3250285988Sdumbbell 3251285988Sdumbbell /* If the object is bigger than the entire aperture, reject it early 3252285988Sdumbbell * before evicting everything in a vain attempt to find space. 3253285988Sdumbbell */ 3254285988Sdumbbell if (obj->base.size > 3255285988Sdumbbell (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 3256285988Sdumbbell DRM_ERROR("Attempting to bind an object larger than the aperture\n"); 3257285988Sdumbbell return -E2BIG; 3258285988Sdumbbell } 3259285988Sdumbbell 3260296548Sdumbbell ret = i915_gem_object_get_pages(obj); 3261296548Sdumbbell if (ret) 3262296548Sdumbbell return ret; 3263296548Sdumbbell 3264296548Sdumbbell i915_gem_object_pin_pages(obj); 3265296548Sdumbbell 3266296681Sdumbbell node = malloc(sizeof(*node), DRM_MEM_MM, M_NOWAIT | M_ZERO); 3267296548Sdumbbell if (node == NULL) { 3268296548Sdumbbell i915_gem_object_unpin_pages(obj); 3269296548Sdumbbell return -ENOMEM; 3270296548Sdumbbell } 3271296548Sdumbbell 3272285988Sdumbbell search_free: 3273285988Sdumbbell if (map_and_fenceable) 3274296548Sdumbbell ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node, 3275296548Sdumbbell size, alignment, obj->cache_level, 3276296548Sdumbbell 0, dev_priv->mm.gtt_mappable_end); 3277285988Sdumbbell else 3278296548Sdumbbell ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node, 3279296548Sdumbbell size, alignment, obj->cache_level); 3280296548Sdumbbell if (ret) { 3281285988Sdumbbell ret = i915_gem_evict_something(dev, size, alignment, 3282296548Sdumbbell obj->cache_level, 3283296548Sdumbbell map_and_fenceable, 3284296548Sdumbbell nonblocking); 3285296548Sdumbbell if (ret == 0) 3286296548Sdumbbell goto search_free; 3287296548Sdumbbell 3288296548Sdumbbell i915_gem_object_unpin_pages(obj); 3289296681Sdumbbell free(node, DRM_MEM_MM); 3290285988Sdumbbell return ret; 3291285988Sdumbbell } 3292296548Sdumbbell if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) { 3293296548Sdumbbell i915_gem_object_unpin_pages(obj); 3294296548Sdumbbell drm_mm_put_block(node); 3295296548Sdumbbell return -EINVAL; 3296296548Sdumbbell } 3297285988Sdumbbell 3298285988Sdumbbell ret = i915_gem_gtt_prepare_object(obj); 3299285988Sdumbbell if (ret) { 3300296548Sdumbbell i915_gem_object_unpin_pages(obj); 3301296548Sdumbbell drm_mm_put_block(node); 3302296548Sdumbbell return ret; 3303285988Sdumbbell } 3304285988Sdumbbell 3305296548Sdumbbell list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list); 3306285988Sdumbbell list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3307285988Sdumbbell 3308296548Sdumbbell obj->gtt_space = node; 3309296548Sdumbbell obj->gtt_offset = node->start; 3310285988Sdumbbell 3311285988Sdumbbell fenceable = 3312296548Sdumbbell node->size == fence_size && 3313296548Sdumbbell (node->start & (fence_alignment - 1)) == 0; 3314285988Sdumbbell 3315285988Sdumbbell mappable = 3316285988Sdumbbell obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 3317285988Sdumbbell 3318285988Sdumbbell obj->map_and_fenceable = mappable && fenceable; 3319285988Sdumbbell 3320296548Sdumbbell i915_gem_object_unpin_pages(obj); 3321285988Sdumbbell CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, 3322285988Sdumbbell obj->base.size, map_and_fenceable); 3323296548Sdumbbell i915_gem_verify_gtt(dev); 3324285988Sdumbbell return 0; 3325235783Skib} 3326235783Skib 3327285988Sdumbbellvoid 3328285988Sdumbbelli915_gem_clflush_object(struct drm_i915_gem_object *obj) 3329235783Skib{ 3330285988Sdumbbell /* If we don't have a page list set up, then we're not pinned 3331285988Sdumbbell * to GPU, and we can ignore the cache flush because it'll happen 3332285988Sdumbbell * again at bind time. 3333285988Sdumbbell */ 3334285988Sdumbbell if (obj->pages == NULL) 3335285988Sdumbbell return; 3336235783Skib 3337285988Sdumbbell /* If the GPU is snooping the contents of the CPU cache, 3338285988Sdumbbell * we do not need to manually clear the CPU cache lines. However, 3339285988Sdumbbell * the caches are only snooped when the render cache is 3340285988Sdumbbell * flushed/invalidated. As we always have to emit invalidations 3341285988Sdumbbell * and flushes when moving into and out of the RENDER domain, correct 3342285988Sdumbbell * snooping behaviour occurs naturally as the result of our domain 3343285988Sdumbbell * tracking. 3344285988Sdumbbell */ 3345285988Sdumbbell if (obj->cache_level != I915_CACHE_NONE) 3346285988Sdumbbell return; 3347285988Sdumbbell 3348285988Sdumbbell CTR1(KTR_DRM, "object_clflush %p", obj); 3349285988Sdumbbell 3350285988Sdumbbell drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 3351235783Skib} 3352235783Skib 3353285988Sdumbbell/** Flushes the GTT write domain for the object if it's dirty. */ 3354235783Skibstatic void 3355285988Sdumbbelli915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 3356235783Skib{ 3357285988Sdumbbell uint32_t old_write_domain; 3358235783Skib 3359285988Sdumbbell if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 3360285988Sdumbbell return; 3361235783Skib 3362285988Sdumbbell /* No actual flushing is required for the GTT write domain. Writes 3363285988Sdumbbell * to it immediately go to main memory as far as we know, so there's 3364285988Sdumbbell * no chipset flush. It also doesn't land in render cache. 3365285988Sdumbbell * 3366285988Sdumbbell * However, we do have to enforce the order so that all writes through 3367285988Sdumbbell * the GTT land before any writes to the device, such as updates to 3368285988Sdumbbell * the GATT itself. 3369285988Sdumbbell */ 3370285988Sdumbbell wmb(); 3371285988Sdumbbell 3372285988Sdumbbell old_write_domain = obj->base.write_domain; 3373285988Sdumbbell obj->base.write_domain = 0; 3374285988Sdumbbell 3375285988Sdumbbell CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, 3376285988Sdumbbell obj->base.read_domains, old_write_domain); 3377285988Sdumbbell} 3378285988Sdumbbell 3379285988Sdumbbell/** Flushes the CPU write domain for the object if it's dirty. */ 3380285988Sdumbbellstatic void 3381285988Sdumbbelli915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 3382285988Sdumbbell{ 3383285988Sdumbbell uint32_t old_write_domain; 3384285988Sdumbbell 3385285988Sdumbbell if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 3386235783Skib return; 3387285988Sdumbbell 3388285988Sdumbbell i915_gem_clflush_object(obj); 3389296548Sdumbbell i915_gem_chipset_flush(obj->base.dev); 3390285988Sdumbbell old_write_domain = obj->base.write_domain; 3391285988Sdumbbell obj->base.write_domain = 0; 3392285988Sdumbbell 3393285988Sdumbbell CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, 3394285988Sdumbbell obj->base.read_domains, old_write_domain); 3395285988Sdumbbell} 3396285988Sdumbbell 3397285988Sdumbbell/** 3398285988Sdumbbell * Moves a single object to the GTT read, and possibly write domain. 3399285988Sdumbbell * 3400285988Sdumbbell * This function returns when the move is complete, including waiting on 3401285988Sdumbbell * flushes to occur. 3402285988Sdumbbell */ 3403285988Sdumbbellint 3404285988Sdumbbelli915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 3405285988Sdumbbell{ 3406285988Sdumbbell drm_i915_private_t *dev_priv = obj->base.dev->dev_private; 3407285988Sdumbbell uint32_t old_write_domain, old_read_domains; 3408285988Sdumbbell int ret; 3409285988Sdumbbell 3410285988Sdumbbell /* Not valid to be called on unbound objects. */ 3411285988Sdumbbell if (obj->gtt_space == NULL) 3412285988Sdumbbell return -EINVAL; 3413285988Sdumbbell 3414285988Sdumbbell if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 3415285988Sdumbbell return 0; 3416285988Sdumbbell 3417296548Sdumbbell ret = i915_gem_object_wait_rendering(obj, !write); 3418285988Sdumbbell if (ret) 3419287174Sbapt return ret; 3420285988Sdumbbell 3421285988Sdumbbell i915_gem_object_flush_cpu_write_domain(obj); 3422235783Skib 3423285988Sdumbbell old_write_domain = obj->base.write_domain; 3424285988Sdumbbell old_read_domains = obj->base.read_domains; 3425235783Skib 3426285988Sdumbbell /* It should now be out of any other write domains, and we can update 3427285988Sdumbbell * the domain values for our changes. 3428235783Skib */ 3429296548Sdumbbell BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 3430285988Sdumbbell obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3431285988Sdumbbell if (write) { 3432285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_GTT; 3433285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_GTT; 3434285988Sdumbbell obj->dirty = 1; 3435285988Sdumbbell } 3436235783Skib 3437285988Sdumbbell CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, 3438285988Sdumbbell old_read_domains, old_write_domain); 3439235783Skib 3440285988Sdumbbell /* And bump the LRU for this access */ 3441285988Sdumbbell if (i915_gem_object_is_inactive(obj)) 3442285988Sdumbbell list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 3443285988Sdumbbell 3444285988Sdumbbell return 0; 3445285988Sdumbbell} 3446285988Sdumbbell 3447285988Sdumbbellint i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 3448285988Sdumbbell enum i915_cache_level cache_level) 3449285988Sdumbbell{ 3450285988Sdumbbell struct drm_device *dev = obj->base.dev; 3451285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 3452285988Sdumbbell int ret; 3453285988Sdumbbell 3454285988Sdumbbell if (obj->cache_level == cache_level) 3455285988Sdumbbell return 0; 3456285988Sdumbbell 3457285988Sdumbbell if (obj->pin_count) { 3458285988Sdumbbell DRM_DEBUG("can not change the cache level of pinned objects\n"); 3459285988Sdumbbell return -EBUSY; 3460285988Sdumbbell } 3461285988Sdumbbell 3462296548Sdumbbell if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) { 3463296548Sdumbbell ret = i915_gem_object_unbind(obj); 3464296548Sdumbbell if (ret) 3465296548Sdumbbell return ret; 3466296548Sdumbbell } 3467296548Sdumbbell 3468285988Sdumbbell if (obj->gtt_space) { 3469285988Sdumbbell ret = i915_gem_object_finish_gpu(obj); 3470285988Sdumbbell if (ret) 3471285988Sdumbbell return ret; 3472285988Sdumbbell 3473285988Sdumbbell i915_gem_object_finish_gtt(obj); 3474285988Sdumbbell 3475285988Sdumbbell /* Before SandyBridge, you could not use tiling or fence 3476285988Sdumbbell * registers with snooped memory, so relinquish any fences 3477285988Sdumbbell * currently pointing to our region in the aperture. 3478285988Sdumbbell */ 3479296548Sdumbbell if (INTEL_INFO(dev)->gen < 6) { 3480285988Sdumbbell ret = i915_gem_object_put_fence(obj); 3481285988Sdumbbell if (ret) 3482285988Sdumbbell return ret; 3483235783Skib } 3484235783Skib 3485285988Sdumbbell if (obj->has_global_gtt_mapping) 3486285988Sdumbbell i915_gem_gtt_bind_object(obj, cache_level); 3487285988Sdumbbell if (obj->has_aliasing_ppgtt_mapping) 3488285988Sdumbbell i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 3489285988Sdumbbell obj, cache_level); 3490296548Sdumbbell 3491296548Sdumbbell obj->gtt_space->color = cache_level; 3492235783Skib } 3493235783Skib 3494285988Sdumbbell if (cache_level == I915_CACHE_NONE) { 3495285988Sdumbbell u32 old_read_domains, old_write_domain; 3496285988Sdumbbell 3497285988Sdumbbell /* If we're coming from LLC cached, then we haven't 3498285988Sdumbbell * actually been tracking whether the data is in the 3499285988Sdumbbell * CPU cache or not, since we only allow one bit set 3500285988Sdumbbell * in obj->write_domain and have been skipping the clflushes. 3501285988Sdumbbell * Just set it to the CPU cache for now. 3502285988Sdumbbell */ 3503296548Sdumbbell WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); 3504296548Sdumbbell WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU); 3505285988Sdumbbell 3506285988Sdumbbell old_read_domains = obj->base.read_domains; 3507285988Sdumbbell old_write_domain = obj->base.write_domain; 3508285988Sdumbbell 3509285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3510285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3511285988Sdumbbell 3512285988Sdumbbell CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", 3513285988Sdumbbell obj, old_read_domains, old_write_domain); 3514285988Sdumbbell } 3515285988Sdumbbell 3516285988Sdumbbell obj->cache_level = cache_level; 3517296548Sdumbbell i915_gem_verify_gtt(dev); 3518285988Sdumbbell return 0; 3519285988Sdumbbell} 3520285988Sdumbbell 3521296548Sdumbbellint i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 3522296548Sdumbbell struct drm_file *file) 3523296548Sdumbbell{ 3524296548Sdumbbell struct drm_i915_gem_caching *args = data; 3525296548Sdumbbell struct drm_i915_gem_object *obj; 3526296548Sdumbbell int ret; 3527296548Sdumbbell 3528296548Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3529296548Sdumbbell if (ret) 3530296548Sdumbbell return ret; 3531296548Sdumbbell 3532296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3533296548Sdumbbell if (&obj->base == NULL) { 3534296548Sdumbbell ret = -ENOENT; 3535296548Sdumbbell goto unlock; 3536296548Sdumbbell } 3537296548Sdumbbell 3538296548Sdumbbell args->caching = obj->cache_level != I915_CACHE_NONE; 3539296548Sdumbbell 3540296548Sdumbbell drm_gem_object_unreference(&obj->base); 3541296548Sdumbbellunlock: 3542296548Sdumbbell DRM_UNLOCK(dev); 3543296548Sdumbbell return ret; 3544296548Sdumbbell} 3545296548Sdumbbell 3546296548Sdumbbellint i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 3547296548Sdumbbell struct drm_file *file) 3548296548Sdumbbell{ 3549296548Sdumbbell struct drm_i915_gem_caching *args = data; 3550296548Sdumbbell struct drm_i915_gem_object *obj; 3551296548Sdumbbell enum i915_cache_level level; 3552296548Sdumbbell int ret; 3553296548Sdumbbell 3554296548Sdumbbell switch (args->caching) { 3555296548Sdumbbell case I915_CACHING_NONE: 3556296548Sdumbbell level = I915_CACHE_NONE; 3557296548Sdumbbell break; 3558296548Sdumbbell case I915_CACHING_CACHED: 3559296548Sdumbbell level = I915_CACHE_LLC; 3560296548Sdumbbell break; 3561296548Sdumbbell default: 3562296548Sdumbbell return -EINVAL; 3563296548Sdumbbell } 3564296548Sdumbbell 3565296548Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3566296548Sdumbbell if (ret) 3567296548Sdumbbell return ret; 3568296548Sdumbbell 3569296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3570296548Sdumbbell if (&obj->base == NULL) { 3571296548Sdumbbell ret = -ENOENT; 3572296548Sdumbbell goto unlock; 3573296548Sdumbbell } 3574296548Sdumbbell 3575296548Sdumbbell ret = i915_gem_object_set_cache_level(obj, level); 3576296548Sdumbbell 3577296548Sdumbbell drm_gem_object_unreference(&obj->base); 3578296548Sdumbbellunlock: 3579296548Sdumbbell DRM_UNLOCK(dev); 3580296548Sdumbbell return ret; 3581296548Sdumbbell} 3582296548Sdumbbell 3583285988Sdumbbellstatic bool is_pin_display(struct drm_i915_gem_object *obj) 3584285988Sdumbbell{ 3585285988Sdumbbell /* There are 3 sources that pin objects: 3586285988Sdumbbell * 1. The display engine (scanouts, sprites, cursors); 3587285988Sdumbbell * 2. Reservations for execbuffer; 3588285988Sdumbbell * 3. The user. 3589285988Sdumbbell * 3590285988Sdumbbell * We can ignore reservations as we hold the struct_mutex and 3591285988Sdumbbell * are only called outside of the reservation path. The user 3592285988Sdumbbell * can only increment pin_count once, and so if after 3593285988Sdumbbell * subtracting the potential reference by the user, any pin_count 3594285988Sdumbbell * remains, it must be due to another use by the display engine. 3595285988Sdumbbell */ 3596285988Sdumbbell return obj->pin_count - !!obj->user_pin_count; 3597285988Sdumbbell} 3598285988Sdumbbell 3599296548Sdumbbell/* 3600296548Sdumbbell * Prepare buffer for display plane (scanout, cursors, etc). 3601296548Sdumbbell * Can be called from an uninterruptible phase (modesetting) and allows 3602296548Sdumbbell * any flushes to be pipelined (for pageflips). 3603296548Sdumbbell */ 3604285988Sdumbbellint 3605285988Sdumbbelli915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 3606285988Sdumbbell u32 alignment, 3607285988Sdumbbell struct intel_ring_buffer *pipelined) 3608285988Sdumbbell{ 3609285988Sdumbbell u32 old_read_domains, old_write_domain; 3610285988Sdumbbell int ret; 3611285988Sdumbbell 3612285988Sdumbbell if (pipelined != obj->ring) { 3613285988Sdumbbell ret = i915_gem_object_sync(obj, pipelined); 3614285988Sdumbbell if (ret) 3615285988Sdumbbell return ret; 3616285988Sdumbbell } 3617285988Sdumbbell 3618285988Sdumbbell /* Mark the pin_display early so that we account for the 3619285988Sdumbbell * display coherency whilst setting up the cache domains. 3620285988Sdumbbell */ 3621285988Sdumbbell obj->pin_display = true; 3622285988Sdumbbell 3623285988Sdumbbell /* The display engine is not coherent with the LLC cache on gen6. As 3624285988Sdumbbell * a result, we make sure that the pinning that is about to occur is 3625285988Sdumbbell * done with uncached PTEs. This is lowest common denominator for all 3626285988Sdumbbell * chipsets. 3627285988Sdumbbell * 3628285988Sdumbbell * However for gen6+, we could do better by using the GFDT bit instead 3629285988Sdumbbell * of uncaching, which would allow us to flush all the LLC-cached data 3630285988Sdumbbell * with that bit in the PTE to main memory with just one PIPE_CONTROL. 3631285988Sdumbbell */ 3632285988Sdumbbell ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 3633285988Sdumbbell if (ret) 3634285988Sdumbbell goto err_unpin_display; 3635285988Sdumbbell 3636285988Sdumbbell /* As the user may map the buffer once pinned in the display plane 3637285988Sdumbbell * (e.g. libkms for the bootup splash), we have to ensure that we 3638285988Sdumbbell * always use map_and_fenceable for all scanout buffers. 3639285988Sdumbbell */ 3640296548Sdumbbell ret = i915_gem_object_pin(obj, alignment, true, false); 3641285988Sdumbbell if (ret) 3642285988Sdumbbell goto err_unpin_display; 3643285988Sdumbbell 3644285988Sdumbbell i915_gem_object_flush_cpu_write_domain(obj); 3645285988Sdumbbell 3646285988Sdumbbell old_write_domain = obj->base.write_domain; 3647285988Sdumbbell old_read_domains = obj->base.read_domains; 3648285988Sdumbbell 3649296548Sdumbbell /* It should now be out of any other write domains, and we can update 3650296548Sdumbbell * the domain values for our changes. 3651296548Sdumbbell */ 3652296548Sdumbbell obj->base.write_domain = 0; 3653285988Sdumbbell obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 3654285988Sdumbbell 3655285988Sdumbbell CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", 3656296548Sdumbbell obj, old_read_domains, old_write_domain); 3657285988Sdumbbell 3658285988Sdumbbell return 0; 3659285988Sdumbbell 3660285988Sdumbbellerr_unpin_display: 3661285988Sdumbbell obj->pin_display = is_pin_display(obj); 3662285988Sdumbbell return ret; 3663285988Sdumbbell} 3664285988Sdumbbell 3665285988Sdumbbellvoid 3666285988Sdumbbelli915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) 3667285988Sdumbbell{ 3668285988Sdumbbell i915_gem_object_unpin(obj); 3669285988Sdumbbell obj->pin_display = is_pin_display(obj); 3670285988Sdumbbell} 3671285988Sdumbbell 3672285988Sdumbbellint 3673285988Sdumbbelli915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 3674285988Sdumbbell{ 3675285988Sdumbbell int ret; 3676285988Sdumbbell 3677285988Sdumbbell if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 3678285988Sdumbbell return 0; 3679285988Sdumbbell 3680296548Sdumbbell ret = i915_gem_object_wait_rendering(obj, false); 3681285988Sdumbbell if (ret) 3682285988Sdumbbell return ret; 3683285988Sdumbbell 3684285988Sdumbbell /* Ensure that we invalidate the GPU's caches and TLBs. */ 3685285988Sdumbbell obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 3686285988Sdumbbell return 0; 3687285988Sdumbbell} 3688285988Sdumbbell 3689287174Sbapt/** 3690287174Sbapt * Moves a single object to the CPU read, and possibly write domain. 3691287174Sbapt * 3692287174Sbapt * This function returns when the move is complete, including waiting on 3693287174Sbapt * flushes to occur. 3694287174Sbapt */ 3695285988Sdumbbellint 3696285988Sdumbbelli915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 3697285988Sdumbbell{ 3698285988Sdumbbell uint32_t old_write_domain, old_read_domains; 3699285988Sdumbbell int ret; 3700285988Sdumbbell 3701285988Sdumbbell if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 3702285988Sdumbbell return 0; 3703285988Sdumbbell 3704296548Sdumbbell ret = i915_gem_object_wait_rendering(obj, !write); 3705285988Sdumbbell if (ret) 3706285988Sdumbbell return ret; 3707285988Sdumbbell 3708285988Sdumbbell i915_gem_object_flush_gtt_write_domain(obj); 3709285988Sdumbbell 3710285988Sdumbbell old_write_domain = obj->base.write_domain; 3711285988Sdumbbell old_read_domains = obj->base.read_domains; 3712285988Sdumbbell 3713285988Sdumbbell /* Flush the CPU cache if it's still invalid. */ 3714285988Sdumbbell if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 3715285988Sdumbbell i915_gem_clflush_object(obj); 3716285988Sdumbbell 3717285988Sdumbbell obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 3718285988Sdumbbell } 3719285988Sdumbbell 3720285988Sdumbbell /* It should now be out of any other write domains, and we can update 3721285988Sdumbbell * the domain values for our changes. 3722285988Sdumbbell */ 3723296548Sdumbbell BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3724285988Sdumbbell 3725285988Sdumbbell /* If we're writing through the CPU, then the GPU read domains will 3726285988Sdumbbell * need to be invalidated at next use. 3727285988Sdumbbell */ 3728285988Sdumbbell if (write) { 3729285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3730285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3731285988Sdumbbell } 3732285988Sdumbbell 3733285988Sdumbbell CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, 3734285988Sdumbbell old_read_domains, old_write_domain); 3735285988Sdumbbell 3736285988Sdumbbell return 0; 3737285988Sdumbbell} 3738285988Sdumbbell 3739285988Sdumbbell/* Throttle our rendering by waiting until the ring has completed our requests 3740285988Sdumbbell * emitted over 20 msec ago. 3741285988Sdumbbell * 3742285988Sdumbbell * Note that if we were to use the current jiffies each time around the loop, 3743285988Sdumbbell * we wouldn't escape the function with any frames outstanding if the time to 3744285988Sdumbbell * render a frame was over 20ms. 3745285988Sdumbbell * 3746285988Sdumbbell * This should get us reasonable parallelism between CPU and GPU but also 3747285988Sdumbbell * relatively low latency when blocking on a particular request to finish. 3748285988Sdumbbell */ 3749285988Sdumbbellstatic int 3750285988Sdumbbelli915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 3751285988Sdumbbell{ 3752285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 3753285988Sdumbbell struct drm_i915_file_private *file_priv = file->driver_priv; 3754296548Sdumbbell unsigned long recent_enough = jiffies - msecs_to_jiffies(20); 3755285988Sdumbbell struct drm_i915_gem_request *request; 3756285988Sdumbbell struct intel_ring_buffer *ring = NULL; 3757285988Sdumbbell u32 seqno = 0; 3758285988Sdumbbell int ret; 3759285988Sdumbbell 3760296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) 3761285988Sdumbbell return -EIO; 3762285988Sdumbbell 3763296548Sdumbbell mtx_lock(&file_priv->mm.lock); 3764285988Sdumbbell list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 3765285988Sdumbbell if (time_after_eq(request->emitted_jiffies, recent_enough)) 3766285988Sdumbbell break; 3767296548Sdumbbell 3768285988Sdumbbell ring = request->ring; 3769285988Sdumbbell seqno = request->seqno; 3770285988Sdumbbell } 3771296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 3772296548Sdumbbell 3773285988Sdumbbell if (seqno == 0) 3774285988Sdumbbell return 0; 3775285988Sdumbbell 3776296548Sdumbbell ret = __wait_seqno(ring, seqno, true, NULL); 3777285988Sdumbbell if (ret == 0) 3778296548Sdumbbell taskqueue_enqueue_timeout(dev_priv->wq, 3779296548Sdumbbell &dev_priv->mm.retire_work, 0); 3780235783Skib 3781285988Sdumbbell return ret; 3782285988Sdumbbell} 3783285988Sdumbbell 3784285988Sdumbbellint 3785285988Sdumbbelli915_gem_object_pin(struct drm_i915_gem_object *obj, 3786285988Sdumbbell uint32_t alignment, 3787296548Sdumbbell bool map_and_fenceable, 3788296548Sdumbbell bool nonblocking) 3789285988Sdumbbell{ 3790285988Sdumbbell int ret; 3791285988Sdumbbell 3792296548Sdumbbell if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) 3793285988Sdumbbell return -EBUSY; 3794285988Sdumbbell 3795285988Sdumbbell if (obj->gtt_space != NULL) { 3796285988Sdumbbell if ((alignment && obj->gtt_offset & (alignment - 1)) || 3797285988Sdumbbell (map_and_fenceable && !obj->map_and_fenceable)) { 3798296548Sdumbbell WARN(obj->pin_count, 3799296548Sdumbbell "bo is already pinned with incorrect alignment:" 3800285988Sdumbbell " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 3801285988Sdumbbell " obj->map_and_fenceable=%d\n", 3802285988Sdumbbell obj->gtt_offset, alignment, 3803285988Sdumbbell map_and_fenceable, 3804285988Sdumbbell obj->map_and_fenceable); 3805285988Sdumbbell ret = i915_gem_object_unbind(obj); 3806285988Sdumbbell if (ret) 3807285988Sdumbbell return ret; 3808285988Sdumbbell } 3809285988Sdumbbell } 3810285988Sdumbbell 3811285988Sdumbbell if (obj->gtt_space == NULL) { 3812296548Sdumbbell struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3813296548Sdumbbell 3814285988Sdumbbell ret = i915_gem_object_bind_to_gtt(obj, alignment, 3815296548Sdumbbell map_and_fenceable, 3816296548Sdumbbell nonblocking); 3817285988Sdumbbell if (ret) 3818285988Sdumbbell return ret; 3819296548Sdumbbell 3820296548Sdumbbell if (!dev_priv->mm.aliasing_ppgtt) 3821296548Sdumbbell i915_gem_gtt_bind_object(obj, obj->cache_level); 3822285988Sdumbbell } 3823285988Sdumbbell 3824285988Sdumbbell if (!obj->has_global_gtt_mapping && map_and_fenceable) 3825285988Sdumbbell i915_gem_gtt_bind_object(obj, obj->cache_level); 3826285988Sdumbbell 3827285988Sdumbbell obj->pin_count++; 3828285988Sdumbbell obj->pin_mappable |= map_and_fenceable; 3829285988Sdumbbell 3830285988Sdumbbell return 0; 3831285988Sdumbbell} 3832285988Sdumbbell 3833285988Sdumbbellvoid 3834285988Sdumbbelli915_gem_object_unpin(struct drm_i915_gem_object *obj) 3835285988Sdumbbell{ 3836296548Sdumbbell BUG_ON(obj->pin_count == 0); 3837296548Sdumbbell BUG_ON(obj->gtt_space == NULL); 3838285988Sdumbbell 3839285988Sdumbbell if (--obj->pin_count == 0) 3840285988Sdumbbell obj->pin_mappable = false; 3841285988Sdumbbell} 3842285988Sdumbbell 3843285988Sdumbbellint 3844285988Sdumbbelli915_gem_pin_ioctl(struct drm_device *dev, void *data, 3845285988Sdumbbell struct drm_file *file) 3846285988Sdumbbell{ 3847285988Sdumbbell struct drm_i915_gem_pin *args = data; 3848285988Sdumbbell struct drm_i915_gem_object *obj; 3849285988Sdumbbell int ret; 3850285988Sdumbbell 3851285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3852285988Sdumbbell if (ret) 3853285988Sdumbbell return ret; 3854285988Sdumbbell 3855296548Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3856296548Sdumbbell if (&obj->base == NULL) { 3857285988Sdumbbell ret = -ENOENT; 3858285988Sdumbbell goto unlock; 3859285988Sdumbbell } 3860285988Sdumbbell 3861285988Sdumbbell if (obj->madv != I915_MADV_WILLNEED) { 3862285988Sdumbbell DRM_ERROR("Attempting to pin a purgeable buffer\n"); 3863285988Sdumbbell ret = -EINVAL; 3864285988Sdumbbell goto out; 3865285988Sdumbbell } 3866285988Sdumbbell 3867285988Sdumbbell if (obj->pin_filp != NULL && obj->pin_filp != file) { 3868285988Sdumbbell DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3869285988Sdumbbell args->handle); 3870285988Sdumbbell ret = -EINVAL; 3871285988Sdumbbell goto out; 3872285988Sdumbbell } 3873285988Sdumbbell 3874296548Sdumbbell if (obj->user_pin_count == 0) { 3875296548Sdumbbell ret = i915_gem_object_pin(obj, args->alignment, true, false); 3876285988Sdumbbell if (ret) 3877285988Sdumbbell goto out; 3878285988Sdumbbell } 3879285988Sdumbbell 3880296548Sdumbbell obj->user_pin_count++; 3881296548Sdumbbell obj->pin_filp = file; 3882296548Sdumbbell 3883285988Sdumbbell /* XXX - flush the CPU caches for pinned objects 3884285988Sdumbbell * as the X server doesn't manage domains yet 3885285988Sdumbbell */ 3886285988Sdumbbell i915_gem_object_flush_cpu_write_domain(obj); 3887285988Sdumbbell args->offset = obj->gtt_offset; 3888285988Sdumbbellout: 3889285988Sdumbbell drm_gem_object_unreference(&obj->base); 3890285988Sdumbbellunlock: 3891235783Skib DRM_UNLOCK(dev); 3892285988Sdumbbell return ret; 3893235783Skib} 3894235783Skib 3895285988Sdumbbellint 3896285988Sdumbbelli915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3897285988Sdumbbell struct drm_file *file) 3898285988Sdumbbell{ 3899285988Sdumbbell struct drm_i915_gem_pin *args = data; 3900285988Sdumbbell struct drm_i915_gem_object *obj; 3901285988Sdumbbell int ret; 3902285988Sdumbbell 3903285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3904285988Sdumbbell if (ret) 3905285988Sdumbbell return ret; 3906285988Sdumbbell 3907285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3908285988Sdumbbell if (&obj->base == NULL) { 3909285988Sdumbbell ret = -ENOENT; 3910285988Sdumbbell goto unlock; 3911285988Sdumbbell } 3912285988Sdumbbell 3913285988Sdumbbell if (obj->pin_filp != file) { 3914285988Sdumbbell DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3915285988Sdumbbell args->handle); 3916285988Sdumbbell ret = -EINVAL; 3917285988Sdumbbell goto out; 3918285988Sdumbbell } 3919285988Sdumbbell obj->user_pin_count--; 3920285988Sdumbbell if (obj->user_pin_count == 0) { 3921285988Sdumbbell obj->pin_filp = NULL; 3922285988Sdumbbell i915_gem_object_unpin(obj); 3923285988Sdumbbell } 3924285988Sdumbbell 3925285988Sdumbbellout: 3926285988Sdumbbell drm_gem_object_unreference(&obj->base); 3927285988Sdumbbellunlock: 3928285988Sdumbbell DRM_UNLOCK(dev); 3929285988Sdumbbell return ret; 3930285988Sdumbbell} 3931285988Sdumbbell 3932285988Sdumbbellint 3933285988Sdumbbelli915_gem_busy_ioctl(struct drm_device *dev, void *data, 3934285988Sdumbbell struct drm_file *file) 3935285988Sdumbbell{ 3936285988Sdumbbell struct drm_i915_gem_busy *args = data; 3937285988Sdumbbell struct drm_i915_gem_object *obj; 3938285988Sdumbbell int ret; 3939285988Sdumbbell 3940285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3941285988Sdumbbell if (ret) 3942285988Sdumbbell return ret; 3943285988Sdumbbell 3944285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 3945285988Sdumbbell if (&obj->base == NULL) { 3946285988Sdumbbell ret = -ENOENT; 3947285988Sdumbbell goto unlock; 3948285988Sdumbbell } 3949285988Sdumbbell 3950296548Sdumbbell /* Count all active objects as busy, even if they are currently not used 3951296548Sdumbbell * by the gpu. Users of this interface expect objects to eventually 3952296548Sdumbbell * become non-busy without any further actions, therefore emit any 3953296548Sdumbbell * necessary flushes here. 3954296548Sdumbbell */ 3955296548Sdumbbell ret = i915_gem_object_flush_active(obj); 3956296548Sdumbbell 3957285988Sdumbbell args->busy = obj->active; 3958296548Sdumbbell if (obj->ring) { 3959296548Sdumbbell BUILD_BUG_ON(I915_NUM_RINGS > 16); 3960296548Sdumbbell args->busy |= intel_ring_flag(obj->ring) << 16; 3961285988Sdumbbell } 3962285988Sdumbbell 3963285988Sdumbbell drm_gem_object_unreference(&obj->base); 3964285988Sdumbbellunlock: 3965285988Sdumbbell DRM_UNLOCK(dev); 3966285988Sdumbbell return ret; 3967285988Sdumbbell} 3968285988Sdumbbell 3969285988Sdumbbellint 3970285988Sdumbbelli915_gem_throttle_ioctl(struct drm_device *dev, void *data, 3971285988Sdumbbell struct drm_file *file_priv) 3972285988Sdumbbell{ 3973285988Sdumbbell return i915_gem_ring_throttle(dev, file_priv); 3974285988Sdumbbell} 3975285988Sdumbbell 3976285988Sdumbbellint 3977285988Sdumbbelli915_gem_madvise_ioctl(struct drm_device *dev, void *data, 3978285988Sdumbbell struct drm_file *file_priv) 3979285988Sdumbbell{ 3980285988Sdumbbell struct drm_i915_gem_madvise *args = data; 3981285988Sdumbbell struct drm_i915_gem_object *obj; 3982285988Sdumbbell int ret; 3983285988Sdumbbell 3984285988Sdumbbell switch (args->madv) { 3985285988Sdumbbell case I915_MADV_DONTNEED: 3986285988Sdumbbell case I915_MADV_WILLNEED: 3987285988Sdumbbell break; 3988285988Sdumbbell default: 3989285988Sdumbbell return -EINVAL; 3990285988Sdumbbell } 3991285988Sdumbbell 3992285988Sdumbbell ret = i915_mutex_lock_interruptible(dev); 3993285988Sdumbbell if (ret) 3994285988Sdumbbell return ret; 3995285988Sdumbbell 3996285988Sdumbbell obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 3997285988Sdumbbell if (&obj->base == NULL) { 3998285988Sdumbbell ret = -ENOENT; 3999285988Sdumbbell goto unlock; 4000285988Sdumbbell } 4001285988Sdumbbell 4002285988Sdumbbell if (obj->pin_count) { 4003285988Sdumbbell ret = -EINVAL; 4004285988Sdumbbell goto out; 4005285988Sdumbbell } 4006285988Sdumbbell 4007296548Sdumbbell if (obj->madv != __I915_MADV_PURGED) 4008285988Sdumbbell obj->madv = args->madv; 4009285988Sdumbbell 4010285988Sdumbbell /* if the object is no longer attached, discard its backing storage */ 4011296548Sdumbbell if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) 4012285988Sdumbbell i915_gem_object_truncate(obj); 4013285988Sdumbbell 4014296548Sdumbbell args->retained = obj->madv != __I915_MADV_PURGED; 4015285988Sdumbbell 4016285988Sdumbbellout: 4017285988Sdumbbell drm_gem_object_unreference(&obj->base); 4018285988Sdumbbellunlock: 4019285988Sdumbbell DRM_UNLOCK(dev); 4020285988Sdumbbell return ret; 4021285988Sdumbbell} 4022285988Sdumbbell 4023296548Sdumbbellvoid i915_gem_object_init(struct drm_i915_gem_object *obj, 4024296548Sdumbbell const struct drm_i915_gem_object_ops *ops) 4025296548Sdumbbell{ 4026296548Sdumbbell INIT_LIST_HEAD(&obj->mm_list); 4027296548Sdumbbell INIT_LIST_HEAD(&obj->gtt_list); 4028296548Sdumbbell INIT_LIST_HEAD(&obj->ring_list); 4029296548Sdumbbell INIT_LIST_HEAD(&obj->exec_list); 4030296548Sdumbbell 4031296548Sdumbbell obj->ops = ops; 4032296548Sdumbbell 4033296548Sdumbbell obj->fence_reg = I915_FENCE_REG_NONE; 4034296548Sdumbbell obj->madv = I915_MADV_WILLNEED; 4035296548Sdumbbell /* Avoid an unnecessary call to unbind on the first bind. */ 4036296548Sdumbbell obj->map_and_fenceable = true; 4037296548Sdumbbell 4038296548Sdumbbell i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); 4039296548Sdumbbell} 4040296548Sdumbbell 4041296548Sdumbbellstatic const struct drm_i915_gem_object_ops i915_gem_object_ops = { 4042296548Sdumbbell .get_pages = i915_gem_object_get_pages_gtt, 4043296548Sdumbbell .put_pages = i915_gem_object_put_pages_gtt, 4044296548Sdumbbell}; 4045296548Sdumbbell 4046285988Sdumbbellstruct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, 4047285988Sdumbbell size_t size) 4048285988Sdumbbell{ 4049285988Sdumbbell struct drm_i915_gem_object *obj; 4050285988Sdumbbell 4051285988Sdumbbell obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 4052296548Sdumbbell if (obj == NULL) 4053296548Sdumbbell return NULL; 4054285988Sdumbbell 4055285988Sdumbbell if (drm_gem_object_init(dev, &obj->base, size) != 0) { 4056285988Sdumbbell free(obj, DRM_I915_GEM); 4057285988Sdumbbell return NULL; 4058285988Sdumbbell } 4059285988Sdumbbell 4060296548Sdumbbell#ifdef FREEBSD_WIP 4061296548Sdumbbell mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 4062296548Sdumbbell if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { 4063296548Sdumbbell /* 965gm cannot relocate objects above 4GiB. */ 4064296548Sdumbbell mask &= ~__GFP_HIGHMEM; 4065296548Sdumbbell mask |= __GFP_DMA32; 4066296548Sdumbbell } 4067296548Sdumbbell 4068296548Sdumbbell mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; 4069296548Sdumbbell mapping_set_gfp_mask(mapping, mask); 4070296548Sdumbbell#endif /* FREEBSD_WIP */ 4071296548Sdumbbell 4072296548Sdumbbell i915_gem_object_init(obj, &i915_gem_object_ops); 4073296548Sdumbbell 4074285988Sdumbbell obj->base.write_domain = I915_GEM_DOMAIN_CPU; 4075285988Sdumbbell obj->base.read_domains = I915_GEM_DOMAIN_CPU; 4076285988Sdumbbell 4077285988Sdumbbell if (HAS_LLC(dev)) { 4078285988Sdumbbell /* On some devices, we can have the GPU use the LLC (the CPU 4079285988Sdumbbell * cache) for about a 10% performance improvement 4080285988Sdumbbell * compared to uncached. Graphics requests other than 4081285988Sdumbbell * display scanout are coherent with the CPU in 4082285988Sdumbbell * accessing this cache. This means in this mode we 4083285988Sdumbbell * don't need to clflush on the CPU side, and on the 4084285988Sdumbbell * GPU side we only need to flush internal caches to 4085285988Sdumbbell * get data visible to the CPU. 4086285988Sdumbbell * 4087285988Sdumbbell * However, we maintain the display planes as UC, and so 4088285988Sdumbbell * need to rebind when first used as such. 4089285988Sdumbbell */ 4090285988Sdumbbell obj->cache_level = I915_CACHE_LLC; 4091285988Sdumbbell } else 4092285988Sdumbbell obj->cache_level = I915_CACHE_NONE; 4093285988Sdumbbell 4094285988Sdumbbell return obj; 4095285988Sdumbbell} 4096285988Sdumbbell 4097285988Sdumbbellint i915_gem_init_object(struct drm_gem_object *obj) 4098285988Sdumbbell{ 4099285988Sdumbbell printf("i915_gem_init_object called\n"); 4100285988Sdumbbell 4101285988Sdumbbell return 0; 4102285988Sdumbbell} 4103285988Sdumbbell 4104285988Sdumbbellvoid i915_gem_free_object(struct drm_gem_object *gem_obj) 4105285988Sdumbbell{ 4106285988Sdumbbell struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 4107285988Sdumbbell struct drm_device *dev = obj->base.dev; 4108285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4109285988Sdumbbell 4110285988Sdumbbell CTR1(KTR_DRM, "object_destroy_tail %p", obj); 4111285988Sdumbbell 4112285988Sdumbbell if (obj->phys_obj) 4113285988Sdumbbell i915_gem_detach_phys_object(dev, obj); 4114285988Sdumbbell 4115285988Sdumbbell obj->pin_count = 0; 4116296548Sdumbbell if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { 4117285988Sdumbbell bool was_interruptible; 4118285988Sdumbbell 4119285988Sdumbbell was_interruptible = dev_priv->mm.interruptible; 4120285988Sdumbbell dev_priv->mm.interruptible = false; 4121285988Sdumbbell 4122296548Sdumbbell WARN_ON(i915_gem_object_unbind(obj)); 4123285988Sdumbbell 4124285988Sdumbbell dev_priv->mm.interruptible = was_interruptible; 4125285988Sdumbbell } 4126285988Sdumbbell 4127296548Sdumbbell obj->pages_pin_count = 0; 4128296548Sdumbbell i915_gem_object_put_pages(obj); 4129296548Sdumbbell i915_gem_object_free_mmap_offset(obj); 4130296548Sdumbbell 4131296548Sdumbbell BUG_ON(obj->pages); 4132296548Sdumbbell 4133296548Sdumbbell#ifdef FREEBSD_WIP 4134296548Sdumbbell if (obj->base.import_attach) 4135296548Sdumbbell drm_prime_gem_destroy(&obj->base, NULL); 4136296548Sdumbbell#endif /* FREEBSD_WIP */ 4137296548Sdumbbell 4138285988Sdumbbell drm_gem_object_release(&obj->base); 4139285988Sdumbbell i915_gem_info_remove_obj(dev_priv, obj->base.size); 4140285988Sdumbbell 4141285988Sdumbbell free(obj->bit_17, DRM_I915_GEM); 4142285988Sdumbbell free(obj, DRM_I915_GEM); 4143285988Sdumbbell} 4144285988Sdumbbell 4145285988Sdumbbellint 4146285988Sdumbbelli915_gem_idle(struct drm_device *dev) 4147285988Sdumbbell{ 4148285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4149285988Sdumbbell int ret; 4150285988Sdumbbell 4151285988Sdumbbell DRM_LOCK(dev); 4152285988Sdumbbell 4153285988Sdumbbell if (dev_priv->mm.suspended) { 4154285988Sdumbbell DRM_UNLOCK(dev); 4155285988Sdumbbell return 0; 4156285988Sdumbbell } 4157285988Sdumbbell 4158285988Sdumbbell ret = i915_gpu_idle(dev); 4159285988Sdumbbell if (ret) { 4160285988Sdumbbell DRM_UNLOCK(dev); 4161285988Sdumbbell return ret; 4162285988Sdumbbell } 4163285988Sdumbbell i915_gem_retire_requests(dev); 4164285988Sdumbbell 4165285988Sdumbbell /* Under UMS, be paranoid and evict. */ 4166296548Sdumbbell if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4167296548Sdumbbell i915_gem_evict_everything(dev); 4168285988Sdumbbell 4169285988Sdumbbell i915_gem_reset_fences(dev); 4170285988Sdumbbell 4171285988Sdumbbell /* Hack! Don't let anybody do execbuf while we don't control the chip. 4172285988Sdumbbell * We need to replace this with a semaphore, or something. 4173285988Sdumbbell * And not confound mm.suspended! 4174285988Sdumbbell */ 4175285988Sdumbbell dev_priv->mm.suspended = 1; 4176285988Sdumbbell callout_stop(&dev_priv->hangcheck_timer); 4177285988Sdumbbell 4178285988Sdumbbell i915_kernel_lost_context(dev); 4179285988Sdumbbell i915_gem_cleanup_ringbuffer(dev); 4180285988Sdumbbell 4181285988Sdumbbell DRM_UNLOCK(dev); 4182285988Sdumbbell 4183285988Sdumbbell /* Cancel the retire work handler, which should be idle now. */ 4184296548Sdumbbell taskqueue_cancel_timeout(dev_priv->wq, &dev_priv->mm.retire_work, NULL); 4185285988Sdumbbell 4186296548Sdumbbell return 0; 4187285988Sdumbbell} 4188285988Sdumbbell 4189296548Sdumbbellvoid i915_gem_l3_remap(struct drm_device *dev) 4190296548Sdumbbell{ 4191296548Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4192296548Sdumbbell u32 misccpctl; 4193296548Sdumbbell int i; 4194296548Sdumbbell 4195296548Sdumbbell if (!HAS_L3_GPU_CACHE(dev)) 4196296548Sdumbbell return; 4197296548Sdumbbell 4198296548Sdumbbell if (!dev_priv->l3_parity.remap_info) 4199296548Sdumbbell return; 4200296548Sdumbbell 4201296548Sdumbbell misccpctl = I915_READ(GEN7_MISCCPCTL); 4202296548Sdumbbell I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); 4203296548Sdumbbell POSTING_READ(GEN7_MISCCPCTL); 4204296548Sdumbbell 4205296548Sdumbbell for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { 4206296548Sdumbbell u32 remap = I915_READ(GEN7_L3LOG_BASE + i); 4207296548Sdumbbell if (remap && remap != dev_priv->l3_parity.remap_info[i/4]) 4208296548Sdumbbell DRM_DEBUG("0x%x was already programmed to %x\n", 4209296548Sdumbbell GEN7_L3LOG_BASE + i, remap); 4210296548Sdumbbell if (remap && !dev_priv->l3_parity.remap_info[i/4]) 4211296548Sdumbbell DRM_DEBUG_DRIVER("Clearing remapped register\n"); 4212296548Sdumbbell I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]); 4213296548Sdumbbell } 4214296548Sdumbbell 4215296548Sdumbbell /* Make sure all the writes land before disabling dop clock gating */ 4216296548Sdumbbell POSTING_READ(GEN7_L3LOG_BASE); 4217296548Sdumbbell 4218296548Sdumbbell I915_WRITE(GEN7_MISCCPCTL, misccpctl); 4219296548Sdumbbell} 4220296548Sdumbbell 4221285988Sdumbbellvoid i915_gem_init_swizzling(struct drm_device *dev) 4222285988Sdumbbell{ 4223285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4224285988Sdumbbell 4225285988Sdumbbell if (INTEL_INFO(dev)->gen < 5 || 4226285988Sdumbbell dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 4227285988Sdumbbell return; 4228285988Sdumbbell 4229285988Sdumbbell I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 4230285988Sdumbbell DISP_TILE_SURFACE_SWIZZLING); 4231285988Sdumbbell 4232285988Sdumbbell if (IS_GEN5(dev)) 4233285988Sdumbbell return; 4234285988Sdumbbell 4235285988Sdumbbell I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 4236285988Sdumbbell if (IS_GEN6(dev)) 4237285988Sdumbbell I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 4238285988Sdumbbell else 4239285988Sdumbbell I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 4240285988Sdumbbell} 4241285988Sdumbbell 4242296548Sdumbbellstatic bool 4243296548Sdumbbellintel_enable_blt(struct drm_device *dev) 4244296548Sdumbbell{ 4245296548Sdumbbell if (!HAS_BLT(dev)) 4246296548Sdumbbell return false; 4247296548Sdumbbell 4248296548Sdumbbell /* The blitter was dysfunctional on early prototypes */ 4249296548Sdumbbell if (IS_GEN6(dev) && pci_get_revid(dev->dev) < 8) { 4250296548Sdumbbell DRM_INFO("BLT not supported on this pre-production hardware;" 4251296548Sdumbbell " graphics performance will be degraded.\n"); 4252296548Sdumbbell return false; 4253296548Sdumbbell } 4254296548Sdumbbell 4255296548Sdumbbell return true; 4256296548Sdumbbell} 4257296548Sdumbbell 4258285988Sdumbbellint 4259285988Sdumbbelli915_gem_init_hw(struct drm_device *dev) 4260285988Sdumbbell{ 4261285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4262285988Sdumbbell int ret; 4263285988Sdumbbell 4264296548Sdumbbell#ifdef FREEBSD_WIP 4265296548Sdumbbell if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) 4266296548Sdumbbell return -EIO; 4267296548Sdumbbell#endif /* FREEBSD_WIP */ 4268296548Sdumbbell 4269296548Sdumbbell if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) 4270296548Sdumbbell I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); 4271296548Sdumbbell 4272296548Sdumbbell i915_gem_l3_remap(dev); 4273296548Sdumbbell 4274285988Sdumbbell i915_gem_init_swizzling(dev); 4275285988Sdumbbell 4276285988Sdumbbell ret = intel_init_render_ring_buffer(dev); 4277285988Sdumbbell if (ret) 4278285988Sdumbbell return ret; 4279285988Sdumbbell 4280285988Sdumbbell if (HAS_BSD(dev)) { 4281285988Sdumbbell ret = intel_init_bsd_ring_buffer(dev); 4282285988Sdumbbell if (ret) 4283285988Sdumbbell goto cleanup_render_ring; 4284285988Sdumbbell } 4285285988Sdumbbell 4286296548Sdumbbell if (intel_enable_blt(dev)) { 4287285988Sdumbbell ret = intel_init_blt_ring_buffer(dev); 4288285988Sdumbbell if (ret) 4289285988Sdumbbell goto cleanup_bsd_ring; 4290285988Sdumbbell } 4291285988Sdumbbell 4292285988Sdumbbell dev_priv->next_seqno = 1; 4293285988Sdumbbell 4294285988Sdumbbell /* 4295285988Sdumbbell * XXX: There was some w/a described somewhere suggesting loading 4296285988Sdumbbell * contexts before PPGTT. 4297285988Sdumbbell */ 4298285988Sdumbbell i915_gem_context_init(dev); 4299285988Sdumbbell i915_gem_init_ppgtt(dev); 4300285988Sdumbbell 4301285988Sdumbbell return 0; 4302285988Sdumbbell 4303285988Sdumbbellcleanup_bsd_ring: 4304296548Sdumbbell intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); 4305285988Sdumbbellcleanup_render_ring: 4306296548Sdumbbell intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); 4307285988Sdumbbell return ret; 4308285988Sdumbbell} 4309285988Sdumbbell 4310285988Sdumbbellstatic bool 4311285988Sdumbbellintel_enable_ppgtt(struct drm_device *dev) 4312285988Sdumbbell{ 4313285988Sdumbbell if (i915_enable_ppgtt >= 0) 4314285988Sdumbbell return i915_enable_ppgtt; 4315285988Sdumbbell 4316296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU 4317285988Sdumbbell /* Disable ppgtt on SNB if VT-d is on. */ 4318296548Sdumbbell if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) 4319285988Sdumbbell return false; 4320296548Sdumbbell#endif 4321285988Sdumbbell 4322285988Sdumbbell return true; 4323285988Sdumbbell} 4324285988Sdumbbell 4325285988Sdumbbellint i915_gem_init(struct drm_device *dev) 4326285988Sdumbbell{ 4327285988Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 4328285988Sdumbbell unsigned long gtt_size, mappable_size; 4329285988Sdumbbell int ret; 4330285988Sdumbbell 4331296548Sdumbbell gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; 4332296548Sdumbbell mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; 4333285988Sdumbbell 4334285988Sdumbbell DRM_LOCK(dev); 4335285988Sdumbbell if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { 4336285988Sdumbbell /* PPGTT pdes are stolen from global gtt ptes, so shrink the 4337285988Sdumbbell * aperture accordingly when using aliasing ppgtt. */ 4338285988Sdumbbell gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; 4339285988Sdumbbell 4340285988Sdumbbell i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); 4341285988Sdumbbell 4342285988Sdumbbell ret = i915_gem_init_aliasing_ppgtt(dev); 4343285988Sdumbbell if (ret) { 4344285988Sdumbbell DRM_UNLOCK(dev); 4345285988Sdumbbell return ret; 4346285988Sdumbbell } 4347285988Sdumbbell } else { 4348285988Sdumbbell /* Let GEM Manage all of the aperture. 4349285988Sdumbbell * 4350285988Sdumbbell * However, leave one page at the end still bound to the scratch 4351285988Sdumbbell * page. There are a number of places where the hardware 4352285988Sdumbbell * apparently prefetches past the end of the object, and we've 4353285988Sdumbbell * seen multiple hangs with the GPU head pointer stuck in a 4354285988Sdumbbell * batchbuffer bound at the last page of the aperture. One page 4355285988Sdumbbell * should be enough to keep any prefetching inside of the 4356285988Sdumbbell * aperture. 4357285988Sdumbbell */ 4358285988Sdumbbell i915_gem_init_global_gtt(dev, 0, mappable_size, 4359285988Sdumbbell gtt_size); 4360285988Sdumbbell } 4361285988Sdumbbell 4362285988Sdumbbell ret = i915_gem_init_hw(dev); 4363285988Sdumbbell DRM_UNLOCK(dev); 4364285988Sdumbbell if (ret) { 4365285988Sdumbbell i915_gem_cleanup_aliasing_ppgtt(dev); 4366285988Sdumbbell return ret; 4367285988Sdumbbell } 4368285988Sdumbbell 4369285988Sdumbbell /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ 4370285988Sdumbbell if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4371285988Sdumbbell dev_priv->dri1.allow_batchbuffer = 1; 4372285988Sdumbbell return 0; 4373285988Sdumbbell} 4374285988Sdumbbell 4375235783Skibvoid 4376285988Sdumbbelli915_gem_cleanup_ringbuffer(struct drm_device *dev) 4377285988Sdumbbell{ 4378285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4379285988Sdumbbell struct intel_ring_buffer *ring; 4380285988Sdumbbell int i; 4381285988Sdumbbell 4382285988Sdumbbell for_each_ring(ring, dev_priv, i) 4383285988Sdumbbell intel_cleanup_ring_buffer(ring); 4384285988Sdumbbell} 4385285988Sdumbbell 4386285988Sdumbbellint 4387285988Sdumbbelli915_gem_entervt_ioctl(struct drm_device *dev, void *data, 4388285988Sdumbbell struct drm_file *file_priv) 4389285988Sdumbbell{ 4390285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4391285988Sdumbbell int ret; 4392285988Sdumbbell 4393285988Sdumbbell if (drm_core_check_feature(dev, DRIVER_MODESET)) 4394285988Sdumbbell return 0; 4395285988Sdumbbell 4396296548Sdumbbell if (atomic_read(&dev_priv->mm.wedged)) { 4397285988Sdumbbell DRM_ERROR("Reenabling wedged hardware, good luck\n"); 4398296548Sdumbbell atomic_set(&dev_priv->mm.wedged, 0); 4399285988Sdumbbell } 4400285988Sdumbbell 4401285988Sdumbbell DRM_LOCK(dev); 4402285988Sdumbbell dev_priv->mm.suspended = 0; 4403285988Sdumbbell 4404285988Sdumbbell ret = i915_gem_init_hw(dev); 4405285988Sdumbbell if (ret != 0) { 4406285988Sdumbbell DRM_UNLOCK(dev); 4407285988Sdumbbell return ret; 4408285988Sdumbbell } 4409285988Sdumbbell 4410296548Sdumbbell BUG_ON(!list_empty(&dev_priv->mm.active_list)); 4411285988Sdumbbell DRM_UNLOCK(dev); 4412285988Sdumbbell 4413285988Sdumbbell ret = drm_irq_install(dev); 4414285988Sdumbbell if (ret) 4415285988Sdumbbell goto cleanup_ringbuffer; 4416285988Sdumbbell 4417285988Sdumbbell return 0; 4418285988Sdumbbell 4419285988Sdumbbellcleanup_ringbuffer: 4420285988Sdumbbell DRM_LOCK(dev); 4421285988Sdumbbell i915_gem_cleanup_ringbuffer(dev); 4422285988Sdumbbell dev_priv->mm.suspended = 1; 4423285988Sdumbbell DRM_UNLOCK(dev); 4424285988Sdumbbell 4425285988Sdumbbell return ret; 4426285988Sdumbbell} 4427285988Sdumbbell 4428285988Sdumbbellint 4429285988Sdumbbelli915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 4430285988Sdumbbell struct drm_file *file_priv) 4431285988Sdumbbell{ 4432285988Sdumbbell if (drm_core_check_feature(dev, DRIVER_MODESET)) 4433285988Sdumbbell return 0; 4434285988Sdumbbell 4435285988Sdumbbell drm_irq_uninstall(dev); 4436285988Sdumbbell return i915_gem_idle(dev); 4437285988Sdumbbell} 4438285988Sdumbbell 4439285988Sdumbbellvoid 4440235783Skibi915_gem_lastclose(struct drm_device *dev) 4441235783Skib{ 4442235783Skib int ret; 4443235783Skib 4444235783Skib if (drm_core_check_feature(dev, DRIVER_MODESET)) 4445235783Skib return; 4446235783Skib 4447235783Skib ret = i915_gem_idle(dev); 4448285988Sdumbbell if (ret) 4449235783Skib DRM_ERROR("failed to idle hardware: %d\n", ret); 4450235783Skib} 4451235783Skib 4452285988Sdumbbellstatic void 4453285988Sdumbbellinit_ring_lists(struct intel_ring_buffer *ring) 4454235783Skib{ 4455285988Sdumbbell INIT_LIST_HEAD(&ring->active_list); 4456285988Sdumbbell INIT_LIST_HEAD(&ring->request_list); 4457285988Sdumbbell} 4458285988Sdumbbell 4459285988Sdumbbellvoid 4460285988Sdumbbelli915_gem_load(struct drm_device *dev) 4461285988Sdumbbell{ 4462285988Sdumbbell int i; 4463285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4464285988Sdumbbell 4465285988Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.active_list); 4466285988Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 4467296548Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.unbound_list); 4468296548Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.bound_list); 4469285988Sdumbbell INIT_LIST_HEAD(&dev_priv->mm.fence_list); 4470285988Sdumbbell for (i = 0; i < I915_NUM_RINGS; i++) 4471296548Sdumbbell init_ring_lists(&dev_priv->ring[i]); 4472285988Sdumbbell for (i = 0; i < I915_MAX_NUM_FENCES; i++) 4473285988Sdumbbell INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 4474296548Sdumbbell TIMEOUT_TASK_INIT(dev_priv->wq, &dev_priv->mm.retire_work, 0, 4475296548Sdumbbell i915_gem_retire_work_handler, dev_priv); 4476296548Sdumbbell init_completion(&dev_priv->error_completion); 4477285988Sdumbbell 4478285988Sdumbbell /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 4479285988Sdumbbell if (IS_GEN3(dev)) { 4480285988Sdumbbell I915_WRITE(MI_ARB_STATE, 4481285988Sdumbbell _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 4482285988Sdumbbell } 4483285988Sdumbbell 4484285988Sdumbbell dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 4485285988Sdumbbell 4486285988Sdumbbell /* Old X drivers will take 0-2 for front, back, depth buffers */ 4487285988Sdumbbell if (!drm_core_check_feature(dev, DRIVER_MODESET)) 4488285988Sdumbbell dev_priv->fence_reg_start = 3; 4489285988Sdumbbell 4490285988Sdumbbell if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 4491285988Sdumbbell dev_priv->num_fence_regs = 16; 4492285988Sdumbbell else 4493285988Sdumbbell dev_priv->num_fence_regs = 8; 4494285988Sdumbbell 4495285988Sdumbbell /* Initialize fence registers to zero */ 4496285988Sdumbbell i915_gem_reset_fences(dev); 4497285988Sdumbbell 4498285988Sdumbbell i915_gem_detect_bit_6_swizzle(dev); 4499296548Sdumbbell DRM_INIT_WAITQUEUE(&dev_priv->pending_flip_queue); 4500296548Sdumbbell 4501285988Sdumbbell dev_priv->mm.interruptible = true; 4502285988Sdumbbell 4503296548Sdumbbell dev_priv->mm.inactive_shrinker = EVENTHANDLER_REGISTER(vm_lowmem, 4504296548Sdumbbell i915_gem_inactive_shrink, dev, EVENTHANDLER_PRI_ANY); 4505285988Sdumbbell} 4506285988Sdumbbell 4507287174Sbapt/* 4508287174Sbapt * Create a physically contiguous memory object for this object 4509287174Sbapt * e.g. for cursor + overlay regs 4510287174Sbapt */ 4511285988Sdumbbellstatic int i915_gem_init_phys_object(struct drm_device *dev, 4512285988Sdumbbell int id, int size, int align) 4513285988Sdumbbell{ 4514285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4515235783Skib struct drm_i915_gem_phys_object *phys_obj; 4516235783Skib int ret; 4517235783Skib 4518285988Sdumbbell if (dev_priv->mm.phys_objs[id - 1] || !size) 4519285988Sdumbbell return 0; 4520235783Skib 4521285988Sdumbbell phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), 4522285988Sdumbbell DRM_I915_GEM, M_WAITOK | M_ZERO); 4523296548Sdumbbell if (!phys_obj) 4524296548Sdumbbell return -ENOMEM; 4525235783Skib 4526235783Skib phys_obj->id = id; 4527235783Skib 4528280183Sdumbbell phys_obj->handle = drm_pci_alloc(dev, size, align, BUS_SPACE_MAXADDR); 4529285988Sdumbbell if (!phys_obj->handle) { 4530235783Skib ret = -ENOMEM; 4531285988Sdumbbell goto kfree_obj; 4532235783Skib } 4533296548Sdumbbell#ifdef CONFIG_X86 4534235783Skib pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 4535235783Skib size / PAGE_SIZE, PAT_WRITE_COMBINING); 4536296548Sdumbbell#endif 4537235783Skib 4538235783Skib dev_priv->mm.phys_objs[id - 1] = phys_obj; 4539235783Skib 4540285988Sdumbbell return 0; 4541285988Sdumbbellkfree_obj: 4542235783Skib free(phys_obj, DRM_I915_GEM); 4543285988Sdumbbell return ret; 4544235783Skib} 4545235783Skib 4546285988Sdumbbellstatic void i915_gem_free_phys_object(struct drm_device *dev, int id) 4547235783Skib{ 4548285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4549235783Skib struct drm_i915_gem_phys_object *phys_obj; 4550235783Skib 4551285988Sdumbbell if (!dev_priv->mm.phys_objs[id - 1]) 4552235783Skib return; 4553235783Skib 4554235783Skib phys_obj = dev_priv->mm.phys_objs[id - 1]; 4555285988Sdumbbell if (phys_obj->cur_obj) { 4556235783Skib i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 4557285988Sdumbbell } 4558235783Skib 4559296548Sdumbbell#ifdef FREEBSD_WIP 4560296548Sdumbbell#ifdef CONFIG_X86 4561296548Sdumbbell set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); 4562296548Sdumbbell#endif 4563296548Sdumbbell#endif /* FREEBSD_WIP */ 4564296548Sdumbbell 4565235783Skib drm_pci_free(dev, phys_obj->handle); 4566235783Skib free(phys_obj, DRM_I915_GEM); 4567235783Skib dev_priv->mm.phys_objs[id - 1] = NULL; 4568235783Skib} 4569235783Skib 4570285988Sdumbbellvoid i915_gem_free_all_phys_object(struct drm_device *dev) 4571235783Skib{ 4572235783Skib int i; 4573235783Skib 4574235783Skib for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 4575235783Skib i915_gem_free_phys_object(dev, i); 4576235783Skib} 4577235783Skib 4578285988Sdumbbellvoid i915_gem_detach_phys_object(struct drm_device *dev, 4579285988Sdumbbell struct drm_i915_gem_object *obj) 4580235783Skib{ 4581235783Skib struct sf_buf *sf; 4582296548Sdumbbell char *vaddr; 4583296548Sdumbbell char *dst; 4584296548Sdumbbell int i; 4585296548Sdumbbell int page_count; 4586235783Skib 4587285988Sdumbbell if (!obj->phys_obj) 4588235783Skib return; 4589235783Skib vaddr = obj->phys_obj->handle->vaddr; 4590235783Skib 4591235783Skib page_count = obj->base.size / PAGE_SIZE; 4592248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4593235783Skib for (i = 0; i < page_count; i++) { 4594296548Sdumbbell vm_page_t page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4595285988Sdumbbell if (page == NULL) 4596235783Skib continue; /* XXX */ 4597235783Skib 4598248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4599285988Sdumbbell sf = sf_buf_alloc(page, 0); 4600235783Skib if (sf != NULL) { 4601235783Skib dst = (char *)sf_buf_kva(sf); 4602235783Skib memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 4603235783Skib sf_buf_free(sf); 4604235783Skib } 4605285988Sdumbbell drm_clflush_pages(&page, 1); 4606235783Skib 4607248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4608285988Sdumbbell vm_page_reference(page); 4609285988Sdumbbell vm_page_lock(page); 4610285988Sdumbbell vm_page_dirty(page); 4611285988Sdumbbell vm_page_unwire(page, PQ_INACTIVE); 4612285988Sdumbbell vm_page_unlock(page); 4613235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4614235783Skib } 4615248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4616296548Sdumbbell i915_gem_chipset_flush(dev); 4617235783Skib 4618235783Skib obj->phys_obj->cur_obj = NULL; 4619235783Skib obj->phys_obj = NULL; 4620235783Skib} 4621235783Skib 4622235783Skibint 4623235783Skibi915_gem_attach_phys_object(struct drm_device *dev, 4624285988Sdumbbell struct drm_i915_gem_object *obj, 4625285988Sdumbbell int id, 4626285988Sdumbbell int align) 4627235783Skib{ 4628285988Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 4629235783Skib struct sf_buf *sf; 4630235783Skib char *dst, *src; 4631285988Sdumbbell int ret = 0; 4632285988Sdumbbell int page_count; 4633285988Sdumbbell int i; 4634235783Skib 4635235783Skib if (id > I915_MAX_PHYS_OBJECT) 4636285988Sdumbbell return -EINVAL; 4637235783Skib 4638285988Sdumbbell if (obj->phys_obj) { 4639235783Skib if (obj->phys_obj->id == id) 4640285988Sdumbbell return 0; 4641235783Skib i915_gem_detach_phys_object(dev, obj); 4642235783Skib } 4643235783Skib 4644285988Sdumbbell /* create a new object */ 4645285988Sdumbbell if (!dev_priv->mm.phys_objs[id - 1]) { 4646285988Sdumbbell ret = i915_gem_init_phys_object(dev, id, 4647285988Sdumbbell obj->base.size, align); 4648285988Sdumbbell if (ret) { 4649235783Skib DRM_ERROR("failed to init phys object %d size: %zu\n", 4650235783Skib id, obj->base.size); 4651285988Sdumbbell return ret; 4652235783Skib } 4653235783Skib } 4654235783Skib 4655235783Skib /* bind to the object */ 4656235783Skib obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4657235783Skib obj->phys_obj->cur_obj = obj; 4658235783Skib 4659235783Skib page_count = obj->base.size / PAGE_SIZE; 4660235783Skib 4661248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4662235783Skib for (i = 0; i < page_count; i++) { 4663296548Sdumbbell vm_page_t page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); 4664285988Sdumbbell if (page == NULL) { 4665235783Skib ret = -EIO; 4666235783Skib break; 4667235783Skib } 4668248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4669285988Sdumbbell sf = sf_buf_alloc(page, 0); 4670235783Skib src = (char *)sf_buf_kva(sf); 4671235783Skib dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 4672235783Skib memcpy(dst, src, PAGE_SIZE); 4673235783Skib sf_buf_free(sf); 4674235783Skib 4675248084Sattilio VM_OBJECT_WLOCK(obj->base.vm_obj); 4676235783Skib 4677285988Sdumbbell vm_page_reference(page); 4678285988Sdumbbell vm_page_lock(page); 4679285988Sdumbbell vm_page_unwire(page, PQ_INACTIVE); 4680285988Sdumbbell vm_page_unlock(page); 4681235783Skib atomic_add_long(&i915_gem_wired_pages_cnt, -1); 4682235783Skib } 4683248084Sattilio VM_OBJECT_WUNLOCK(obj->base.vm_obj); 4684235783Skib 4685285988Sdumbbell return ret; 4686235783Skib} 4687235783Skib 4688235783Skibstatic int 4689285988Sdumbbelli915_gem_phys_pwrite(struct drm_device *dev, 4690285988Sdumbbell struct drm_i915_gem_object *obj, 4691285988Sdumbbell struct drm_i915_gem_pwrite *args, 4692285988Sdumbbell struct drm_file *file_priv) 4693235783Skib{ 4694285988Sdumbbell void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset; 4695285988Sdumbbell char __user *user_data = to_user_ptr(args->data_ptr); 4696235783Skib 4697285988Sdumbbell if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 4698285988Sdumbbell unsigned long unwritten; 4699285988Sdumbbell 4700285988Sdumbbell /* The physical object once assigned is fixed for the lifetime 4701285988Sdumbbell * of the obj, so we can safely drop the lock and continue 4702285988Sdumbbell * to access vaddr. 4703285988Sdumbbell */ 4704285988Sdumbbell DRM_UNLOCK(dev); 4705285988Sdumbbell unwritten = copy_from_user(vaddr, user_data, args->size); 4706285988Sdumbbell DRM_LOCK(dev); 4707285988Sdumbbell if (unwritten) 4708285988Sdumbbell return -EFAULT; 4709285988Sdumbbell } 4710285988Sdumbbell 4711285988Sdumbbell i915_gem_chipset_flush(dev); 4712285988Sdumbbell return 0; 4713235783Skib} 4714235783Skib 4715285988Sdumbbellvoid i915_gem_release(struct drm_device *dev, struct drm_file *file) 4716235783Skib{ 4717285988Sdumbbell struct drm_i915_file_private *file_priv = file->driver_priv; 4718235783Skib 4719285988Sdumbbell /* Clean up our request list when the client is going away, so that 4720285988Sdumbbell * later retire_requests won't dereference our soon-to-be-gone 4721285988Sdumbbell * file_priv. 4722285988Sdumbbell */ 4723296548Sdumbbell mtx_lock(&file_priv->mm.lock); 4724285988Sdumbbell while (!list_empty(&file_priv->mm.request_list)) { 4725285988Sdumbbell struct drm_i915_gem_request *request; 4726235783Skib 4727285988Sdumbbell request = list_first_entry(&file_priv->mm.request_list, 4728285988Sdumbbell struct drm_i915_gem_request, 4729285988Sdumbbell client_list); 4730285988Sdumbbell list_del(&request->client_list); 4731285988Sdumbbell request->file_priv = NULL; 4732235783Skib } 4733296548Sdumbbell mtx_unlock(&file_priv->mm.lock); 4734285988Sdumbbell} 4735235783Skib 4736296548Sdumbbellstatic void 4737296548Sdumbbelli915_gem_inactive_shrink(void *arg) 4738296548Sdumbbell{ 4739296548Sdumbbell struct drm_device *dev = arg; 4740296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 4741296548Sdumbbell int pass1, pass2; 4742296548Sdumbbell 4743296548Sdumbbell if (!sx_try_xlock(&dev->dev_struct_lock)) { 4744296548Sdumbbell return; 4745296548Sdumbbell } 4746296548Sdumbbell 4747296548Sdumbbell CTR0(KTR_DRM, "gem_lowmem"); 4748296548Sdumbbell 4749296548Sdumbbell pass1 = i915_gem_purge(dev_priv, -1); 4750296548Sdumbbell pass2 = __i915_gem_shrink(dev_priv, -1, false); 4751296548Sdumbbell 4752296548Sdumbbell if (pass2 <= pass1 / 100) 4753296548Sdumbbell i915_gem_shrink_all(dev_priv); 4754296548Sdumbbell 4755296548Sdumbbell DRM_UNLOCK(dev); 4756296548Sdumbbell} 4757296548Sdumbbell 4758285988Sdumbbellstatic vm_page_t 4759285988Sdumbbelli915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh) 4760285988Sdumbbell{ 4761285988Sdumbbell vm_page_t page; 4762285988Sdumbbell int rv; 4763235783Skib 4764285988Sdumbbell VM_OBJECT_ASSERT_WLOCKED(object); 4765285988Sdumbbell page = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); 4766285988Sdumbbell if (page->valid != VM_PAGE_BITS_ALL) { 4767285988Sdumbbell if (vm_pager_has_page(object, pindex, NULL, NULL)) { 4768292373Sglebius rv = vm_pager_get_pages(object, &page, 1, NULL, NULL); 4769285988Sdumbbell if (rv != VM_PAGER_OK) { 4770285988Sdumbbell vm_page_lock(page); 4771285988Sdumbbell vm_page_free(page); 4772285988Sdumbbell vm_page_unlock(page); 4773285988Sdumbbell return (NULL); 4774285988Sdumbbell } 4775285988Sdumbbell if (fresh != NULL) 4776285988Sdumbbell *fresh = true; 4777285988Sdumbbell } else { 4778285988Sdumbbell pmap_zero_page(page); 4779285988Sdumbbell page->valid = VM_PAGE_BITS_ALL; 4780285988Sdumbbell page->dirty = 0; 4781285988Sdumbbell if (fresh != NULL) 4782285988Sdumbbell *fresh = false; 4783285988Sdumbbell } 4784285988Sdumbbell } else if (fresh != NULL) { 4785285988Sdumbbell *fresh = false; 4786235783Skib } 4787285988Sdumbbell vm_page_lock(page); 4788285988Sdumbbell vm_page_wire(page); 4789285988Sdumbbell vm_page_unlock(page); 4790285988Sdumbbell vm_page_xunbusy(page); 4791285988Sdumbbell atomic_add_long(&i915_gem_wired_pages_cnt, 1); 4792285988Sdumbbell return (page); 4793235783Skib} 4794