1235783Skib/* 2235783Skib * Copyright �� 2008,2010 Intel Corporation 3235783Skib * 4235783Skib * Permission is hereby granted, free of charge, to any person obtaining a 5235783Skib * copy of this software and associated documentation files (the "Software"), 6235783Skib * to deal in the Software without restriction, including without limitation 7235783Skib * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8235783Skib * and/or sell copies of the Software, and to permit persons to whom the 9235783Skib * Software is furnished to do so, subject to the following conditions: 10235783Skib * 11235783Skib * The above copyright notice and this permission notice (including the next 12235783Skib * paragraph) shall be included in all copies or substantial portions of the 13235783Skib * Software. 14235783Skib * 15235783Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16235783Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17235783Skib * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18235783Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19235783Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20235783Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21235783Skib * IN THE SOFTWARE. 22235783Skib * 23235783Skib * Authors: 24235783Skib * Eric Anholt <eric@anholt.net> 25235783Skib * Chris Wilson <chris@chris-wilson.co.uk> 26235783Skib * 27235783Skib */ 28235783Skib 29235783Skib#include <sys/cdefs.h> 30235783Skib__FBSDID("$FreeBSD$"); 31235783Skib 32235783Skib#include <dev/drm2/drmP.h> 33235783Skib#include <dev/drm2/i915/i915_drm.h> 34235783Skib#include <dev/drm2/i915/i915_drv.h> 35235783Skib#include <dev/drm2/i915/intel_drv.h> 36296548Sdumbbell 37235783Skib#include <sys/limits.h> 38235783Skib#include <sys/sf_buf.h> 39235783Skib 40235783Skibstruct eb_objects { 41296548Sdumbbell int and; 42296548Sdumbbell struct hlist_head buckets[0]; 43235783Skib}; 44235783Skib 45235783Skibstatic struct eb_objects * 46235783Skibeb_create(int size) 47235783Skib{ 48235783Skib struct eb_objects *eb; 49296548Sdumbbell int count = PAGE_SIZE / sizeof(struct hlist_head) / 2; 50296548Sdumbbell BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head)); 51296548Sdumbbell while (count > size) 52296548Sdumbbell count >>= 1; 53296548Sdumbbell eb = malloc(count*sizeof(struct hlist_head) + 54296548Sdumbbell sizeof(struct eb_objects), 55296548Sdumbbell DRM_I915_GEM, M_WAITOK | M_ZERO); 56296548Sdumbbell if (eb == NULL) 57296548Sdumbbell return eb; 58235783Skib 59296548Sdumbbell eb->and = count - 1; 60290228Sdumbbell return eb; 61235783Skib} 62235783Skib 63235783Skibstatic void 64235783Skibeb_reset(struct eb_objects *eb) 65235783Skib{ 66296548Sdumbbell memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head)); 67235783Skib} 68235783Skib 69235783Skibstatic void 70235783Skibeb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj) 71235783Skib{ 72296548Sdumbbell hlist_add_head(&obj->exec_node, 73296548Sdumbbell &eb->buckets[obj->exec_handle & eb->and]); 74235783Skib} 75235783Skib 76235783Skibstatic struct drm_i915_gem_object * 77235783Skibeb_get_object(struct eb_objects *eb, unsigned long handle) 78235783Skib{ 79296548Sdumbbell struct hlist_head *head; 80296548Sdumbbell struct hlist_node *node; 81235783Skib struct drm_i915_gem_object *obj; 82235783Skib 83296548Sdumbbell head = &eb->buckets[handle & eb->and]; 84296548Sdumbbell hlist_for_each(node, head) { 85296548Sdumbbell obj = hlist_entry(node, struct drm_i915_gem_object, exec_node); 86235783Skib if (obj->exec_handle == handle) 87290228Sdumbbell return obj; 88235783Skib } 89290228Sdumbbell 90290228Sdumbbell return NULL; 91235783Skib} 92235783Skib 93235783Skibstatic void 94235783Skibeb_destroy(struct eb_objects *eb) 95235783Skib{ 96235783Skib free(eb, DRM_I915_GEM); 97235783Skib} 98235783Skib 99277487Skibstatic inline int use_cpu_reloc(struct drm_i915_gem_object *obj) 100277487Skib{ 101277487Skib return (obj->base.write_domain == I915_GEM_DOMAIN_CPU || 102296548Sdumbbell !obj->map_and_fenceable || 103277487Skib obj->cache_level != I915_CACHE_NONE); 104277487Skib} 105277487Skib 106235783Skibstatic int 107235783Skibi915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, 108235783Skib struct eb_objects *eb, 109235783Skib struct drm_i915_gem_relocation_entry *reloc) 110235783Skib{ 111235783Skib struct drm_device *dev = obj->base.dev; 112235783Skib struct drm_gem_object *target_obj; 113277487Skib struct drm_i915_gem_object *target_i915_obj; 114235783Skib uint32_t target_offset; 115235783Skib int ret = -EINVAL; 116235783Skib 117235783Skib /* we've already hold a reference to all valid objects */ 118235783Skib target_obj = &eb_get_object(eb, reloc->target_handle)->base; 119235783Skib if (unlikely(target_obj == NULL)) 120235783Skib return -ENOENT; 121235783Skib 122277487Skib target_i915_obj = to_intel_bo(target_obj); 123277487Skib target_offset = target_i915_obj->gtt_offset; 124235783Skib 125296548Sdumbbell /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and 126296548Sdumbbell * pipe_control writes because the gpu doesn't properly redirect them 127296548Sdumbbell * through the ppgtt for non_secure batchbuffers. */ 128296548Sdumbbell if (unlikely(IS_GEN6(dev) && 129296548Sdumbbell reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && 130296548Sdumbbell !target_i915_obj->has_global_gtt_mapping)) { 131296548Sdumbbell i915_gem_gtt_bind_object(target_i915_obj, 132296548Sdumbbell target_i915_obj->cache_level); 133235783Skib } 134235783Skib 135235783Skib /* Validate that the target is in a valid r/w GPU domain */ 136235783Skib if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { 137235783Skib DRM_DEBUG("reloc with multiple write domains: " 138235783Skib "obj %p target %d offset %d " 139235783Skib "read %08x write %08x", 140235783Skib obj, reloc->target_handle, 141235783Skib (int) reloc->offset, 142235783Skib reloc->read_domains, 143235783Skib reloc->write_domain); 144235783Skib return ret; 145235783Skib } 146235783Skib if (unlikely((reloc->write_domain | reloc->read_domains) 147235783Skib & ~I915_GEM_GPU_DOMAINS)) { 148235783Skib DRM_DEBUG("reloc with read/write non-GPU domains: " 149235783Skib "obj %p target %d offset %d " 150235783Skib "read %08x write %08x", 151235783Skib obj, reloc->target_handle, 152235783Skib (int) reloc->offset, 153235783Skib reloc->read_domains, 154235783Skib reloc->write_domain); 155235783Skib return ret; 156235783Skib } 157235783Skib if (unlikely(reloc->write_domain && target_obj->pending_write_domain && 158235783Skib reloc->write_domain != target_obj->pending_write_domain)) { 159235783Skib DRM_DEBUG("Write domain conflict: " 160235783Skib "obj %p target %d offset %d " 161235783Skib "new %08x old %08x\n", 162235783Skib obj, reloc->target_handle, 163235783Skib (int) reloc->offset, 164235783Skib reloc->write_domain, 165235783Skib target_obj->pending_write_domain); 166235783Skib return ret; 167235783Skib } 168235783Skib 169235783Skib target_obj->pending_read_domains |= reloc->read_domains; 170235783Skib target_obj->pending_write_domain |= reloc->write_domain; 171235783Skib 172235783Skib /* If the relocation already has the right value in it, no 173235783Skib * more work needs to be done. 174235783Skib */ 175235783Skib if (target_offset == reloc->presumed_offset) 176235783Skib return 0; 177235783Skib 178235783Skib /* Check that the relocation address is valid... */ 179235783Skib if (unlikely(reloc->offset > obj->base.size - 4)) { 180235783Skib DRM_DEBUG("Relocation beyond object bounds: " 181235783Skib "obj %p target %d offset %d size %d.\n", 182235783Skib obj, reloc->target_handle, 183235783Skib (int) reloc->offset, 184235783Skib (int) obj->base.size); 185235783Skib return ret; 186235783Skib } 187235783Skib if (unlikely(reloc->offset & 3)) { 188235783Skib DRM_DEBUG("Relocation not 4-byte aligned: " 189235783Skib "obj %p target %d offset %d.\n", 190235783Skib obj, reloc->target_handle, 191235783Skib (int) reloc->offset); 192235783Skib return ret; 193235783Skib } 194235783Skib 195277487Skib /* We can't wait for rendering with pagefaults disabled */ 196277487Skib if (obj->active && (curthread->td_pflags & TDP_NOFAULTING) != 0) 197290228Sdumbbell return -EFAULT; 198277487Skib 199235783Skib reloc->delta += target_offset; 200277487Skib if (use_cpu_reloc(obj)) { 201235783Skib uint32_t page_offset = reloc->offset & PAGE_MASK; 202235783Skib char *vaddr; 203235783Skib struct sf_buf *sf; 204235783Skib 205277487Skib ret = i915_gem_object_set_to_cpu_domain(obj, 1); 206277487Skib if (ret) 207277487Skib return ret; 208277487Skib 209235783Skib sf = sf_buf_alloc(obj->pages[OFF_TO_IDX(reloc->offset)], 210235783Skib SFB_NOWAIT); 211235783Skib if (sf == NULL) 212290228Sdumbbell return -ENOMEM; 213235783Skib vaddr = (void *)sf_buf_kva(sf); 214235783Skib *(uint32_t *)(vaddr + page_offset) = reloc->delta; 215235783Skib sf_buf_free(sf); 216235783Skib } else { 217296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 218296548Sdumbbell uint32_t __iomem *reloc_entry; 219296548Sdumbbell char __iomem *reloc_page; 220235783Skib 221277487Skib ret = i915_gem_object_set_to_gtt_domain(obj, true); 222235783Skib if (ret) 223235783Skib return ret; 224235783Skib 225277487Skib ret = i915_gem_object_put_fence(obj); 226277487Skib if (ret) 227277487Skib return ret; 228277487Skib 229287174Sbapt /* Map the page containing the relocation we're going to perform. */ 230235783Skib reloc->offset += obj->gtt_offset; 231296548Sdumbbell reloc_page = pmap_mapdev_attr(dev_priv->mm.gtt_base_addr + (reloc->offset & 232235783Skib ~PAGE_MASK), PAGE_SIZE, PAT_WRITE_COMBINING); 233296548Sdumbbell reloc_entry = (uint32_t __iomem *) 234293837Sdumbbell (reloc_page + (reloc->offset & PAGE_MASK)); 235235783Skib *(volatile uint32_t *)reloc_entry = reloc->delta; 236235783Skib pmap_unmapdev((vm_offset_t)reloc_page, PAGE_SIZE); 237235783Skib } 238235783Skib 239235783Skib /* and update the user's relocation entry */ 240235783Skib reloc->presumed_offset = target_offset; 241235783Skib 242235783Skib return 0; 243235783Skib} 244235783Skib 245235783Skibstatic int 246235783Skibi915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj, 247287174Sbapt struct eb_objects *eb) 248235783Skib{ 249277487Skib#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) 250277487Skib struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)]; 251296548Sdumbbell struct drm_i915_gem_relocation_entry __user *user_relocs; 252235783Skib struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 253277487Skib int remain, ret; 254235783Skib 255296548Sdumbbell user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr; 256296548Sdumbbell 257277487Skib remain = entry->relocation_count; 258277487Skib while (remain) { 259277487Skib struct drm_i915_gem_relocation_entry *r = stack_reloc; 260277487Skib int count = remain; 261296548Sdumbbell if (count > ARRAY_SIZE(stack_reloc)) 262296548Sdumbbell count = ARRAY_SIZE(stack_reloc); 263277487Skib remain -= count; 264235783Skib 265296548Sdumbbell if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]))) 266296548Sdumbbell return -EFAULT; 267235783Skib 268277487Skib do { 269277487Skib u64 offset = r->presumed_offset; 270287174Sbapt 271277487Skib ret = i915_gem_execbuffer_relocate_entry(obj, eb, r); 272277487Skib if (ret) 273277487Skib return ret; 274277487Skib 275277487Skib if (r->presumed_offset != offset && 276296548Sdumbbell __copy_to_user_inatomic(&user_relocs->presumed_offset, 277296548Sdumbbell &r->presumed_offset, 278296548Sdumbbell sizeof(r->presumed_offset))) { 279277487Skib return -EFAULT; 280277487Skib } 281277487Skib 282277487Skib user_relocs++; 283277487Skib r++; 284277487Skib } while (--count); 285235783Skib } 286287174Sbapt 287287174Sbapt return 0; 288277487Skib#undef N_RELOC 289235783Skib} 290235783Skib 291235783Skibstatic int 292235783Skibi915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj, 293287174Sbapt struct eb_objects *eb, 294287174Sbapt struct drm_i915_gem_relocation_entry *relocs) 295235783Skib{ 296235783Skib const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 297235783Skib int i, ret; 298235783Skib 299235783Skib for (i = 0; i < entry->relocation_count; i++) { 300235783Skib ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]); 301235783Skib if (ret) 302235783Skib return ret; 303235783Skib } 304235783Skib 305235783Skib return 0; 306235783Skib} 307235783Skib 308235783Skibstatic int 309235783Skibi915_gem_execbuffer_relocate(struct drm_device *dev, 310235783Skib struct eb_objects *eb, 311235783Skib struct list_head *objects) 312235783Skib{ 313235783Skib struct drm_i915_gem_object *obj; 314293837Sdumbbell int ret = 0, pflags; 315235783Skib 316235783Skib /* This is the fast path and we cannot handle a pagefault whilst 317296548Sdumbbell * holding the struct mutex lest the user pass in the relocations 318235783Skib * contained within a mmaped bo. For in such a case we, the page 319235783Skib * fault handler would call i915_gem_fault() and we would try to 320296548Sdumbbell * acquire the struct mutex again. Obviously this is bad and so 321296548Sdumbbell * lockdep complains vehemently. 322235783Skib */ 323290228Sdumbbell pflags = vm_fault_disable_pagefaults(); 324235783Skib list_for_each_entry(obj, objects, exec_list) { 325235783Skib ret = i915_gem_execbuffer_relocate_object(obj, eb); 326287174Sbapt if (ret) 327235783Skib break; 328235783Skib } 329235783Skib vm_fault_enable_pagefaults(pflags); 330287174Sbapt 331287174Sbapt return ret; 332235783Skib} 333235783Skib 334296548Sdumbbell#define __EXEC_OBJECT_HAS_PIN (1<<31) 335296548Sdumbbell#define __EXEC_OBJECT_HAS_FENCE (1<<30) 336235783Skib 337235783Skibstatic int 338277487Skibneed_reloc_mappable(struct drm_i915_gem_object *obj) 339277487Skib{ 340277487Skib struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 341277487Skib return entry->relocation_count && !use_cpu_reloc(obj); 342277487Skib} 343277487Skib 344277487Skibstatic int 345296548Sdumbbelli915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj, 346296548Sdumbbell struct intel_ring_buffer *ring) 347235783Skib{ 348296548Sdumbbell struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 349235783Skib struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 350235783Skib bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; 351235783Skib bool need_fence, need_mappable; 352235783Skib int ret; 353235783Skib 354235783Skib need_fence = 355235783Skib has_fenced_gpu_access && 356235783Skib entry->flags & EXEC_OBJECT_NEEDS_FENCE && 357235783Skib obj->tiling_mode != I915_TILING_NONE; 358277487Skib need_mappable = need_fence || need_reloc_mappable(obj); 359235783Skib 360296548Sdumbbell ret = i915_gem_object_pin(obj, entry->alignment, need_mappable, false); 361235783Skib if (ret) 362235783Skib return ret; 363235783Skib 364296548Sdumbbell entry->flags |= __EXEC_OBJECT_HAS_PIN; 365296548Sdumbbell 366235783Skib if (has_fenced_gpu_access) { 367235783Skib if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) { 368277487Skib ret = i915_gem_object_get_fence(obj); 369277487Skib if (ret) 370296548Sdumbbell return ret; 371235783Skib 372277487Skib if (i915_gem_object_pin_fence(obj)) 373235783Skib entry->flags |= __EXEC_OBJECT_HAS_FENCE; 374277487Skib 375235783Skib obj->pending_fenced_gpu_access = true; 376235783Skib } 377235783Skib } 378235783Skib 379296548Sdumbbell /* Ensure ppgtt mapping exists if needed */ 380296548Sdumbbell if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) { 381296548Sdumbbell i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 382296548Sdumbbell obj, obj->cache_level); 383296548Sdumbbell 384296548Sdumbbell obj->has_aliasing_ppgtt_mapping = 1; 385296548Sdumbbell } 386296548Sdumbbell 387235783Skib entry->offset = obj->gtt_offset; 388235783Skib return 0; 389296548Sdumbbell} 390235783Skib 391296548Sdumbbellstatic void 392296548Sdumbbelli915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj) 393296548Sdumbbell{ 394296548Sdumbbell struct drm_i915_gem_exec_object2 *entry; 395296548Sdumbbell 396296548Sdumbbell if (!obj->gtt_space) 397296548Sdumbbell return; 398296548Sdumbbell 399296548Sdumbbell entry = obj->exec_entry; 400296548Sdumbbell 401296548Sdumbbell if (entry->flags & __EXEC_OBJECT_HAS_FENCE) 402296548Sdumbbell i915_gem_object_unpin_fence(obj); 403296548Sdumbbell 404296548Sdumbbell if (entry->flags & __EXEC_OBJECT_HAS_PIN) 405296548Sdumbbell i915_gem_object_unpin(obj); 406296548Sdumbbell 407296548Sdumbbell entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN); 408235783Skib} 409235783Skib 410235783Skibstatic int 411235783Skibi915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, 412235783Skib struct drm_file *file, 413235783Skib struct list_head *objects) 414235783Skib{ 415235783Skib struct drm_i915_gem_object *obj; 416287174Sbapt struct list_head ordered_objects; 417287174Sbapt bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; 418290228Sdumbbell int retry; 419235783Skib 420235783Skib INIT_LIST_HEAD(&ordered_objects); 421235783Skib while (!list_empty(objects)) { 422235783Skib struct drm_i915_gem_exec_object2 *entry; 423235783Skib bool need_fence, need_mappable; 424235783Skib 425235783Skib obj = list_first_entry(objects, 426235783Skib struct drm_i915_gem_object, 427235783Skib exec_list); 428235783Skib entry = obj->exec_entry; 429235783Skib 430235783Skib need_fence = 431235783Skib has_fenced_gpu_access && 432235783Skib entry->flags & EXEC_OBJECT_NEEDS_FENCE && 433235783Skib obj->tiling_mode != I915_TILING_NONE; 434277487Skib need_mappable = need_fence || need_reloc_mappable(obj); 435235783Skib 436235783Skib if (need_mappable) 437235783Skib list_move(&obj->exec_list, &ordered_objects); 438235783Skib else 439235783Skib list_move_tail(&obj->exec_list, &ordered_objects); 440235783Skib 441235783Skib obj->base.pending_read_domains = 0; 442235783Skib obj->base.pending_write_domain = 0; 443296548Sdumbbell obj->pending_fenced_gpu_access = false; 444235783Skib } 445235783Skib list_splice(&ordered_objects, objects); 446235783Skib 447235783Skib /* Attempt to pin all of the buffers into the GTT. 448235783Skib * This is done in 3 phases: 449235783Skib * 450235783Skib * 1a. Unbind all objects that do not match the GTT constraints for 451235783Skib * the execbuffer (fenceable, mappable, alignment etc). 452287174Sbapt * 1b. Increment pin count for already bound objects. 453235783Skib * 2. Bind new objects. 454235783Skib * 3. Decrement pin count. 455235783Skib * 456287174Sbapt * This avoid unnecessary unbinding of later objects in order to make 457235783Skib * room for the earlier objects *unless* we need to defragment. 458235783Skib */ 459235783Skib retry = 0; 460235783Skib do { 461296548Sdumbbell int ret = 0; 462235783Skib 463235783Skib /* Unbind any ill-fitting objects or pin. */ 464235783Skib list_for_each_entry(obj, objects, exec_list) { 465235783Skib struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; 466235783Skib bool need_fence, need_mappable; 467235783Skib 468235783Skib if (!obj->gtt_space) 469235783Skib continue; 470235783Skib 471235783Skib need_fence = 472235783Skib has_fenced_gpu_access && 473235783Skib entry->flags & EXEC_OBJECT_NEEDS_FENCE && 474235783Skib obj->tiling_mode != I915_TILING_NONE; 475277487Skib need_mappable = need_fence || need_reloc_mappable(obj); 476235783Skib 477235783Skib if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) || 478235783Skib (need_mappable && !obj->map_and_fenceable)) 479235783Skib ret = i915_gem_object_unbind(obj); 480235783Skib else 481296548Sdumbbell ret = i915_gem_execbuffer_reserve_object(obj, ring); 482235783Skib if (ret) 483235783Skib goto err; 484235783Skib } 485235783Skib 486235783Skib /* Bind fresh objects */ 487235783Skib list_for_each_entry(obj, objects, exec_list) { 488235783Skib if (obj->gtt_space) 489235783Skib continue; 490235783Skib 491296548Sdumbbell ret = i915_gem_execbuffer_reserve_object(obj, ring); 492296548Sdumbbell if (ret) 493296548Sdumbbell goto err; 494235783Skib } 495235783Skib 496296548Sdumbbellerr: /* Decrement pin count for bound objects */ 497296548Sdumbbell list_for_each_entry(obj, objects, exec_list) 498296548Sdumbbell i915_gem_execbuffer_unreserve_object(obj); 499235783Skib 500296548Sdumbbell if (ret != -ENOSPC || retry++) 501235783Skib return ret; 502235783Skib 503296548Sdumbbell ret = i915_gem_evict_everything(ring->dev); 504235783Skib if (ret) 505235783Skib return ret; 506235783Skib } while (1); 507235783Skib} 508235783Skib 509235783Skibstatic int 510235783Skibi915_gem_execbuffer_relocate_slow(struct drm_device *dev, 511287174Sbapt struct drm_file *file, 512287174Sbapt struct intel_ring_buffer *ring, 513287174Sbapt struct list_head *objects, 514287174Sbapt struct eb_objects *eb, 515287174Sbapt struct drm_i915_gem_exec_object2 *exec, 516287174Sbapt int count) 517235783Skib{ 518235783Skib struct drm_i915_gem_relocation_entry *reloc; 519235783Skib struct drm_i915_gem_object *obj; 520235783Skib int *reloc_offset; 521235783Skib int i, total, ret; 522235783Skib 523235783Skib /* We may process another execbuffer during the unlock... */ 524235783Skib while (!list_empty(objects)) { 525235783Skib obj = list_first_entry(objects, 526235783Skib struct drm_i915_gem_object, 527235783Skib exec_list); 528235783Skib list_del_init(&obj->exec_list); 529235783Skib drm_gem_object_unreference(&obj->base); 530235783Skib } 531235783Skib 532235783Skib DRM_UNLOCK(dev); 533235783Skib 534235783Skib total = 0; 535235783Skib for (i = 0; i < count; i++) 536235783Skib total += exec[i].relocation_count; 537235783Skib 538296548Sdumbbell reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset)); 539296548Sdumbbell reloc = drm_malloc_ab(total, sizeof(*reloc)); 540296548Sdumbbell if (reloc == NULL || reloc_offset == NULL) { 541296548Sdumbbell drm_free_large(reloc); 542296548Sdumbbell drm_free_large(reloc_offset); 543296548Sdumbbell DRM_LOCK(dev); 544296548Sdumbbell return -ENOMEM; 545296548Sdumbbell } 546235783Skib 547235783Skib total = 0; 548235783Skib for (i = 0; i < count; i++) { 549296548Sdumbbell struct drm_i915_gem_relocation_entry __user *user_relocs; 550296548Sdumbbell u64 invalid_offset = (u64)-1; 551296548Sdumbbell int j; 552235783Skib 553296548Sdumbbell user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr; 554296548Sdumbbell 555296548Sdumbbell if (copy_from_user(reloc+total, user_relocs, 556296548Sdumbbell exec[i].relocation_count * sizeof(*reloc))) { 557296548Sdumbbell ret = -EFAULT; 558235783Skib DRM_LOCK(dev); 559235783Skib goto err; 560235783Skib } 561235783Skib 562296548Sdumbbell /* As we do not update the known relocation offsets after 563296548Sdumbbell * relocating (due to the complexities in lock handling), 564296548Sdumbbell * we need to mark them as invalid now so that we force the 565296548Sdumbbell * relocation processing next time. Just in case the target 566296548Sdumbbell * object is evicted and then rebound into its old 567296548Sdumbbell * presumed_offset before the next execbuffer - if that 568296548Sdumbbell * happened we would make the mistake of assuming that the 569296548Sdumbbell * relocations were valid. 570296548Sdumbbell */ 571296548Sdumbbell for (j = 0; j < exec[i].relocation_count; j++) { 572296548Sdumbbell if (copy_to_user(&user_relocs[j].presumed_offset, 573296548Sdumbbell &invalid_offset, 574296548Sdumbbell sizeof(invalid_offset))) { 575296548Sdumbbell ret = -EFAULT; 576296548Sdumbbell DRM_LOCK(dev); 577296548Sdumbbell goto err; 578296548Sdumbbell } 579296548Sdumbbell } 580296548Sdumbbell 581235783Skib reloc_offset[i] = total; 582235783Skib total += exec[i].relocation_count; 583235783Skib } 584235783Skib 585235783Skib ret = i915_mutex_lock_interruptible(dev); 586235783Skib if (ret) { 587235783Skib DRM_LOCK(dev); 588235783Skib goto err; 589235783Skib } 590235783Skib 591235783Skib /* reacquire the objects */ 592235783Skib eb_reset(eb); 593235783Skib for (i = 0; i < count; i++) { 594235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, 595235783Skib exec[i].handle)); 596235783Skib if (&obj->base == NULL) { 597235783Skib DRM_DEBUG("Invalid object handle %d at index %d\n", 598235783Skib exec[i].handle, i); 599235783Skib ret = -ENOENT; 600235783Skib goto err; 601235783Skib } 602235783Skib 603235783Skib list_add_tail(&obj->exec_list, objects); 604235783Skib obj->exec_handle = exec[i].handle; 605235783Skib obj->exec_entry = &exec[i]; 606235783Skib eb_add_object(eb, obj); 607235783Skib } 608235783Skib 609235783Skib ret = i915_gem_execbuffer_reserve(ring, file, objects); 610235783Skib if (ret) 611235783Skib goto err; 612235783Skib 613235783Skib list_for_each_entry(obj, objects, exec_list) { 614235783Skib int offset = obj->exec_entry - exec; 615235783Skib ret = i915_gem_execbuffer_relocate_object_slow(obj, eb, 616287174Sbapt reloc + reloc_offset[offset]); 617235783Skib if (ret) 618235783Skib goto err; 619235783Skib } 620235783Skib 621235783Skib /* Leave the user relocations as are, this is the painfully slow path, 622235783Skib * and we want to avoid the complication of dropping the lock whilst 623235783Skib * having buffers reserved in the aperture and so causing spurious 624235783Skib * ENOSPC for random operations. 625235783Skib */ 626235783Skib 627235783Skiberr: 628296548Sdumbbell drm_free_large(reloc); 629296548Sdumbbell drm_free_large(reloc_offset); 630235783Skib return ret; 631235783Skib} 632235783Skib 633235783Skibstatic int 634235783Skibi915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips) 635235783Skib{ 636235783Skib u32 plane, flip_mask; 637235783Skib int ret; 638235783Skib 639235783Skib /* Check for any pending flips. As we only maintain a flip queue depth 640235783Skib * of 1, we can simply insert a WAIT for the next display flip prior 641235783Skib * to executing the batch and avoid stalling the CPU. 642235783Skib */ 643235783Skib 644235783Skib for (plane = 0; flips >> plane; plane++) { 645235783Skib if (((flips >> plane) & 1) == 0) 646235783Skib continue; 647235783Skib 648235783Skib if (plane) 649235783Skib flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; 650235783Skib else 651235783Skib flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; 652235783Skib 653235783Skib ret = intel_ring_begin(ring, 2); 654235783Skib if (ret) 655235783Skib return ret; 656235783Skib 657235783Skib intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); 658235783Skib intel_ring_emit(ring, MI_NOOP); 659235783Skib intel_ring_advance(ring); 660235783Skib } 661235783Skib 662235783Skib return 0; 663235783Skib} 664235783Skib 665235783Skibstatic int 666235783Skibi915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring, 667235783Skib struct list_head *objects) 668235783Skib{ 669235783Skib struct drm_i915_gem_object *obj; 670296548Sdumbbell uint32_t flush_domains = 0; 671296548Sdumbbell uint32_t flips = 0; 672235783Skib int ret; 673235783Skib 674296548Sdumbbell list_for_each_entry(obj, objects, exec_list) { 675296548Sdumbbell ret = i915_gem_object_sync(obj, ring); 676235783Skib if (ret) 677235783Skib return ret; 678235783Skib 679296548Sdumbbell if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) 680296548Sdumbbell i915_gem_clflush_object(obj); 681296548Sdumbbell 682296548Sdumbbell if (obj->base.pending_write_domain) 683296548Sdumbbell flips |= atomic_read(&obj->pending_flip); 684296548Sdumbbell 685296548Sdumbbell flush_domains |= obj->base.write_domain; 686235783Skib } 687235783Skib 688296548Sdumbbell if (flips) { 689296548Sdumbbell ret = i915_gem_execbuffer_wait_for_flips(ring, flips); 690235783Skib if (ret) 691235783Skib return ret; 692235783Skib } 693235783Skib 694296548Sdumbbell if (flush_domains & I915_GEM_DOMAIN_CPU) 695296548Sdumbbell i915_gem_chipset_flush(ring->dev); 696296548Sdumbbell 697296548Sdumbbell if (flush_domains & I915_GEM_DOMAIN_GTT) 698296548Sdumbbell wmb(); 699296548Sdumbbell 700296548Sdumbbell /* Unconditionally invalidate gpu caches and ensure that we do flush 701296548Sdumbbell * any residual writes from the previous batch. 702296548Sdumbbell */ 703296548Sdumbbell return intel_ring_invalidate_all_caches(ring); 704235783Skib} 705235783Skib 706235783Skibstatic bool 707235783Skibi915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) 708235783Skib{ 709235783Skib return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0; 710235783Skib} 711235783Skib 712235783Skibstatic int 713296548Sdumbbellvalidate_exec_list(struct drm_i915_gem_exec_object2 *exec, 714296548Sdumbbell int count, vm_page_t ***map, int **maplen) 715235783Skib{ 716296548Sdumbbell int i; 717296548Sdumbbell int relocs_total = 0; 718296548Sdumbbell int relocs_max = INT_MAX / sizeof(struct drm_i915_gem_relocation_entry); 719235783Skib vm_page_t *ma; 720235783Skib 721235783Skib /* XXXKIB various limits checking is missing there */ 722235783Skib *map = malloc(count * sizeof(*ma), DRM_I915_GEM, M_WAITOK | M_ZERO); 723289719Sjhb *maplen = malloc(count * sizeof(*maplen), DRM_I915_GEM, M_WAITOK | 724289719Sjhb M_ZERO); 725293837Sdumbbell 726235783Skib for (i = 0; i < count; i++) { 727296548Sdumbbell char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr; 728296548Sdumbbell int length; /* limited by fault_in_pages_readable() */ 729296548Sdumbbell 730296548Sdumbbell /* First check for malicious input causing overflow in 731296548Sdumbbell * the worst case where we need to allocate the entire 732296548Sdumbbell * relocation tree as a single array. 733296548Sdumbbell */ 734296548Sdumbbell if (exec[i].relocation_count > relocs_max - relocs_total) 735235783Skib return -EINVAL; 736296548Sdumbbell relocs_total += exec[i].relocation_count; 737235783Skib 738235783Skib length = exec[i].relocation_count * 739290228Sdumbbell sizeof(struct drm_i915_gem_relocation_entry); 740235783Skib if (length == 0) { 741235783Skib (*map)[i] = NULL; 742235783Skib continue; 743235783Skib } 744290228Sdumbbell 745235783Skib /* 746235783Skib * Since both start and end of the relocation region 747235783Skib * may be not aligned on the page boundary, be 748235783Skib * conservative and request a page slot for each 749235783Skib * partial page. Thus +2. 750235783Skib */ 751296548Sdumbbell int page_count; 752296548Sdumbbell 753235783Skib page_count = howmany(length, PAGE_SIZE) + 2; 754235783Skib ma = (*map)[i] = malloc(page_count * sizeof(vm_page_t), 755235783Skib DRM_I915_GEM, M_WAITOK | M_ZERO); 756289719Sjhb (*maplen)[i] = vm_fault_quick_hold_pages( 757296548Sdumbbell &curproc->p_vmspace->vm_map, (vm_offset_t)ptr, length, 758289719Sjhb VM_PROT_READ | VM_PROT_WRITE, ma, page_count); 759289719Sjhb if ((*maplen)[i] == -1) { 760235783Skib free(ma, DRM_I915_GEM); 761235783Skib (*map)[i] = NULL; 762290228Sdumbbell return -EFAULT; 763235783Skib } 764235783Skib } 765235783Skib 766235783Skib return 0; 767235783Skib} 768235783Skib 769235783Skibstatic void 770235783Skibi915_gem_execbuffer_move_to_active(struct list_head *objects, 771296548Sdumbbell struct intel_ring_buffer *ring) 772235783Skib{ 773235783Skib struct drm_i915_gem_object *obj; 774235783Skib 775235783Skib list_for_each_entry(obj, objects, exec_list) { 776296548Sdumbbell#if defined(KTR) 777296548Sdumbbell u32 old_read = obj->base.read_domains; 778296548Sdumbbell u32 old_write = obj->base.write_domain; 779296548Sdumbbell#endif 780235783Skib 781235783Skib obj->base.read_domains = obj->base.pending_read_domains; 782235783Skib obj->base.write_domain = obj->base.pending_write_domain; 783235783Skib obj->fenced_gpu_access = obj->pending_fenced_gpu_access; 784235783Skib 785296548Sdumbbell i915_gem_object_move_to_active(obj, ring); 786235783Skib if (obj->base.write_domain) { 787235783Skib obj->dirty = 1; 788296548Sdumbbell obj->last_write_seqno = intel_ring_get_seqno(ring); 789277487Skib if (obj->pin_count) /* check for potential scanout */ 790296548Sdumbbell intel_mark_fb_busy(obj); 791235783Skib } 792296548Sdumbbell 793235783Skib CTR3(KTR_DRM, "object_change_domain move_to_active %p %x %x", 794235783Skib obj, old_read, old_write); 795235783Skib } 796235783Skib} 797235783Skib 798235783Skibstatic void 799235783Skibi915_gem_execbuffer_retire_commands(struct drm_device *dev, 800235783Skib struct drm_file *file, 801235783Skib struct intel_ring_buffer *ring) 802235783Skib{ 803296548Sdumbbell /* Unconditionally force add_request to emit a full flush. */ 804296548Sdumbbell ring->gpu_caches_dirty = true; 805235783Skib 806235783Skib /* Add a breadcrumb for the completion of the batch buffer */ 807296548Sdumbbell (void)i915_add_request(ring, file, NULL); 808235783Skib} 809235783Skib 810290228Sdumbbellstatic int 811235783Skibi915_reset_gen7_sol_offsets(struct drm_device *dev, 812235783Skib struct intel_ring_buffer *ring) 813235783Skib{ 814235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 815235783Skib int ret, i; 816235783Skib 817296548Sdumbbell if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS]) 818235783Skib return 0; 819235783Skib 820235783Skib ret = intel_ring_begin(ring, 4 * 3); 821235783Skib if (ret) 822235783Skib return ret; 823235783Skib 824235783Skib for (i = 0; i < 4; i++) { 825235783Skib intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 826235783Skib intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i)); 827235783Skib intel_ring_emit(ring, 0); 828235783Skib } 829235783Skib 830235783Skib intel_ring_advance(ring); 831235783Skib 832235783Skib return 0; 833235783Skib} 834235783Skib 835235783Skibstatic int 836235783Skibi915_gem_do_execbuffer(struct drm_device *dev, void *data, 837235783Skib struct drm_file *file, 838235783Skib struct drm_i915_gem_execbuffer2 *args, 839235783Skib struct drm_i915_gem_exec_object2 *exec) 840235783Skib{ 841235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 842235783Skib struct list_head objects; 843235783Skib struct eb_objects *eb; 844235783Skib struct drm_i915_gem_object *batch_obj; 845235783Skib struct drm_clip_rect *cliprects = NULL; 846235783Skib struct intel_ring_buffer *ring; 847271705Sdumbbell u32 ctx_id = i915_execbuffer2_get_context_id(*args); 848235783Skib u32 exec_start, exec_len; 849235783Skib u32 mask; 850296548Sdumbbell u32 flags; 851235783Skib int ret, mode, i; 852290228Sdumbbell vm_page_t **relocs_ma; 853290228Sdumbbell int *relocs_len; 854235783Skib 855235783Skib if (!i915_gem_check_execbuffer(args)) { 856235783Skib DRM_DEBUG("execbuf with invalid offset/length\n"); 857235783Skib return -EINVAL; 858235783Skib } 859235783Skib 860290228Sdumbbell ret = validate_exec_list(exec, args->buffer_count, 861290228Sdumbbell &relocs_ma, &relocs_len); 862290228Sdumbbell if (ret) 863290228Sdumbbell goto pre_mutex_err; 864235783Skib 865296548Sdumbbell flags = 0; 866296548Sdumbbell if (args->flags & I915_EXEC_SECURE) { 867296548Sdumbbell if (!file->is_master || !capable(CAP_SYS_ADMIN)) { 868296548Sdumbbell ret = -EPERM; 869296548Sdumbbell goto pre_mutex_err; 870296548Sdumbbell } 871296548Sdumbbell 872296548Sdumbbell flags |= I915_DISPATCH_SECURE; 873296548Sdumbbell } 874296548Sdumbbell if (args->flags & I915_EXEC_IS_PINNED) 875296548Sdumbbell flags |= I915_DISPATCH_PINNED; 876296548Sdumbbell 877235783Skib switch (args->flags & I915_EXEC_RING_MASK) { 878235783Skib case I915_EXEC_DEFAULT: 879235783Skib case I915_EXEC_RENDER: 880296548Sdumbbell ring = &dev_priv->ring[RCS]; 881235783Skib break; 882235783Skib case I915_EXEC_BSD: 883296548Sdumbbell ring = &dev_priv->ring[VCS]; 884271705Sdumbbell if (ctx_id != 0) { 885271705Sdumbbell DRM_DEBUG("Ring %s doesn't support contexts\n", 886271705Sdumbbell ring->name); 887288452Sjhb ret = -EPERM; 888290228Sdumbbell goto pre_mutex_err; 889271705Sdumbbell } 890235783Skib break; 891235783Skib case I915_EXEC_BLT: 892296548Sdumbbell ring = &dev_priv->ring[BCS]; 893271705Sdumbbell if (ctx_id != 0) { 894271705Sdumbbell DRM_DEBUG("Ring %s doesn't support contexts\n", 895271705Sdumbbell ring->name); 896288452Sjhb ret = -EPERM; 897290228Sdumbbell goto pre_mutex_err; 898271705Sdumbbell } 899235783Skib break; 900235783Skib default: 901235783Skib DRM_DEBUG("execbuf with unknown ring: %d\n", 902235783Skib (int)(args->flags & I915_EXEC_RING_MASK)); 903235783Skib ret = -EINVAL; 904290228Sdumbbell goto pre_mutex_err; 905235783Skib } 906277487Skib if (!intel_ring_initialized(ring)) { 907277487Skib DRM_DEBUG("execbuf with invalid ring: %d\n", 908277487Skib (int)(args->flags & I915_EXEC_RING_MASK)); 909288452Sjhb ret = -EINVAL; 910290228Sdumbbell goto pre_mutex_err; 911277487Skib } 912235783Skib 913235783Skib mode = args->flags & I915_EXEC_CONSTANTS_MASK; 914235783Skib mask = I915_EXEC_CONSTANTS_MASK; 915235783Skib switch (mode) { 916235783Skib case I915_EXEC_CONSTANTS_REL_GENERAL: 917235783Skib case I915_EXEC_CONSTANTS_ABSOLUTE: 918235783Skib case I915_EXEC_CONSTANTS_REL_SURFACE: 919296548Sdumbbell if (ring == &dev_priv->ring[RCS] && 920235783Skib mode != dev_priv->relative_constants_mode) { 921235783Skib if (INTEL_INFO(dev)->gen < 4) { 922235783Skib ret = -EINVAL; 923290228Sdumbbell goto pre_mutex_err; 924235783Skib } 925235783Skib 926235783Skib if (INTEL_INFO(dev)->gen > 5 && 927235783Skib mode == I915_EXEC_CONSTANTS_REL_SURFACE) { 928235783Skib ret = -EINVAL; 929290228Sdumbbell goto pre_mutex_err; 930235783Skib } 931235783Skib 932235783Skib /* The HW changed the meaning on this bit on gen6 */ 933235783Skib if (INTEL_INFO(dev)->gen >= 6) 934235783Skib mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE; 935235783Skib } 936235783Skib break; 937235783Skib default: 938235783Skib DRM_DEBUG("execbuf with unknown constants: %d\n", mode); 939235783Skib ret = -EINVAL; 940290228Sdumbbell goto pre_mutex_err; 941235783Skib } 942235783Skib 943235783Skib if (args->buffer_count < 1) { 944235783Skib DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count); 945235783Skib ret = -EINVAL; 946290228Sdumbbell goto pre_mutex_err; 947235783Skib } 948235783Skib 949235783Skib if (args->num_cliprects != 0) { 950296548Sdumbbell if (ring != &dev_priv->ring[RCS]) { 951287174Sbapt DRM_DEBUG("clip rectangles are only valid with the render ring\n"); 952235783Skib ret = -EINVAL; 953290228Sdumbbell goto pre_mutex_err; 954235783Skib } 955235783Skib 956277487Skib if (INTEL_INFO(dev)->gen >= 5) { 957277487Skib DRM_DEBUG("clip rectangles are only valid on pre-gen5\n"); 958277487Skib ret = -EINVAL; 959290228Sdumbbell goto pre_mutex_err; 960277487Skib } 961277487Skib 962235783Skib if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) { 963235783Skib DRM_DEBUG("execbuf with %u cliprects\n", 964235783Skib args->num_cliprects); 965235783Skib ret = -EINVAL; 966290228Sdumbbell goto pre_mutex_err; 967235783Skib } 968296548Sdumbbell 969290228Sdumbbell cliprects = malloc(args->num_cliprects * sizeof(*cliprects), 970296548Sdumbbell DRM_I915_GEM, M_WAITOK); 971296548Sdumbbell if (cliprects == NULL) { 972296548Sdumbbell ret = -ENOMEM; 973290228Sdumbbell goto pre_mutex_err; 974296548Sdumbbell } 975296548Sdumbbell 976296548Sdumbbell if (copy_from_user(cliprects, 977296548Sdumbbell (struct drm_clip_rect __user *)(uintptr_t) 978296548Sdumbbell args->cliprects_ptr, 979296548Sdumbbell sizeof(*cliprects)*args->num_cliprects)) { 980296548Sdumbbell ret = -EFAULT; 981296548Sdumbbell goto pre_mutex_err; 982296548Sdumbbell } 983235783Skib } 984235783Skib 985235783Skib ret = i915_mutex_lock_interruptible(dev); 986235783Skib if (ret) 987290228Sdumbbell goto pre_mutex_err; 988235783Skib 989235783Skib if (dev_priv->mm.suspended) { 990280183Sdumbbell DRM_UNLOCK(dev); 991235783Skib ret = -EBUSY; 992290228Sdumbbell goto pre_mutex_err; 993235783Skib } 994235783Skib 995235783Skib eb = eb_create(args->buffer_count); 996235783Skib if (eb == NULL) { 997280183Sdumbbell DRM_UNLOCK(dev); 998235783Skib ret = -ENOMEM; 999290228Sdumbbell goto pre_mutex_err; 1000235783Skib } 1001235783Skib 1002235783Skib /* Look up object handles */ 1003235783Skib INIT_LIST_HEAD(&objects); 1004235783Skib for (i = 0; i < args->buffer_count; i++) { 1005235783Skib struct drm_i915_gem_object *obj; 1006287174Sbapt 1007235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, 1008235783Skib exec[i].handle)); 1009235783Skib if (&obj->base == NULL) { 1010235783Skib DRM_DEBUG("Invalid object handle %d at index %d\n", 1011235783Skib exec[i].handle, i); 1012235783Skib /* prevent error path from reading uninitialized data */ 1013235783Skib ret = -ENOENT; 1014235783Skib goto err; 1015235783Skib } 1016235783Skib 1017235783Skib if (!list_empty(&obj->exec_list)) { 1018235783Skib DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n", 1019235783Skib obj, exec[i].handle, i); 1020235783Skib ret = -EINVAL; 1021235783Skib goto err; 1022235783Skib } 1023235783Skib 1024235783Skib list_add_tail(&obj->exec_list, &objects); 1025235783Skib obj->exec_handle = exec[i].handle; 1026235783Skib obj->exec_entry = &exec[i]; 1027235783Skib eb_add_object(eb, obj); 1028235783Skib } 1029235783Skib 1030235783Skib /* take note of the batch buffer before we might reorder the lists */ 1031235783Skib batch_obj = list_entry(objects.prev, 1032235783Skib struct drm_i915_gem_object, 1033235783Skib exec_list); 1034235783Skib 1035235783Skib /* Move the objects en-masse into the GTT, evicting if necessary. */ 1036235783Skib ret = i915_gem_execbuffer_reserve(ring, file, &objects); 1037235783Skib if (ret) 1038235783Skib goto err; 1039235783Skib 1040235783Skib /* The objects are in their final locations, apply the relocations. */ 1041235783Skib ret = i915_gem_execbuffer_relocate(dev, eb, &objects); 1042235783Skib if (ret) { 1043235783Skib if (ret == -EFAULT) { 1044235783Skib ret = i915_gem_execbuffer_relocate_slow(dev, file, ring, 1045287174Sbapt &objects, eb, 1046287174Sbapt exec, 1047287174Sbapt args->buffer_count); 1048235783Skib DRM_LOCK_ASSERT(dev); 1049235783Skib } 1050235783Skib if (ret) 1051235783Skib goto err; 1052235783Skib } 1053235783Skib 1054235783Skib /* Set the pending read domains for the batch buffer to COMMAND */ 1055235783Skib if (batch_obj->base.pending_write_domain) { 1056235783Skib DRM_DEBUG("Attempting to use self-modifying batch buffer\n"); 1057235783Skib ret = -EINVAL; 1058235783Skib goto err; 1059235783Skib } 1060235783Skib batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; 1061235783Skib 1062296548Sdumbbell /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure 1063296548Sdumbbell * batch" bit. Hence we need to pin secure batches into the global gtt. 1064296548Sdumbbell * hsw should have this fixed, but let's be paranoid and do it 1065296548Sdumbbell * unconditionally for now. */ 1066296548Sdumbbell if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping) 1067296548Sdumbbell i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level); 1068296548Sdumbbell 1069235783Skib ret = i915_gem_execbuffer_move_to_gpu(ring, &objects); 1070235783Skib if (ret) 1071235783Skib goto err; 1072235783Skib 1073271705Sdumbbell ret = i915_switch_context(ring, file, ctx_id); 1074271705Sdumbbell if (ret) 1075271705Sdumbbell goto err; 1076271705Sdumbbell 1077296548Sdumbbell if (ring == &dev_priv->ring[RCS] && 1078235783Skib mode != dev_priv->relative_constants_mode) { 1079235783Skib ret = intel_ring_begin(ring, 4); 1080235783Skib if (ret) 1081290228Sdumbbell goto err; 1082235783Skib 1083235783Skib intel_ring_emit(ring, MI_NOOP); 1084235783Skib intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 1085235783Skib intel_ring_emit(ring, INSTPM); 1086235783Skib intel_ring_emit(ring, mask << 16 | mode); 1087235783Skib intel_ring_advance(ring); 1088235783Skib 1089235783Skib dev_priv->relative_constants_mode = mode; 1090235783Skib } 1091235783Skib 1092235783Skib if (args->flags & I915_EXEC_GEN7_SOL_RESET) { 1093235783Skib ret = i915_reset_gen7_sol_offsets(dev, ring); 1094235783Skib if (ret) 1095235783Skib goto err; 1096235783Skib } 1097235783Skib 1098235783Skib exec_start = batch_obj->gtt_offset + args->batch_start_offset; 1099235783Skib exec_len = args->batch_len; 1100235783Skib if (cliprects) { 1101235783Skib for (i = 0; i < args->num_cliprects; i++) { 1102287177Sbapt ret = i915_emit_box(dev, &cliprects[i], 1103287174Sbapt args->DR1, args->DR4); 1104235783Skib if (ret) 1105235783Skib goto err; 1106235783Skib 1107287174Sbapt ret = ring->dispatch_execbuffer(ring, 1108296548Sdumbbell exec_start, exec_len, 1109296548Sdumbbell flags); 1110235783Skib if (ret) 1111235783Skib goto err; 1112235783Skib } 1113235783Skib } else { 1114287174Sbapt ret = ring->dispatch_execbuffer(ring, 1115296548Sdumbbell exec_start, exec_len, 1116296548Sdumbbell flags); 1117235783Skib if (ret) 1118235783Skib goto err; 1119235783Skib } 1120235783Skib 1121296548Sdumbbell CTR3(KTR_DRM, "ring_dispatch ring=%s seqno=%d flags=%u", ring->name, 1122296548Sdumbbell intel_ring_get_seqno(ring), flags); 1123290228Sdumbbell 1124296548Sdumbbell i915_gem_execbuffer_move_to_active(&objects, ring); 1125235783Skib i915_gem_execbuffer_retire_commands(dev, file, ring); 1126235783Skib 1127235783Skiberr: 1128235783Skib eb_destroy(eb); 1129235783Skib while (!list_empty(&objects)) { 1130235783Skib struct drm_i915_gem_object *obj; 1131235783Skib 1132287174Sbapt obj = list_first_entry(&objects, 1133287174Sbapt struct drm_i915_gem_object, 1134287174Sbapt exec_list); 1135235783Skib list_del_init(&obj->exec_list); 1136235783Skib drm_gem_object_unreference(&obj->base); 1137235783Skib } 1138290228Sdumbbell 1139235783Skib DRM_UNLOCK(dev); 1140235783Skib 1141290228Sdumbbellpre_mutex_err: 1142235783Skib for (i = 0; i < args->buffer_count; i++) { 1143235783Skib if (relocs_ma[i] != NULL) { 1144289719Sjhb vm_page_unhold_pages(relocs_ma[i], relocs_len[i]); 1145235783Skib free(relocs_ma[i], DRM_I915_GEM); 1146235783Skib } 1147235783Skib } 1148289719Sjhb free(relocs_len, DRM_I915_GEM); 1149235783Skib free(relocs_ma, DRM_I915_GEM); 1150235783Skib free(cliprects, DRM_I915_GEM); 1151235783Skib return ret; 1152235783Skib} 1153235783Skib 1154235783Skib/* 1155235783Skib * Legacy execbuffer just creates an exec2 list from the original exec object 1156235783Skib * list array and passes it to the real function. 1157235783Skib */ 1158235783Skibint 1159235783Skibi915_gem_execbuffer(struct drm_device *dev, void *data, 1160235783Skib struct drm_file *file) 1161235783Skib{ 1162235783Skib struct drm_i915_gem_execbuffer *args = data; 1163235783Skib struct drm_i915_gem_execbuffer2 exec2; 1164235783Skib struct drm_i915_gem_exec_object *exec_list = NULL; 1165235783Skib struct drm_i915_gem_exec_object2 *exec2_list = NULL; 1166235783Skib int ret, i; 1167235783Skib 1168235783Skib if (args->buffer_count < 1) { 1169235783Skib DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count); 1170235783Skib return -EINVAL; 1171235783Skib } 1172235783Skib 1173235783Skib /* Copy in the exec list from userland */ 1174235783Skib /* XXXKIB user-controlled malloc size */ 1175296548Sdumbbell exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count); 1176296548Sdumbbell exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count); 1177296548Sdumbbell if (exec_list == NULL || exec2_list == NULL) { 1178296548Sdumbbell DRM_DEBUG("Failed to allocate exec list for %d buffers\n", 1179296548Sdumbbell args->buffer_count); 1180296548Sdumbbell drm_free_large(exec_list); 1181296548Sdumbbell drm_free_large(exec2_list); 1182296548Sdumbbell return -ENOMEM; 1183296548Sdumbbell } 1184296548Sdumbbell ret = copy_from_user(exec_list, 1185296548Sdumbbell (void __user *)(uintptr_t)args->buffers_ptr, 1186296548Sdumbbell sizeof(*exec_list) * args->buffer_count); 1187235783Skib if (ret != 0) { 1188235783Skib DRM_DEBUG("copy %d exec entries failed %d\n", 1189235783Skib args->buffer_count, ret); 1190296548Sdumbbell drm_free_large(exec_list); 1191296548Sdumbbell drm_free_large(exec2_list); 1192296548Sdumbbell return -EFAULT; 1193235783Skib } 1194235783Skib 1195235783Skib for (i = 0; i < args->buffer_count; i++) { 1196235783Skib exec2_list[i].handle = exec_list[i].handle; 1197235783Skib exec2_list[i].relocation_count = exec_list[i].relocation_count; 1198235783Skib exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; 1199235783Skib exec2_list[i].alignment = exec_list[i].alignment; 1200235783Skib exec2_list[i].offset = exec_list[i].offset; 1201235783Skib if (INTEL_INFO(dev)->gen < 4) 1202235783Skib exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; 1203235783Skib else 1204235783Skib exec2_list[i].flags = 0; 1205235783Skib } 1206235783Skib 1207235783Skib exec2.buffers_ptr = args->buffers_ptr; 1208235783Skib exec2.buffer_count = args->buffer_count; 1209235783Skib exec2.batch_start_offset = args->batch_start_offset; 1210235783Skib exec2.batch_len = args->batch_len; 1211235783Skib exec2.DR1 = args->DR1; 1212235783Skib exec2.DR4 = args->DR4; 1213235783Skib exec2.num_cliprects = args->num_cliprects; 1214235783Skib exec2.cliprects_ptr = args->cliprects_ptr; 1215235783Skib exec2.flags = I915_EXEC_RENDER; 1216271705Sdumbbell i915_execbuffer2_set_context_id(exec2, 0); 1217235783Skib 1218235783Skib ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list); 1219235783Skib if (!ret) { 1220235783Skib /* Copy the new buffer offsets back to the user's exec list. */ 1221235783Skib for (i = 0; i < args->buffer_count; i++) 1222235783Skib exec_list[i].offset = exec2_list[i].offset; 1223235783Skib /* ... and back out to userspace */ 1224296548Sdumbbell ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr, 1225296548Sdumbbell exec_list, 1226296548Sdumbbell sizeof(*exec_list) * args->buffer_count); 1227296548Sdumbbell if (ret) { 1228296548Sdumbbell ret = -EFAULT; 1229235783Skib DRM_DEBUG("failed to copy %d exec entries " 1230235783Skib "back to user (%d)\n", 1231235783Skib args->buffer_count, ret); 1232235783Skib } 1233235783Skib } 1234235783Skib 1235296548Sdumbbell drm_free_large(exec_list); 1236296548Sdumbbell drm_free_large(exec2_list); 1237235783Skib return ret; 1238235783Skib} 1239235783Skib 1240235783Skibint 1241235783Skibi915_gem_execbuffer2(struct drm_device *dev, void *data, 1242235783Skib struct drm_file *file) 1243235783Skib{ 1244235783Skib struct drm_i915_gem_execbuffer2 *args = data; 1245235783Skib struct drm_i915_gem_exec_object2 *exec2_list = NULL; 1246235783Skib int ret; 1247235783Skib 1248235783Skib if (args->buffer_count < 1 || 1249235783Skib args->buffer_count > UINT_MAX / sizeof(*exec2_list)) { 1250235783Skib DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count); 1251235783Skib return -EINVAL; 1252235783Skib } 1253235783Skib 1254235783Skib /* XXXKIB user-controllable malloc size */ 1255290228Sdumbbell exec2_list = malloc(sizeof(*exec2_list)*args->buffer_count, 1256290228Sdumbbell DRM_I915_GEM, M_WAITOK); 1257296548Sdumbbell if (exec2_list == NULL) { 1258296548Sdumbbell DRM_DEBUG("Failed to allocate exec list for %d buffers\n", 1259296548Sdumbbell args->buffer_count); 1260296548Sdumbbell return -ENOMEM; 1261296548Sdumbbell } 1262296548Sdumbbell ret = copy_from_user(exec2_list, 1263296548Sdumbbell (struct drm_i915_relocation_entry __user *) 1264296548Sdumbbell (uintptr_t) args->buffers_ptr, 1265296548Sdumbbell sizeof(*exec2_list) * args->buffer_count); 1266235783Skib if (ret != 0) { 1267235783Skib DRM_DEBUG("copy %d exec entries failed %d\n", 1268235783Skib args->buffer_count, ret); 1269235783Skib free(exec2_list, DRM_I915_GEM); 1270287174Sbapt return -EFAULT; 1271235783Skib } 1272235783Skib 1273235783Skib ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list); 1274235783Skib if (!ret) { 1275235783Skib /* Copy the new buffer offsets back to the user's exec list. */ 1276296548Sdumbbell ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr, 1277296548Sdumbbell exec2_list, 1278296548Sdumbbell sizeof(*exec2_list) * args->buffer_count); 1279235783Skib if (ret) { 1280296548Sdumbbell ret = -EFAULT; 1281235783Skib DRM_DEBUG("failed to copy %d exec entries " 1282235783Skib "back to user (%d)\n", 1283235783Skib args->buffer_count, ret); 1284235783Skib } 1285235783Skib } 1286235783Skib 1287235783Skib free(exec2_list, DRM_I915_GEM); 1288235783Skib return ret; 1289235783Skib} 1290