1/* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright �� 2011-2012 Intel Corporation 5 */ 6 7/* 8 * This file implements HW context support. On gen5+ a HW context consists of an 9 * opaque GPU object which is referenced at times of context saves and restores. 10 * With RC6 enabled, the context is also referenced as the GPU enters and exists 11 * from RC6 (GPU has it's own internal power context, except on gen5). Though 12 * something like a context does exist for the media ring, the code only 13 * supports contexts for the render ring. 14 * 15 * In software, there is a distinction between contexts created by the user, 16 * and the default HW context. The default HW context is used by GPU clients 17 * that do not request setup of their own hardware context. The default 18 * context's state is never restored to help prevent programming errors. This 19 * would happen if a client ran and piggy-backed off another clients GPU state. 20 * The default context only exists to give the GPU some offset to load as the 21 * current to invoke a save of the context we actually care about. In fact, the 22 * code could likely be constructed, albeit in a more complicated fashion, to 23 * never use the default context, though that limits the driver's ability to 24 * swap out, and/or destroy other contexts. 25 * 26 * All other contexts are created as a request by the GPU client. These contexts 27 * store GPU state, and thus allow GPU clients to not re-emit state (and 28 * potentially query certain state) at any time. The kernel driver makes 29 * certain that the appropriate commands are inserted. 30 * 31 * The context life cycle is semi-complicated in that context BOs may live 32 * longer than the context itself because of the way the hardware, and object 33 * tracking works. Below is a very crude representation of the state machine 34 * describing the context life. 35 * refcount pincount active 36 * S0: initial state 0 0 0 37 * S1: context created 1 0 0 38 * S2: context is currently running 2 1 X 39 * S3: GPU referenced, but not current 2 0 1 40 * S4: context is current, but destroyed 1 1 0 41 * S5: like S3, but destroyed 1 0 1 42 * 43 * The most common (but not all) transitions: 44 * S0->S1: client creates a context 45 * S1->S2: client submits execbuf with context 46 * S2->S3: other clients submits execbuf with context 47 * S3->S1: context object was retired 48 * S3->S2: clients submits another execbuf 49 * S2->S4: context destroy called with current context 50 * S3->S5->S0: destroy path 51 * S4->S5->S0: destroy path on current context 52 * 53 * There are two confusing terms used above: 54 * The "current context" means the context which is currently running on the 55 * GPU. The GPU has loaded its state already and has stored away the gtt 56 * offset of the BO. The GPU is not actively referencing the data at this 57 * offset, but it will on the next context switch. The only way to avoid this 58 * is to do a GPU reset. 59 * 60 * An "active context' is one which was previously the "current context" and is 61 * on the active list waiting for the next context switch to occur. Until this 62 * happens, the object must remain at the same gtt offset. It is therefore 63 * possible to destroy a context, but it is still active. 64 * 65 */ 66 67#include <linux/highmem.h> 68#include <linux/log2.h> 69#include <linux/nospec.h> 70 71#include <drm/drm_cache.h> 72#include <drm/drm_syncobj.h> 73 74#include "gt/gen6_ppgtt.h" 75#include "gt/intel_context.h" 76#include "gt/intel_context_param.h" 77#include "gt/intel_engine_heartbeat.h" 78#include "gt/intel_engine_user.h" 79#include "gt/intel_gpu_commands.h" 80#include "gt/intel_ring.h" 81 82#include "pxp/intel_pxp.h" 83 84#include "i915_file_private.h" 85#include "i915_gem_context.h" 86#include "i915_trace.h" 87#include "i915_user_extensions.h" 88 89#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 90 91static struct pool slab_luts; 92 93struct i915_lut_handle *i915_lut_handle_alloc(void) 94{ 95#ifdef __linux__ 96 return kmem_cache_alloc(slab_luts, GFP_KERNEL); 97#else 98 return pool_get(&slab_luts, PR_WAITOK); 99#endif 100} 101 102void i915_lut_handle_free(struct i915_lut_handle *lut) 103{ 104#ifdef __linux__ 105 return kmem_cache_free(slab_luts, lut); 106#else 107 pool_put(&slab_luts, lut); 108#endif 109} 110 111static void lut_close(struct i915_gem_context *ctx) 112{ 113 struct radix_tree_iter iter; 114 void __rcu **slot; 115 116 mutex_lock(&ctx->lut_mutex); 117 rcu_read_lock(); 118 radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { 119 struct i915_vma *vma = rcu_dereference_raw(*slot); 120 struct drm_i915_gem_object *obj = vma->obj; 121 struct i915_lut_handle *lut; 122 123 if (!kref_get_unless_zero(&obj->base.refcount)) 124 continue; 125 126 spin_lock(&obj->lut_lock); 127 list_for_each_entry(lut, &obj->lut_list, obj_link) { 128 if (lut->ctx != ctx) 129 continue; 130 131 if (lut->handle != iter.index) 132 continue; 133 134 list_del(&lut->obj_link); 135 break; 136 } 137 spin_unlock(&obj->lut_lock); 138 139 if (&lut->obj_link != &obj->lut_list) { 140 i915_lut_handle_free(lut); 141 radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); 142 i915_vma_close(vma); 143 i915_gem_object_put(obj); 144 } 145 146 i915_gem_object_put(obj); 147 } 148 rcu_read_unlock(); 149 mutex_unlock(&ctx->lut_mutex); 150} 151 152static struct intel_context * 153lookup_user_engine(struct i915_gem_context *ctx, 154 unsigned long flags, 155 const struct i915_engine_class_instance *ci) 156#define LOOKUP_USER_INDEX BIT(0) 157{ 158 int idx; 159 160 if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) 161 return ERR_PTR(-EINVAL); 162 163 if (!i915_gem_context_user_engines(ctx)) { 164 struct intel_engine_cs *engine; 165 166 engine = intel_engine_lookup_user(ctx->i915, 167 ci->engine_class, 168 ci->engine_instance); 169 if (!engine) 170 return ERR_PTR(-EINVAL); 171 172 idx = engine->legacy_idx; 173 } else { 174 idx = ci->engine_instance; 175 } 176 177 return i915_gem_context_get_engine(ctx, idx); 178} 179 180static int validate_priority(struct drm_i915_private *i915, 181 const struct drm_i915_gem_context_param *args) 182{ 183 s64 priority = args->value; 184 185 if (args->size) 186 return -EINVAL; 187 188 if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 189 return -ENODEV; 190 191 if (priority > I915_CONTEXT_MAX_USER_PRIORITY || 192 priority < I915_CONTEXT_MIN_USER_PRIORITY) 193 return -EINVAL; 194 195 if (priority > I915_CONTEXT_DEFAULT_PRIORITY && 196 !capable(CAP_SYS_NICE)) 197 return -EPERM; 198 199 return 0; 200} 201 202static void proto_context_close(struct drm_i915_private *i915, 203 struct i915_gem_proto_context *pc) 204{ 205 int i; 206 207 if (pc->pxp_wakeref) 208 intel_runtime_pm_put(&i915->runtime_pm, pc->pxp_wakeref); 209 if (pc->vm) 210 i915_vm_put(pc->vm); 211 if (pc->user_engines) { 212 for (i = 0; i < pc->num_user_engines; i++) 213 kfree(pc->user_engines[i].siblings); 214 kfree(pc->user_engines); 215 } 216 kfree(pc); 217} 218 219static int proto_context_set_persistence(struct drm_i915_private *i915, 220 struct i915_gem_proto_context *pc, 221 bool persist) 222{ 223 if (persist) { 224 /* 225 * Only contexts that are short-lived [that will expire or be 226 * reset] are allowed to survive past termination. We require 227 * hangcheck to ensure that the persistent requests are healthy. 228 */ 229 if (!i915->params.enable_hangcheck) 230 return -EINVAL; 231 232 pc->user_flags |= BIT(UCONTEXT_PERSISTENCE); 233 } else { 234 /* To cancel a context we use "preempt-to-idle" */ 235 if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 236 return -ENODEV; 237 238 /* 239 * If the cancel fails, we then need to reset, cleanly! 240 * 241 * If the per-engine reset fails, all hope is lost! We resort 242 * to a full GPU reset in that unlikely case, but realistically 243 * if the engine could not reset, the full reset does not fare 244 * much better. The damage has been done. 245 * 246 * However, if we cannot reset an engine by itself, we cannot 247 * cleanup a hanging persistent context without causing 248 * colateral damage, and we should not pretend we can by 249 * exposing the interface. 250 */ 251 if (!intel_has_reset_engine(to_gt(i915))) 252 return -ENODEV; 253 254 pc->user_flags &= ~BIT(UCONTEXT_PERSISTENCE); 255 } 256 257 return 0; 258} 259 260static int proto_context_set_protected(struct drm_i915_private *i915, 261 struct i915_gem_proto_context *pc, 262 bool protected) 263{ 264 int ret = 0; 265 266 if (!protected) { 267 pc->uses_protected_content = false; 268 } else if (!intel_pxp_is_enabled(i915->pxp)) { 269 ret = -ENODEV; 270 } else if ((pc->user_flags & BIT(UCONTEXT_RECOVERABLE)) || 271 !(pc->user_flags & BIT(UCONTEXT_BANNABLE))) { 272 ret = -EPERM; 273 } else { 274 pc->uses_protected_content = true; 275 276 /* 277 * protected context usage requires the PXP session to be up, 278 * which in turn requires the device to be active. 279 */ 280 pc->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm); 281 282 if (!intel_pxp_is_active(i915->pxp)) 283 ret = intel_pxp_start(i915->pxp); 284 } 285 286 return ret; 287} 288 289static struct i915_gem_proto_context * 290proto_context_create(struct drm_i915_private *i915, unsigned int flags) 291{ 292 struct i915_gem_proto_context *pc, *err; 293 294 pc = kzalloc(sizeof(*pc), GFP_KERNEL); 295 if (!pc) 296 return ERR_PTR(-ENOMEM); 297 298 pc->num_user_engines = -1; 299 pc->user_engines = NULL; 300 pc->user_flags = BIT(UCONTEXT_BANNABLE) | 301 BIT(UCONTEXT_RECOVERABLE); 302 if (i915->params.enable_hangcheck) 303 pc->user_flags |= BIT(UCONTEXT_PERSISTENCE); 304 pc->sched.priority = I915_PRIORITY_NORMAL; 305 306 if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { 307 if (!HAS_EXECLISTS(i915)) { 308 err = ERR_PTR(-EINVAL); 309 goto proto_close; 310 } 311 pc->single_timeline = true; 312 } 313 314 return pc; 315 316proto_close: 317 proto_context_close(i915, pc); 318 return err; 319} 320 321static int proto_context_register_locked(struct drm_i915_file_private *fpriv, 322 struct i915_gem_proto_context *pc, 323 u32 *id) 324{ 325 int ret; 326 void *old; 327 328 lockdep_assert_held(&fpriv->proto_context_lock); 329 330 ret = xa_alloc(&fpriv->context_xa, id, NULL, xa_limit_32b, GFP_KERNEL); 331 if (ret) 332 return ret; 333 334 old = xa_store(&fpriv->proto_context_xa, *id, pc, GFP_KERNEL); 335 if (xa_is_err(old)) { 336 xa_erase(&fpriv->context_xa, *id); 337 return xa_err(old); 338 } 339 WARN_ON(old); 340 341 return 0; 342} 343 344static int proto_context_register(struct drm_i915_file_private *fpriv, 345 struct i915_gem_proto_context *pc, 346 u32 *id) 347{ 348 int ret; 349 350 mutex_lock(&fpriv->proto_context_lock); 351 ret = proto_context_register_locked(fpriv, pc, id); 352 mutex_unlock(&fpriv->proto_context_lock); 353 354 return ret; 355} 356 357static struct i915_address_space * 358i915_gem_vm_lookup(struct drm_i915_file_private *file_priv, u32 id) 359{ 360 struct i915_address_space *vm; 361 362 xa_lock(&file_priv->vm_xa); 363 vm = xa_load(&file_priv->vm_xa, id); 364 if (vm) 365 kref_get(&vm->ref); 366 xa_unlock(&file_priv->vm_xa); 367 368 return vm; 369} 370 371static int set_proto_ctx_vm(struct drm_i915_file_private *fpriv, 372 struct i915_gem_proto_context *pc, 373 const struct drm_i915_gem_context_param *args) 374{ 375 struct drm_i915_private *i915 = fpriv->i915; 376 struct i915_address_space *vm; 377 378 if (args->size) 379 return -EINVAL; 380 381 if (!HAS_FULL_PPGTT(i915)) 382 return -ENODEV; 383 384 if (upper_32_bits(args->value)) 385 return -ENOENT; 386 387 vm = i915_gem_vm_lookup(fpriv, args->value); 388 if (!vm) 389 return -ENOENT; 390 391 if (pc->vm) 392 i915_vm_put(pc->vm); 393 pc->vm = vm; 394 395 return 0; 396} 397 398struct set_proto_ctx_engines { 399 struct drm_i915_private *i915; 400 unsigned num_engines; 401 struct i915_gem_proto_engine *engines; 402}; 403 404static int 405set_proto_ctx_engines_balance(struct i915_user_extension __user *base, 406 void *data) 407{ 408 struct i915_context_engines_load_balance __user *ext = 409 container_of_user(base, typeof(*ext), base); 410 const struct set_proto_ctx_engines *set = data; 411 struct drm_i915_private *i915 = set->i915; 412 struct intel_engine_cs **siblings; 413 u16 num_siblings, idx; 414 unsigned int n; 415 int err; 416 417 if (!HAS_EXECLISTS(i915)) 418 return -ENODEV; 419 420 if (get_user(idx, &ext->engine_index)) 421 return -EFAULT; 422 423 if (idx >= set->num_engines) { 424 drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", 425 idx, set->num_engines); 426 return -EINVAL; 427 } 428 429 idx = array_index_nospec(idx, set->num_engines); 430 if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_INVALID) { 431 drm_dbg(&i915->drm, 432 "Invalid placement[%d], already occupied\n", idx); 433 return -EEXIST; 434 } 435 436 if (get_user(num_siblings, &ext->num_siblings)) 437 return -EFAULT; 438 439 err = check_user_mbz(&ext->flags); 440 if (err) 441 return err; 442 443 err = check_user_mbz(&ext->mbz64); 444 if (err) 445 return err; 446 447 if (num_siblings == 0) 448 return 0; 449 450 siblings = kmalloc_array(num_siblings, sizeof(*siblings), GFP_KERNEL); 451 if (!siblings) 452 return -ENOMEM; 453 454 for (n = 0; n < num_siblings; n++) { 455 struct i915_engine_class_instance ci; 456 457 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 458 err = -EFAULT; 459 goto err_siblings; 460 } 461 462 siblings[n] = intel_engine_lookup_user(i915, 463 ci.engine_class, 464 ci.engine_instance); 465 if (!siblings[n]) { 466 drm_dbg(&i915->drm, 467 "Invalid sibling[%d]: { class:%d, inst:%d }\n", 468 n, ci.engine_class, ci.engine_instance); 469 err = -EINVAL; 470 goto err_siblings; 471 } 472 } 473 474 if (num_siblings == 1) { 475 set->engines[idx].type = I915_GEM_ENGINE_TYPE_PHYSICAL; 476 set->engines[idx].engine = siblings[0]; 477 kfree(siblings); 478 } else { 479 set->engines[idx].type = I915_GEM_ENGINE_TYPE_BALANCED; 480 set->engines[idx].num_siblings = num_siblings; 481 set->engines[idx].siblings = siblings; 482 } 483 484 return 0; 485 486err_siblings: 487 kfree(siblings); 488 489 return err; 490} 491 492static int 493set_proto_ctx_engines_bond(struct i915_user_extension __user *base, void *data) 494{ 495 struct i915_context_engines_bond __user *ext = 496 container_of_user(base, typeof(*ext), base); 497 const struct set_proto_ctx_engines *set = data; 498 struct drm_i915_private *i915 = set->i915; 499 struct i915_engine_class_instance ci; 500 struct intel_engine_cs *master; 501 u16 idx, num_bonds; 502 int err, n; 503 504 if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915) && 505 !IS_ROCKETLAKE(i915) && !IS_ALDERLAKE_S(i915)) { 506 drm_dbg(&i915->drm, 507 "Bonding not supported on this platform\n"); 508 return -ENODEV; 509 } 510 511 if (get_user(idx, &ext->virtual_index)) 512 return -EFAULT; 513 514 if (idx >= set->num_engines) { 515 drm_dbg(&i915->drm, 516 "Invalid index for virtual engine: %d >= %d\n", 517 idx, set->num_engines); 518 return -EINVAL; 519 } 520 521 idx = array_index_nospec(idx, set->num_engines); 522 if (set->engines[idx].type == I915_GEM_ENGINE_TYPE_INVALID) { 523 drm_dbg(&i915->drm, "Invalid engine at %d\n", idx); 524 return -EINVAL; 525 } 526 527 if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_PHYSICAL) { 528 drm_dbg(&i915->drm, 529 "Bonding with virtual engines not allowed\n"); 530 return -EINVAL; 531 } 532 533 err = check_user_mbz(&ext->flags); 534 if (err) 535 return err; 536 537 for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 538 err = check_user_mbz(&ext->mbz64[n]); 539 if (err) 540 return err; 541 } 542 543 if (copy_from_user(&ci, &ext->master, sizeof(ci))) 544 return -EFAULT; 545 546 master = intel_engine_lookup_user(i915, 547 ci.engine_class, 548 ci.engine_instance); 549 if (!master) { 550 drm_dbg(&i915->drm, 551 "Unrecognised master engine: { class:%u, instance:%u }\n", 552 ci.engine_class, ci.engine_instance); 553 return -EINVAL; 554 } 555 556 if (intel_engine_uses_guc(master)) { 557 drm_dbg(&i915->drm, "bonding extension not supported with GuC submission"); 558 return -ENODEV; 559 } 560 561 if (get_user(num_bonds, &ext->num_bonds)) 562 return -EFAULT; 563 564 for (n = 0; n < num_bonds; n++) { 565 struct intel_engine_cs *bond; 566 567 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) 568 return -EFAULT; 569 570 bond = intel_engine_lookup_user(i915, 571 ci.engine_class, 572 ci.engine_instance); 573 if (!bond) { 574 drm_dbg(&i915->drm, 575 "Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", 576 n, ci.engine_class, ci.engine_instance); 577 return -EINVAL; 578 } 579 } 580 581 return 0; 582} 583 584static int 585set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, 586 void *data) 587{ 588 struct i915_context_engines_parallel_submit __user *ext = 589 container_of_user(base, typeof(*ext), base); 590 const struct set_proto_ctx_engines *set = data; 591 struct drm_i915_private *i915 = set->i915; 592 struct i915_engine_class_instance prev_engine; 593 u64 flags; 594 int err = 0, n, i, j; 595 u16 slot, width, num_siblings; 596 struct intel_engine_cs **siblings = NULL; 597 intel_engine_mask_t prev_mask; 598 599 if (get_user(slot, &ext->engine_index)) 600 return -EFAULT; 601 602 if (get_user(width, &ext->width)) 603 return -EFAULT; 604 605 if (get_user(num_siblings, &ext->num_siblings)) 606 return -EFAULT; 607 608 if (!intel_uc_uses_guc_submission(&to_gt(i915)->uc) && 609 num_siblings != 1) { 610 drm_dbg(&i915->drm, "Only 1 sibling (%d) supported in non-GuC mode\n", 611 num_siblings); 612 return -EINVAL; 613 } 614 615 if (slot >= set->num_engines) { 616 drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", 617 slot, set->num_engines); 618 return -EINVAL; 619 } 620 621 if (set->engines[slot].type != I915_GEM_ENGINE_TYPE_INVALID) { 622 drm_dbg(&i915->drm, 623 "Invalid placement[%d], already occupied\n", slot); 624 return -EINVAL; 625 } 626 627 if (get_user(flags, &ext->flags)) 628 return -EFAULT; 629 630 if (flags) { 631 drm_dbg(&i915->drm, "Unknown flags 0x%02llx", flags); 632 return -EINVAL; 633 } 634 635 for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 636 err = check_user_mbz(&ext->mbz64[n]); 637 if (err) 638 return err; 639 } 640 641 if (width < 2) { 642 drm_dbg(&i915->drm, "Width (%d) < 2\n", width); 643 return -EINVAL; 644 } 645 646 if (num_siblings < 1) { 647 drm_dbg(&i915->drm, "Number siblings (%d) < 1\n", 648 num_siblings); 649 return -EINVAL; 650 } 651 652 siblings = kmalloc_array(num_siblings * width, 653 sizeof(*siblings), 654 GFP_KERNEL); 655 if (!siblings) 656 return -ENOMEM; 657 658 /* Create contexts / engines */ 659 for (i = 0; i < width; ++i) { 660 intel_engine_mask_t current_mask = 0; 661 662 for (j = 0; j < num_siblings; ++j) { 663 struct i915_engine_class_instance ci; 664 665 n = i * num_siblings + j; 666 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 667 err = -EFAULT; 668 goto out_err; 669 } 670 671 siblings[n] = 672 intel_engine_lookup_user(i915, ci.engine_class, 673 ci.engine_instance); 674 if (!siblings[n]) { 675 drm_dbg(&i915->drm, 676 "Invalid sibling[%d]: { class:%d, inst:%d }\n", 677 n, ci.engine_class, ci.engine_instance); 678 err = -EINVAL; 679 goto out_err; 680 } 681 682 /* 683 * We don't support breadcrumb handshake on these 684 * classes 685 */ 686 if (siblings[n]->class == RENDER_CLASS || 687 siblings[n]->class == COMPUTE_CLASS) { 688 err = -EINVAL; 689 goto out_err; 690 } 691 692 if (n) { 693 if (prev_engine.engine_class != 694 ci.engine_class) { 695 drm_dbg(&i915->drm, 696 "Mismatched class %d, %d\n", 697 prev_engine.engine_class, 698 ci.engine_class); 699 err = -EINVAL; 700 goto out_err; 701 } 702 } 703 704 prev_engine = ci; 705 current_mask |= siblings[n]->logical_mask; 706 } 707 708 if (i > 0) { 709 if (current_mask != prev_mask << 1) { 710 drm_dbg(&i915->drm, 711 "Non contiguous logical mask 0x%x, 0x%x\n", 712 prev_mask, current_mask); 713 err = -EINVAL; 714 goto out_err; 715 } 716 } 717 prev_mask = current_mask; 718 } 719 720 set->engines[slot].type = I915_GEM_ENGINE_TYPE_PARALLEL; 721 set->engines[slot].num_siblings = num_siblings; 722 set->engines[slot].width = width; 723 set->engines[slot].siblings = siblings; 724 725 return 0; 726 727out_err: 728 kfree(siblings); 729 730 return err; 731} 732 733static const i915_user_extension_fn set_proto_ctx_engines_extensions[] = { 734 [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_proto_ctx_engines_balance, 735 [I915_CONTEXT_ENGINES_EXT_BOND] = set_proto_ctx_engines_bond, 736 [I915_CONTEXT_ENGINES_EXT_PARALLEL_SUBMIT] = 737 set_proto_ctx_engines_parallel_submit, 738}; 739 740static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv, 741 struct i915_gem_proto_context *pc, 742 const struct drm_i915_gem_context_param *args) 743{ 744 struct drm_i915_private *i915 = fpriv->i915; 745 struct set_proto_ctx_engines set = { .i915 = i915 }; 746 struct i915_context_param_engines __user *user = 747 u64_to_user_ptr(args->value); 748 unsigned int n; 749 u64 extensions; 750 int err; 751 752 if (pc->num_user_engines >= 0) { 753 drm_dbg(&i915->drm, "Cannot set engines twice"); 754 return -EINVAL; 755 } 756 757 if (args->size < sizeof(*user) || 758 !IS_ALIGNED(args->size - sizeof(*user), sizeof(*user->engines))) { 759 drm_dbg(&i915->drm, "Invalid size for engine array: %d\n", 760 args->size); 761 return -EINVAL; 762 } 763 764 set.num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); 765 /* RING_MASK has no shift so we can use it directly here */ 766 if (set.num_engines > I915_EXEC_RING_MASK + 1) 767 return -EINVAL; 768 769 set.engines = kmalloc_array(set.num_engines, sizeof(*set.engines), GFP_KERNEL); 770 if (!set.engines) 771 return -ENOMEM; 772 773 for (n = 0; n < set.num_engines; n++) { 774 struct i915_engine_class_instance ci; 775 struct intel_engine_cs *engine; 776 777 if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { 778 kfree(set.engines); 779 return -EFAULT; 780 } 781 782 memset(&set.engines[n], 0, sizeof(set.engines[n])); 783 784 if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && 785 ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) 786 continue; 787 788 engine = intel_engine_lookup_user(i915, 789 ci.engine_class, 790 ci.engine_instance); 791 if (!engine) { 792 drm_dbg(&i915->drm, 793 "Invalid engine[%d]: { class:%d, instance:%d }\n", 794 n, ci.engine_class, ci.engine_instance); 795 kfree(set.engines); 796 return -ENOENT; 797 } 798 799 set.engines[n].type = I915_GEM_ENGINE_TYPE_PHYSICAL; 800 set.engines[n].engine = engine; 801 } 802 803 err = -EFAULT; 804 if (!get_user(extensions, &user->extensions)) 805 err = i915_user_extensions(u64_to_user_ptr(extensions), 806 set_proto_ctx_engines_extensions, 807 ARRAY_SIZE(set_proto_ctx_engines_extensions), 808 &set); 809 if (err) { 810 kfree(set.engines); 811 return err; 812 } 813 814 pc->num_user_engines = set.num_engines; 815 pc->user_engines = set.engines; 816 817 return 0; 818} 819 820static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv, 821 struct i915_gem_proto_context *pc, 822 struct drm_i915_gem_context_param *args) 823{ 824 struct drm_i915_private *i915 = fpriv->i915; 825 struct drm_i915_gem_context_param_sseu user_sseu; 826 struct intel_sseu *sseu; 827 int ret; 828 829 if (args->size < sizeof(user_sseu)) 830 return -EINVAL; 831 832 if (GRAPHICS_VER(i915) != 11) 833 return -ENODEV; 834 835 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 836 sizeof(user_sseu))) 837 return -EFAULT; 838 839 if (user_sseu.rsvd) 840 return -EINVAL; 841 842 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 843 return -EINVAL; 844 845 if (!!(user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) != (pc->num_user_engines >= 0)) 846 return -EINVAL; 847 848 if (pc->num_user_engines >= 0) { 849 int idx = user_sseu.engine.engine_instance; 850 struct i915_gem_proto_engine *pe; 851 852 if (idx >= pc->num_user_engines) 853 return -EINVAL; 854 855 idx = array_index_nospec(idx, pc->num_user_engines); 856 pe = &pc->user_engines[idx]; 857 858 /* Only render engine supports RPCS configuration. */ 859 if (pe->engine->class != RENDER_CLASS) 860 return -EINVAL; 861 862 sseu = &pe->sseu; 863 } else { 864 /* Only render engine supports RPCS configuration. */ 865 if (user_sseu.engine.engine_class != I915_ENGINE_CLASS_RENDER) 866 return -EINVAL; 867 868 /* There is only one render engine */ 869 if (user_sseu.engine.engine_instance != 0) 870 return -EINVAL; 871 872 sseu = &pc->legacy_rcs_sseu; 873 } 874 875 ret = i915_gem_user_to_context_sseu(to_gt(i915), &user_sseu, sseu); 876 if (ret) 877 return ret; 878 879 args->size = sizeof(user_sseu); 880 881 return 0; 882} 883 884static int set_proto_ctx_param(struct drm_i915_file_private *fpriv, 885 struct i915_gem_proto_context *pc, 886 struct drm_i915_gem_context_param *args) 887{ 888 int ret = 0; 889 890 switch (args->param) { 891 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 892 if (args->size) 893 ret = -EINVAL; 894 else if (args->value) 895 pc->user_flags |= BIT(UCONTEXT_NO_ERROR_CAPTURE); 896 else 897 pc->user_flags &= ~BIT(UCONTEXT_NO_ERROR_CAPTURE); 898 break; 899 900 case I915_CONTEXT_PARAM_BANNABLE: 901 if (args->size) 902 ret = -EINVAL; 903 else if (!capable(CAP_SYS_ADMIN) && !args->value) 904 ret = -EPERM; 905 else if (args->value) 906 pc->user_flags |= BIT(UCONTEXT_BANNABLE); 907 else if (pc->uses_protected_content) 908 ret = -EPERM; 909 else 910 pc->user_flags &= ~BIT(UCONTEXT_BANNABLE); 911 break; 912 913 case I915_CONTEXT_PARAM_RECOVERABLE: 914 if (args->size) 915 ret = -EINVAL; 916 else if (!args->value) 917 pc->user_flags &= ~BIT(UCONTEXT_RECOVERABLE); 918 else if (pc->uses_protected_content) 919 ret = -EPERM; 920 else 921 pc->user_flags |= BIT(UCONTEXT_RECOVERABLE); 922 break; 923 924 case I915_CONTEXT_PARAM_PRIORITY: 925 ret = validate_priority(fpriv->i915, args); 926 if (!ret) 927 pc->sched.priority = args->value; 928 break; 929 930 case I915_CONTEXT_PARAM_SSEU: 931 ret = set_proto_ctx_sseu(fpriv, pc, args); 932 break; 933 934 case I915_CONTEXT_PARAM_VM: 935 ret = set_proto_ctx_vm(fpriv, pc, args); 936 break; 937 938 case I915_CONTEXT_PARAM_ENGINES: 939 ret = set_proto_ctx_engines(fpriv, pc, args); 940 break; 941 942 case I915_CONTEXT_PARAM_PERSISTENCE: 943 if (args->size) 944 ret = -EINVAL; 945 else 946 ret = proto_context_set_persistence(fpriv->i915, pc, 947 args->value); 948 break; 949 950 case I915_CONTEXT_PARAM_PROTECTED_CONTENT: 951 ret = proto_context_set_protected(fpriv->i915, pc, 952 args->value); 953 break; 954 955 case I915_CONTEXT_PARAM_NO_ZEROMAP: 956 case I915_CONTEXT_PARAM_BAN_PERIOD: 957 case I915_CONTEXT_PARAM_RINGSIZE: 958 default: 959 ret = -EINVAL; 960 break; 961 } 962 963 return ret; 964} 965 966static int intel_context_set_gem(struct intel_context *ce, 967 struct i915_gem_context *ctx, 968 struct intel_sseu sseu) 969{ 970 int ret = 0; 971 972 GEM_BUG_ON(rcu_access_pointer(ce->gem_context)); 973 RCU_INIT_POINTER(ce->gem_context, ctx); 974 975 GEM_BUG_ON(intel_context_is_pinned(ce)); 976 977 if (ce->engine->class == COMPUTE_CLASS) 978 ce->ring_size = SZ_512K; 979 else 980 ce->ring_size = SZ_16K; 981 982 i915_vm_put(ce->vm); 983 ce->vm = i915_gem_context_get_eb_vm(ctx); 984 985 if (ctx->sched.priority >= I915_PRIORITY_NORMAL && 986 intel_engine_has_timeslices(ce->engine) && 987 intel_engine_has_semaphores(ce->engine)) 988 __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 989 990 if (CONFIG_DRM_I915_REQUEST_TIMEOUT && 991 ctx->i915->params.request_timeout_ms) { 992 unsigned int timeout_ms = ctx->i915->params.request_timeout_ms; 993 994 intel_context_set_watchdog_us(ce, (u64)timeout_ms * 1000); 995 } 996 997 /* A valid SSEU has no zero fields */ 998 if (sseu.slice_mask && !WARN_ON(ce->engine->class != RENDER_CLASS)) 999 ret = intel_context_reconfigure_sseu(ce, sseu); 1000 1001 return ret; 1002} 1003 1004static void __unpin_engines(struct i915_gem_engines *e, unsigned int count) 1005{ 1006 while (count--) { 1007 struct intel_context *ce = e->engines[count], *child; 1008 1009 if (!ce || !test_bit(CONTEXT_PERMA_PIN, &ce->flags)) 1010 continue; 1011 1012 for_each_child(ce, child) 1013 intel_context_unpin(child); 1014 intel_context_unpin(ce); 1015 } 1016} 1017 1018static void unpin_engines(struct i915_gem_engines *e) 1019{ 1020 __unpin_engines(e, e->num_engines); 1021} 1022 1023static void __free_engines(struct i915_gem_engines *e, unsigned int count) 1024{ 1025 while (count--) { 1026 if (!e->engines[count]) 1027 continue; 1028 1029 intel_context_put(e->engines[count]); 1030 } 1031 kfree(e); 1032} 1033 1034static void free_engines(struct i915_gem_engines *e) 1035{ 1036 __free_engines(e, e->num_engines); 1037} 1038 1039static void free_engines_rcu(struct rcu_head *rcu) 1040{ 1041 struct i915_gem_engines *engines = 1042 container_of(rcu, struct i915_gem_engines, rcu); 1043 1044 i915_sw_fence_fini(&engines->fence); 1045 free_engines(engines); 1046} 1047 1048static void accumulate_runtime(struct i915_drm_client *client, 1049 struct i915_gem_engines *engines) 1050{ 1051 struct i915_gem_engines_iter it; 1052 struct intel_context *ce; 1053 1054 if (!client) 1055 return; 1056 1057 /* Transfer accumulated runtime to the parent GEM context. */ 1058 for_each_gem_engine(ce, engines, it) { 1059 unsigned int class = ce->engine->uabi_class; 1060 1061 GEM_BUG_ON(class >= ARRAY_SIZE(client->past_runtime)); 1062 atomic64_add(intel_context_get_total_runtime_ns(ce), 1063 &client->past_runtime[class]); 1064 } 1065} 1066 1067static int 1068engines_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) 1069{ 1070 struct i915_gem_engines *engines = 1071 container_of(fence, typeof(*engines), fence); 1072 struct i915_gem_context *ctx = engines->ctx; 1073 1074 switch (state) { 1075 case FENCE_COMPLETE: 1076 if (!list_empty(&engines->link)) { 1077 unsigned long flags; 1078 1079 spin_lock_irqsave(&ctx->stale.lock, flags); 1080 list_del(&engines->link); 1081 spin_unlock_irqrestore(&ctx->stale.lock, flags); 1082 } 1083 accumulate_runtime(ctx->client, engines); 1084 i915_gem_context_put(ctx); 1085 1086 break; 1087 1088 case FENCE_FREE: 1089 init_rcu_head(&engines->rcu); 1090 call_rcu(&engines->rcu, free_engines_rcu); 1091 break; 1092 } 1093 1094 return NOTIFY_DONE; 1095} 1096 1097static struct i915_gem_engines *alloc_engines(unsigned int count) 1098{ 1099 struct i915_gem_engines *e; 1100 1101 e = kzalloc(struct_size(e, engines, count), GFP_KERNEL); 1102 if (!e) 1103 return NULL; 1104 1105 i915_sw_fence_init(&e->fence, engines_notify); 1106 return e; 1107} 1108 1109static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx, 1110 struct intel_sseu rcs_sseu) 1111{ 1112 const unsigned int max = I915_NUM_ENGINES; 1113 struct intel_engine_cs *engine; 1114 struct i915_gem_engines *e, *err; 1115 1116 e = alloc_engines(max); 1117 if (!e) 1118 return ERR_PTR(-ENOMEM); 1119 1120 for_each_uabi_engine(engine, ctx->i915) { 1121 struct intel_context *ce; 1122 struct intel_sseu sseu = {}; 1123 int ret; 1124 1125 if (engine->legacy_idx == INVALID_ENGINE) 1126 continue; 1127 1128 GEM_BUG_ON(engine->legacy_idx >= max); 1129 GEM_BUG_ON(e->engines[engine->legacy_idx]); 1130 1131 ce = intel_context_create(engine); 1132 if (IS_ERR(ce)) { 1133 err = ERR_CAST(ce); 1134 goto free_engines; 1135 } 1136 1137 e->engines[engine->legacy_idx] = ce; 1138 e->num_engines = max(e->num_engines, engine->legacy_idx + 1); 1139 1140 if (engine->class == RENDER_CLASS) 1141 sseu = rcs_sseu; 1142 1143 ret = intel_context_set_gem(ce, ctx, sseu); 1144 if (ret) { 1145 err = ERR_PTR(ret); 1146 goto free_engines; 1147 } 1148 1149 } 1150 1151 return e; 1152 1153free_engines: 1154 free_engines(e); 1155 return err; 1156} 1157 1158static int perma_pin_contexts(struct intel_context *ce) 1159{ 1160 struct intel_context *child; 1161 int i = 0, j = 0, ret; 1162 1163 GEM_BUG_ON(!intel_context_is_parent(ce)); 1164 1165 ret = intel_context_pin(ce); 1166 if (unlikely(ret)) 1167 return ret; 1168 1169 for_each_child(ce, child) { 1170 ret = intel_context_pin(child); 1171 if (unlikely(ret)) 1172 goto unwind; 1173 ++i; 1174 } 1175 1176 set_bit(CONTEXT_PERMA_PIN, &ce->flags); 1177 1178 return 0; 1179 1180unwind: 1181 intel_context_unpin(ce); 1182 for_each_child(ce, child) { 1183 if (j++ < i) 1184 intel_context_unpin(child); 1185 else 1186 break; 1187 } 1188 1189 return ret; 1190} 1191 1192static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx, 1193 unsigned int num_engines, 1194 struct i915_gem_proto_engine *pe) 1195{ 1196 struct i915_gem_engines *e, *err; 1197 unsigned int n; 1198 1199 e = alloc_engines(num_engines); 1200 if (!e) 1201 return ERR_PTR(-ENOMEM); 1202 e->num_engines = num_engines; 1203 1204 for (n = 0; n < num_engines; n++) { 1205 struct intel_context *ce, *child; 1206 int ret; 1207 1208 switch (pe[n].type) { 1209 case I915_GEM_ENGINE_TYPE_PHYSICAL: 1210 ce = intel_context_create(pe[n].engine); 1211 break; 1212 1213 case I915_GEM_ENGINE_TYPE_BALANCED: 1214 ce = intel_engine_create_virtual(pe[n].siblings, 1215 pe[n].num_siblings, 0); 1216 break; 1217 1218 case I915_GEM_ENGINE_TYPE_PARALLEL: 1219 ce = intel_engine_create_parallel(pe[n].siblings, 1220 pe[n].num_siblings, 1221 pe[n].width); 1222 break; 1223 1224 case I915_GEM_ENGINE_TYPE_INVALID: 1225 default: 1226 GEM_WARN_ON(pe[n].type != I915_GEM_ENGINE_TYPE_INVALID); 1227 continue; 1228 } 1229 1230 if (IS_ERR(ce)) { 1231 err = ERR_CAST(ce); 1232 goto free_engines; 1233 } 1234 1235 e->engines[n] = ce; 1236 1237 ret = intel_context_set_gem(ce, ctx, pe->sseu); 1238 if (ret) { 1239 err = ERR_PTR(ret); 1240 goto free_engines; 1241 } 1242 for_each_child(ce, child) { 1243 ret = intel_context_set_gem(child, ctx, pe->sseu); 1244 if (ret) { 1245 err = ERR_PTR(ret); 1246 goto free_engines; 1247 } 1248 } 1249 1250 /* 1251 * XXX: Must be done after calling intel_context_set_gem as that 1252 * function changes the ring size. The ring is allocated when 1253 * the context is pinned. If the ring size is changed after 1254 * allocation we have a mismatch of the ring size and will cause 1255 * the context to hang. Presumably with a bit of reordering we 1256 * could move the perma-pin step to the backend function 1257 * intel_engine_create_parallel. 1258 */ 1259 if (pe[n].type == I915_GEM_ENGINE_TYPE_PARALLEL) { 1260 ret = perma_pin_contexts(ce); 1261 if (ret) { 1262 err = ERR_PTR(ret); 1263 goto free_engines; 1264 } 1265 } 1266 } 1267 1268 return e; 1269 1270free_engines: 1271 free_engines(e); 1272 return err; 1273} 1274 1275static void i915_gem_context_release_work(struct work_struct *work) 1276{ 1277 struct i915_gem_context *ctx = container_of(work, typeof(*ctx), 1278 release_work); 1279 struct i915_address_space *vm; 1280 1281 trace_i915_context_free(ctx); 1282 GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 1283 1284 spin_lock(&ctx->i915->gem.contexts.lock); 1285 list_del(&ctx->link); 1286 spin_unlock(&ctx->i915->gem.contexts.lock); 1287 1288 if (ctx->syncobj) 1289 drm_syncobj_put(ctx->syncobj); 1290 1291 vm = ctx->vm; 1292 if (vm) 1293 i915_vm_put(vm); 1294 1295 if (ctx->pxp_wakeref) 1296 intel_runtime_pm_put(&ctx->i915->runtime_pm, ctx->pxp_wakeref); 1297 1298 if (ctx->client) 1299 i915_drm_client_put(ctx->client); 1300 1301 mutex_destroy(&ctx->engines_mutex); 1302 mutex_destroy(&ctx->lut_mutex); 1303 1304 put_pid(ctx->pid); 1305 mutex_destroy(&ctx->mutex); 1306 1307 kfree_rcu(ctx, rcu); 1308} 1309 1310void i915_gem_context_release(struct kref *ref) 1311{ 1312 struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); 1313 1314 queue_work(ctx->i915->wq, &ctx->release_work); 1315} 1316 1317static inline struct i915_gem_engines * 1318__context_engines_static(const struct i915_gem_context *ctx) 1319{ 1320 return rcu_dereference_protected(ctx->engines, true); 1321} 1322 1323static void __reset_context(struct i915_gem_context *ctx, 1324 struct intel_engine_cs *engine) 1325{ 1326 intel_gt_handle_error(engine->gt, engine->mask, 0, 1327 "context closure in %s", ctx->name); 1328} 1329 1330static bool __cancel_engine(struct intel_engine_cs *engine) 1331{ 1332 /* 1333 * Send a "high priority pulse" down the engine to cause the 1334 * current request to be momentarily preempted. (If it fails to 1335 * be preempted, it will be reset). As we have marked our context 1336 * as banned, any incomplete request, including any running, will 1337 * be skipped following the preemption. 1338 * 1339 * If there is no hangchecking (one of the reasons why we try to 1340 * cancel the context) and no forced preemption, there may be no 1341 * means by which we reset the GPU and evict the persistent hog. 1342 * Ergo if we are unable to inject a preemptive pulse that can 1343 * kill the banned context, we fallback to doing a local reset 1344 * instead. 1345 */ 1346 return intel_engine_pulse(engine) == 0; 1347} 1348 1349static struct intel_engine_cs *active_engine(struct intel_context *ce) 1350{ 1351 struct intel_engine_cs *engine = NULL; 1352 struct i915_request *rq; 1353 1354 if (intel_context_has_inflight(ce)) 1355 return intel_context_inflight(ce); 1356 1357 if (!ce->timeline) 1358 return NULL; 1359 1360 /* 1361 * rq->link is only SLAB_TYPESAFE_BY_RCU, we need to hold a reference 1362 * to the request to prevent it being transferred to a new timeline 1363 * (and onto a new timeline->requests list). 1364 */ 1365 rcu_read_lock(); 1366 list_for_each_entry_reverse(rq, &ce->timeline->requests, link) { 1367 bool found; 1368 1369 /* timeline is already completed upto this point? */ 1370 if (!i915_request_get_rcu(rq)) 1371 break; 1372 1373 /* Check with the backend if the request is inflight */ 1374 found = true; 1375 if (likely(rcu_access_pointer(rq->timeline) == ce->timeline)) 1376 found = i915_request_active_engine(rq, &engine); 1377 1378 i915_request_put(rq); 1379 if (found) 1380 break; 1381 } 1382 rcu_read_unlock(); 1383 1384 return engine; 1385} 1386 1387static void 1388kill_engines(struct i915_gem_engines *engines, bool exit, bool persistent) 1389{ 1390 struct i915_gem_engines_iter it; 1391 struct intel_context *ce; 1392 1393 /* 1394 * Map the user's engine back to the actual engines; one virtual 1395 * engine will be mapped to multiple engines, and using ctx->engine[] 1396 * the same engine may be have multiple instances in the user's map. 1397 * However, we only care about pending requests, so only include 1398 * engines on which there are incomplete requests. 1399 */ 1400 for_each_gem_engine(ce, engines, it) { 1401 struct intel_engine_cs *engine; 1402 1403 if ((exit || !persistent) && intel_context_revoke(ce)) 1404 continue; /* Already marked. */ 1405 1406 /* 1407 * Check the current active state of this context; if we 1408 * are currently executing on the GPU we need to evict 1409 * ourselves. On the other hand, if we haven't yet been 1410 * submitted to the GPU or if everything is complete, 1411 * we have nothing to do. 1412 */ 1413 engine = active_engine(ce); 1414 1415 /* First attempt to gracefully cancel the context */ 1416 if (engine && !__cancel_engine(engine) && (exit || !persistent)) 1417 /* 1418 * If we are unable to send a preemptive pulse to bump 1419 * the context from the GPU, we have to resort to a full 1420 * reset. We hope the collateral damage is worth it. 1421 */ 1422 __reset_context(engines->ctx, engine); 1423 } 1424} 1425 1426static void kill_context(struct i915_gem_context *ctx) 1427{ 1428 struct i915_gem_engines *pos, *next; 1429 static int warn = 1; 1430 1431 spin_lock_irq(&ctx->stale.lock); 1432 GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 1433 list_for_each_entry_safe(pos, next, &ctx->stale.engines, link) { 1434 if (!i915_sw_fence_await(&pos->fence)) { 1435 list_del_init(&pos->link); 1436 continue; 1437 } 1438 1439 /* 1440 * XXX don't incorrectly reset chip on 1441 * gm45/vlv/ivb/hsw/bdw cause unknown 1442 */ 1443 if (IS_GRAPHICS_VER(ctx->i915, 4, 8)) { 1444 if (warn) { 1445 DRM_DEBUG("%s XXX skipping reset pos %p\n", __func__, pos); 1446 warn = 0; 1447 } 1448 continue; 1449 } 1450 1451 spin_unlock_irq(&ctx->stale.lock); 1452 1453 kill_engines(pos, !ctx->i915->params.enable_hangcheck, 1454 i915_gem_context_is_persistent(ctx)); 1455 1456 spin_lock_irq(&ctx->stale.lock); 1457 GEM_BUG_ON(i915_sw_fence_signaled(&pos->fence)); 1458 list_safe_reset_next(pos, next, link); 1459 list_del_init(&pos->link); /* decouple from FENCE_COMPLETE */ 1460 1461 i915_sw_fence_complete(&pos->fence); 1462 } 1463 spin_unlock_irq(&ctx->stale.lock); 1464} 1465 1466static void engines_idle_release(struct i915_gem_context *ctx, 1467 struct i915_gem_engines *engines) 1468{ 1469 struct i915_gem_engines_iter it; 1470 struct intel_context *ce; 1471 1472 INIT_LIST_HEAD(&engines->link); 1473 1474 engines->ctx = i915_gem_context_get(ctx); 1475 1476 for_each_gem_engine(ce, engines, it) { 1477 int err; 1478 1479 /* serialises with execbuf */ 1480 intel_context_close(ce); 1481 if (!intel_context_pin_if_active(ce)) 1482 continue; 1483 1484 /* Wait until context is finally scheduled out and retired */ 1485 err = i915_sw_fence_await_active(&engines->fence, 1486 &ce->active, 1487 I915_ACTIVE_AWAIT_BARRIER); 1488 intel_context_unpin(ce); 1489 if (err) 1490 goto kill; 1491 } 1492 1493 spin_lock_irq(&ctx->stale.lock); 1494 if (!i915_gem_context_is_closed(ctx)) 1495 list_add_tail(&engines->link, &ctx->stale.engines); 1496 spin_unlock_irq(&ctx->stale.lock); 1497 1498kill: 1499 if (list_empty(&engines->link)) /* raced, already closed */ 1500 kill_engines(engines, true, 1501 i915_gem_context_is_persistent(ctx)); 1502 1503 i915_sw_fence_commit(&engines->fence); 1504} 1505 1506static void set_closed_name(struct i915_gem_context *ctx) 1507{ 1508 char *s; 1509 1510 /* Replace '[]' with '<>' to indicate closed in debug prints */ 1511 1512 s = strrchr(ctx->name, '['); 1513 if (!s) 1514 return; 1515 1516 *s = '<'; 1517 1518 s = strchr(s + 1, ']'); 1519 if (s) 1520 *s = '>'; 1521} 1522 1523static void context_close(struct i915_gem_context *ctx) 1524{ 1525 struct i915_drm_client *client; 1526 1527 /* Flush any concurrent set_engines() */ 1528 mutex_lock(&ctx->engines_mutex); 1529 unpin_engines(__context_engines_static(ctx)); 1530 engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1)); 1531 i915_gem_context_set_closed(ctx); 1532 mutex_unlock(&ctx->engines_mutex); 1533 1534 mutex_lock(&ctx->mutex); 1535 1536 set_closed_name(ctx); 1537 1538 /* 1539 * The LUT uses the VMA as a backpointer to unref the object, 1540 * so we need to clear the LUT before we close all the VMA (inside 1541 * the ppgtt). 1542 */ 1543 lut_close(ctx); 1544 1545 ctx->file_priv = ERR_PTR(-EBADF); 1546 1547 client = ctx->client; 1548 if (client) { 1549 spin_lock(&client->ctx_lock); 1550 list_del_rcu(&ctx->client_link); 1551 spin_unlock(&client->ctx_lock); 1552 } 1553 1554 mutex_unlock(&ctx->mutex); 1555 1556 /* 1557 * If the user has disabled hangchecking, we can not be sure that 1558 * the batches will ever complete after the context is closed, 1559 * keeping the context and all resources pinned forever. So in this 1560 * case we opt to forcibly kill off all remaining requests on 1561 * context close. 1562 */ 1563 kill_context(ctx); 1564 1565 i915_gem_context_put(ctx); 1566} 1567 1568static int __context_set_persistence(struct i915_gem_context *ctx, bool state) 1569{ 1570 if (i915_gem_context_is_persistent(ctx) == state) 1571 return 0; 1572 1573 if (state) { 1574 /* 1575 * Only contexts that are short-lived [that will expire or be 1576 * reset] are allowed to survive past termination. We require 1577 * hangcheck to ensure that the persistent requests are healthy. 1578 */ 1579 if (!ctx->i915->params.enable_hangcheck) 1580 return -EINVAL; 1581 1582 i915_gem_context_set_persistence(ctx); 1583 } else { 1584 /* To cancel a context we use "preempt-to-idle" */ 1585 if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 1586 return -ENODEV; 1587 1588 /* 1589 * If the cancel fails, we then need to reset, cleanly! 1590 * 1591 * If the per-engine reset fails, all hope is lost! We resort 1592 * to a full GPU reset in that unlikely case, but realistically 1593 * if the engine could not reset, the full reset does not fare 1594 * much better. The damage has been done. 1595 * 1596 * However, if we cannot reset an engine by itself, we cannot 1597 * cleanup a hanging persistent context without causing 1598 * colateral damage, and we should not pretend we can by 1599 * exposing the interface. 1600 */ 1601 if (!intel_has_reset_engine(to_gt(ctx->i915))) 1602 return -ENODEV; 1603 1604 i915_gem_context_clear_persistence(ctx); 1605 } 1606 1607 return 0; 1608} 1609 1610static struct i915_gem_context * 1611i915_gem_create_context(struct drm_i915_private *i915, 1612 const struct i915_gem_proto_context *pc) 1613{ 1614 struct i915_gem_context *ctx; 1615 struct i915_address_space *vm = NULL; 1616 struct i915_gem_engines *e; 1617 int err; 1618 int i; 1619 1620 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 1621 if (!ctx) 1622 return ERR_PTR(-ENOMEM); 1623 1624 kref_init(&ctx->ref); 1625 ctx->i915 = i915; 1626 ctx->sched = pc->sched; 1627 rw_init(&ctx->mutex, "gemctx"); 1628 INIT_LIST_HEAD(&ctx->link); 1629 INIT_WORK(&ctx->release_work, i915_gem_context_release_work); 1630 1631 mtx_init(&ctx->stale.lock, IPL_TTY); 1632 INIT_LIST_HEAD(&ctx->stale.engines); 1633 1634 if (pc->vm) { 1635 vm = i915_vm_get(pc->vm); 1636 } else if (HAS_FULL_PPGTT(i915)) { 1637 struct i915_ppgtt *ppgtt; 1638 1639 ppgtt = i915_ppgtt_create(to_gt(i915), 0); 1640 if (IS_ERR(ppgtt)) { 1641 drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n", 1642 PTR_ERR(ppgtt)); 1643 err = PTR_ERR(ppgtt); 1644 goto err_ctx; 1645 } 1646 vm = &ppgtt->vm; 1647 } 1648 if (vm) 1649 ctx->vm = vm; 1650 1651 rw_init(&ctx->engines_mutex, "gemeng"); 1652 if (pc->num_user_engines >= 0) { 1653 i915_gem_context_set_user_engines(ctx); 1654 e = user_engines(ctx, pc->num_user_engines, pc->user_engines); 1655 } else { 1656 i915_gem_context_clear_user_engines(ctx); 1657 e = default_engines(ctx, pc->legacy_rcs_sseu); 1658 } 1659 if (IS_ERR(e)) { 1660 err = PTR_ERR(e); 1661 goto err_vm; 1662 } 1663 RCU_INIT_POINTER(ctx->engines, e); 1664 1665 INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); 1666 rw_init(&ctx->lut_mutex, "lutrw"); 1667 1668 /* NB: Mark all slices as needing a remap so that when the context first 1669 * loads it will restore whatever remap state already exists. If there 1670 * is no remap info, it will be a NOP. */ 1671 ctx->remap_slice = ALL_L3_SLICES(i915); 1672 1673 ctx->user_flags = pc->user_flags; 1674 1675 for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 1676 ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 1677 1678 if (pc->single_timeline) { 1679 err = drm_syncobj_create(&ctx->syncobj, 1680 DRM_SYNCOBJ_CREATE_SIGNALED, 1681 NULL); 1682 if (err) 1683 goto err_engines; 1684 } 1685 1686 if (pc->uses_protected_content) { 1687 ctx->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1688 ctx->uses_protected_content = true; 1689 } 1690 1691 trace_i915_context_create(ctx); 1692 1693 return ctx; 1694 1695err_engines: 1696 free_engines(e); 1697err_vm: 1698 if (ctx->vm) 1699 i915_vm_put(ctx->vm); 1700err_ctx: 1701 kfree(ctx); 1702 return ERR_PTR(err); 1703} 1704 1705static void init_contexts(struct i915_gem_contexts *gc) 1706{ 1707 mtx_init(&gc->lock, IPL_NONE); 1708 INIT_LIST_HEAD(&gc->list); 1709} 1710 1711void i915_gem_init__contexts(struct drm_i915_private *i915) 1712{ 1713 init_contexts(&i915->gem.contexts); 1714} 1715 1716/* 1717 * Note that this implicitly consumes the ctx reference, by placing 1718 * the ctx in the context_xa. 1719 */ 1720static void gem_context_register(struct i915_gem_context *ctx, 1721 struct drm_i915_file_private *fpriv, 1722 u32 id) 1723{ 1724 struct drm_i915_private *i915 = ctx->i915; 1725 void *old; 1726 1727 ctx->file_priv = fpriv; 1728 1729#ifdef __linux__ 1730 ctx->pid = get_task_pid(current, PIDTYPE_PID); 1731 ctx->client = i915_drm_client_get(fpriv->client); 1732 1733 snprintf(ctx->name, sizeof(ctx->name), "%s[%d]", 1734 current->comm, pid_nr(ctx->pid)); 1735#else 1736 ctx->pid = curproc->p_p->ps_pid; 1737 ctx->client = i915_drm_client_get(fpriv->client); 1738 1739 snprintf(ctx->name, sizeof(ctx->name), "%s[%d]", 1740 curproc->p_p->ps_comm, ctx->pid); 1741#endif 1742 1743 spin_lock(&ctx->client->ctx_lock); 1744 list_add_tail_rcu(&ctx->client_link, &ctx->client->ctx_list); 1745 spin_unlock(&ctx->client->ctx_lock); 1746 1747 spin_lock(&i915->gem.contexts.lock); 1748 list_add_tail(&ctx->link, &i915->gem.contexts.list); 1749 spin_unlock(&i915->gem.contexts.lock); 1750 1751 /* And finally expose ourselves to userspace via the idr */ 1752 old = xa_store(&fpriv->context_xa, id, ctx, GFP_KERNEL); 1753 WARN_ON(old); 1754} 1755 1756int i915_gem_context_open(struct drm_i915_private *i915, 1757 struct drm_file *file) 1758{ 1759 struct drm_i915_file_private *file_priv = file->driver_priv; 1760 struct i915_gem_proto_context *pc; 1761 struct i915_gem_context *ctx; 1762 int err; 1763 1764 rw_init(&file_priv->proto_context_lock, "pctxlk"); 1765 xa_init_flags(&file_priv->proto_context_xa, XA_FLAGS_ALLOC); 1766 1767 /* 0 reserved for the default context */ 1768 xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC1); 1769 1770 /* 0 reserved for invalid/unassigned ppgtt */ 1771 xa_init_flags(&file_priv->vm_xa, XA_FLAGS_ALLOC1); 1772 1773 pc = proto_context_create(i915, 0); 1774 if (IS_ERR(pc)) { 1775 err = PTR_ERR(pc); 1776 goto err; 1777 } 1778 1779 ctx = i915_gem_create_context(i915, pc); 1780 proto_context_close(i915, pc); 1781 if (IS_ERR(ctx)) { 1782 err = PTR_ERR(ctx); 1783 goto err; 1784 } 1785 1786 gem_context_register(ctx, file_priv, 0); 1787 1788 return 0; 1789 1790err: 1791 xa_destroy(&file_priv->vm_xa); 1792 xa_destroy(&file_priv->context_xa); 1793 xa_destroy(&file_priv->proto_context_xa); 1794 mutex_destroy(&file_priv->proto_context_lock); 1795 return err; 1796} 1797 1798void i915_gem_context_close(struct drm_file *file) 1799{ 1800 struct drm_i915_file_private *file_priv = file->driver_priv; 1801 struct i915_gem_proto_context *pc; 1802 struct i915_address_space *vm; 1803 struct i915_gem_context *ctx; 1804 unsigned long idx; 1805 1806 xa_for_each(&file_priv->proto_context_xa, idx, pc) 1807 proto_context_close(file_priv->i915, pc); 1808 xa_destroy(&file_priv->proto_context_xa); 1809 mutex_destroy(&file_priv->proto_context_lock); 1810 1811 xa_for_each(&file_priv->context_xa, idx, ctx) 1812 context_close(ctx); 1813 xa_destroy(&file_priv->context_xa); 1814 1815 xa_for_each(&file_priv->vm_xa, idx, vm) 1816 i915_vm_put(vm); 1817 xa_destroy(&file_priv->vm_xa); 1818} 1819 1820int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, 1821 struct drm_file *file) 1822{ 1823 struct drm_i915_private *i915 = to_i915(dev); 1824 struct drm_i915_gem_vm_control *args = data; 1825 struct drm_i915_file_private *file_priv = file->driver_priv; 1826 struct i915_ppgtt *ppgtt; 1827 u32 id; 1828 int err; 1829 1830 if (!HAS_FULL_PPGTT(i915)) 1831 return -ENODEV; 1832 1833 if (args->flags) 1834 return -EINVAL; 1835 1836 ppgtt = i915_ppgtt_create(to_gt(i915), 0); 1837 if (IS_ERR(ppgtt)) 1838 return PTR_ERR(ppgtt); 1839 1840 if (args->extensions) { 1841 err = i915_user_extensions(u64_to_user_ptr(args->extensions), 1842 NULL, 0, 1843 ppgtt); 1844 if (err) 1845 goto err_put; 1846 } 1847 1848 err = xa_alloc(&file_priv->vm_xa, &id, &ppgtt->vm, 1849 xa_limit_32b, GFP_KERNEL); 1850 if (err) 1851 goto err_put; 1852 1853 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 1854 args->vm_id = id; 1855 return 0; 1856 1857err_put: 1858 i915_vm_put(&ppgtt->vm); 1859 return err; 1860} 1861 1862int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, 1863 struct drm_file *file) 1864{ 1865 struct drm_i915_file_private *file_priv = file->driver_priv; 1866 struct drm_i915_gem_vm_control *args = data; 1867 struct i915_address_space *vm; 1868 1869 if (args->flags) 1870 return -EINVAL; 1871 1872 if (args->extensions) 1873 return -EINVAL; 1874 1875 vm = xa_erase(&file_priv->vm_xa, args->vm_id); 1876 if (!vm) 1877 return -ENOENT; 1878 1879 i915_vm_put(vm); 1880 return 0; 1881} 1882 1883static int get_ppgtt(struct drm_i915_file_private *file_priv, 1884 struct i915_gem_context *ctx, 1885 struct drm_i915_gem_context_param *args) 1886{ 1887 struct i915_address_space *vm; 1888 int err; 1889 u32 id; 1890 1891 if (!i915_gem_context_has_full_ppgtt(ctx)) 1892 return -ENODEV; 1893 1894 vm = ctx->vm; 1895 GEM_BUG_ON(!vm); 1896 1897 /* 1898 * Get a reference for the allocated handle. Once the handle is 1899 * visible in the vm_xa table, userspace could try to close it 1900 * from under our feet, so we need to hold the extra reference 1901 * first. 1902 */ 1903 i915_vm_get(vm); 1904 1905 err = xa_alloc(&file_priv->vm_xa, &id, vm, xa_limit_32b, GFP_KERNEL); 1906 if (err) { 1907 i915_vm_put(vm); 1908 return err; 1909 } 1910 1911 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 1912 args->value = id; 1913 args->size = 0; 1914 1915 return err; 1916} 1917 1918int 1919i915_gem_user_to_context_sseu(struct intel_gt *gt, 1920 const struct drm_i915_gem_context_param_sseu *user, 1921 struct intel_sseu *context) 1922{ 1923 const struct sseu_dev_info *device = >->info.sseu; 1924 struct drm_i915_private *i915 = gt->i915; 1925 unsigned int dev_subslice_mask = intel_sseu_get_hsw_subslices(device, 0); 1926 1927 /* No zeros in any field. */ 1928 if (!user->slice_mask || !user->subslice_mask || 1929 !user->min_eus_per_subslice || !user->max_eus_per_subslice) 1930 return -EINVAL; 1931 1932 /* Max > min. */ 1933 if (user->max_eus_per_subslice < user->min_eus_per_subslice) 1934 return -EINVAL; 1935 1936 /* 1937 * Some future proofing on the types since the uAPI is wider than the 1938 * current internal implementation. 1939 */ 1940 if (overflows_type(user->slice_mask, context->slice_mask) || 1941 overflows_type(user->subslice_mask, context->subslice_mask) || 1942 overflows_type(user->min_eus_per_subslice, 1943 context->min_eus_per_subslice) || 1944 overflows_type(user->max_eus_per_subslice, 1945 context->max_eus_per_subslice)) 1946 return -EINVAL; 1947 1948 /* Check validity against hardware. */ 1949 if (user->slice_mask & ~device->slice_mask) 1950 return -EINVAL; 1951 1952 if (user->subslice_mask & ~dev_subslice_mask) 1953 return -EINVAL; 1954 1955 if (user->max_eus_per_subslice > device->max_eus_per_subslice) 1956 return -EINVAL; 1957 1958 context->slice_mask = user->slice_mask; 1959 context->subslice_mask = user->subslice_mask; 1960 context->min_eus_per_subslice = user->min_eus_per_subslice; 1961 context->max_eus_per_subslice = user->max_eus_per_subslice; 1962 1963 /* Part specific restrictions. */ 1964 if (GRAPHICS_VER(i915) == 11) { 1965 unsigned int hw_s = hweight8(device->slice_mask); 1966 unsigned int hw_ss_per_s = hweight8(dev_subslice_mask); 1967 unsigned int req_s = hweight8(context->slice_mask); 1968 unsigned int req_ss = hweight8(context->subslice_mask); 1969 1970 /* 1971 * Only full subslice enablement is possible if more than one 1972 * slice is turned on. 1973 */ 1974 if (req_s > 1 && req_ss != hw_ss_per_s) 1975 return -EINVAL; 1976 1977 /* 1978 * If more than four (SScount bitfield limit) subslices are 1979 * requested then the number has to be even. 1980 */ 1981 if (req_ss > 4 && (req_ss & 1)) 1982 return -EINVAL; 1983 1984 /* 1985 * If only one slice is enabled and subslice count is below the 1986 * device full enablement, it must be at most half of the all 1987 * available subslices. 1988 */ 1989 if (req_s == 1 && req_ss < hw_ss_per_s && 1990 req_ss > (hw_ss_per_s / 2)) 1991 return -EINVAL; 1992 1993 /* ABI restriction - VME use case only. */ 1994 1995 /* All slices or one slice only. */ 1996 if (req_s != 1 && req_s != hw_s) 1997 return -EINVAL; 1998 1999 /* 2000 * Half subslices or full enablement only when one slice is 2001 * enabled. 2002 */ 2003 if (req_s == 1 && 2004 (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) 2005 return -EINVAL; 2006 2007 /* No EU configuration changes. */ 2008 if ((user->min_eus_per_subslice != 2009 device->max_eus_per_subslice) || 2010 (user->max_eus_per_subslice != 2011 device->max_eus_per_subslice)) 2012 return -EINVAL; 2013 } 2014 2015 return 0; 2016} 2017 2018static int set_sseu(struct i915_gem_context *ctx, 2019 struct drm_i915_gem_context_param *args) 2020{ 2021 struct drm_i915_private *i915 = ctx->i915; 2022 struct drm_i915_gem_context_param_sseu user_sseu; 2023 struct intel_context *ce; 2024 struct intel_sseu sseu; 2025 unsigned long lookup; 2026 int ret; 2027 2028 if (args->size < sizeof(user_sseu)) 2029 return -EINVAL; 2030 2031 if (GRAPHICS_VER(i915) != 11) 2032 return -ENODEV; 2033 2034 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 2035 sizeof(user_sseu))) 2036 return -EFAULT; 2037 2038 if (user_sseu.rsvd) 2039 return -EINVAL; 2040 2041 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 2042 return -EINVAL; 2043 2044 lookup = 0; 2045 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 2046 lookup |= LOOKUP_USER_INDEX; 2047 2048 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 2049 if (IS_ERR(ce)) 2050 return PTR_ERR(ce); 2051 2052 /* Only render engine supports RPCS configuration. */ 2053 if (ce->engine->class != RENDER_CLASS) { 2054 ret = -ENODEV; 2055 goto out_ce; 2056 } 2057 2058 ret = i915_gem_user_to_context_sseu(ce->engine->gt, &user_sseu, &sseu); 2059 if (ret) 2060 goto out_ce; 2061 2062 ret = intel_context_reconfigure_sseu(ce, sseu); 2063 if (ret) 2064 goto out_ce; 2065 2066 args->size = sizeof(user_sseu); 2067 2068out_ce: 2069 intel_context_put(ce); 2070 return ret; 2071} 2072 2073static int 2074set_persistence(struct i915_gem_context *ctx, 2075 const struct drm_i915_gem_context_param *args) 2076{ 2077 if (args->size) 2078 return -EINVAL; 2079 2080 return __context_set_persistence(ctx, args->value); 2081} 2082 2083static int set_priority(struct i915_gem_context *ctx, 2084 const struct drm_i915_gem_context_param *args) 2085{ 2086 struct i915_gem_engines_iter it; 2087 struct intel_context *ce; 2088 int err; 2089 2090 err = validate_priority(ctx->i915, args); 2091 if (err) 2092 return err; 2093 2094 ctx->sched.priority = args->value; 2095 2096 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 2097 if (!intel_engine_has_timeslices(ce->engine)) 2098 continue; 2099 2100 if (ctx->sched.priority >= I915_PRIORITY_NORMAL && 2101 intel_engine_has_semaphores(ce->engine)) 2102 intel_context_set_use_semaphores(ce); 2103 else 2104 intel_context_clear_use_semaphores(ce); 2105 } 2106 i915_gem_context_unlock_engines(ctx); 2107 2108 return 0; 2109} 2110 2111static int get_protected(struct i915_gem_context *ctx, 2112 struct drm_i915_gem_context_param *args) 2113{ 2114 args->size = 0; 2115 args->value = i915_gem_context_uses_protected_content(ctx); 2116 2117 return 0; 2118} 2119 2120static int ctx_setparam(struct drm_i915_file_private *fpriv, 2121 struct i915_gem_context *ctx, 2122 struct drm_i915_gem_context_param *args) 2123{ 2124 int ret = 0; 2125 2126 switch (args->param) { 2127 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 2128 if (args->size) 2129 ret = -EINVAL; 2130 else if (args->value) 2131 i915_gem_context_set_no_error_capture(ctx); 2132 else 2133 i915_gem_context_clear_no_error_capture(ctx); 2134 break; 2135 2136 case I915_CONTEXT_PARAM_BANNABLE: 2137 if (args->size) 2138 ret = -EINVAL; 2139 else if (!capable(CAP_SYS_ADMIN) && !args->value) 2140 ret = -EPERM; 2141 else if (args->value) 2142 i915_gem_context_set_bannable(ctx); 2143 else if (i915_gem_context_uses_protected_content(ctx)) 2144 ret = -EPERM; /* can't clear this for protected contexts */ 2145 else 2146 i915_gem_context_clear_bannable(ctx); 2147 break; 2148 2149 case I915_CONTEXT_PARAM_RECOVERABLE: 2150 if (args->size) 2151 ret = -EINVAL; 2152 else if (!args->value) 2153 i915_gem_context_clear_recoverable(ctx); 2154 else if (i915_gem_context_uses_protected_content(ctx)) 2155 ret = -EPERM; /* can't set this for protected contexts */ 2156 else 2157 i915_gem_context_set_recoverable(ctx); 2158 break; 2159 2160 case I915_CONTEXT_PARAM_PRIORITY: 2161 ret = set_priority(ctx, args); 2162 break; 2163 2164 case I915_CONTEXT_PARAM_SSEU: 2165 ret = set_sseu(ctx, args); 2166 break; 2167 2168 case I915_CONTEXT_PARAM_PERSISTENCE: 2169 ret = set_persistence(ctx, args); 2170 break; 2171 2172 case I915_CONTEXT_PARAM_PROTECTED_CONTENT: 2173 case I915_CONTEXT_PARAM_NO_ZEROMAP: 2174 case I915_CONTEXT_PARAM_BAN_PERIOD: 2175 case I915_CONTEXT_PARAM_RINGSIZE: 2176 case I915_CONTEXT_PARAM_VM: 2177 case I915_CONTEXT_PARAM_ENGINES: 2178 default: 2179 ret = -EINVAL; 2180 break; 2181 } 2182 2183 return ret; 2184} 2185 2186struct create_ext { 2187 struct i915_gem_proto_context *pc; 2188 struct drm_i915_file_private *fpriv; 2189}; 2190 2191static int create_setparam(struct i915_user_extension __user *ext, void *data) 2192{ 2193 struct drm_i915_gem_context_create_ext_setparam local; 2194 const struct create_ext *arg = data; 2195 2196 if (copy_from_user(&local, ext, sizeof(local))) 2197 return -EFAULT; 2198 2199 if (local.param.ctx_id) 2200 return -EINVAL; 2201 2202 return set_proto_ctx_param(arg->fpriv, arg->pc, &local.param); 2203} 2204 2205static int invalid_ext(struct i915_user_extension __user *ext, void *data) 2206{ 2207 return -EINVAL; 2208} 2209 2210static const i915_user_extension_fn create_extensions[] = { 2211 [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, 2212 [I915_CONTEXT_CREATE_EXT_CLONE] = invalid_ext, 2213}; 2214 2215static bool client_is_banned(struct drm_i915_file_private *file_priv) 2216{ 2217 return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; 2218} 2219 2220static inline struct i915_gem_context * 2221__context_lookup(struct drm_i915_file_private *file_priv, u32 id) 2222{ 2223 struct i915_gem_context *ctx; 2224 2225 rcu_read_lock(); 2226 ctx = xa_load(&file_priv->context_xa, id); 2227 if (ctx && !kref_get_unless_zero(&ctx->ref)) 2228 ctx = NULL; 2229 rcu_read_unlock(); 2230 2231 return ctx; 2232} 2233 2234static struct i915_gem_context * 2235finalize_create_context_locked(struct drm_i915_file_private *file_priv, 2236 struct i915_gem_proto_context *pc, u32 id) 2237{ 2238 struct i915_gem_context *ctx; 2239 void *old; 2240 2241 lockdep_assert_held(&file_priv->proto_context_lock); 2242 2243 ctx = i915_gem_create_context(file_priv->i915, pc); 2244 if (IS_ERR(ctx)) 2245 return ctx; 2246 2247 /* 2248 * One for the xarray and one for the caller. We need to grab 2249 * the reference *prior* to making the ctx visble to userspace 2250 * in gem_context_register(), as at any point after that 2251 * userspace can try to race us with another thread destroying 2252 * the context under our feet. 2253 */ 2254 i915_gem_context_get(ctx); 2255 2256 gem_context_register(ctx, file_priv, id); 2257 2258 old = xa_erase(&file_priv->proto_context_xa, id); 2259 GEM_BUG_ON(old != pc); 2260 proto_context_close(file_priv->i915, pc); 2261 2262 return ctx; 2263} 2264 2265struct i915_gem_context * 2266i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) 2267{ 2268 struct i915_gem_proto_context *pc; 2269 struct i915_gem_context *ctx; 2270 2271 ctx = __context_lookup(file_priv, id); 2272 if (ctx) 2273 return ctx; 2274 2275 mutex_lock(&file_priv->proto_context_lock); 2276 /* Try one more time under the lock */ 2277 ctx = __context_lookup(file_priv, id); 2278 if (!ctx) { 2279 pc = xa_load(&file_priv->proto_context_xa, id); 2280 if (!pc) 2281 ctx = ERR_PTR(-ENOENT); 2282 else 2283 ctx = finalize_create_context_locked(file_priv, pc, id); 2284 } 2285 mutex_unlock(&file_priv->proto_context_lock); 2286 2287 return ctx; 2288} 2289 2290int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 2291 struct drm_file *file) 2292{ 2293 struct drm_i915_private *i915 = to_i915(dev); 2294 struct drm_i915_gem_context_create_ext *args = data; 2295 struct create_ext ext_data; 2296 int ret; 2297 u32 id; 2298 2299 if (!DRIVER_CAPS(i915)->has_logical_contexts) 2300 return -ENODEV; 2301 2302 if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) 2303 return -EINVAL; 2304 2305 ret = intel_gt_terminally_wedged(to_gt(i915)); 2306 if (ret) 2307 return ret; 2308 2309 ext_data.fpriv = file->driver_priv; 2310 if (client_is_banned(ext_data.fpriv)) { 2311#ifdef __linux__ 2312 drm_dbg(&i915->drm, 2313 "client %s[%d] banned from creating ctx\n", 2314 current->comm, task_pid_nr(current)); 2315#else 2316 drm_dbg(&i915->drm, 2317 "client %s[%d] banned from creating ctx\n", 2318 curproc->p_p->ps_comm, curproc->p_p->ps_pid); 2319#endif 2320 return -EIO; 2321 } 2322 2323 ext_data.pc = proto_context_create(i915, args->flags); 2324 if (IS_ERR(ext_data.pc)) 2325 return PTR_ERR(ext_data.pc); 2326 2327 if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { 2328 ret = i915_user_extensions(u64_to_user_ptr(args->extensions), 2329 create_extensions, 2330 ARRAY_SIZE(create_extensions), 2331 &ext_data); 2332 if (ret) 2333 goto err_pc; 2334 } 2335 2336 if (GRAPHICS_VER(i915) > 12) { 2337 struct i915_gem_context *ctx; 2338 2339 /* Get ourselves a context ID */ 2340 ret = xa_alloc(&ext_data.fpriv->context_xa, &id, NULL, 2341 xa_limit_32b, GFP_KERNEL); 2342 if (ret) 2343 goto err_pc; 2344 2345 ctx = i915_gem_create_context(i915, ext_data.pc); 2346 if (IS_ERR(ctx)) { 2347 ret = PTR_ERR(ctx); 2348 goto err_pc; 2349 } 2350 2351 proto_context_close(i915, ext_data.pc); 2352 gem_context_register(ctx, ext_data.fpriv, id); 2353 } else { 2354 ret = proto_context_register(ext_data.fpriv, ext_data.pc, &id); 2355 if (ret < 0) 2356 goto err_pc; 2357 } 2358 2359 args->ctx_id = id; 2360 2361 return 0; 2362 2363err_pc: 2364 proto_context_close(i915, ext_data.pc); 2365 return ret; 2366} 2367 2368int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 2369 struct drm_file *file) 2370{ 2371 struct drm_i915_gem_context_destroy *args = data; 2372 struct drm_i915_file_private *file_priv = file->driver_priv; 2373 struct i915_gem_proto_context *pc; 2374 struct i915_gem_context *ctx; 2375 2376 if (args->pad != 0) 2377 return -EINVAL; 2378 2379 if (!args->ctx_id) 2380 return -ENOENT; 2381 2382 /* We need to hold the proto-context lock here to prevent races 2383 * with finalize_create_context_locked(). 2384 */ 2385 mutex_lock(&file_priv->proto_context_lock); 2386 ctx = xa_erase(&file_priv->context_xa, args->ctx_id); 2387 pc = xa_erase(&file_priv->proto_context_xa, args->ctx_id); 2388 mutex_unlock(&file_priv->proto_context_lock); 2389 2390 if (!ctx && !pc) 2391 return -ENOENT; 2392 GEM_WARN_ON(ctx && pc); 2393 2394 if (pc) 2395 proto_context_close(file_priv->i915, pc); 2396 2397 if (ctx) 2398 context_close(ctx); 2399 2400 return 0; 2401} 2402 2403static int get_sseu(struct i915_gem_context *ctx, 2404 struct drm_i915_gem_context_param *args) 2405{ 2406 struct drm_i915_gem_context_param_sseu user_sseu; 2407 struct intel_context *ce; 2408 unsigned long lookup; 2409 int err; 2410 2411 if (args->size == 0) 2412 goto out; 2413 else if (args->size < sizeof(user_sseu)) 2414 return -EINVAL; 2415 2416 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 2417 sizeof(user_sseu))) 2418 return -EFAULT; 2419 2420 if (user_sseu.rsvd) 2421 return -EINVAL; 2422 2423 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 2424 return -EINVAL; 2425 2426 lookup = 0; 2427 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 2428 lookup |= LOOKUP_USER_INDEX; 2429 2430 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 2431 if (IS_ERR(ce)) 2432 return PTR_ERR(ce); 2433 2434 err = intel_context_lock_pinned(ce); /* serialises with set_sseu */ 2435 if (err) { 2436 intel_context_put(ce); 2437 return err; 2438 } 2439 2440 user_sseu.slice_mask = ce->sseu.slice_mask; 2441 user_sseu.subslice_mask = ce->sseu.subslice_mask; 2442 user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; 2443 user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; 2444 2445 intel_context_unlock_pinned(ce); 2446 intel_context_put(ce); 2447 2448 if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, 2449 sizeof(user_sseu))) 2450 return -EFAULT; 2451 2452out: 2453 args->size = sizeof(user_sseu); 2454 2455 return 0; 2456} 2457 2458int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 2459 struct drm_file *file) 2460{ 2461 struct drm_i915_file_private *file_priv = file->driver_priv; 2462 struct drm_i915_gem_context_param *args = data; 2463 struct i915_gem_context *ctx; 2464 struct i915_address_space *vm; 2465 int ret = 0; 2466 2467 ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 2468 if (IS_ERR(ctx)) 2469 return PTR_ERR(ctx); 2470 2471 switch (args->param) { 2472 case I915_CONTEXT_PARAM_GTT_SIZE: 2473 args->size = 0; 2474 vm = i915_gem_context_get_eb_vm(ctx); 2475 args->value = vm->total; 2476 i915_vm_put(vm); 2477 2478 break; 2479 2480 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 2481 args->size = 0; 2482 args->value = i915_gem_context_no_error_capture(ctx); 2483 break; 2484 2485 case I915_CONTEXT_PARAM_BANNABLE: 2486 args->size = 0; 2487 args->value = i915_gem_context_is_bannable(ctx); 2488 break; 2489 2490 case I915_CONTEXT_PARAM_RECOVERABLE: 2491 args->size = 0; 2492 args->value = i915_gem_context_is_recoverable(ctx); 2493 break; 2494 2495 case I915_CONTEXT_PARAM_PRIORITY: 2496 args->size = 0; 2497 args->value = ctx->sched.priority; 2498 break; 2499 2500 case I915_CONTEXT_PARAM_SSEU: 2501 ret = get_sseu(ctx, args); 2502 break; 2503 2504 case I915_CONTEXT_PARAM_VM: 2505 ret = get_ppgtt(file_priv, ctx, args); 2506 break; 2507 2508 case I915_CONTEXT_PARAM_PERSISTENCE: 2509 args->size = 0; 2510 args->value = i915_gem_context_is_persistent(ctx); 2511 break; 2512 2513 case I915_CONTEXT_PARAM_PROTECTED_CONTENT: 2514 ret = get_protected(ctx, args); 2515 break; 2516 2517 case I915_CONTEXT_PARAM_NO_ZEROMAP: 2518 case I915_CONTEXT_PARAM_BAN_PERIOD: 2519 case I915_CONTEXT_PARAM_ENGINES: 2520 case I915_CONTEXT_PARAM_RINGSIZE: 2521 default: 2522 ret = -EINVAL; 2523 break; 2524 } 2525 2526 i915_gem_context_put(ctx); 2527 return ret; 2528} 2529 2530int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 2531 struct drm_file *file) 2532{ 2533 struct drm_i915_file_private *file_priv = file->driver_priv; 2534 struct drm_i915_gem_context_param *args = data; 2535 struct i915_gem_proto_context *pc; 2536 struct i915_gem_context *ctx; 2537 int ret = 0; 2538 2539 mutex_lock(&file_priv->proto_context_lock); 2540 ctx = __context_lookup(file_priv, args->ctx_id); 2541 if (!ctx) { 2542 pc = xa_load(&file_priv->proto_context_xa, args->ctx_id); 2543 if (pc) { 2544 /* Contexts should be finalized inside 2545 * GEM_CONTEXT_CREATE starting with graphics 2546 * version 13. 2547 */ 2548 WARN_ON(GRAPHICS_VER(file_priv->i915) > 12); 2549 ret = set_proto_ctx_param(file_priv, pc, args); 2550 } else { 2551 ret = -ENOENT; 2552 } 2553 } 2554 mutex_unlock(&file_priv->proto_context_lock); 2555 2556 if (ctx) { 2557 ret = ctx_setparam(file_priv, ctx, args); 2558 i915_gem_context_put(ctx); 2559 } 2560 2561 return ret; 2562} 2563 2564int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 2565 void *data, struct drm_file *file) 2566{ 2567 struct drm_i915_private *i915 = to_i915(dev); 2568 struct drm_i915_reset_stats *args = data; 2569 struct i915_gem_context *ctx; 2570 2571 if (args->flags || args->pad) 2572 return -EINVAL; 2573 2574 ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id); 2575 if (IS_ERR(ctx)) 2576 return PTR_ERR(ctx); 2577 2578 /* 2579 * We opt for unserialised reads here. This may result in tearing 2580 * in the extremely unlikely event of a GPU hang on this context 2581 * as we are querying them. If we need that extra layer of protection, 2582 * we should wrap the hangstats with a seqlock. 2583 */ 2584 2585 if (capable(CAP_SYS_ADMIN)) 2586 args->reset_count = i915_reset_count(&i915->gpu_error); 2587 else 2588 args->reset_count = 0; 2589 2590 args->batch_active = atomic_read(&ctx->guilty_count); 2591 args->batch_pending = atomic_read(&ctx->active_count); 2592 2593 i915_gem_context_put(ctx); 2594 return 0; 2595} 2596 2597/* GEM context-engines iterator: for_each_gem_engine() */ 2598struct intel_context * 2599i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) 2600{ 2601 const struct i915_gem_engines *e = it->engines; 2602 struct intel_context *ctx; 2603 2604 if (unlikely(!e)) 2605 return NULL; 2606 2607 do { 2608 if (it->idx >= e->num_engines) 2609 return NULL; 2610 2611 ctx = e->engines[it->idx++]; 2612 } while (!ctx); 2613 2614 return ctx; 2615} 2616 2617#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2618#include "selftests/mock_context.c" 2619#include "selftests/i915_gem_context.c" 2620#endif 2621 2622void i915_gem_context_module_exit(void) 2623{ 2624#ifdef __linux__ 2625 kmem_cache_destroy(slab_luts); 2626#else 2627 pool_destroy(&slab_luts); 2628#endif 2629} 2630 2631int __init i915_gem_context_module_init(void) 2632{ 2633#ifdef __linux__ 2634 slab_luts = KMEM_CACHE(i915_lut_handle, 0); 2635 if (!slab_luts) 2636 return -ENOMEM; 2637#else 2638 pool_init(&slab_luts , sizeof(struct i915_lut_handle), 2639 0, IPL_NONE, 0, "drmlut", NULL); 2640#endif 2641 2642 return 0; 2643} 2644