1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2018 Intel Corporation 4 */ 5 6#include "gem/i915_gem_internal.h" 7#include "gem/i915_gem_pm.h" 8#include "gt/intel_engine_user.h" 9#include "gt/intel_gt.h" 10#include "i915_selftest.h" 11#include "intel_reset.h" 12 13#include "selftests/igt_flush_test.h" 14#include "selftests/igt_reset.h" 15#include "selftests/igt_spinner.h" 16#include "selftests/intel_scheduler_helpers.h" 17#include "selftests/mock_drm.h" 18 19#include "gem/selftests/igt_gem_utils.h" 20#include "gem/selftests/mock_context.h" 21 22static const struct wo_register { 23 enum intel_platform platform; 24 u32 reg; 25} wo_registers[] = { 26 { INTEL_GEMINILAKE, 0x731c } 27}; 28 29struct wa_lists { 30 struct i915_wa_list gt_wa_list; 31 struct { 32 struct i915_wa_list wa_list; 33 struct i915_wa_list ctx_wa_list; 34 } engine[I915_NUM_ENGINES]; 35}; 36 37static int request_add_sync(struct i915_request *rq, int err) 38{ 39 i915_request_get(rq); 40 i915_request_add(rq); 41 if (i915_request_wait(rq, 0, HZ / 5) < 0) 42 err = -EIO; 43 i915_request_put(rq); 44 45 return err; 46} 47 48static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 49{ 50 int err = 0; 51 52 i915_request_get(rq); 53 i915_request_add(rq); 54 if (spin && !igt_wait_for_spinner(spin, rq)) 55 err = -ETIMEDOUT; 56 i915_request_put(rq); 57 58 return err; 59} 60 61static void 62reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 63{ 64 struct intel_engine_cs *engine; 65 enum intel_engine_id id; 66 67 memset(lists, 0, sizeof(*lists)); 68 69 wa_init_start(&lists->gt_wa_list, gt, "GT_REF", "global"); 70 gt_init_workarounds(gt, &lists->gt_wa_list); 71 wa_init_finish(&lists->gt_wa_list); 72 73 for_each_engine(engine, gt, id) { 74 struct i915_wa_list *wal = &lists->engine[id].wa_list; 75 76 wa_init_start(wal, gt, "REF", engine->name); 77 engine_init_workarounds(engine, wal); 78 wa_init_finish(wal); 79 80 __intel_engine_init_ctx_wa(engine, 81 &lists->engine[id].ctx_wa_list, 82 "CTX_REF"); 83 } 84} 85 86static void 87reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 88{ 89 struct intel_engine_cs *engine; 90 enum intel_engine_id id; 91 92 for_each_engine(engine, gt, id) 93 intel_wa_list_free(&lists->engine[id].wa_list); 94 95 intel_wa_list_free(&lists->gt_wa_list); 96} 97 98static struct drm_i915_gem_object * 99read_nonprivs(struct intel_context *ce) 100{ 101 struct intel_engine_cs *engine = ce->engine; 102 const u32 base = engine->mmio_base; 103 struct drm_i915_gem_object *result; 104 struct i915_request *rq; 105 struct i915_vma *vma; 106 u32 srm, *cs; 107 int err; 108 int i; 109 110 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 111 if (IS_ERR(result)) 112 return result; 113 114 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 115 116 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB); 117 if (IS_ERR(cs)) { 118 err = PTR_ERR(cs); 119 goto err_obj; 120 } 121 memset(cs, 0xc5, PAGE_SIZE); 122 i915_gem_object_flush_map(result); 123 i915_gem_object_unpin_map(result); 124 125 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 126 if (IS_ERR(vma)) { 127 err = PTR_ERR(vma); 128 goto err_obj; 129 } 130 131 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 132 if (err) 133 goto err_obj; 134 135 rq = intel_context_create_request(ce); 136 if (IS_ERR(rq)) { 137 err = PTR_ERR(rq); 138 goto err_pin; 139 } 140 141 err = igt_vma_move_to_active_unlocked(vma, rq, EXEC_OBJECT_WRITE); 142 if (err) 143 goto err_req; 144 145 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 146 if (GRAPHICS_VER(engine->i915) >= 8) 147 srm++; 148 149 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 150 if (IS_ERR(cs)) { 151 err = PTR_ERR(cs); 152 goto err_req; 153 } 154 155 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 156 *cs++ = srm; 157 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 158 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 159 *cs++ = 0; 160 } 161 intel_ring_advance(rq, cs); 162 163 i915_request_add(rq); 164 i915_vma_unpin(vma); 165 166 return result; 167 168err_req: 169 i915_request_add(rq); 170err_pin: 171 i915_vma_unpin(vma); 172err_obj: 173 i915_gem_object_put(result); 174 return ERR_PTR(err); 175} 176 177static u32 178get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 179{ 180 i915_reg_t reg = i < engine->whitelist.count ? 181 engine->whitelist.list[i].reg : 182 RING_NOPID(engine->mmio_base); 183 184 return i915_mmio_reg_offset(reg); 185} 186 187static void 188print_results(const struct intel_engine_cs *engine, const u32 *results) 189{ 190 unsigned int i; 191 192 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 193 u32 expected = get_whitelist_reg(engine, i); 194 u32 actual = results[i]; 195 196 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 197 i, expected, actual); 198 } 199} 200 201static int check_whitelist(struct intel_context *ce) 202{ 203 struct intel_engine_cs *engine = ce->engine; 204 struct drm_i915_gem_object *results; 205 struct intel_wedge_me wedge; 206 u32 *vaddr; 207 int err; 208 int i; 209 210 results = read_nonprivs(ce); 211 if (IS_ERR(results)) 212 return PTR_ERR(results); 213 214 err = 0; 215 i915_gem_object_lock(results, NULL); 216 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 217 err = i915_gem_object_set_to_cpu_domain(results, false); 218 219 if (intel_gt_is_wedged(engine->gt)) 220 err = -EIO; 221 if (err) 222 goto out_put; 223 224 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 225 if (IS_ERR(vaddr)) { 226 err = PTR_ERR(vaddr); 227 goto out_put; 228 } 229 230 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 231 u32 expected = get_whitelist_reg(engine, i); 232 u32 actual = vaddr[i]; 233 234 if (expected != actual) { 235 print_results(engine, vaddr); 236 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 237 i, expected, actual); 238 239 err = -EINVAL; 240 break; 241 } 242 } 243 244 i915_gem_object_unpin_map(results); 245out_put: 246 i915_gem_object_unlock(results); 247 i915_gem_object_put(results); 248 return err; 249} 250 251static int do_device_reset(struct intel_engine_cs *engine) 252{ 253 intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 254 return 0; 255} 256 257static int do_engine_reset(struct intel_engine_cs *engine) 258{ 259 return intel_engine_reset(engine, "live_workarounds"); 260} 261 262static int do_guc_reset(struct intel_engine_cs *engine) 263{ 264 /* Currently a no-op as the reset is handled by GuC */ 265 return 0; 266} 267 268static int 269switch_to_scratch_context(struct intel_engine_cs *engine, 270 struct igt_spinner *spin, 271 struct i915_request **rq) 272{ 273 struct intel_context *ce; 274 int err = 0; 275 276 ce = intel_context_create(engine); 277 if (IS_ERR(ce)) 278 return PTR_ERR(ce); 279 280 *rq = igt_spinner_create_request(spin, ce, MI_NOOP); 281 intel_context_put(ce); 282 283 if (IS_ERR(*rq)) { 284 spin = NULL; 285 err = PTR_ERR(*rq); 286 goto err; 287 } 288 289 err = request_add_spin(*rq, spin); 290err: 291 if (err && spin) 292 igt_spinner_end(spin); 293 294 return err; 295} 296 297static int check_whitelist_across_reset(struct intel_engine_cs *engine, 298 int (*reset)(struct intel_engine_cs *), 299 const char *name) 300{ 301 struct intel_context *ce, *tmp; 302 struct igt_spinner spin; 303 struct i915_request *rq; 304 intel_wakeref_t wakeref; 305 int err; 306 307 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 308 engine->whitelist.count, engine->name, name); 309 310 ce = intel_context_create(engine); 311 if (IS_ERR(ce)) 312 return PTR_ERR(ce); 313 314 err = igt_spinner_init(&spin, engine->gt); 315 if (err) 316 goto out_ctx; 317 318 err = check_whitelist(ce); 319 if (err) { 320 pr_err("Invalid whitelist *before* %s reset!\n", name); 321 goto out_spin; 322 } 323 324 err = switch_to_scratch_context(engine, &spin, &rq); 325 if (err) 326 goto out_spin; 327 328 /* Ensure the spinner hasn't aborted */ 329 if (i915_request_completed(rq)) { 330 pr_err("%s spinner failed to start\n", name); 331 err = -ETIMEDOUT; 332 goto out_spin; 333 } 334 335 with_intel_runtime_pm(engine->uncore->rpm, wakeref) 336 err = reset(engine); 337 338 /* Ensure the reset happens and kills the engine */ 339 if (err == 0) 340 err = intel_selftest_wait_for_rq(rq); 341 342 igt_spinner_end(&spin); 343 344 if (err) { 345 pr_err("%s reset failed\n", name); 346 goto out_spin; 347 } 348 349 err = check_whitelist(ce); 350 if (err) { 351 pr_err("Whitelist not preserved in context across %s reset!\n", 352 name); 353 goto out_spin; 354 } 355 356 tmp = intel_context_create(engine); 357 if (IS_ERR(tmp)) { 358 err = PTR_ERR(tmp); 359 goto out_spin; 360 } 361 intel_context_put(ce); 362 ce = tmp; 363 364 err = check_whitelist(ce); 365 if (err) { 366 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 367 name); 368 goto out_spin; 369 } 370 371out_spin: 372 igt_spinner_fini(&spin); 373out_ctx: 374 intel_context_put(ce); 375 return err; 376} 377 378static struct i915_vma *create_batch(struct i915_address_space *vm) 379{ 380 struct drm_i915_gem_object *obj; 381 struct i915_vma *vma; 382 int err; 383 384 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 385 if (IS_ERR(obj)) 386 return ERR_CAST(obj); 387 388 vma = i915_vma_instance(obj, vm, NULL); 389 if (IS_ERR(vma)) { 390 err = PTR_ERR(vma); 391 goto err_obj; 392 } 393 394 err = i915_vma_pin(vma, 0, 0, PIN_USER); 395 if (err) 396 goto err_obj; 397 398 return vma; 399 400err_obj: 401 i915_gem_object_put(obj); 402 return ERR_PTR(err); 403} 404 405static u32 reg_write(u32 old, u32 new, u32 rsvd) 406{ 407 if (rsvd == 0x0000ffff) { 408 old &= ~(new >> 16); 409 old |= new & (new >> 16); 410 } else { 411 old &= ~rsvd; 412 old |= new & rsvd; 413 } 414 415 return old; 416} 417 418static bool wo_register(struct intel_engine_cs *engine, u32 reg) 419{ 420 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 421 int i; 422 423 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 424 RING_FORCE_TO_NONPRIV_ACCESS_WR) 425 return true; 426 427 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 428 if (wo_registers[i].platform == platform && 429 wo_registers[i].reg == reg) 430 return true; 431 } 432 433 return false; 434} 435 436static bool timestamp(const struct intel_engine_cs *engine, u32 reg) 437{ 438 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK; 439 switch (reg) { 440 case 0x358: 441 case 0x35c: 442 case 0x3a8: 443 return true; 444 445 default: 446 return false; 447 } 448} 449 450static bool ro_register(u32 reg) 451{ 452 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 453 RING_FORCE_TO_NONPRIV_ACCESS_RD) 454 return true; 455 456 return false; 457} 458 459static int whitelist_writable_count(struct intel_engine_cs *engine) 460{ 461 int count = engine->whitelist.count; 462 int i; 463 464 for (i = 0; i < engine->whitelist.count; i++) { 465 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 466 467 if (ro_register(reg)) 468 count--; 469 } 470 471 return count; 472} 473 474static int check_dirty_whitelist(struct intel_context *ce) 475{ 476 const u32 values[] = { 477 0x00000000, 478 0x01010101, 479 0x10100101, 480 0x03030303, 481 0x30300303, 482 0x05050505, 483 0x50500505, 484 0x0f0f0f0f, 485 0xf00ff00f, 486 0x10101010, 487 0xf0f01010, 488 0x30303030, 489 0xa0a03030, 490 0x50505050, 491 0xc0c05050, 492 0xf0f0f0f0, 493 0x11111111, 494 0x33333333, 495 0x55555555, 496 0x0000ffff, 497 0x00ff00ff, 498 0xff0000ff, 499 0xffff00ff, 500 0xffffffff, 501 }; 502 struct intel_engine_cs *engine = ce->engine; 503 struct i915_vma *scratch; 504 struct i915_vma *batch; 505 int err = 0, i, v, sz; 506 u32 *cs, *results; 507 508 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32); 509 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz); 510 if (IS_ERR(scratch)) 511 return PTR_ERR(scratch); 512 513 batch = create_batch(ce->vm); 514 if (IS_ERR(batch)) { 515 err = PTR_ERR(batch); 516 goto out_scratch; 517 } 518 519 for (i = 0; i < engine->whitelist.count; i++) { 520 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 521 struct i915_gem_ww_ctx ww; 522 u64 addr = i915_vma_offset(scratch); 523 struct i915_request *rq; 524 u32 srm, lrm, rsvd; 525 u32 expect; 526 int idx; 527 bool ro_reg; 528 529 if (wo_register(engine, reg)) 530 continue; 531 532 if (timestamp(engine, reg)) 533 continue; /* timestamps are expected to autoincrement */ 534 535 ro_reg = ro_register(reg); 536 537 i915_gem_ww_ctx_init(&ww, false); 538retry: 539 cs = NULL; 540 err = i915_gem_object_lock(scratch->obj, &ww); 541 if (!err) 542 err = i915_gem_object_lock(batch->obj, &ww); 543 if (!err) 544 err = intel_context_pin_ww(ce, &ww); 545 if (err) 546 goto out; 547 548 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 549 if (IS_ERR(cs)) { 550 err = PTR_ERR(cs); 551 goto out_ctx; 552 } 553 554 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 555 if (IS_ERR(results)) { 556 err = PTR_ERR(results); 557 goto out_unmap_batch; 558 } 559 560 /* Clear non priv flags */ 561 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 562 563 srm = MI_STORE_REGISTER_MEM; 564 lrm = MI_LOAD_REGISTER_MEM; 565 if (GRAPHICS_VER(engine->i915) >= 8) 566 lrm++, srm++; 567 568 pr_debug("%s: Writing garbage to %x\n", 569 engine->name, reg); 570 571 /* SRM original */ 572 *cs++ = srm; 573 *cs++ = reg; 574 *cs++ = lower_32_bits(addr); 575 *cs++ = upper_32_bits(addr); 576 577 idx = 1; 578 for (v = 0; v < ARRAY_SIZE(values); v++) { 579 /* LRI garbage */ 580 *cs++ = MI_LOAD_REGISTER_IMM(1); 581 *cs++ = reg; 582 *cs++ = values[v]; 583 584 /* SRM result */ 585 *cs++ = srm; 586 *cs++ = reg; 587 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 588 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 589 idx++; 590 } 591 for (v = 0; v < ARRAY_SIZE(values); v++) { 592 /* LRI garbage */ 593 *cs++ = MI_LOAD_REGISTER_IMM(1); 594 *cs++ = reg; 595 *cs++ = ~values[v]; 596 597 /* SRM result */ 598 *cs++ = srm; 599 *cs++ = reg; 600 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 601 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 602 idx++; 603 } 604 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 605 606 /* LRM original -- don't leave garbage in the context! */ 607 *cs++ = lrm; 608 *cs++ = reg; 609 *cs++ = lower_32_bits(addr); 610 *cs++ = upper_32_bits(addr); 611 612 *cs++ = MI_BATCH_BUFFER_END; 613 614 i915_gem_object_flush_map(batch->obj); 615 i915_gem_object_unpin_map(batch->obj); 616 intel_gt_chipset_flush(engine->gt); 617 cs = NULL; 618 619 rq = i915_request_create(ce); 620 if (IS_ERR(rq)) { 621 err = PTR_ERR(rq); 622 goto out_unmap_scratch; 623 } 624 625 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 626 err = engine->emit_init_breadcrumb(rq); 627 if (err) 628 goto err_request; 629 } 630 631 err = i915_vma_move_to_active(batch, rq, 0); 632 if (err) 633 goto err_request; 634 635 err = i915_vma_move_to_active(scratch, rq, 636 EXEC_OBJECT_WRITE); 637 if (err) 638 goto err_request; 639 640 err = engine->emit_bb_start(rq, 641 i915_vma_offset(batch), PAGE_SIZE, 642 0); 643 if (err) 644 goto err_request; 645 646err_request: 647 err = request_add_sync(rq, err); 648 if (err) { 649 pr_err("%s: Futzing %x timedout; cancelling test\n", 650 engine->name, reg); 651 intel_gt_set_wedged(engine->gt); 652 goto out_unmap_scratch; 653 } 654 655 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 656 if (!ro_reg) { 657 /* detect write masking */ 658 rsvd = results[ARRAY_SIZE(values)]; 659 if (!rsvd) { 660 pr_err("%s: Unable to write to whitelisted register %x\n", 661 engine->name, reg); 662 err = -EINVAL; 663 goto out_unmap_scratch; 664 } 665 } else { 666 rsvd = 0; 667 } 668 669 expect = results[0]; 670 idx = 1; 671 for (v = 0; v < ARRAY_SIZE(values); v++) { 672 if (ro_reg) 673 expect = results[0]; 674 else 675 expect = reg_write(expect, values[v], rsvd); 676 677 if (results[idx] != expect) 678 err++; 679 idx++; 680 } 681 for (v = 0; v < ARRAY_SIZE(values); v++) { 682 if (ro_reg) 683 expect = results[0]; 684 else 685 expect = reg_write(expect, ~values[v], rsvd); 686 687 if (results[idx] != expect) 688 err++; 689 idx++; 690 } 691 if (err) { 692 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 693 engine->name, err, reg); 694 695 if (ro_reg) 696 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 697 engine->name, reg, results[0]); 698 else 699 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 700 engine->name, reg, results[0], rsvd); 701 702 expect = results[0]; 703 idx = 1; 704 for (v = 0; v < ARRAY_SIZE(values); v++) { 705 u32 w = values[v]; 706 707 if (ro_reg) 708 expect = results[0]; 709 else 710 expect = reg_write(expect, w, rsvd); 711 pr_info("Wrote %08x, read %08x, expect %08x\n", 712 w, results[idx], expect); 713 idx++; 714 } 715 for (v = 0; v < ARRAY_SIZE(values); v++) { 716 u32 w = ~values[v]; 717 718 if (ro_reg) 719 expect = results[0]; 720 else 721 expect = reg_write(expect, w, rsvd); 722 pr_info("Wrote %08x, read %08x, expect %08x\n", 723 w, results[idx], expect); 724 idx++; 725 } 726 727 err = -EINVAL; 728 } 729out_unmap_scratch: 730 i915_gem_object_unpin_map(scratch->obj); 731out_unmap_batch: 732 if (cs) 733 i915_gem_object_unpin_map(batch->obj); 734out_ctx: 735 intel_context_unpin(ce); 736out: 737 if (err == -EDEADLK) { 738 err = i915_gem_ww_ctx_backoff(&ww); 739 if (!err) 740 goto retry; 741 } 742 i915_gem_ww_ctx_fini(&ww); 743 if (err) 744 break; 745 } 746 747 if (igt_flush_test(engine->i915)) 748 err = -EIO; 749 750 i915_vma_unpin_and_release(&batch, 0); 751out_scratch: 752 i915_vma_unpin_and_release(&scratch, 0); 753 return err; 754} 755 756static int live_dirty_whitelist(void *arg) 757{ 758 struct intel_gt *gt = arg; 759 struct intel_engine_cs *engine; 760 enum intel_engine_id id; 761 762 /* Can the user write to the whitelisted registers? */ 763 764 if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 765 return 0; 766 767 for_each_engine(engine, gt, id) { 768 struct intel_context *ce; 769 int err; 770 771 if (engine->whitelist.count == 0) 772 continue; 773 774 ce = intel_context_create(engine); 775 if (IS_ERR(ce)) 776 return PTR_ERR(ce); 777 778 err = check_dirty_whitelist(ce); 779 intel_context_put(ce); 780 if (err) 781 return err; 782 } 783 784 return 0; 785} 786 787static int live_reset_whitelist(void *arg) 788{ 789 struct intel_gt *gt = arg; 790 struct intel_engine_cs *engine; 791 enum intel_engine_id id; 792 int err = 0; 793 794 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 795 igt_global_reset_lock(gt); 796 797 for_each_engine(engine, gt, id) { 798 if (engine->whitelist.count == 0) 799 continue; 800 801 if (intel_has_reset_engine(gt)) { 802 if (intel_engine_uses_guc(engine)) { 803 struct intel_selftest_saved_policy saved; 804 int err2; 805 806 err = intel_selftest_modify_policy(engine, &saved, 807 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 808 if (err) 809 goto out; 810 811 err = check_whitelist_across_reset(engine, 812 do_guc_reset, 813 "guc"); 814 815 err2 = intel_selftest_restore_policy(engine, &saved); 816 if (err == 0) 817 err = err2; 818 } else { 819 err = check_whitelist_across_reset(engine, 820 do_engine_reset, 821 "engine"); 822 } 823 824 if (err) 825 goto out; 826 } 827 828 if (intel_has_gpu_reset(gt)) { 829 err = check_whitelist_across_reset(engine, 830 do_device_reset, 831 "device"); 832 if (err) 833 goto out; 834 } 835 } 836 837out: 838 igt_global_reset_unlock(gt); 839 return err; 840} 841 842static int read_whitelisted_registers(struct intel_context *ce, 843 struct i915_vma *results) 844{ 845 struct intel_engine_cs *engine = ce->engine; 846 struct i915_request *rq; 847 int i, err = 0; 848 u32 srm, *cs; 849 850 rq = intel_context_create_request(ce); 851 if (IS_ERR(rq)) 852 return PTR_ERR(rq); 853 854 err = igt_vma_move_to_active_unlocked(results, rq, EXEC_OBJECT_WRITE); 855 if (err) 856 goto err_req; 857 858 srm = MI_STORE_REGISTER_MEM; 859 if (GRAPHICS_VER(engine->i915) >= 8) 860 srm++; 861 862 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 863 if (IS_ERR(cs)) { 864 err = PTR_ERR(cs); 865 goto err_req; 866 } 867 868 for (i = 0; i < engine->whitelist.count; i++) { 869 u64 offset = i915_vma_offset(results) + sizeof(u32) * i; 870 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 871 872 /* Clear non priv flags */ 873 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 874 875 *cs++ = srm; 876 *cs++ = reg; 877 *cs++ = lower_32_bits(offset); 878 *cs++ = upper_32_bits(offset); 879 } 880 intel_ring_advance(rq, cs); 881 882err_req: 883 return request_add_sync(rq, err); 884} 885 886static int scrub_whitelisted_registers(struct intel_context *ce) 887{ 888 struct intel_engine_cs *engine = ce->engine; 889 struct i915_request *rq; 890 struct i915_vma *batch; 891 int i, err = 0; 892 u32 *cs; 893 894 batch = create_batch(ce->vm); 895 if (IS_ERR(batch)) 896 return PTR_ERR(batch); 897 898 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 899 if (IS_ERR(cs)) { 900 err = PTR_ERR(cs); 901 goto err_batch; 902 } 903 904 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 905 for (i = 0; i < engine->whitelist.count; i++) { 906 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 907 908 if (ro_register(reg)) 909 continue; 910 911 /* Clear non priv flags */ 912 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 913 914 *cs++ = reg; 915 *cs++ = 0xffffffff; 916 } 917 *cs++ = MI_BATCH_BUFFER_END; 918 919 i915_gem_object_flush_map(batch->obj); 920 intel_gt_chipset_flush(engine->gt); 921 922 rq = intel_context_create_request(ce); 923 if (IS_ERR(rq)) { 924 err = PTR_ERR(rq); 925 goto err_unpin; 926 } 927 928 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 929 err = engine->emit_init_breadcrumb(rq); 930 if (err) 931 goto err_request; 932 } 933 934 err = igt_vma_move_to_active_unlocked(batch, rq, 0); 935 if (err) 936 goto err_request; 937 938 /* Perform the writes from an unprivileged "user" batch */ 939 err = engine->emit_bb_start(rq, i915_vma_offset(batch), 0, 0); 940 941err_request: 942 err = request_add_sync(rq, err); 943 944err_unpin: 945 i915_gem_object_unpin_map(batch->obj); 946err_batch: 947 i915_vma_unpin_and_release(&batch, 0); 948 return err; 949} 950 951struct regmask { 952 i915_reg_t reg; 953 u8 graphics_ver; 954}; 955 956static bool find_reg(struct drm_i915_private *i915, 957 i915_reg_t reg, 958 const struct regmask *tbl, 959 unsigned long count) 960{ 961 u32 offset = i915_mmio_reg_offset(reg); 962 963 while (count--) { 964 if (GRAPHICS_VER(i915) == tbl->graphics_ver && 965 i915_mmio_reg_offset(tbl->reg) == offset) 966 return true; 967 tbl++; 968 } 969 970 return false; 971} 972 973static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 974{ 975 /* Alas, we must pardon some whitelists. Mistakes already made */ 976 static const struct regmask pardon[] = { 977 { GEN9_CTX_PREEMPT_REG, 9 }, 978 { _MMIO(0xb118), 9 }, /* GEN8_L3SQCREG4 */ 979 }; 980 981 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 982} 983 984static bool result_eq(struct intel_engine_cs *engine, 985 u32 a, u32 b, i915_reg_t reg) 986{ 987 if (a != b && !pardon_reg(engine->i915, reg)) { 988 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 989 i915_mmio_reg_offset(reg), a, b); 990 return false; 991 } 992 993 return true; 994} 995 996static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 997{ 998 /* Some registers do not seem to behave and our writes unreadable */ 999 static const struct regmask wo[] = { 1000 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 }, 1001 }; 1002 1003 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 1004} 1005 1006static bool result_neq(struct intel_engine_cs *engine, 1007 u32 a, u32 b, i915_reg_t reg) 1008{ 1009 if (a == b && !writeonly_reg(engine->i915, reg)) { 1010 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 1011 i915_mmio_reg_offset(reg), a); 1012 return false; 1013 } 1014 1015 return true; 1016} 1017 1018static int 1019check_whitelisted_registers(struct intel_engine_cs *engine, 1020 struct i915_vma *A, 1021 struct i915_vma *B, 1022 bool (*fn)(struct intel_engine_cs *engine, 1023 u32 a, u32 b, 1024 i915_reg_t reg)) 1025{ 1026 u32 *a, *b; 1027 int i, err; 1028 1029 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB); 1030 if (IS_ERR(a)) 1031 return PTR_ERR(a); 1032 1033 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB); 1034 if (IS_ERR(b)) { 1035 err = PTR_ERR(b); 1036 goto err_a; 1037 } 1038 1039 err = 0; 1040 for (i = 0; i < engine->whitelist.count; i++) { 1041 const struct i915_wa *wa = &engine->whitelist.list[i]; 1042 1043 if (i915_mmio_reg_offset(wa->reg) & 1044 RING_FORCE_TO_NONPRIV_ACCESS_RD) 1045 continue; 1046 1047 if (!fn(engine, a[i], b[i], wa->reg)) 1048 err = -EINVAL; 1049 } 1050 1051 i915_gem_object_unpin_map(B->obj); 1052err_a: 1053 i915_gem_object_unpin_map(A->obj); 1054 return err; 1055} 1056 1057static int live_isolated_whitelist(void *arg) 1058{ 1059 struct intel_gt *gt = arg; 1060 struct { 1061 struct i915_vma *scratch[2]; 1062 } client[2] = {}; 1063 struct intel_engine_cs *engine; 1064 enum intel_engine_id id; 1065 int i, err = 0; 1066 1067 /* 1068 * Check that a write into a whitelist register works, but 1069 * invisible to a second context. 1070 */ 1071 1072 if (!intel_engines_has_context_isolation(gt->i915)) 1073 return 0; 1074 1075 for (i = 0; i < ARRAY_SIZE(client); i++) { 1076 client[i].scratch[0] = 1077 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1078 if (IS_ERR(client[i].scratch[0])) { 1079 err = PTR_ERR(client[i].scratch[0]); 1080 goto err; 1081 } 1082 1083 client[i].scratch[1] = 1084 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1085 if (IS_ERR(client[i].scratch[1])) { 1086 err = PTR_ERR(client[i].scratch[1]); 1087 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1088 goto err; 1089 } 1090 } 1091 1092 for_each_engine(engine, gt, id) { 1093 struct intel_context *ce[2]; 1094 1095 if (!engine->kernel_context->vm) 1096 continue; 1097 1098 if (!whitelist_writable_count(engine)) 1099 continue; 1100 1101 ce[0] = intel_context_create(engine); 1102 if (IS_ERR(ce[0])) { 1103 err = PTR_ERR(ce[0]); 1104 break; 1105 } 1106 ce[1] = intel_context_create(engine); 1107 if (IS_ERR(ce[1])) { 1108 err = PTR_ERR(ce[1]); 1109 intel_context_put(ce[0]); 1110 break; 1111 } 1112 1113 /* Read default values */ 1114 err = read_whitelisted_registers(ce[0], client[0].scratch[0]); 1115 if (err) 1116 goto err_ce; 1117 1118 /* Try to overwrite registers (should only affect ctx0) */ 1119 err = scrub_whitelisted_registers(ce[0]); 1120 if (err) 1121 goto err_ce; 1122 1123 /* Read values from ctx1, we expect these to be defaults */ 1124 err = read_whitelisted_registers(ce[1], client[1].scratch[0]); 1125 if (err) 1126 goto err_ce; 1127 1128 /* Verify that both reads return the same default values */ 1129 err = check_whitelisted_registers(engine, 1130 client[0].scratch[0], 1131 client[1].scratch[0], 1132 result_eq); 1133 if (err) 1134 goto err_ce; 1135 1136 /* Read back the updated values in ctx0 */ 1137 err = read_whitelisted_registers(ce[0], client[0].scratch[1]); 1138 if (err) 1139 goto err_ce; 1140 1141 /* User should be granted privilege to overwhite regs */ 1142 err = check_whitelisted_registers(engine, 1143 client[0].scratch[0], 1144 client[0].scratch[1], 1145 result_neq); 1146err_ce: 1147 intel_context_put(ce[1]); 1148 intel_context_put(ce[0]); 1149 if (err) 1150 break; 1151 } 1152 1153err: 1154 for (i = 0; i < ARRAY_SIZE(client); i++) { 1155 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1156 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1157 } 1158 1159 if (igt_flush_test(gt->i915)) 1160 err = -EIO; 1161 1162 return err; 1163} 1164 1165static bool 1166verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, 1167 const char *str) 1168{ 1169 struct intel_engine_cs *engine; 1170 enum intel_engine_id id; 1171 bool ok = true; 1172 1173 ok &= wa_list_verify(gt, &lists->gt_wa_list, str); 1174 1175 for_each_engine(engine, gt, id) { 1176 struct intel_context *ce; 1177 1178 ce = intel_context_create(engine); 1179 if (IS_ERR(ce)) 1180 return false; 1181 1182 ok &= engine_wa_list_verify(ce, 1183 &lists->engine[id].wa_list, 1184 str) == 0; 1185 1186 ok &= engine_wa_list_verify(ce, 1187 &lists->engine[id].ctx_wa_list, 1188 str) == 0; 1189 1190 intel_context_put(ce); 1191 } 1192 1193 return ok; 1194} 1195 1196static int 1197live_gpu_reset_workarounds(void *arg) 1198{ 1199 struct intel_gt *gt = arg; 1200 intel_wakeref_t wakeref; 1201 struct wa_lists *lists; 1202 bool ok; 1203 1204 if (!intel_has_gpu_reset(gt)) 1205 return 0; 1206 1207 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1208 if (!lists) 1209 return -ENOMEM; 1210 1211 pr_info("Verifying after GPU reset...\n"); 1212 1213 igt_global_reset_lock(gt); 1214 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1215 1216 reference_lists_init(gt, lists); 1217 1218 ok = verify_wa_lists(gt, lists, "before reset"); 1219 if (!ok) 1220 goto out; 1221 1222 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1223 1224 ok = verify_wa_lists(gt, lists, "after reset"); 1225 1226out: 1227 reference_lists_fini(gt, lists); 1228 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1229 igt_global_reset_unlock(gt); 1230 kfree(lists); 1231 1232 return ok ? 0 : -ESRCH; 1233} 1234 1235static int 1236live_engine_reset_workarounds(void *arg) 1237{ 1238 struct intel_gt *gt = arg; 1239 struct intel_engine_cs *engine; 1240 enum intel_engine_id id; 1241 struct intel_context *ce; 1242 struct igt_spinner spin; 1243 struct i915_request *rq; 1244 intel_wakeref_t wakeref; 1245 struct wa_lists *lists; 1246 int ret = 0; 1247 1248 if (!intel_has_reset_engine(gt)) 1249 return 0; 1250 1251 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1252 if (!lists) 1253 return -ENOMEM; 1254 1255 igt_global_reset_lock(gt); 1256 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1257 1258 reference_lists_init(gt, lists); 1259 1260 for_each_engine(engine, gt, id) { 1261 struct intel_selftest_saved_policy saved; 1262 bool using_guc = intel_engine_uses_guc(engine); 1263 bool ok; 1264 int ret2; 1265 1266 pr_info("Verifying after %s reset...\n", engine->name); 1267 ret = intel_selftest_modify_policy(engine, &saved, 1268 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 1269 if (ret) 1270 break; 1271 1272 ce = intel_context_create(engine); 1273 if (IS_ERR(ce)) { 1274 ret = PTR_ERR(ce); 1275 goto restore; 1276 } 1277 1278 if (!using_guc) { 1279 ok = verify_wa_lists(gt, lists, "before reset"); 1280 if (!ok) { 1281 ret = -ESRCH; 1282 goto err; 1283 } 1284 1285 ret = intel_engine_reset(engine, "live_workarounds:idle"); 1286 if (ret) { 1287 pr_err("%s: Reset failed while idle\n", engine->name); 1288 goto err; 1289 } 1290 1291 ok = verify_wa_lists(gt, lists, "after idle reset"); 1292 if (!ok) { 1293 ret = -ESRCH; 1294 goto err; 1295 } 1296 } 1297 1298 ret = igt_spinner_init(&spin, engine->gt); 1299 if (ret) 1300 goto err; 1301 1302 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1303 if (IS_ERR(rq)) { 1304 ret = PTR_ERR(rq); 1305 igt_spinner_fini(&spin); 1306 goto err; 1307 } 1308 1309 ret = request_add_spin(rq, &spin); 1310 if (ret) { 1311 pr_err("%s: Spinner failed to start\n", engine->name); 1312 igt_spinner_fini(&spin); 1313 goto err; 1314 } 1315 1316 /* Ensure the spinner hasn't aborted */ 1317 if (i915_request_completed(rq)) { 1318 ret = -ETIMEDOUT; 1319 goto skip; 1320 } 1321 1322 if (!using_guc) { 1323 ret = intel_engine_reset(engine, "live_workarounds:active"); 1324 if (ret) { 1325 pr_err("%s: Reset failed on an active spinner\n", 1326 engine->name); 1327 igt_spinner_fini(&spin); 1328 goto err; 1329 } 1330 } 1331 1332 /* Ensure the reset happens and kills the engine */ 1333 if (ret == 0) 1334 ret = intel_selftest_wait_for_rq(rq); 1335 1336skip: 1337 igt_spinner_end(&spin); 1338 igt_spinner_fini(&spin); 1339 1340 ok = verify_wa_lists(gt, lists, "after busy reset"); 1341 if (!ok) 1342 ret = -ESRCH; 1343 1344err: 1345 intel_context_put(ce); 1346 1347restore: 1348 ret2 = intel_selftest_restore_policy(engine, &saved); 1349 if (ret == 0) 1350 ret = ret2; 1351 if (ret) 1352 break; 1353 } 1354 1355 reference_lists_fini(gt, lists); 1356 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1357 igt_global_reset_unlock(gt); 1358 kfree(lists); 1359 1360 igt_flush_test(gt->i915); 1361 1362 return ret; 1363} 1364 1365int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1366{ 1367 static const struct i915_subtest tests[] = { 1368 SUBTEST(live_dirty_whitelist), 1369 SUBTEST(live_reset_whitelist), 1370 SUBTEST(live_isolated_whitelist), 1371 SUBTEST(live_gpu_reset_workarounds), 1372 SUBTEST(live_engine_reset_workarounds), 1373 }; 1374 1375 if (intel_gt_is_wedged(to_gt(i915))) 1376 return 0; 1377 1378 return intel_gt_live_subtests(tests, to_gt(i915)); 1379} 1380