1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2020 Intel Corporation 4 */ 5 6#include <linux/sort.h> 7 8#include "gem/i915_gem_internal.h" 9#include "gem/i915_gem_lmem.h" 10 11#include "selftests/igt_spinner.h" 12#include "selftests/i915_random.h" 13 14static const unsigned int sizes[] = { 15 SZ_4K, 16 SZ_64K, 17 SZ_2M, 18 CHUNK_SZ - SZ_4K, 19 CHUNK_SZ, 20 CHUNK_SZ + SZ_4K, 21 SZ_64M, 22}; 23 24static struct drm_i915_gem_object * 25create_lmem_or_internal(struct drm_i915_private *i915, size_t size) 26{ 27 struct drm_i915_gem_object *obj; 28 29 obj = i915_gem_object_create_lmem(i915, size, 0); 30 if (!IS_ERR(obj)) 31 return obj; 32 33 return i915_gem_object_create_internal(i915, size); 34} 35 36static int copy(struct intel_migrate *migrate, 37 int (*fn)(struct intel_migrate *migrate, 38 struct i915_gem_ww_ctx *ww, 39 struct drm_i915_gem_object *src, 40 struct drm_i915_gem_object *dst, 41 struct i915_request **out), 42 u32 sz, struct rnd_state *prng) 43{ 44 struct drm_i915_private *i915 = migrate->context->engine->i915; 45 struct drm_i915_gem_object *src, *dst; 46 struct i915_request *rq; 47 struct i915_gem_ww_ctx ww; 48 u32 *vaddr; 49 int err = 0; 50 int i; 51 52 src = create_lmem_or_internal(i915, sz); 53 if (IS_ERR(src)) 54 return 0; 55 56 sz = src->base.size; 57 dst = i915_gem_object_create_internal(i915, sz); 58 if (IS_ERR(dst)) 59 goto err_free_src; 60 61 for_i915_gem_ww(&ww, err, true) { 62 err = i915_gem_object_lock(src, &ww); 63 if (err) 64 continue; 65 66 err = i915_gem_object_lock(dst, &ww); 67 if (err) 68 continue; 69 70 vaddr = i915_gem_object_pin_map(src, I915_MAP_WC); 71 if (IS_ERR(vaddr)) { 72 err = PTR_ERR(vaddr); 73 continue; 74 } 75 76 for (i = 0; i < sz / sizeof(u32); i++) 77 vaddr[i] = i; 78 i915_gem_object_flush_map(src); 79 80 vaddr = i915_gem_object_pin_map(dst, I915_MAP_WC); 81 if (IS_ERR(vaddr)) { 82 err = PTR_ERR(vaddr); 83 goto unpin_src; 84 } 85 86 for (i = 0; i < sz / sizeof(u32); i++) 87 vaddr[i] = ~i; 88 i915_gem_object_flush_map(dst); 89 90 err = fn(migrate, &ww, src, dst, &rq); 91 if (!err) 92 continue; 93 94 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS) 95 pr_err("%ps failed, size: %u\n", fn, sz); 96 if (rq) { 97 i915_request_wait(rq, 0, HZ); 98 i915_request_put(rq); 99 } 100 i915_gem_object_unpin_map(dst); 101unpin_src: 102 i915_gem_object_unpin_map(src); 103 } 104 if (err) 105 goto err_out; 106 107 if (rq) { 108 if (i915_request_wait(rq, 0, HZ) < 0) { 109 pr_err("%ps timed out, size: %u\n", fn, sz); 110 err = -ETIME; 111 } 112 i915_request_put(rq); 113 } 114 115 for (i = 0; !err && i < sz / PAGE_SIZE; i++) { 116 int x = i * 1024 + i915_prandom_u32_max_state(1024, prng); 117 118 if (vaddr[x] != x) { 119 pr_err("%ps failed, size: %u, offset: %zu\n", 120 fn, sz, x * sizeof(u32)); 121 igt_hexdump(vaddr + i * 1024, 4096); 122 err = -EINVAL; 123 } 124 } 125 126 i915_gem_object_unpin_map(dst); 127 i915_gem_object_unpin_map(src); 128 129err_out: 130 i915_gem_object_put(dst); 131err_free_src: 132 i915_gem_object_put(src); 133 134 return err; 135} 136 137static int intel_context_copy_ccs(struct intel_context *ce, 138 const struct i915_deps *deps, 139 struct scatterlist *sg, 140 unsigned int pat_index, 141 bool write_to_ccs, 142 struct i915_request **out) 143{ 144 u8 src_access = write_to_ccs ? DIRECT_ACCESS : INDIRECT_ACCESS; 145 u8 dst_access = write_to_ccs ? INDIRECT_ACCESS : DIRECT_ACCESS; 146 struct sgt_dma it = sg_sgt(sg); 147 struct i915_request *rq; 148 u32 offset; 149 int err; 150 151 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); 152 *out = NULL; 153 154 GEM_BUG_ON(ce->ring->size < SZ_64K); 155 156 offset = 0; 157 if (HAS_64K_PAGES(ce->engine->i915)) 158 offset = CHUNK_SZ; 159 160 do { 161 int len; 162 163 rq = i915_request_create(ce); 164 if (IS_ERR(rq)) { 165 err = PTR_ERR(rq); 166 goto out_ce; 167 } 168 169 if (deps) { 170 err = i915_request_await_deps(rq, deps); 171 if (err) 172 goto out_rq; 173 174 if (rq->engine->emit_init_breadcrumb) { 175 err = rq->engine->emit_init_breadcrumb(rq); 176 if (err) 177 goto out_rq; 178 } 179 180 deps = NULL; 181 } 182 183 /* The PTE updates + clear must not be interrupted. */ 184 err = emit_no_arbitration(rq); 185 if (err) 186 goto out_rq; 187 188 len = emit_pte(rq, &it, pat_index, true, offset, CHUNK_SZ); 189 if (len <= 0) { 190 err = len; 191 goto out_rq; 192 } 193 194 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 195 if (err) 196 goto out_rq; 197 198 err = emit_copy_ccs(rq, offset, dst_access, 199 offset, src_access, len); 200 if (err) 201 goto out_rq; 202 203 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); 204 205 /* Arbitration is re-enabled between requests. */ 206out_rq: 207 if (*out) 208 i915_request_put(*out); 209 *out = i915_request_get(rq); 210 i915_request_add(rq); 211 if (err || !it.sg || !sg_dma_len(it.sg)) 212 break; 213 214 cond_resched(); 215 } while (1); 216 217out_ce: 218 return err; 219} 220 221static int 222intel_migrate_ccs_copy(struct intel_migrate *m, 223 struct i915_gem_ww_ctx *ww, 224 const struct i915_deps *deps, 225 struct scatterlist *sg, 226 unsigned int pat_index, 227 bool write_to_ccs, 228 struct i915_request **out) 229{ 230 struct intel_context *ce; 231 int err; 232 233 *out = NULL; 234 if (!m->context) 235 return -ENODEV; 236 237 ce = intel_migrate_create_context(m); 238 if (IS_ERR(ce)) 239 ce = intel_context_get(m->context); 240 GEM_BUG_ON(IS_ERR(ce)); 241 242 err = intel_context_pin_ww(ce, ww); 243 if (err) 244 goto out; 245 246 err = intel_context_copy_ccs(ce, deps, sg, pat_index, 247 write_to_ccs, out); 248 249 intel_context_unpin(ce); 250out: 251 intel_context_put(ce); 252 return err; 253} 254 255static int clear(struct intel_migrate *migrate, 256 int (*fn)(struct intel_migrate *migrate, 257 struct i915_gem_ww_ctx *ww, 258 struct drm_i915_gem_object *obj, 259 u32 value, 260 struct i915_request **out), 261 u32 sz, struct rnd_state *prng) 262{ 263 struct drm_i915_private *i915 = migrate->context->engine->i915; 264 struct drm_i915_gem_object *obj; 265 struct i915_request *rq; 266 struct i915_gem_ww_ctx ww; 267 u32 *vaddr, val = 0; 268 bool ccs_cap = false; 269 int err = 0; 270 int i; 271 272 obj = create_lmem_or_internal(i915, sz); 273 if (IS_ERR(obj)) 274 return 0; 275 276 /* Consider the rounded up memory too */ 277 sz = obj->base.size; 278 279 if (HAS_FLAT_CCS(i915) && i915_gem_object_is_lmem(obj)) 280 ccs_cap = true; 281 282 for_i915_gem_ww(&ww, err, true) { 283 int ccs_bytes, ccs_bytes_per_chunk; 284 285 err = i915_gem_object_lock(obj, &ww); 286 if (err) 287 continue; 288 289 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC); 290 if (IS_ERR(vaddr)) { 291 err = PTR_ERR(vaddr); 292 continue; 293 } 294 295 for (i = 0; i < sz / sizeof(u32); i++) 296 vaddr[i] = ~i; 297 i915_gem_object_flush_map(obj); 298 299 if (ccs_cap && !val) { 300 /* Write the obj data into ccs surface */ 301 err = intel_migrate_ccs_copy(migrate, &ww, NULL, 302 obj->mm.pages->sgl, 303 obj->pat_index, 304 true, &rq); 305 if (rq && !err) { 306 if (i915_request_wait(rq, 0, HZ) < 0) { 307 pr_err("%ps timed out, size: %u\n", 308 fn, sz); 309 err = -ETIME; 310 } 311 i915_request_put(rq); 312 rq = NULL; 313 } 314 if (err) 315 continue; 316 } 317 318 err = fn(migrate, &ww, obj, val, &rq); 319 if (rq && !err) { 320 if (i915_request_wait(rq, 0, HZ) < 0) { 321 pr_err("%ps timed out, size: %u\n", fn, sz); 322 err = -ETIME; 323 } 324 i915_request_put(rq); 325 rq = NULL; 326 } 327 if (err) 328 continue; 329 330 i915_gem_object_flush_map(obj); 331 332 /* Verify the set/clear of the obj mem */ 333 for (i = 0; !err && i < sz / PAGE_SIZE; i++) { 334 int x = i * 1024 + 335 i915_prandom_u32_max_state(1024, prng); 336 337 if (vaddr[x] != val) { 338 pr_err("%ps failed, (%u != %u), offset: %zu\n", 339 fn, vaddr[x], val, x * sizeof(u32)); 340 igt_hexdump(vaddr + i * 1024, 4096); 341 err = -EINVAL; 342 } 343 } 344 if (err) 345 continue; 346 347 if (ccs_cap && !val) { 348 for (i = 0; i < sz / sizeof(u32); i++) 349 vaddr[i] = ~i; 350 i915_gem_object_flush_map(obj); 351 352 err = intel_migrate_ccs_copy(migrate, &ww, NULL, 353 obj->mm.pages->sgl, 354 obj->pat_index, 355 false, &rq); 356 if (rq && !err) { 357 if (i915_request_wait(rq, 0, HZ) < 0) { 358 pr_err("%ps timed out, size: %u\n", 359 fn, sz); 360 err = -ETIME; 361 } 362 i915_request_put(rq); 363 rq = NULL; 364 } 365 if (err) 366 continue; 367 368 ccs_bytes = GET_CCS_BYTES(i915, sz); 369 ccs_bytes_per_chunk = GET_CCS_BYTES(i915, CHUNK_SZ); 370 i915_gem_object_flush_map(obj); 371 372 for (i = 0; !err && i < DIV_ROUND_UP(ccs_bytes, PAGE_SIZE); i++) { 373 int offset = ((i * PAGE_SIZE) / 374 ccs_bytes_per_chunk) * CHUNK_SZ / sizeof(u32); 375 int ccs_bytes_left = (ccs_bytes - i * PAGE_SIZE) / sizeof(u32); 376 int x = i915_prandom_u32_max_state(min_t(int, 1024, 377 ccs_bytes_left), prng); 378 379 if (vaddr[offset + x]) { 380 pr_err("%ps ccs clearing failed, offset: %ld/%d\n", 381 fn, i * PAGE_SIZE + x * sizeof(u32), ccs_bytes); 382 igt_hexdump(vaddr + offset, 383 min_t(int, 4096, 384 ccs_bytes_left * sizeof(u32))); 385 err = -EINVAL; 386 } 387 } 388 389 if (err) 390 continue; 391 } 392 i915_gem_object_unpin_map(obj); 393 } 394 395 if (err) { 396 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS) 397 pr_err("%ps failed, size: %u\n", fn, sz); 398 if (rq && err != -EINVAL) { 399 i915_request_wait(rq, 0, HZ); 400 i915_request_put(rq); 401 } 402 403 i915_gem_object_unpin_map(obj); 404 } 405 406 i915_gem_object_put(obj); 407 return err; 408} 409 410static int __migrate_copy(struct intel_migrate *migrate, 411 struct i915_gem_ww_ctx *ww, 412 struct drm_i915_gem_object *src, 413 struct drm_i915_gem_object *dst, 414 struct i915_request **out) 415{ 416 return intel_migrate_copy(migrate, ww, NULL, 417 src->mm.pages->sgl, src->pat_index, 418 i915_gem_object_is_lmem(src), 419 dst->mm.pages->sgl, dst->pat_index, 420 i915_gem_object_is_lmem(dst), 421 out); 422} 423 424static int __global_copy(struct intel_migrate *migrate, 425 struct i915_gem_ww_ctx *ww, 426 struct drm_i915_gem_object *src, 427 struct drm_i915_gem_object *dst, 428 struct i915_request **out) 429{ 430 return intel_context_migrate_copy(migrate->context, NULL, 431 src->mm.pages->sgl, src->pat_index, 432 i915_gem_object_is_lmem(src), 433 dst->mm.pages->sgl, dst->pat_index, 434 i915_gem_object_is_lmem(dst), 435 out); 436} 437 438static int 439migrate_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 440{ 441 return copy(migrate, __migrate_copy, sz, prng); 442} 443 444static int 445global_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 446{ 447 return copy(migrate, __global_copy, sz, prng); 448} 449 450static int __migrate_clear(struct intel_migrate *migrate, 451 struct i915_gem_ww_ctx *ww, 452 struct drm_i915_gem_object *obj, 453 u32 value, 454 struct i915_request **out) 455{ 456 return intel_migrate_clear(migrate, ww, NULL, 457 obj->mm.pages->sgl, 458 obj->pat_index, 459 i915_gem_object_is_lmem(obj), 460 value, out); 461} 462 463static int __global_clear(struct intel_migrate *migrate, 464 struct i915_gem_ww_ctx *ww, 465 struct drm_i915_gem_object *obj, 466 u32 value, 467 struct i915_request **out) 468{ 469 return intel_context_migrate_clear(migrate->context, NULL, 470 obj->mm.pages->sgl, 471 obj->pat_index, 472 i915_gem_object_is_lmem(obj), 473 value, out); 474} 475 476static int 477migrate_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 478{ 479 return clear(migrate, __migrate_clear, sz, prng); 480} 481 482static int 483global_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng) 484{ 485 return clear(migrate, __global_clear, sz, prng); 486} 487 488static int live_migrate_copy(void *arg) 489{ 490 struct intel_gt *gt = arg; 491 struct intel_migrate *migrate = >->migrate; 492 struct drm_i915_private *i915 = migrate->context->engine->i915; 493 I915_RND_STATE(prng); 494 int i; 495 496 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 497 int err; 498 499 err = migrate_copy(migrate, sizes[i], &prng); 500 if (err == 0) 501 err = global_copy(migrate, sizes[i], &prng); 502 i915_gem_drain_freed_objects(i915); 503 if (err) 504 return err; 505 } 506 507 return 0; 508} 509 510static int live_migrate_clear(void *arg) 511{ 512 struct intel_gt *gt = arg; 513 struct intel_migrate *migrate = >->migrate; 514 struct drm_i915_private *i915 = migrate->context->engine->i915; 515 I915_RND_STATE(prng); 516 int i; 517 518 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 519 int err; 520 521 err = migrate_clear(migrate, sizes[i], &prng); 522 if (err == 0) 523 err = global_clear(migrate, sizes[i], &prng); 524 525 i915_gem_drain_freed_objects(i915); 526 if (err) 527 return err; 528 } 529 530 return 0; 531} 532 533struct spinner_timer { 534 struct timer_list timer; 535 struct igt_spinner spin; 536}; 537 538static void spinner_kill(struct timer_list *timer) 539{ 540 struct spinner_timer *st = from_timer(st, timer, timer); 541 542 igt_spinner_end(&st->spin); 543 pr_info("%s\n", __func__); 544} 545 546static int live_emit_pte_full_ring(void *arg) 547{ 548 struct intel_gt *gt = arg; 549 struct intel_migrate *migrate = >->migrate; 550 struct drm_i915_private *i915 = migrate->context->engine->i915; 551 struct drm_i915_gem_object *obj; 552 struct intel_context *ce; 553 struct i915_request *rq, *prev; 554 struct spinner_timer st; 555 struct sgt_dma it; 556 int len, sz, err; 557 u32 *cs; 558 559 /* 560 * Simple regression test to check that we don't trample the 561 * rq->reserved_space when returning from emit_pte(), if the ring is 562 * nearly full. 563 */ 564 565 if (igt_spinner_init(&st.spin, to_gt(i915))) 566 return -ENOMEM; 567 568 obj = i915_gem_object_create_internal(i915, 2 * PAGE_SIZE); 569 if (IS_ERR(obj)) { 570 err = PTR_ERR(obj); 571 goto out_spinner; 572 } 573 574 err = i915_gem_object_pin_pages_unlocked(obj); 575 if (err) 576 goto out_obj; 577 578 ce = intel_migrate_create_context(migrate); 579 if (IS_ERR(ce)) { 580 err = PTR_ERR(ce); 581 goto out_obj; 582 } 583 584 ce->ring_size = SZ_4K; /* Not too big */ 585 586 err = intel_context_pin(ce); 587 if (err) 588 goto out_put; 589 590 rq = igt_spinner_create_request(&st.spin, ce, MI_ARB_CHECK); 591 if (IS_ERR(rq)) { 592 err = PTR_ERR(rq); 593 goto out_unpin; 594 } 595 596 i915_request_add(rq); 597 if (!igt_wait_for_spinner(&st.spin, rq)) { 598 err = -EIO; 599 goto out_unpin; 600 } 601 602 /* 603 * Fill the rest of the ring leaving I915_EMIT_PTE_NUM_DWORDS + 604 * ring->reserved_space at the end. To actually emit the PTEs we require 605 * slightly more than I915_EMIT_PTE_NUM_DWORDS, since our object size is 606 * greater than PAGE_SIZE. The correct behaviour is to wait for more 607 * ring space in emit_pte(), otherwise we trample on the reserved_space 608 * resulting in crashes when later submitting the rq. 609 */ 610 611 prev = NULL; 612 do { 613 if (prev) 614 i915_request_add(rq); 615 616 rq = i915_request_create(ce); 617 if (IS_ERR(rq)) { 618 err = PTR_ERR(rq); 619 goto out_unpin; 620 } 621 622 sz = (rq->ring->space - rq->reserved_space) / sizeof(u32) - 623 I915_EMIT_PTE_NUM_DWORDS; 624 sz = min_t(u32, sz, (SZ_1K - rq->reserved_space) / sizeof(u32) - 625 I915_EMIT_PTE_NUM_DWORDS); 626 cs = intel_ring_begin(rq, sz); 627 if (IS_ERR(cs)) { 628 err = PTR_ERR(cs); 629 goto out_rq; 630 } 631 632 memset32(cs, MI_NOOP, sz); 633 cs += sz; 634 intel_ring_advance(rq, cs); 635 636 pr_info("%s emit=%u sz=%d\n", __func__, rq->ring->emit, sz); 637 638 prev = rq; 639 } while (rq->ring->space > (rq->reserved_space + 640 I915_EMIT_PTE_NUM_DWORDS * sizeof(u32))); 641 642 timer_setup_on_stack(&st.timer, spinner_kill, 0); 643 mod_timer(&st.timer, jiffies + 2 * HZ); 644 645 /* 646 * This should wait for the spinner to be killed, otherwise we should go 647 * down in flames when doing i915_request_add(). 648 */ 649 pr_info("%s emite_pte ring space=%u\n", __func__, rq->ring->space); 650 it = sg_sgt(obj->mm.pages->sgl); 651 len = emit_pte(rq, &it, obj->pat_index, false, 0, CHUNK_SZ); 652 if (!len) { 653 err = -EINVAL; 654 goto out_rq; 655 } 656 if (len < 0) { 657 err = len; 658 goto out_rq; 659 } 660 661out_rq: 662 i915_request_add(rq); /* GEM_BUG_ON(rq->reserved_space > ring->space)? */ 663 del_timer_sync(&st.timer); 664 destroy_timer_on_stack(&st.timer); 665out_unpin: 666 intel_context_unpin(ce); 667out_put: 668 intel_context_put(ce); 669out_obj: 670 i915_gem_object_put(obj); 671out_spinner: 672 igt_spinner_fini(&st.spin); 673 return err; 674} 675 676struct threaded_migrate { 677 struct intel_migrate *migrate; 678 struct task_struct *tsk; 679 struct rnd_state prng; 680}; 681 682static int threaded_migrate(struct intel_migrate *migrate, 683 int (*fn)(void *arg), 684 unsigned int flags) 685{ 686 const unsigned int n_cpus = num_online_cpus() + 1; 687 struct threaded_migrate *thread; 688 I915_RND_STATE(prng); 689 unsigned int i; 690 int err = 0; 691 692 thread = kcalloc(n_cpus, sizeof(*thread), GFP_KERNEL); 693 if (!thread) 694 return 0; 695 696 for (i = 0; i < n_cpus; ++i) { 697 struct task_struct *tsk; 698 699 thread[i].migrate = migrate; 700 thread[i].prng = 701 I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng)); 702 703 tsk = kthread_run(fn, &thread[i], "igt-%d", i); 704 if (IS_ERR(tsk)) { 705 err = PTR_ERR(tsk); 706 break; 707 } 708 709 get_task_struct(tsk); 710 thread[i].tsk = tsk; 711 } 712 713 msleep(10 * n_cpus); /* start all threads before we kthread_stop() */ 714 715 for (i = 0; i < n_cpus; ++i) { 716 struct task_struct *tsk = thread[i].tsk; 717 int status; 718 719 if (IS_ERR_OR_NULL(tsk)) 720 continue; 721 722 status = kthread_stop_put(tsk); 723 if (status && !err) 724 err = status; 725 } 726 727 kfree(thread); 728 return err; 729} 730 731static int __thread_migrate_copy(void *arg) 732{ 733 struct threaded_migrate *tm = arg; 734 735 return migrate_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 736} 737 738static int thread_migrate_copy(void *arg) 739{ 740 struct intel_gt *gt = arg; 741 struct intel_migrate *migrate = >->migrate; 742 743 return threaded_migrate(migrate, __thread_migrate_copy, 0); 744} 745 746static int __thread_global_copy(void *arg) 747{ 748 struct threaded_migrate *tm = arg; 749 750 return global_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 751} 752 753static int thread_global_copy(void *arg) 754{ 755 struct intel_gt *gt = arg; 756 struct intel_migrate *migrate = >->migrate; 757 758 return threaded_migrate(migrate, __thread_global_copy, 0); 759} 760 761static int __thread_migrate_clear(void *arg) 762{ 763 struct threaded_migrate *tm = arg; 764 765 return migrate_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 766} 767 768static int __thread_global_clear(void *arg) 769{ 770 struct threaded_migrate *tm = arg; 771 772 return global_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng); 773} 774 775static int thread_migrate_clear(void *arg) 776{ 777 struct intel_gt *gt = arg; 778 struct intel_migrate *migrate = >->migrate; 779 780 return threaded_migrate(migrate, __thread_migrate_clear, 0); 781} 782 783static int thread_global_clear(void *arg) 784{ 785 struct intel_gt *gt = arg; 786 struct intel_migrate *migrate = >->migrate; 787 788 return threaded_migrate(migrate, __thread_global_clear, 0); 789} 790 791int intel_migrate_live_selftests(struct drm_i915_private *i915) 792{ 793 static const struct i915_subtest tests[] = { 794 SUBTEST(live_migrate_copy), 795 SUBTEST(live_migrate_clear), 796 SUBTEST(live_emit_pte_full_ring), 797 SUBTEST(thread_migrate_copy), 798 SUBTEST(thread_migrate_clear), 799 SUBTEST(thread_global_copy), 800 SUBTEST(thread_global_clear), 801 }; 802 struct intel_gt *gt = to_gt(i915); 803 804 if (!gt->migrate.context) 805 return 0; 806 807 return intel_gt_live_subtests(tests, gt); 808} 809 810static struct drm_i915_gem_object * 811create_init_lmem_internal(struct intel_gt *gt, size_t sz, bool try_lmem) 812{ 813 struct drm_i915_gem_object *obj = NULL; 814 int err; 815 816 if (try_lmem) 817 obj = i915_gem_object_create_lmem(gt->i915, sz, 0); 818 819 if (IS_ERR_OR_NULL(obj)) { 820 obj = i915_gem_object_create_internal(gt->i915, sz); 821 if (IS_ERR(obj)) 822 return obj; 823 } 824 825 i915_gem_object_trylock(obj, NULL); 826 err = i915_gem_object_pin_pages(obj); 827 if (err) { 828 i915_gem_object_unlock(obj); 829 i915_gem_object_put(obj); 830 return ERR_PTR(err); 831 } 832 833 return obj; 834} 835 836static int wrap_ktime_compare(const void *A, const void *B) 837{ 838 const ktime_t *a = A, *b = B; 839 840 return ktime_compare(*a, *b); 841} 842 843static int __perf_clear_blt(struct intel_context *ce, 844 struct scatterlist *sg, 845 unsigned int pat_index, 846 bool is_lmem, 847 size_t sz) 848{ 849 ktime_t t[5]; 850 int pass; 851 int err = 0; 852 853 for (pass = 0; pass < ARRAY_SIZE(t); pass++) { 854 struct i915_request *rq; 855 ktime_t t0, t1; 856 857 t0 = ktime_get(); 858 859 err = intel_context_migrate_clear(ce, NULL, sg, pat_index, 860 is_lmem, 0, &rq); 861 if (rq) { 862 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0) 863 err = -EIO; 864 i915_request_put(rq); 865 } 866 if (err) 867 break; 868 869 t1 = ktime_get(); 870 t[pass] = ktime_sub(t1, t0); 871 } 872 if (err) 873 return err; 874 875 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); 876 pr_info("%s: %zd KiB fill: %lld MiB/s\n", 877 ce->engine->name, sz >> 10, 878 div64_u64(mul_u32_u32(4 * sz, 879 1000 * 1000 * 1000), 880 t[1] + 2 * t[2] + t[3]) >> 20); 881 return 0; 882} 883 884static int perf_clear_blt(void *arg) 885{ 886 struct intel_gt *gt = arg; 887 static const unsigned long sizes[] = { 888 SZ_4K, 889 SZ_64K, 890 SZ_2M, 891 SZ_64M 892 }; 893 int i; 894 895 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 896 struct drm_i915_gem_object *dst; 897 int err; 898 899 dst = create_init_lmem_internal(gt, sizes[i], true); 900 if (IS_ERR(dst)) 901 return PTR_ERR(dst); 902 903 err = __perf_clear_blt(gt->migrate.context, 904 dst->mm.pages->sgl, 905 i915_gem_get_pat_index(gt->i915, 906 I915_CACHE_NONE), 907 i915_gem_object_is_lmem(dst), 908 sizes[i]); 909 910 i915_gem_object_unlock(dst); 911 i915_gem_object_put(dst); 912 if (err) 913 return err; 914 } 915 916 return 0; 917} 918 919static int __perf_copy_blt(struct intel_context *ce, 920 struct scatterlist *src, 921 unsigned int src_pat_index, 922 bool src_is_lmem, 923 struct scatterlist *dst, 924 unsigned int dst_pat_index, 925 bool dst_is_lmem, 926 size_t sz) 927{ 928 ktime_t t[5]; 929 int pass; 930 int err = 0; 931 932 for (pass = 0; pass < ARRAY_SIZE(t); pass++) { 933 struct i915_request *rq; 934 ktime_t t0, t1; 935 936 t0 = ktime_get(); 937 938 err = intel_context_migrate_copy(ce, NULL, 939 src, src_pat_index, 940 src_is_lmem, 941 dst, dst_pat_index, 942 dst_is_lmem, 943 &rq); 944 if (rq) { 945 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0) 946 err = -EIO; 947 i915_request_put(rq); 948 } 949 if (err) 950 break; 951 952 t1 = ktime_get(); 953 t[pass] = ktime_sub(t1, t0); 954 } 955 if (err) 956 return err; 957 958 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL); 959 pr_info("%s: %zd KiB copy: %lld MiB/s\n", 960 ce->engine->name, sz >> 10, 961 div64_u64(mul_u32_u32(4 * sz, 962 1000 * 1000 * 1000), 963 t[1] + 2 * t[2] + t[3]) >> 20); 964 return 0; 965} 966 967static int perf_copy_blt(void *arg) 968{ 969 struct intel_gt *gt = arg; 970 static const unsigned long sizes[] = { 971 SZ_4K, 972 SZ_64K, 973 SZ_2M, 974 SZ_64M 975 }; 976 int i; 977 978 for (i = 0; i < ARRAY_SIZE(sizes); i++) { 979 struct drm_i915_gem_object *src, *dst; 980 size_t sz; 981 int err; 982 983 src = create_init_lmem_internal(gt, sizes[i], true); 984 if (IS_ERR(src)) 985 return PTR_ERR(src); 986 987 sz = src->base.size; 988 dst = create_init_lmem_internal(gt, sz, false); 989 if (IS_ERR(dst)) { 990 err = PTR_ERR(dst); 991 goto err_src; 992 } 993 994 err = __perf_copy_blt(gt->migrate.context, 995 src->mm.pages->sgl, 996 i915_gem_get_pat_index(gt->i915, 997 I915_CACHE_NONE), 998 i915_gem_object_is_lmem(src), 999 dst->mm.pages->sgl, 1000 i915_gem_get_pat_index(gt->i915, 1001 I915_CACHE_NONE), 1002 i915_gem_object_is_lmem(dst), 1003 sz); 1004 1005 i915_gem_object_unlock(dst); 1006 i915_gem_object_put(dst); 1007err_src: 1008 i915_gem_object_unlock(src); 1009 i915_gem_object_put(src); 1010 if (err) 1011 return err; 1012 } 1013 1014 return 0; 1015} 1016 1017int intel_migrate_perf_selftests(struct drm_i915_private *i915) 1018{ 1019 static const struct i915_subtest tests[] = { 1020 SUBTEST(perf_clear_blt), 1021 SUBTEST(perf_copy_blt), 1022 }; 1023 struct intel_gt *gt = to_gt(i915); 1024 1025 if (intel_gt_is_wedged(gt)) 1026 return 0; 1027 1028 if (!gt->migrate.context) 1029 return 0; 1030 1031 return intel_gt_live_subtests(tests, gt); 1032} 1033