1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2020 Intel Corporation 4 */ 5 6#include "xe_migrate.h" 7 8#include <linux/bitfield.h> 9#include <linux/sizes.h> 10 11#include <drm/drm_managed.h> 12#include <drm/ttm/ttm_tt.h> 13#include <drm/xe_drm.h> 14 15#include <generated/xe_wa_oob.h> 16 17#include "instructions/xe_mi_commands.h" 18#include "regs/xe_gpu_commands.h" 19#include "regs/xe_gtt_defs.h" 20#include "tests/xe_test.h" 21#include "xe_assert.h" 22#include "xe_bb.h" 23#include "xe_bo.h" 24#include "xe_exec_queue.h" 25#include "xe_ggtt.h" 26#include "xe_gt.h" 27#include "xe_hw_engine.h" 28#include "xe_lrc.h" 29#include "xe_map.h" 30#include "xe_mocs.h" 31#include "xe_pt.h" 32#include "xe_res_cursor.h" 33#include "xe_sched_job.h" 34#include "xe_sync.h" 35#include "xe_trace.h" 36#include "xe_vm.h" 37 38/** 39 * struct xe_migrate - migrate context. 40 */ 41struct xe_migrate { 42 /** @q: Default exec queue used for migration */ 43 struct xe_exec_queue *q; 44 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ 45 struct xe_tile *tile; 46 /** @job_mutex: Timeline mutex for @eng. */ 47 struct mutex job_mutex; 48 /** @pt_bo: Page-table buffer object. */ 49 struct xe_bo *pt_bo; 50 /** @batch_base_ofs: VM offset of the migration batch buffer */ 51 u64 batch_base_ofs; 52 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ 53 u64 usm_batch_base_ofs; 54 /** @cleared_mem_ofs: VM offset of @cleared_bo. */ 55 u64 cleared_mem_ofs; 56 /** 57 * @fence: dma-fence representing the last migration job batch. 58 * Protected by @job_mutex. 59 */ 60 struct dma_fence *fence; 61 /** 62 * @vm_update_sa: For integrated, used to suballocate page-tables 63 * out of the pt_bo. 64 */ 65 struct drm_suballoc_manager vm_update_sa; 66 /** @min_chunk_size: For dgfx, Minimum chunk size */ 67 u64 min_chunk_size; 68}; 69 70#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ 71#define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ 72#define NUM_KERNEL_PDE 17 73#define NUM_PT_SLOTS 32 74#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M 75#define MAX_NUM_PTE 512 76 77/* 78 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest 79 * legal value accepted. Since that instruction field is always stored in 80 * (val-2) format, this translates to 0x400 dwords for the true maximum length 81 * of the instruction. Subtracting the instruction header (1 dword) and 82 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values. 83 */ 84#define MAX_PTE_PER_SDI 0x1FE 85 86/** 87 * xe_tile_migrate_engine() - Get this tile's migrate engine. 88 * @tile: The tile. 89 * 90 * Returns the default migrate engine of this tile. 91 * TODO: Perhaps this function is slightly misplaced, and even unneeded? 92 * 93 * Return: The default migrate engine 94 */ 95struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile) 96{ 97 return tile->migrate->q; 98} 99 100static void xe_migrate_fini(struct drm_device *dev, void *arg) 101{ 102 struct xe_migrate *m = arg; 103 104 xe_vm_lock(m->q->vm, false); 105 xe_bo_unpin(m->pt_bo); 106 xe_vm_unlock(m->q->vm); 107 108 dma_fence_put(m->fence); 109 xe_bo_put(m->pt_bo); 110 drm_suballoc_manager_fini(&m->vm_update_sa); 111 mutex_destroy(&m->job_mutex); 112 xe_vm_close_and_put(m->q->vm); 113 xe_exec_queue_put(m->q); 114} 115 116static u64 xe_migrate_vm_addr(u64 slot, u32 level) 117{ 118 XE_WARN_ON(slot >= NUM_PT_SLOTS); 119 120 /* First slot is reserved for mapping of PT bo and bb, start from 1 */ 121 return (slot + 1ULL) << xe_pt_shift(level + 1); 122} 123 124static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr) 125{ 126 /* 127 * Remove the DPA to get a correct offset into identity table for the 128 * migrate offset 129 */ 130 addr -= xe->mem.vram.dpa_base; 131 return addr + (256ULL << xe_pt_shift(2)); 132} 133 134static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, 135 struct xe_vm *vm) 136{ 137 struct xe_device *xe = tile_to_xe(tile); 138 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 139 u8 id = tile->id; 140 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; 141 u32 map_ofs, level, i; 142 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; 143 u64 entry; 144 145 /* Can't bump NUM_PT_SLOTS too high */ 146 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); 147 /* Must be a multiple of 64K to support all platforms */ 148 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); 149 /* And one slot reserved for the 4KiB page table updates */ 150 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); 151 152 /* Need to be sure everything fits in the first PT, or create more */ 153 xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M); 154 155 bo = xe_bo_create_pin_map(vm->xe, tile, vm, 156 num_entries * XE_PAGE_SIZE, 157 ttm_bo_type_kernel, 158 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 159 XE_BO_FLAG_PINNED); 160 if (IS_ERR(bo)) 161 return PTR_ERR(bo); 162 163 entry = vm->pt_ops->pde_encode_bo(bo, bo->size - XE_PAGE_SIZE, pat_index); 164 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); 165 166 map_ofs = (num_entries - num_level) * XE_PAGE_SIZE; 167 168 /* Map the entire BO in our level 0 pt */ 169 for (i = 0, level = 0; i < num_entries; level++) { 170 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE, 171 pat_index, 0); 172 173 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); 174 175 if (vm->flags & XE_VM_FLAG_64K) 176 i += 16; 177 else 178 i += 1; 179 } 180 181 if (!IS_DGFX(xe)) { 182 /* Write out batch too */ 183 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE; 184 for (i = 0; i < batch->size; 185 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 186 XE_PAGE_SIZE) { 187 entry = vm->pt_ops->pte_encode_bo(batch, i, 188 pat_index, 0); 189 190 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 191 entry); 192 level++; 193 } 194 if (xe->info.has_usm) { 195 xe_tile_assert(tile, batch->size == SZ_1M); 196 197 batch = tile->primary_gt->usm.bb_pool->bo; 198 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M; 199 xe_tile_assert(tile, batch->size == SZ_512K); 200 201 for (i = 0; i < batch->size; 202 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 203 XE_PAGE_SIZE) { 204 entry = vm->pt_ops->pte_encode_bo(batch, i, 205 pat_index, 0); 206 207 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 208 entry); 209 level++; 210 } 211 } 212 } else { 213 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 214 215 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr); 216 217 if (xe->info.has_usm) { 218 batch = tile->primary_gt->usm.bb_pool->bo; 219 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 220 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr); 221 } 222 } 223 224 for (level = 1; level < num_level; level++) { 225 u32 flags = 0; 226 227 if (vm->flags & XE_VM_FLAG_64K && level == 1) 228 flags = XE_PDE_64K; 229 230 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * 231 XE_PAGE_SIZE, pat_index); 232 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, 233 entry | flags); 234 } 235 236 /* Write PDE's that point to our BO. */ 237 for (i = 0; i < num_entries - num_level; i++) { 238 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE, 239 pat_index); 240 241 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + 242 (i + 1) * 8, u64, entry); 243 } 244 245 /* Set up a 1GiB NULL mapping at 255GiB offset. */ 246 level = 2; 247 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64, 248 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) 249 | XE_PTE_NULL); 250 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level)); 251 252 /* Identity map the entire vram at 256GiB offset */ 253 if (IS_DGFX(xe)) { 254 u64 pos, ofs, flags; 255 256 level = 2; 257 ofs = map_ofs + XE_PAGE_SIZE * level + 256 * 8; 258 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, 259 true, 0); 260 261 /* 262 * Use 1GB pages, it shouldn't matter the physical amount of 263 * vram is less, when we don't access it. 264 */ 265 for (pos = xe->mem.vram.dpa_base; 266 pos < xe->mem.vram.actual_physical_size + xe->mem.vram.dpa_base; 267 pos += SZ_1G, ofs += 8) 268 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 269 } 270 271 /* 272 * Example layout created above, with root level = 3: 273 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's 274 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's 275 * [PT9...PT28]: Userspace PT's for VM_BIND, 4 KiB PTE's 276 * [PT29 = PDE 0] [PT30 = PDE 1] [PT31 = PDE 2] 277 * 278 * This makes the lowest part of the VM point to the pagetables. 279 * Hence the lowest 2M in the vm should point to itself, with a few writes 280 * and flushes, other parts of the VM can be used either for copying and 281 * clearing. 282 * 283 * For performance, the kernel reserves PDE's, so about 20 are left 284 * for async VM updates. 285 * 286 * To make it easier to work, each scratch PT is put in slot (1 + PT #) 287 * everywhere, this allows lockless updates to scratch pages by using 288 * the different addresses in VM. 289 */ 290#define NUM_VMUSA_UNIT_PER_PAGE 32 291#define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) 292#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) 293 drm_suballoc_manager_init(&m->vm_update_sa, 294 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * 295 NUM_VMUSA_UNIT_PER_PAGE, 0); 296 297 m->pt_bo = bo; 298 return 0; 299} 300 301/* 302 * Including the reserved copy engine is required to avoid deadlocks due to 303 * migrate jobs servicing the faults gets stuck behind the job that faulted. 304 */ 305static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt) 306{ 307 u32 logical_mask = 0; 308 struct xe_hw_engine *hwe; 309 enum xe_hw_engine_id id; 310 311 for_each_hw_engine(hwe, gt, id) { 312 if (hwe->class != XE_ENGINE_CLASS_COPY) 313 continue; 314 315 if (xe_gt_is_usm_hwe(gt, hwe)) 316 logical_mask |= BIT(hwe->logical_instance); 317 } 318 319 return logical_mask; 320} 321 322/** 323 * xe_migrate_init() - Initialize a migrate context 324 * @tile: Back-pointer to the tile we're initializing for. 325 * 326 * Return: Pointer to a migrate context on success. Error pointer on error. 327 */ 328struct xe_migrate *xe_migrate_init(struct xe_tile *tile) 329{ 330 struct xe_device *xe = tile_to_xe(tile); 331 struct xe_gt *primary_gt = tile->primary_gt; 332 struct xe_migrate *m; 333 struct xe_vm *vm; 334 int err; 335 336 m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL); 337 if (!m) 338 return ERR_PTR(-ENOMEM); 339 340 m->tile = tile; 341 342 /* Special layout, prepared below.. */ 343 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | 344 XE_VM_FLAG_SET_TILE_ID(tile)); 345 if (IS_ERR(vm)) 346 return ERR_CAST(vm); 347 348 xe_vm_lock(vm, false); 349 err = xe_migrate_prepare_vm(tile, m, vm); 350 xe_vm_unlock(vm); 351 if (err) { 352 xe_vm_close_and_put(vm); 353 return ERR_PTR(err); 354 } 355 356 if (xe->info.has_usm) { 357 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt, 358 XE_ENGINE_CLASS_COPY, 359 primary_gt->usm.reserved_bcs_instance, 360 false); 361 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt); 362 363 if (!hwe || !logical_mask) 364 return ERR_PTR(-EINVAL); 365 366 /* 367 * XXX: Currently only reserving 1 (likely slow) BCS instance on 368 * PVC, may want to revisit if performance is needed. 369 */ 370 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe, 371 EXEC_QUEUE_FLAG_KERNEL | 372 EXEC_QUEUE_FLAG_PERMANENT | 373 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0); 374 } else { 375 m->q = xe_exec_queue_create_class(xe, primary_gt, vm, 376 XE_ENGINE_CLASS_COPY, 377 EXEC_QUEUE_FLAG_KERNEL | 378 EXEC_QUEUE_FLAG_PERMANENT); 379 } 380 if (IS_ERR(m->q)) { 381 xe_vm_close_and_put(vm); 382 return ERR_CAST(m->q); 383 } 384 385 mutex_init(&m->job_mutex); 386 387 err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m); 388 if (err) 389 return ERR_PTR(err); 390 391 if (IS_DGFX(xe)) { 392 if (xe_device_has_flat_ccs(xe)) 393 /* min chunk size corresponds to 4K of CCS Metadata */ 394 m->min_chunk_size = SZ_4K * SZ_64K / 395 xe_device_ccs_bytes(xe, SZ_64K); 396 else 397 /* Somewhat arbitrary to avoid a huge amount of blits */ 398 m->min_chunk_size = SZ_64K; 399 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size); 400 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n", 401 (unsigned long long)m->min_chunk_size); 402 } 403 404 return m; 405} 406 407static u64 max_mem_transfer_per_pass(struct xe_device *xe) 408{ 409 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 410 return MAX_CCS_LIMITED_TRANSFER; 411 412 return MAX_PREEMPTDISABLE_TRANSFER; 413} 414 415static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur) 416{ 417 struct xe_device *xe = tile_to_xe(m->tile); 418 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining); 419 420 if (mem_type_is_vram(cur->mem_type)) { 421 /* 422 * VRAM we want to blit in chunks with sizes aligned to 423 * min_chunk_size in order for the offset to CCS metadata to be 424 * page-aligned. If it's the last chunk it may be smaller. 425 * 426 * Another constraint is that we need to limit the blit to 427 * the VRAM block size, unless size is smaller than 428 * min_chunk_size. 429 */ 430 u64 chunk = max_t(u64, cur->size, m->min_chunk_size); 431 432 size = min_t(u64, size, chunk); 433 if (size > m->min_chunk_size) 434 size = round_down(size, m->min_chunk_size); 435 } 436 437 return size; 438} 439 440static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) 441{ 442 /* If the chunk is not fragmented, allow identity map. */ 443 return cur->size >= size; 444} 445 446static u32 pte_update_size(struct xe_migrate *m, 447 bool is_vram, 448 struct ttm_resource *res, 449 struct xe_res_cursor *cur, 450 u64 *L0, u64 *L0_ofs, u32 *L0_pt, 451 u32 cmd_size, u32 pt_ofs, u32 avail_pts) 452{ 453 u32 cmds = 0; 454 455 *L0_pt = pt_ofs; 456 if (is_vram && xe_migrate_allow_identity(*L0, cur)) { 457 /* Offset into identity map. */ 458 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile), 459 cur->start + vram_region_gpu_offset(res)); 460 cmds += cmd_size; 461 } else { 462 /* Clip L0 to available size */ 463 u64 size = min(*L0, (u64)avail_pts * SZ_2M); 464 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT; 465 466 *L0 = size; 467 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); 468 469 /* MI_STORE_DATA_IMM */ 470 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI); 471 472 /* PDE qwords */ 473 cmds += num_4k_pages * 2; 474 475 /* Each chunk has a single blit command */ 476 cmds += cmd_size; 477 } 478 479 return cmds; 480} 481 482static void emit_pte(struct xe_migrate *m, 483 struct xe_bb *bb, u32 at_pt, 484 bool is_vram, bool is_comp_pte, 485 struct xe_res_cursor *cur, 486 u32 size, struct ttm_resource *res) 487{ 488 struct xe_device *xe = tile_to_xe(m->tile); 489 struct xe_vm *vm = m->q->vm; 490 u16 pat_index; 491 u32 ptes; 492 u64 ofs = (u64)at_pt * XE_PAGE_SIZE; 493 u64 cur_ofs; 494 495 /* Indirect access needs compression enabled uncached PAT index */ 496 if (GRAPHICS_VERx100(xe) >= 2000) 497 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : 498 xe->pat.idx[XE_CACHE_WB]; 499 else 500 pat_index = xe->pat.idx[XE_CACHE_WB]; 501 502 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); 503 504 while (ptes) { 505 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 506 507 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 508 bb->cs[bb->len++] = ofs; 509 bb->cs[bb->len++] = 0; 510 511 cur_ofs = ofs; 512 ofs += chunk * 8; 513 ptes -= chunk; 514 515 while (chunk--) { 516 u64 addr, flags = 0; 517 bool devmem = false; 518 519 addr = xe_res_dma(cur) & PAGE_MASK; 520 if (is_vram) { 521 if (vm->flags & XE_VM_FLAG_64K) { 522 u64 va = cur_ofs * XE_PAGE_SIZE / 8; 523 524 xe_assert(xe, (va & (SZ_64K - 1)) == 525 (addr & (SZ_64K - 1))); 526 527 flags |= XE_PTE_PS64; 528 } 529 530 addr += vram_region_gpu_offset(res); 531 devmem = true; 532 } 533 534 addr = vm->pt_ops->pte_encode_addr(m->tile->xe, 535 addr, pat_index, 536 0, devmem, flags); 537 bb->cs[bb->len++] = lower_32_bits(addr); 538 bb->cs[bb->len++] = upper_32_bits(addr); 539 540 xe_res_next(cur, min_t(u32, size, PAGE_SIZE)); 541 cur_ofs += 8; 542 } 543 } 544} 545 546#define EMIT_COPY_CCS_DW 5 547static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, 548 u64 dst_ofs, bool dst_is_indirect, 549 u64 src_ofs, bool src_is_indirect, 550 u32 size) 551{ 552 struct xe_device *xe = gt_to_xe(gt); 553 u32 *cs = bb->cs + bb->len; 554 u32 num_ccs_blks; 555 u32 num_pages; 556 u32 ccs_copy_size; 557 u32 mocs; 558 559 if (GRAPHICS_VERx100(xe) >= 2000) { 560 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE); 561 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1)); 562 563 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1); 564 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index); 565 566 } else { 567 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), 568 NUM_CCS_BYTES_PER_BLOCK); 569 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1)); 570 571 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1); 572 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index); 573 } 574 575 *cs++ = XY_CTRL_SURF_COPY_BLT | 576 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | 577 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | 578 ccs_copy_size; 579 *cs++ = lower_32_bits(src_ofs); 580 *cs++ = upper_32_bits(src_ofs) | mocs; 581 *cs++ = lower_32_bits(dst_ofs); 582 *cs++ = upper_32_bits(dst_ofs) | mocs; 583 584 bb->len = cs - bb->cs; 585} 586 587#define EMIT_COPY_DW 10 588static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, 589 u64 src_ofs, u64 dst_ofs, unsigned int size, 590 unsigned int pitch) 591{ 592 struct xe_device *xe = gt_to_xe(gt); 593 u32 mocs = 0; 594 u32 tile_y = 0; 595 596 xe_gt_assert(gt, size / pitch <= S16_MAX); 597 xe_gt_assert(gt, pitch / 4 <= S16_MAX); 598 xe_gt_assert(gt, pitch <= U16_MAX); 599 600 if (GRAPHICS_VER(xe) >= 20) 601 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index); 602 603 if (GRAPHICS_VERx100(xe) >= 1250) 604 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; 605 606 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); 607 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; 608 bb->cs[bb->len++] = 0; 609 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; 610 bb->cs[bb->len++] = lower_32_bits(dst_ofs); 611 bb->cs[bb->len++] = upper_32_bits(dst_ofs); 612 bb->cs[bb->len++] = 0; 613 bb->cs[bb->len++] = pitch | mocs; 614 bb->cs[bb->len++] = lower_32_bits(src_ofs); 615 bb->cs[bb->len++] = upper_32_bits(src_ofs); 616} 617 618static int job_add_deps(struct xe_sched_job *job, struct dma_resv *resv, 619 enum dma_resv_usage usage) 620{ 621 return drm_sched_job_add_resv_dependencies(&job->drm, resv, usage); 622} 623 624static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) 625{ 626 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; 627} 628 629static u32 xe_migrate_ccs_copy(struct xe_migrate *m, 630 struct xe_bb *bb, 631 u64 src_ofs, bool src_is_indirect, 632 u64 dst_ofs, bool dst_is_indirect, u32 dst_size, 633 u64 ccs_ofs, bool copy_ccs) 634{ 635 struct xe_gt *gt = m->tile->primary_gt; 636 u32 flush_flags = 0; 637 638 if (xe_device_has_flat_ccs(gt_to_xe(gt)) && !copy_ccs && dst_is_indirect) { 639 /* 640 * If the src is already in vram, then it should already 641 * have been cleared by us, or has been populated by the 642 * user. Make sure we copy the CCS aux state as-is. 643 * 644 * Otherwise if the bo doesn't have any CCS metadata attached, 645 * we still need to clear it for security reasons. 646 */ 647 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs; 648 649 emit_copy_ccs(gt, bb, 650 dst_ofs, true, 651 ccs_src_ofs, src_is_indirect, dst_size); 652 653 flush_flags = MI_FLUSH_DW_CCS; 654 } else if (copy_ccs) { 655 if (!src_is_indirect) 656 src_ofs = ccs_ofs; 657 else if (!dst_is_indirect) 658 dst_ofs = ccs_ofs; 659 660 xe_gt_assert(gt, src_is_indirect || dst_is_indirect); 661 662 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs, 663 src_is_indirect, dst_size); 664 if (dst_is_indirect) 665 flush_flags = MI_FLUSH_DW_CCS; 666 } 667 668 return flush_flags; 669} 670 671/** 672 * xe_migrate_copy() - Copy content of TTM resources. 673 * @m: The migration context. 674 * @src_bo: The buffer object @src is currently bound to. 675 * @dst_bo: If copying between resources created for the same bo, set this to 676 * the same value as @src_bo. If copying between buffer objects, set it to 677 * the buffer object @dst is currently bound to. 678 * @src: The source TTM resource. 679 * @dst: The dst TTM resource. 680 * @copy_only_ccs: If true copy only CCS metadata 681 * 682 * Copies the contents of @src to @dst: On flat CCS devices, 683 * the CCS metadata is copied as well if needed, or if not present, 684 * the CCS metadata of @dst is cleared for security reasons. 685 * 686 * Return: Pointer to a dma_fence representing the last copy batch, or 687 * an error pointer on failure. If there is a failure, any copy operation 688 * started by the function call has been synced. 689 */ 690struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 691 struct xe_bo *src_bo, 692 struct xe_bo *dst_bo, 693 struct ttm_resource *src, 694 struct ttm_resource *dst, 695 bool copy_only_ccs) 696{ 697 struct xe_gt *gt = m->tile->primary_gt; 698 struct xe_device *xe = gt_to_xe(gt); 699 struct dma_fence *fence = NULL; 700 u64 size = src_bo->size; 701 struct xe_res_cursor src_it, dst_it, ccs_it; 702 u64 src_L0_ofs, dst_L0_ofs; 703 u32 src_L0_pt, dst_L0_pt; 704 u64 src_L0, dst_L0; 705 int pass = 0; 706 int err; 707 bool src_is_pltt = src->mem_type == XE_PL_TT; 708 bool dst_is_pltt = dst->mem_type == XE_PL_TT; 709 bool src_is_vram = mem_type_is_vram(src->mem_type); 710 bool dst_is_vram = mem_type_is_vram(dst->mem_type); 711 bool copy_ccs = xe_device_has_flat_ccs(xe) && 712 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); 713 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); 714 715 /* Copying CCS between two different BOs is not supported yet. */ 716 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) 717 return ERR_PTR(-EINVAL); 718 719 if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size)) 720 return ERR_PTR(-EINVAL); 721 722 if (!src_is_vram) 723 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); 724 else 725 xe_res_first(src, 0, size, &src_it); 726 if (!dst_is_vram) 727 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); 728 else 729 xe_res_first(dst, 0, size, &dst_it); 730 731 if (copy_system_ccs) 732 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), 733 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), 734 &ccs_it); 735 736 while (size) { 737 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ 738 struct xe_sched_job *job; 739 struct xe_bb *bb; 740 u32 flush_flags; 741 u32 update_idx; 742 u64 ccs_ofs, ccs_size; 743 u32 ccs_pt; 744 745 bool usm = xe->info.has_usm; 746 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 747 748 src_L0 = xe_migrate_res_sizes(m, &src_it); 749 dst_L0 = xe_migrate_res_sizes(m, &dst_it); 750 751 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", 752 pass++, src_L0, dst_L0); 753 754 src_L0 = min(src_L0, dst_L0); 755 756 batch_size += pte_update_size(m, src_is_vram, src, &src_it, &src_L0, 757 &src_L0_ofs, &src_L0_pt, 0, 0, 758 avail_pts); 759 760 batch_size += pte_update_size(m, dst_is_vram, dst, &dst_it, &src_L0, 761 &dst_L0_ofs, &dst_L0_pt, 0, 762 avail_pts, avail_pts); 763 764 if (copy_system_ccs) { 765 ccs_size = xe_device_ccs_bytes(xe, src_L0); 766 batch_size += pte_update_size(m, false, NULL, &ccs_it, &ccs_size, 767 &ccs_ofs, &ccs_pt, 0, 768 2 * avail_pts, 769 avail_pts); 770 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 771 } 772 773 /* Add copy commands size here */ 774 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + 775 ((xe_device_has_flat_ccs(xe) ? EMIT_COPY_CCS_DW : 0)); 776 777 bb = xe_bb_new(gt, batch_size, usm); 778 if (IS_ERR(bb)) { 779 err = PTR_ERR(bb); 780 goto err_sync; 781 } 782 783 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it)) 784 xe_res_next(&src_it, src_L0); 785 else 786 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs, 787 &src_it, src_L0, src); 788 789 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it)) 790 xe_res_next(&dst_it, src_L0); 791 else 792 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs, 793 &dst_it, src_L0, dst); 794 795 if (copy_system_ccs) 796 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); 797 798 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 799 update_idx = bb->len; 800 801 if (!copy_only_ccs) 802 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); 803 804 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, 805 IS_DGFX(xe) ? src_is_vram : src_is_pltt, 806 dst_L0_ofs, 807 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt, 808 src_L0, ccs_ofs, copy_ccs); 809 810 mutex_lock(&m->job_mutex); 811 job = xe_bb_create_migration_job(m->q, bb, 812 xe_migrate_batch_base(m, usm), 813 update_idx); 814 if (IS_ERR(job)) { 815 err = PTR_ERR(job); 816 goto err; 817 } 818 819 xe_sched_job_add_migrate_flush(job, flush_flags); 820 if (!fence) { 821 err = job_add_deps(job, src_bo->ttm.base.resv, 822 DMA_RESV_USAGE_BOOKKEEP); 823 if (!err && src_bo != dst_bo) 824 err = job_add_deps(job, dst_bo->ttm.base.resv, 825 DMA_RESV_USAGE_BOOKKEEP); 826 if (err) 827 goto err_job; 828 } 829 830 xe_sched_job_arm(job); 831 dma_fence_put(fence); 832 fence = dma_fence_get(&job->drm.s_fence->finished); 833 xe_sched_job_push(job); 834 835 dma_fence_put(m->fence); 836 m->fence = dma_fence_get(fence); 837 838 mutex_unlock(&m->job_mutex); 839 840 xe_bb_free(bb, fence); 841 size -= src_L0; 842 continue; 843 844err_job: 845 xe_sched_job_put(job); 846err: 847 mutex_unlock(&m->job_mutex); 848 xe_bb_free(bb, NULL); 849 850err_sync: 851 /* Sync partial copy if any. FIXME: under job_mutex? */ 852 if (fence) { 853 dma_fence_wait(fence, false); 854 dma_fence_put(fence); 855 } 856 857 return ERR_PTR(err); 858 } 859 860 return fence; 861} 862 863static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 864 u32 size, u32 pitch) 865{ 866 struct xe_device *xe = gt_to_xe(gt); 867 u32 *cs = bb->cs + bb->len; 868 u32 len = PVC_MEM_SET_CMD_LEN_DW; 869 870 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); 871 *cs++ = pitch - 1; 872 *cs++ = (size / pitch) - 1; 873 *cs++ = pitch - 1; 874 *cs++ = lower_32_bits(src_ofs); 875 *cs++ = upper_32_bits(src_ofs); 876 if (GRAPHICS_VERx100(xe) >= 2000) 877 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 878 else 879 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 880 881 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 882 883 bb->len += len; 884} 885 886static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, 887 u64 src_ofs, u32 size, u32 pitch, bool is_vram) 888{ 889 struct xe_device *xe = gt_to_xe(gt); 890 u32 *cs = bb->cs + bb->len; 891 u32 len = XY_FAST_COLOR_BLT_DW; 892 893 if (GRAPHICS_VERx100(xe) < 1250) 894 len = 11; 895 896 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | 897 (len - 2); 898 if (GRAPHICS_VERx100(xe) >= 2000) 899 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) | 900 (pitch - 1); 901 else 902 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) | 903 (pitch - 1); 904 *cs++ = 0; 905 *cs++ = (size / pitch) << 16 | pitch / 4; 906 *cs++ = lower_32_bits(src_ofs); 907 *cs++ = upper_32_bits(src_ofs); 908 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; 909 *cs++ = 0; 910 *cs++ = 0; 911 *cs++ = 0; 912 *cs++ = 0; 913 914 if (len > 11) { 915 *cs++ = 0; 916 *cs++ = 0; 917 *cs++ = 0; 918 *cs++ = 0; 919 *cs++ = 0; 920 } 921 922 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 923 924 bb->len += len; 925} 926 927static bool has_service_copy_support(struct xe_gt *gt) 928{ 929 /* 930 * What we care about is whether the architecture was designed with 931 * service copy functionality (specifically the new MEM_SET / MEM_COPY 932 * instructions) so check the architectural engine list rather than the 933 * actual list since these instructions are usable on BCS0 even if 934 * all of the actual service copy engines (BCS1-BCS8) have been fused 935 * off. 936 */ 937 return gt->info.__engine_mask & GENMASK(XE_HW_ENGINE_BCS8, 938 XE_HW_ENGINE_BCS1); 939} 940 941static u32 emit_clear_cmd_len(struct xe_gt *gt) 942{ 943 if (has_service_copy_support(gt)) 944 return PVC_MEM_SET_CMD_LEN_DW; 945 else 946 return XY_FAST_COLOR_BLT_DW; 947} 948 949static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 950 u32 size, u32 pitch, bool is_vram) 951{ 952 if (has_service_copy_support(gt)) 953 emit_clear_link_copy(gt, bb, src_ofs, size, pitch); 954 else 955 emit_clear_main_copy(gt, bb, src_ofs, size, pitch, 956 is_vram); 957} 958 959/** 960 * xe_migrate_clear() - Copy content of TTM resources. 961 * @m: The migration context. 962 * @bo: The buffer object @dst is currently bound to. 963 * @dst: The dst TTM resource to be cleared. 964 * 965 * Clear the contents of @dst to zero. On flat CCS devices, 966 * the CCS metadata is cleared to zero as well on VRAM destinations. 967 * TODO: Eliminate the @bo argument. 968 * 969 * Return: Pointer to a dma_fence representing the last clear batch, or 970 * an error pointer on failure. If there is a failure, any clear operation 971 * started by the function call has been synced. 972 */ 973struct dma_fence *xe_migrate_clear(struct xe_migrate *m, 974 struct xe_bo *bo, 975 struct ttm_resource *dst) 976{ 977 bool clear_vram = mem_type_is_vram(dst->mem_type); 978 struct xe_gt *gt = m->tile->primary_gt; 979 struct xe_device *xe = gt_to_xe(gt); 980 bool clear_system_ccs = (xe_bo_needs_ccs_pages(bo) && !IS_DGFX(xe)) ? true : false; 981 struct dma_fence *fence = NULL; 982 u64 size = bo->size; 983 struct xe_res_cursor src_it; 984 struct ttm_resource *src = dst; 985 int err; 986 987 if (!clear_vram) 988 xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it); 989 else 990 xe_res_first(src, 0, bo->size, &src_it); 991 992 while (size) { 993 u64 clear_L0_ofs; 994 u32 clear_L0_pt; 995 u32 flush_flags = 0; 996 u64 clear_L0; 997 struct xe_sched_job *job; 998 struct xe_bb *bb; 999 u32 batch_size, update_idx; 1000 1001 bool usm = xe->info.has_usm; 1002 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1003 1004 clear_L0 = xe_migrate_res_sizes(m, &src_it); 1005 1006 /* Calculate final sizes and batch size.. */ 1007 batch_size = 2 + 1008 pte_update_size(m, clear_vram, src, &src_it, 1009 &clear_L0, &clear_L0_ofs, &clear_L0_pt, 1010 clear_system_ccs ? 0 : emit_clear_cmd_len(gt), 0, 1011 avail_pts); 1012 1013 if (xe_device_has_flat_ccs(xe)) 1014 batch_size += EMIT_COPY_CCS_DW; 1015 1016 /* Clear commands */ 1017 1018 if (WARN_ON_ONCE(!clear_L0)) 1019 break; 1020 1021 bb = xe_bb_new(gt, batch_size, usm); 1022 if (IS_ERR(bb)) { 1023 err = PTR_ERR(bb); 1024 goto err_sync; 1025 } 1026 1027 size -= clear_L0; 1028 /* Preemption is enabled again by the ring ops. */ 1029 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) 1030 xe_res_next(&src_it, clear_L0); 1031 else 1032 emit_pte(m, bb, clear_L0_pt, clear_vram, clear_system_ccs, 1033 &src_it, clear_L0, dst); 1034 1035 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1036 update_idx = bb->len; 1037 1038 if (!clear_system_ccs) 1039 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram); 1040 1041 if (xe_device_has_flat_ccs(xe)) { 1042 emit_copy_ccs(gt, bb, clear_L0_ofs, true, 1043 m->cleared_mem_ofs, false, clear_L0); 1044 flush_flags = MI_FLUSH_DW_CCS; 1045 } 1046 1047 mutex_lock(&m->job_mutex); 1048 job = xe_bb_create_migration_job(m->q, bb, 1049 xe_migrate_batch_base(m, usm), 1050 update_idx); 1051 if (IS_ERR(job)) { 1052 err = PTR_ERR(job); 1053 goto err; 1054 } 1055 1056 xe_sched_job_add_migrate_flush(job, flush_flags); 1057 if (!fence) { 1058 /* 1059 * There can't be anything userspace related at this 1060 * point, so we just need to respect any potential move 1061 * fences, which are always tracked as 1062 * DMA_RESV_USAGE_KERNEL. 1063 */ 1064 err = job_add_deps(job, bo->ttm.base.resv, 1065 DMA_RESV_USAGE_KERNEL); 1066 if (err) 1067 goto err_job; 1068 } 1069 1070 xe_sched_job_arm(job); 1071 dma_fence_put(fence); 1072 fence = dma_fence_get(&job->drm.s_fence->finished); 1073 xe_sched_job_push(job); 1074 1075 dma_fence_put(m->fence); 1076 m->fence = dma_fence_get(fence); 1077 1078 mutex_unlock(&m->job_mutex); 1079 1080 xe_bb_free(bb, fence); 1081 continue; 1082 1083err_job: 1084 xe_sched_job_put(job); 1085err: 1086 mutex_unlock(&m->job_mutex); 1087 xe_bb_free(bb, NULL); 1088err_sync: 1089 /* Sync partial copies if any. FIXME: job_mutex? */ 1090 if (fence) { 1091 dma_fence_wait(m->fence, false); 1092 dma_fence_put(fence); 1093 } 1094 1095 return ERR_PTR(err); 1096 } 1097 1098 if (clear_system_ccs) 1099 bo->ccs_cleared = true; 1100 1101 return fence; 1102} 1103 1104static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, 1105 const struct xe_vm_pgtable_update *update, 1106 struct xe_migrate_pt_update *pt_update) 1107{ 1108 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1109 u32 chunk; 1110 u32 ofs = update->ofs, size = update->qwords; 1111 1112 /* 1113 * If we have 512 entries (max), we would populate it ourselves, 1114 * and update the PDE above it to the new pointer. 1115 * The only time this can only happen if we have to update the top 1116 * PDE. This requires a BO that is almost vm->size big. 1117 * 1118 * This shouldn't be possible in practice.. might change when 16K 1119 * pages are used. Hence the assert. 1120 */ 1121 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); 1122 if (!ppgtt_ofs) 1123 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), 1124 xe_bo_addr(update->pt_bo, 0, 1125 XE_PAGE_SIZE)); 1126 1127 do { 1128 u64 addr = ppgtt_ofs + ofs * 8; 1129 1130 chunk = min(size, MAX_PTE_PER_SDI); 1131 1132 /* Ensure populatefn can do memset64 by aligning bb->cs */ 1133 if (!(bb->len & 1)) 1134 bb->cs[bb->len++] = MI_NOOP; 1135 1136 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 1137 bb->cs[bb->len++] = lower_32_bits(addr); 1138 bb->cs[bb->len++] = upper_32_bits(addr); 1139 ops->populate(pt_update, tile, NULL, bb->cs + bb->len, ofs, chunk, 1140 update); 1141 1142 bb->len += chunk * 2; 1143 ofs += chunk; 1144 size -= chunk; 1145 } while (size); 1146} 1147 1148struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) 1149{ 1150 return xe_vm_get(m->q->vm); 1151} 1152 1153#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1154struct migrate_test_params { 1155 struct xe_test_priv base; 1156 bool force_gpu; 1157}; 1158 1159#define to_migrate_test_params(_priv) \ 1160 container_of(_priv, struct migrate_test_params, base) 1161#endif 1162 1163static struct dma_fence * 1164xe_migrate_update_pgtables_cpu(struct xe_migrate *m, 1165 struct xe_vm *vm, struct xe_bo *bo, 1166 const struct xe_vm_pgtable_update *updates, 1167 u32 num_updates, bool wait_vm, 1168 struct xe_migrate_pt_update *pt_update) 1169{ 1170 XE_TEST_DECLARE(struct migrate_test_params *test = 1171 to_migrate_test_params 1172 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) 1173 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1174 struct dma_fence *fence; 1175 int err; 1176 u32 i; 1177 1178 if (XE_TEST_ONLY(test && test->force_gpu)) 1179 return ERR_PTR(-ETIME); 1180 1181 if (bo && !dma_resv_test_signaled(bo->ttm.base.resv, 1182 DMA_RESV_USAGE_KERNEL)) 1183 return ERR_PTR(-ETIME); 1184 1185 if (wait_vm && !dma_resv_test_signaled(xe_vm_resv(vm), 1186 DMA_RESV_USAGE_BOOKKEEP)) 1187 return ERR_PTR(-ETIME); 1188 1189 if (ops->pre_commit) { 1190 pt_update->job = NULL; 1191 err = ops->pre_commit(pt_update); 1192 if (err) 1193 return ERR_PTR(err); 1194 } 1195 for (i = 0; i < num_updates; i++) { 1196 const struct xe_vm_pgtable_update *update = &updates[i]; 1197 1198 ops->populate(pt_update, m->tile, &update->pt_bo->vmap, NULL, 1199 update->ofs, update->qwords, update); 1200 } 1201 1202 if (vm) { 1203 trace_xe_vm_cpu_bind(vm); 1204 xe_device_wmb(vm->xe); 1205 } 1206 1207 fence = dma_fence_get_stub(); 1208 1209 return fence; 1210} 1211 1212static bool no_in_syncs(struct xe_vm *vm, struct xe_exec_queue *q, 1213 struct xe_sync_entry *syncs, u32 num_syncs) 1214{ 1215 struct dma_fence *fence; 1216 int i; 1217 1218 for (i = 0; i < num_syncs; i++) { 1219 fence = syncs[i].fence; 1220 1221 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 1222 &fence->flags)) 1223 return false; 1224 } 1225 if (q) { 1226 fence = xe_exec_queue_last_fence_get(q, vm); 1227 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { 1228 dma_fence_put(fence); 1229 return false; 1230 } 1231 dma_fence_put(fence); 1232 } 1233 1234 return true; 1235} 1236 1237/** 1238 * xe_migrate_update_pgtables() - Pipelined page-table update 1239 * @m: The migrate context. 1240 * @vm: The vm we'll be updating. 1241 * @bo: The bo whose dma-resv we will await before updating, or NULL if userptr. 1242 * @q: The exec queue to be used for the update or NULL if the default 1243 * migration engine is to be used. 1244 * @updates: An array of update descriptors. 1245 * @num_updates: Number of descriptors in @updates. 1246 * @syncs: Array of xe_sync_entry to await before updating. Note that waits 1247 * will block the engine timeline. 1248 * @num_syncs: Number of entries in @syncs. 1249 * @pt_update: Pointer to a struct xe_migrate_pt_update, which contains 1250 * pointers to callback functions and, if subclassed, private arguments to 1251 * those. 1252 * 1253 * Perform a pipelined page-table update. The update descriptors are typically 1254 * built under the same lock critical section as a call to this function. If 1255 * using the default engine for the updates, they will be performed in the 1256 * order they grab the job_mutex. If different engines are used, external 1257 * synchronization is needed for overlapping updates to maintain page-table 1258 * consistency. Note that the meaing of "overlapping" is that the updates 1259 * touch the same page-table, which might be a higher-level page-directory. 1260 * If no pipelining is needed, then updates may be performed by the cpu. 1261 * 1262 * Return: A dma_fence that, when signaled, indicates the update completion. 1263 */ 1264struct dma_fence * 1265xe_migrate_update_pgtables(struct xe_migrate *m, 1266 struct xe_vm *vm, 1267 struct xe_bo *bo, 1268 struct xe_exec_queue *q, 1269 const struct xe_vm_pgtable_update *updates, 1270 u32 num_updates, 1271 struct xe_sync_entry *syncs, u32 num_syncs, 1272 struct xe_migrate_pt_update *pt_update) 1273{ 1274 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1275 struct xe_tile *tile = m->tile; 1276 struct xe_gt *gt = tile->primary_gt; 1277 struct xe_device *xe = tile_to_xe(tile); 1278 struct xe_sched_job *job; 1279 struct dma_fence *fence; 1280 struct drm_suballoc *sa_bo = NULL; 1281 struct xe_vma *vma = pt_update->vma; 1282 struct xe_bb *bb; 1283 u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0; 1284 u64 addr; 1285 int err = 0; 1286 bool usm = !q && xe->info.has_usm; 1287 bool first_munmap_rebind = vma && 1288 vma->gpuva.flags & XE_VMA_FIRST_REBIND; 1289 struct xe_exec_queue *q_override = !q ? m->q : q; 1290 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 1291 1292 /* Use the CPU if no in syncs and engine is idle */ 1293 if (no_in_syncs(vm, q, syncs, num_syncs) && xe_exec_queue_is_idle(q_override)) { 1294 fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates, 1295 num_updates, 1296 first_munmap_rebind, 1297 pt_update); 1298 if (!IS_ERR(fence) || fence == ERR_PTR(-EAGAIN)) 1299 return fence; 1300 } 1301 1302 /* fixed + PTE entries */ 1303 if (IS_DGFX(xe)) 1304 batch_size = 2; 1305 else 1306 batch_size = 6 + num_updates * 2; 1307 1308 for (i = 0; i < num_updates; i++) { 1309 u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, MAX_PTE_PER_SDI); 1310 1311 /* align noop + MI_STORE_DATA_IMM cmd prefix */ 1312 batch_size += 4 * num_cmds + updates[i].qwords * 2; 1313 } 1314 1315 /* 1316 * XXX: Create temp bo to copy from, if batch_size becomes too big? 1317 * 1318 * Worst case: Sum(2 * (each lower level page size) + (top level page size)) 1319 * Should be reasonably bound.. 1320 */ 1321 xe_tile_assert(tile, batch_size < SZ_128K); 1322 1323 bb = xe_bb_new(gt, batch_size, !q && xe->info.has_usm); 1324 if (IS_ERR(bb)) 1325 return ERR_CAST(bb); 1326 1327 /* For sysmem PTE's, need to map them in our hole.. */ 1328 if (!IS_DGFX(xe)) { 1329 ppgtt_ofs = NUM_KERNEL_PDE - 1; 1330 if (q) { 1331 xe_tile_assert(tile, num_updates <= NUM_VMUSA_WRITES_PER_UNIT); 1332 1333 sa_bo = drm_suballoc_new(&m->vm_update_sa, 1, 1334 GFP_KERNEL, true, 0); 1335 if (IS_ERR(sa_bo)) { 1336 err = PTR_ERR(sa_bo); 1337 goto err; 1338 } 1339 1340 ppgtt_ofs = NUM_KERNEL_PDE + 1341 (drm_suballoc_soffset(sa_bo) / 1342 NUM_VMUSA_UNIT_PER_PAGE); 1343 page_ofs = (drm_suballoc_soffset(sa_bo) % 1344 NUM_VMUSA_UNIT_PER_PAGE) * 1345 VM_SA_UPDATE_UNIT_SIZE; 1346 } 1347 1348 /* Map our PT's to gtt */ 1349 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(num_updates); 1350 bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; 1351 bb->cs[bb->len++] = 0; /* upper_32_bits */ 1352 1353 for (i = 0; i < num_updates; i++) { 1354 struct xe_bo *pt_bo = updates[i].pt_bo; 1355 1356 xe_tile_assert(tile, pt_bo->size == SZ_4K); 1357 1358 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, pat_index, 0); 1359 bb->cs[bb->len++] = lower_32_bits(addr); 1360 bb->cs[bb->len++] = upper_32_bits(addr); 1361 } 1362 1363 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1364 update_idx = bb->len; 1365 1366 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + 1367 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; 1368 for (i = 0; i < num_updates; i++) 1369 write_pgtable(tile, bb, addr + i * XE_PAGE_SIZE, 1370 &updates[i], pt_update); 1371 } else { 1372 /* phys pages, no preamble required */ 1373 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1374 update_idx = bb->len; 1375 1376 for (i = 0; i < num_updates; i++) 1377 write_pgtable(tile, bb, 0, &updates[i], pt_update); 1378 } 1379 1380 if (!q) 1381 mutex_lock(&m->job_mutex); 1382 1383 job = xe_bb_create_migration_job(q ?: m->q, bb, 1384 xe_migrate_batch_base(m, usm), 1385 update_idx); 1386 if (IS_ERR(job)) { 1387 err = PTR_ERR(job); 1388 goto err_bb; 1389 } 1390 1391 /* Wait on BO move */ 1392 if (bo) { 1393 err = job_add_deps(job, bo->ttm.base.resv, 1394 DMA_RESV_USAGE_KERNEL); 1395 if (err) 1396 goto err_job; 1397 } 1398 1399 /* 1400 * Munmap style VM unbind, need to wait for all jobs to be complete / 1401 * trigger preempts before moving forward 1402 */ 1403 if (first_munmap_rebind) { 1404 err = job_add_deps(job, xe_vm_resv(vm), 1405 DMA_RESV_USAGE_BOOKKEEP); 1406 if (err) 1407 goto err_job; 1408 } 1409 1410 err = xe_sched_job_last_fence_add_dep(job, vm); 1411 for (i = 0; !err && i < num_syncs; i++) 1412 err = xe_sync_entry_add_deps(&syncs[i], job); 1413 1414 if (err) 1415 goto err_job; 1416 1417 if (ops->pre_commit) { 1418 pt_update->job = job; 1419 err = ops->pre_commit(pt_update); 1420 if (err) 1421 goto err_job; 1422 } 1423 xe_sched_job_arm(job); 1424 fence = dma_fence_get(&job->drm.s_fence->finished); 1425 xe_sched_job_push(job); 1426 1427 if (!q) 1428 mutex_unlock(&m->job_mutex); 1429 1430 xe_bb_free(bb, fence); 1431 drm_suballoc_free(sa_bo, fence); 1432 1433 return fence; 1434 1435err_job: 1436 xe_sched_job_put(job); 1437err_bb: 1438 if (!q) 1439 mutex_unlock(&m->job_mutex); 1440 xe_bb_free(bb, NULL); 1441err: 1442 drm_suballoc_free(sa_bo, NULL); 1443 return ERR_PTR(err); 1444} 1445 1446/** 1447 * xe_migrate_wait() - Complete all operations using the xe_migrate context 1448 * @m: Migrate context to wait for. 1449 * 1450 * Waits until the GPU no longer uses the migrate context's default engine 1451 * or its page-table objects. FIXME: What about separate page-table update 1452 * engines? 1453 */ 1454void xe_migrate_wait(struct xe_migrate *m) 1455{ 1456 if (m->fence) 1457 dma_fence_wait(m->fence, false); 1458} 1459 1460#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1461#include "tests/xe_migrate.c" 1462#endif 1463