1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2022 Intel Corporation 4 */ 5 6#include "xe_ring_ops.h" 7 8#include <generated/xe_wa_oob.h> 9 10#include "instructions/xe_mi_commands.h" 11#include "regs/xe_engine_regs.h" 12#include "regs/xe_gpu_commands.h" 13#include "regs/xe_gt_regs.h" 14#include "regs/xe_lrc_layout.h" 15#include "xe_exec_queue_types.h" 16#include "xe_gt.h" 17#include "xe_lrc.h" 18#include "xe_macros.h" 19#include "xe_sched_job.h" 20#include "xe_vm_types.h" 21#include "xe_vm.h" 22#include "xe_wa.h" 23 24/* 25 * 3D-related flags that can't be set on _engines_ that lack access to the 3D 26 * pipeline (i.e., CCS engines). 27 */ 28#define PIPE_CONTROL_3D_ENGINE_FLAGS (\ 29 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ 30 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ 31 PIPE_CONTROL_TILE_CACHE_FLUSH | \ 32 PIPE_CONTROL_DEPTH_STALL | \ 33 PIPE_CONTROL_STALL_AT_SCOREBOARD | \ 34 PIPE_CONTROL_PSD_SYNC | \ 35 PIPE_CONTROL_AMFS_FLUSH | \ 36 PIPE_CONTROL_VF_CACHE_INVALIDATE | \ 37 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) 38 39/* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ 40#define PIPE_CONTROL_3D_ARCH_FLAGS ( \ 41 PIPE_CONTROL_3D_ENGINE_FLAGS | \ 42 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ 43 PIPE_CONTROL_FLUSH_ENABLE | \ 44 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ 45 PIPE_CONTROL_DC_FLUSH_ENABLE) 46 47static u32 preparser_disable(bool state) 48{ 49 return MI_ARB_CHECK | BIT(8) | state; 50} 51 52static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg, 53 u32 *dw, int i) 54{ 55 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN; 56 dw[i++] = reg.addr + gt->mmio.adj_offset; 57 dw[i++] = AUX_INV; 58 dw[i++] = MI_NOOP; 59 60 return i; 61} 62 63static int emit_user_interrupt(u32 *dw, int i) 64{ 65 dw[i++] = MI_USER_INTERRUPT; 66 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; 67 dw[i++] = MI_ARB_CHECK; 68 69 return i; 70} 71 72static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) 73{ 74 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 75 dw[i++] = addr; 76 dw[i++] = 0; 77 dw[i++] = value; 78 79 return i; 80} 81 82static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb, 83 u32 *dw, int i) 84{ 85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 86 (invalidate_tlb ? MI_INVALIDATE_TLB : 0); 87 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 88 dw[i++] = 0; 89 dw[i++] = value; 90 91 return i; 92} 93 94static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) 95{ 96 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3); 97 dw[i++] = lower_32_bits(batch_addr); 98 dw[i++] = upper_32_bits(batch_addr); 99 100 return i; 101} 102 103static int emit_flush_invalidate(u32 flag, u32 *dw, int i) 104{ 105 dw[i] = MI_FLUSH_DW; 106 dw[i] |= flag; 107 dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 108 MI_FLUSH_DW_STORE_INDEX; 109 110 dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 111 dw[i++] = 0; 112 dw[i++] = ~0U; 113 114 return i; 115} 116 117static int 118emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value) 119{ 120 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0; 121 dw[i++] = bit_group_1; 122 dw[i++] = offset; 123 dw[i++] = 0; 124 dw[i++] = value; 125 dw[i++] = 0; 126 127 return i; 128} 129 130static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw, 131 int i) 132{ 133 u32 flags = PIPE_CONTROL_CS_STALL | 134 PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | 135 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | 136 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | 137 PIPE_CONTROL_VF_CACHE_INVALIDATE | 138 PIPE_CONTROL_CONST_CACHE_INVALIDATE | 139 PIPE_CONTROL_STATE_CACHE_INVALIDATE | 140 PIPE_CONTROL_QW_WRITE | 141 PIPE_CONTROL_STORE_DATA_INDEX; 142 143 if (invalidate_tlb) 144 flags |= PIPE_CONTROL_TLB_INVALIDATE; 145 146 flags &= ~mask_flags; 147 148 return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0); 149} 150 151static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, 152 u32 *dw, int i) 153{ 154 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1); 155 dw[i++] = lower_32_bits(addr); 156 dw[i++] = upper_32_bits(addr); 157 dw[i++] = lower_32_bits(value); 158 dw[i++] = upper_32_bits(value); 159 160 return i; 161} 162 163static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) 164{ 165 struct xe_gt *gt = job->q->gt; 166 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 167 u32 flags; 168 169 flags = (PIPE_CONTROL_CS_STALL | 170 PIPE_CONTROL_TILE_CACHE_FLUSH | 171 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 172 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 173 PIPE_CONTROL_DC_FLUSH_ENABLE | 174 PIPE_CONTROL_FLUSH_ENABLE); 175 176 if (XE_WA(gt, 1409600907)) 177 flags |= PIPE_CONTROL_DEPTH_STALL; 178 179 if (lacks_render) 180 flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS; 181 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 182 flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; 183 184 return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0); 185} 186 187static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i) 188{ 189 if (hwe->class != XE_ENGINE_CLASS_RENDER) 190 return i; 191 192 if (XE_WA(hwe->gt, 16020292621)) 193 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC, 194 RING_NOPID(hwe->mmio_base).addr, 0); 195 196 return i; 197} 198 199static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw, 200 int i) 201{ 202 u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB | 203 PIPE_CONTROL_QW_WRITE; 204 205 if (!stall_only) 206 flags |= PIPE_CONTROL_FLUSH_ENABLE; 207 208 return emit_pipe_control(dw, i, 0, flags, addr, value); 209} 210 211static u32 get_ppgtt_flag(struct xe_sched_job *job) 212{ 213 return job->q->vm ? BIT(8) : 0; 214} 215 216/* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */ 217static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc, 218 u64 batch_addr, u32 seqno) 219{ 220 u32 dw[MAX_JOB_SIZE_DW], i = 0; 221 u32 ppgtt_flag = get_ppgtt_flag(job); 222 struct xe_gt *gt = job->q->gt; 223 224 if (job->ring_ops_flush_tlb) { 225 dw[i++] = preparser_disable(true); 226 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 227 seqno, true, dw, i); 228 dw[i++] = preparser_disable(false); 229 } else { 230 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 231 seqno, dw, i); 232 } 233 234 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 235 236 if (job->user_fence.used) 237 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 238 job->user_fence.value, 239 dw, i); 240 241 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i); 242 243 i = emit_user_interrupt(dw, i); 244 245 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 246 247 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 248} 249 250static bool has_aux_ccs(struct xe_device *xe) 251{ 252 /* 253 * PVC is a special case that has no compression of either type 254 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2 255 * onward, so any future platforms with no FlatCCS will not have 256 * AuxCCS either. 257 */ 258 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) 259 return false; 260 261 return !xe->info.has_flat_ccs; 262} 263 264static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, 265 u64 batch_addr, u32 seqno) 266{ 267 u32 dw[MAX_JOB_SIZE_DW], i = 0; 268 u32 ppgtt_flag = get_ppgtt_flag(job); 269 struct xe_gt *gt = job->q->gt; 270 struct xe_device *xe = gt_to_xe(gt); 271 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE; 272 273 dw[i++] = preparser_disable(true); 274 275 /* hsdes: 1809175790 */ 276 if (has_aux_ccs(xe)) { 277 if (decode) 278 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i); 279 else 280 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i); 281 } 282 283 if (job->ring_ops_flush_tlb) 284 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 285 seqno, true, dw, i); 286 287 dw[i++] = preparser_disable(false); 288 289 if (!job->ring_ops_flush_tlb) 290 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 291 seqno, dw, i); 292 293 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 294 295 if (job->user_fence.used) 296 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 297 job->user_fence.value, 298 dw, i); 299 300 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i); 301 302 i = emit_user_interrupt(dw, i); 303 304 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 305 306 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 307} 308 309static void __emit_job_gen12_render_compute(struct xe_sched_job *job, 310 struct xe_lrc *lrc, 311 u64 batch_addr, u32 seqno) 312{ 313 u32 dw[MAX_JOB_SIZE_DW], i = 0; 314 u32 ppgtt_flag = get_ppgtt_flag(job); 315 struct xe_gt *gt = job->q->gt; 316 struct xe_device *xe = gt_to_xe(gt); 317 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 318 u32 mask_flags = 0; 319 320 dw[i++] = preparser_disable(true); 321 if (lacks_render) 322 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; 323 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 324 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; 325 326 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ 327 i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i); 328 329 /* hsdes: 1809175790 */ 330 if (has_aux_ccs(xe)) 331 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i); 332 333 dw[i++] = preparser_disable(false); 334 335 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 336 seqno, dw, i); 337 338 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 339 340 i = emit_render_cache_flush(job, dw, i); 341 342 if (job->user_fence.used) 343 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 344 job->user_fence.value, 345 dw, i); 346 347 i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); 348 349 i = emit_user_interrupt(dw, i); 350 351 i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i); 352 353 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 354 355 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 356} 357 358static void emit_migration_job_gen12(struct xe_sched_job *job, 359 struct xe_lrc *lrc, u32 seqno) 360{ 361 u32 dw[MAX_JOB_SIZE_DW], i = 0; 362 363 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 364 seqno, dw, i); 365 366 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ 367 368 i = emit_bb_start(job->batch_addr[0], BIT(8), dw, i); 369 370 /* XXX: Do we need this? Leaving for now. */ 371 dw[i++] = preparser_disable(true); 372 i = emit_flush_invalidate(0, dw, i); 373 dw[i++] = preparser_disable(false); 374 375 i = emit_bb_start(job->batch_addr[1], BIT(8), dw, i); 376 377 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags | 378 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW; 379 dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT; 380 dw[i++] = 0; 381 dw[i++] = seqno; /* value */ 382 383 i = emit_user_interrupt(dw, i); 384 385 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); 386 387 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 388} 389 390static void emit_job_gen12_gsc(struct xe_sched_job *job) 391{ 392 struct xe_gt *gt = job->q->gt; 393 394 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */ 395 396 __emit_job_gen12_simple(job, job->q->lrc, 397 job->batch_addr[0], 398 xe_sched_job_seqno(job)); 399} 400 401static void emit_job_gen12_copy(struct xe_sched_job *job) 402{ 403 int i; 404 405 if (xe_sched_job_is_migration(job->q)) { 406 emit_migration_job_gen12(job, job->q->lrc, 407 xe_sched_job_seqno(job)); 408 return; 409 } 410 411 for (i = 0; i < job->q->width; ++i) 412 __emit_job_gen12_simple(job, job->q->lrc + i, 413 job->batch_addr[i], 414 xe_sched_job_seqno(job)); 415} 416 417static void emit_job_gen12_video(struct xe_sched_job *job) 418{ 419 int i; 420 421 /* FIXME: Not doing parallel handshake for now */ 422 for (i = 0; i < job->q->width; ++i) 423 __emit_job_gen12_video(job, job->q->lrc + i, 424 job->batch_addr[i], 425 xe_sched_job_seqno(job)); 426} 427 428static void emit_job_gen12_render_compute(struct xe_sched_job *job) 429{ 430 int i; 431 432 for (i = 0; i < job->q->width; ++i) 433 __emit_job_gen12_render_compute(job, job->q->lrc + i, 434 job->batch_addr[i], 435 xe_sched_job_seqno(job)); 436} 437 438static const struct xe_ring_ops ring_ops_gen12_gsc = { 439 .emit_job = emit_job_gen12_gsc, 440}; 441 442static const struct xe_ring_ops ring_ops_gen12_copy = { 443 .emit_job = emit_job_gen12_copy, 444}; 445 446static const struct xe_ring_ops ring_ops_gen12_video = { 447 .emit_job = emit_job_gen12_video, 448}; 449 450static const struct xe_ring_ops ring_ops_gen12_render_compute = { 451 .emit_job = emit_job_gen12_render_compute, 452}; 453 454const struct xe_ring_ops * 455xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) 456{ 457 switch (class) { 458 case XE_ENGINE_CLASS_OTHER: 459 return &ring_ops_gen12_gsc; 460 case XE_ENGINE_CLASS_COPY: 461 return &ring_ops_gen12_copy; 462 case XE_ENGINE_CLASS_VIDEO_DECODE: 463 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 464 return &ring_ops_gen12_video; 465 case XE_ENGINE_CLASS_RENDER: 466 case XE_ENGINE_CLASS_COMPUTE: 467 return &ring_ops_gen12_render_compute; 468 default: 469 return NULL; 470 } 471} 472