1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2019 Intel Corporation 4 * 5 */ 6 7#include "i915_drv.h" 8#include "i915_irq.h" 9#include "i915_reg.h" 10#include "intel_crtc.h" 11#include "intel_de.h" 12#include "intel_display_types.h" 13#include "intel_dsb.h" 14#include "intel_dsb_buffer.h" 15#include "intel_dsb_regs.h" 16#include "intel_vblank.h" 17#include "intel_vrr.h" 18#include "skl_watermark.h" 19 20#define CACHELINE_BYTES 64 21 22enum dsb_id { 23 INVALID_DSB = -1, 24 DSB1, 25 DSB2, 26 DSB3, 27 MAX_DSB_PER_PIPE 28}; 29 30struct intel_dsb { 31 enum dsb_id id; 32 33 struct intel_dsb_buffer dsb_buf; 34 struct intel_crtc *crtc; 35 36 /* 37 * maximum number of dwords the buffer will hold. 38 */ 39 unsigned int size; 40 41 /* 42 * free_pos will point the first free dword and 43 * help in calculating tail of command buffer. 44 */ 45 unsigned int free_pos; 46 47 /* 48 * ins_start_offset will help to store start dword of the dsb 49 * instuction and help in identifying the batch of auto-increment 50 * register. 51 */ 52 unsigned int ins_start_offset; 53 54 int dewake_scanline; 55}; 56 57/** 58 * DOC: DSB 59 * 60 * A DSB (Display State Buffer) is a queue of MMIO instructions in the memory 61 * which can be offloaded to DSB HW in Display Controller. DSB HW is a DMA 62 * engine that can be programmed to download the DSB from memory. 63 * It allows driver to batch submit display HW programming. This helps to 64 * reduce loading time and CPU activity, thereby making the context switch 65 * faster. DSB Support added from Gen12 Intel graphics based platform. 66 * 67 * DSB's can access only the pipe, plane, and transcoder Data Island Packet 68 * registers. 69 * 70 * DSB HW can support only register writes (both indexed and direct MMIO 71 * writes). There are no registers reads possible with DSB HW engine. 72 */ 73 74/* DSB opcodes. */ 75#define DSB_OPCODE_SHIFT 24 76#define DSB_OPCODE_NOOP 0x0 77#define DSB_OPCODE_MMIO_WRITE 0x1 78#define DSB_BYTE_EN 0xf 79#define DSB_BYTE_EN_SHIFT 20 80#define DSB_REG_VALUE_MASK 0xfffff 81#define DSB_OPCODE_WAIT_USEC 0x2 82#define DSB_OPCODE_WAIT_SCANLINE 0x3 83#define DSB_OPCODE_WAIT_VBLANKS 0x4 84#define DSB_OPCODE_WAIT_DSL_IN 0x5 85#define DSB_OPCODE_WAIT_DSL_OUT 0x6 86#define DSB_SCANLINE_UPPER_SHIFT 20 87#define DSB_SCANLINE_LOWER_SHIFT 0 88#define DSB_OPCODE_INTERRUPT 0x7 89#define DSB_OPCODE_INDEXED_WRITE 0x9 90/* see DSB_REG_VALUE_MASK */ 91#define DSB_OPCODE_POLL 0xA 92/* see DSB_REG_VALUE_MASK */ 93 94static bool assert_dsb_has_room(struct intel_dsb *dsb) 95{ 96 struct intel_crtc *crtc = dsb->crtc; 97 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 98 99 /* each instruction is 2 dwords */ 100 return !drm_WARN(&i915->drm, dsb->free_pos > dsb->size - 2, 101 "[CRTC:%d:%s] DSB %d buffer overflow\n", 102 crtc->base.base.id, crtc->base.name, dsb->id); 103} 104 105static void intel_dsb_dump(struct intel_dsb *dsb) 106{ 107 struct intel_crtc *crtc = dsb->crtc; 108 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 109 int i; 110 111 drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] DSB %d commands {\n", 112 crtc->base.base.id, crtc->base.name, dsb->id); 113 for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4) 114 drm_dbg_kms(&i915->drm, 115 " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4, 116 intel_dsb_buffer_read(&dsb->dsb_buf, i), 117 intel_dsb_buffer_read(&dsb->dsb_buf, i + 1), 118 intel_dsb_buffer_read(&dsb->dsb_buf, i + 2), 119 intel_dsb_buffer_read(&dsb->dsb_buf, i + 3)); 120 drm_dbg_kms(&i915->drm, "}\n"); 121} 122 123static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe, 124 enum dsb_id id) 125{ 126 return intel_de_read_fw(i915, DSB_CTRL(pipe, id)) & DSB_STATUS_BUSY; 127} 128 129static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw) 130{ 131 if (!assert_dsb_has_room(dsb)) 132 return; 133 134 /* Every instruction should be 8 byte aligned. */ 135 dsb->free_pos = ALIGN(dsb->free_pos, 2); 136 137 dsb->ins_start_offset = dsb->free_pos; 138 139 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, ldw); 140 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, udw); 141} 142 143static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb, 144 u32 opcode, i915_reg_t reg) 145{ 146 u32 prev_opcode, prev_reg; 147 148 /* 149 * Nothing emitted yet? Must check before looking 150 * at the actual data since i915_gem_object_create_internal() 151 * does *not* give you zeroed memory! 152 */ 153 if (dsb->free_pos == 0) 154 return false; 155 156 prev_opcode = intel_dsb_buffer_read(&dsb->dsb_buf, 157 dsb->ins_start_offset + 1) & ~DSB_REG_VALUE_MASK; 158 prev_reg = intel_dsb_buffer_read(&dsb->dsb_buf, 159 dsb->ins_start_offset + 1) & DSB_REG_VALUE_MASK; 160 161 return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg); 162} 163 164static bool intel_dsb_prev_ins_is_mmio_write(struct intel_dsb *dsb, i915_reg_t reg) 165{ 166 /* only full byte-enables can be converted to indexed writes */ 167 return intel_dsb_prev_ins_is_write(dsb, 168 DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT | 169 DSB_BYTE_EN << DSB_BYTE_EN_SHIFT, 170 reg); 171} 172 173static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg) 174{ 175 return intel_dsb_prev_ins_is_write(dsb, 176 DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT, 177 reg); 178} 179 180/** 181 * intel_dsb_reg_write() - Emit register wriite to the DSB context 182 * @dsb: DSB context 183 * @reg: register address. 184 * @val: value. 185 * 186 * This function is used for writing register-value pair in command 187 * buffer of DSB. 188 */ 189void intel_dsb_reg_write(struct intel_dsb *dsb, 190 i915_reg_t reg, u32 val) 191{ 192 u32 old_val; 193 194 /* 195 * For example the buffer will look like below for 3 dwords for auto 196 * increment register: 197 * +--------------------------------------------------------+ 198 * | size = 3 | offset &| value1 | value2 | value3 | zero | 199 * | | opcode | | | | | 200 * +--------------------------------------------------------+ 201 * + + + + + + + 202 * 0 4 8 12 16 20 24 203 * Byte 204 * 205 * As every instruction is 8 byte aligned the index of dsb instruction 206 * will start always from even number while dealing with u32 array. If 207 * we are writing odd no of dwords, Zeros will be added in the end for 208 * padding. 209 */ 210 if (!intel_dsb_prev_ins_is_mmio_write(dsb, reg) && 211 !intel_dsb_prev_ins_is_indexed_write(dsb, reg)) { 212 intel_dsb_emit(dsb, val, 213 (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) | 214 (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) | 215 i915_mmio_reg_offset(reg)); 216 } else { 217 if (!assert_dsb_has_room(dsb)) 218 return; 219 220 /* convert to indexed write? */ 221 if (intel_dsb_prev_ins_is_mmio_write(dsb, reg)) { 222 u32 prev_val = intel_dsb_buffer_read(&dsb->dsb_buf, 223 dsb->ins_start_offset + 0); 224 225 intel_dsb_buffer_write(&dsb->dsb_buf, 226 dsb->ins_start_offset + 0, 1); /* count */ 227 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 1, 228 (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) | 229 i915_mmio_reg_offset(reg)); 230 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 2, prev_val); 231 232 dsb->free_pos++; 233 } 234 235 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, val); 236 /* Update the count */ 237 old_val = intel_dsb_buffer_read(&dsb->dsb_buf, dsb->ins_start_offset); 238 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset, old_val + 1); 239 240 /* if number of data words is odd, then the last dword should be 0.*/ 241 if (dsb->free_pos & 0x1) 242 intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos, 0); 243 } 244} 245 246static u32 intel_dsb_mask_to_byte_en(u32 mask) 247{ 248 return (!!(mask & 0xff000000) << 3 | 249 !!(mask & 0x00ff0000) << 2 | 250 !!(mask & 0x0000ff00) << 1 | 251 !!(mask & 0x000000ff) << 0); 252} 253 254/* Note: mask implemented via byte enables! */ 255void intel_dsb_reg_write_masked(struct intel_dsb *dsb, 256 i915_reg_t reg, u32 mask, u32 val) 257{ 258 intel_dsb_emit(dsb, val, 259 (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) | 260 (intel_dsb_mask_to_byte_en(mask) << DSB_BYTE_EN_SHIFT) | 261 i915_mmio_reg_offset(reg)); 262} 263 264void intel_dsb_noop(struct intel_dsb *dsb, int count) 265{ 266 int i; 267 268 for (i = 0; i < count; i++) 269 intel_dsb_emit(dsb, 0, 270 DSB_OPCODE_NOOP << DSB_OPCODE_SHIFT); 271} 272 273void intel_dsb_nonpost_start(struct intel_dsb *dsb) 274{ 275 struct intel_crtc *crtc = dsb->crtc; 276 enum pipe pipe = crtc->pipe; 277 278 intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id), 279 DSB_NON_POSTED, DSB_NON_POSTED); 280 intel_dsb_noop(dsb, 4); 281} 282 283void intel_dsb_nonpost_end(struct intel_dsb *dsb) 284{ 285 struct intel_crtc *crtc = dsb->crtc; 286 enum pipe pipe = crtc->pipe; 287 288 intel_dsb_reg_write_masked(dsb, DSB_CTRL(pipe, dsb->id), 289 DSB_NON_POSTED, 0); 290 intel_dsb_noop(dsb, 4); 291} 292 293static void intel_dsb_align_tail(struct intel_dsb *dsb) 294{ 295 u32 aligned_tail, tail; 296 297 tail = dsb->free_pos * 4; 298 aligned_tail = ALIGN(tail, CACHELINE_BYTES); 299 300 if (aligned_tail > tail) 301 intel_dsb_buffer_memset(&dsb->dsb_buf, dsb->free_pos, 0, 302 aligned_tail - tail); 303 304 dsb->free_pos = aligned_tail / 4; 305} 306 307void intel_dsb_finish(struct intel_dsb *dsb) 308{ 309 struct intel_crtc *crtc = dsb->crtc; 310 311 /* 312 * DSB_FORCE_DEWAKE remains active even after DSB is 313 * disabled, so make sure to clear it (if set during 314 * intel_dsb_commit()). 315 */ 316 intel_dsb_reg_write_masked(dsb, DSB_PMCTRL_2(crtc->pipe, dsb->id), 317 DSB_FORCE_DEWAKE, 0); 318 319 intel_dsb_align_tail(dsb); 320 321 intel_dsb_buffer_flush_map(&dsb->dsb_buf); 322} 323 324static int intel_dsb_dewake_scanline(const struct intel_crtc_state *crtc_state) 325{ 326 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); 327 const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; 328 unsigned int latency = skl_watermark_max_latency(i915, 0); 329 int vblank_start; 330 331 if (crtc_state->vrr.enable) { 332 vblank_start = intel_vrr_vmin_vblank_start(crtc_state); 333 } else { 334 vblank_start = adjusted_mode->crtc_vblank_start; 335 336 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) 337 vblank_start = DIV_ROUND_UP(vblank_start, 2); 338 } 339 340 return max(0, vblank_start - intel_usecs_to_scanlines(adjusted_mode, latency)); 341} 342 343static u32 dsb_chicken(struct intel_crtc *crtc) 344{ 345 if (crtc->mode_flags & I915_MODE_FLAG_VRR) 346 return DSB_CTRL_WAIT_SAFE_WINDOW | 347 DSB_CTRL_NO_WAIT_VBLANK | 348 DSB_INST_WAIT_SAFE_WINDOW | 349 DSB_INST_NO_WAIT_VBLANK; 350 else 351 return 0; 352} 353 354static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl, 355 int dewake_scanline) 356{ 357 struct intel_crtc *crtc = dsb->crtc; 358 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 359 enum pipe pipe = crtc->pipe; 360 u32 tail; 361 362 tail = dsb->free_pos * 4; 363 if (drm_WARN_ON(&dev_priv->drm, !IS_ALIGNED(tail, CACHELINE_BYTES))) 364 return; 365 366 if (is_dsb_busy(dev_priv, pipe, dsb->id)) { 367 drm_err(&dev_priv->drm, "[CRTC:%d:%s] DSB %d is busy\n", 368 crtc->base.base.id, crtc->base.name, dsb->id); 369 return; 370 } 371 372 intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id), 373 ctrl | DSB_ENABLE); 374 375 intel_de_write_fw(dev_priv, DSB_CHICKEN(pipe, dsb->id), 376 dsb_chicken(crtc)); 377 378 intel_de_write_fw(dev_priv, DSB_HEAD(pipe, dsb->id), 379 intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf)); 380 381 if (dewake_scanline >= 0) { 382 int diff, hw_dewake_scanline; 383 384 hw_dewake_scanline = intel_crtc_scanline_to_hw(crtc, dewake_scanline); 385 386 intel_de_write_fw(dev_priv, DSB_PMCTRL(pipe, dsb->id), 387 DSB_ENABLE_DEWAKE | 388 DSB_SCANLINE_FOR_DEWAKE(hw_dewake_scanline)); 389 390 /* 391 * Force DEwake immediately if we're already past 392 * or close to racing past the target scanline. 393 */ 394 diff = dewake_scanline - intel_get_crtc_scanline(crtc); 395 intel_de_write_fw(dev_priv, DSB_PMCTRL_2(pipe, dsb->id), 396 (diff >= 0 && diff < 5 ? DSB_FORCE_DEWAKE : 0) | 397 DSB_BLOCK_DEWAKE_EXTENSION); 398 } 399 400 intel_de_write_fw(dev_priv, DSB_TAIL(pipe, dsb->id), 401 intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf) + tail); 402} 403 404/** 405 * intel_dsb_commit() - Trigger workload execution of DSB. 406 * @dsb: DSB context 407 * @wait_for_vblank: wait for vblank before executing 408 * 409 * This function is used to do actual write to hardware using DSB. 410 */ 411void intel_dsb_commit(struct intel_dsb *dsb, 412 bool wait_for_vblank) 413{ 414 _intel_dsb_commit(dsb, 415 wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0, 416 wait_for_vblank ? dsb->dewake_scanline : -1); 417} 418 419void intel_dsb_wait(struct intel_dsb *dsb) 420{ 421 struct intel_crtc *crtc = dsb->crtc; 422 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 423 enum pipe pipe = crtc->pipe; 424 425 if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1)) { 426 u32 offset = intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf); 427 428 intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id), 429 DSB_ENABLE | DSB_HALT); 430 431 drm_err(&dev_priv->drm, 432 "[CRTC:%d:%s] DSB %d timed out waiting for idle (current head=0x%x, head=0x%x, tail=0x%x)\n", 433 crtc->base.base.id, crtc->base.name, dsb->id, 434 intel_de_read_fw(dev_priv, DSB_CURRENT_HEAD(pipe, dsb->id)) - offset, 435 intel_de_read_fw(dev_priv, DSB_HEAD(pipe, dsb->id)) - offset, 436 intel_de_read_fw(dev_priv, DSB_TAIL(pipe, dsb->id)) - offset); 437 438 intel_dsb_dump(dsb); 439 } 440 441 /* Attempt to reset it */ 442 dsb->free_pos = 0; 443 dsb->ins_start_offset = 0; 444 intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id), 0); 445} 446 447/** 448 * intel_dsb_prepare() - Allocate, pin and map the DSB command buffer. 449 * @crtc_state: the CRTC state 450 * @max_cmds: number of commands we need to fit into command buffer 451 * 452 * This function prepare the command buffer which is used to store dsb 453 * instructions with data. 454 * 455 * Returns: 456 * DSB context, NULL on failure 457 */ 458struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state, 459 unsigned int max_cmds) 460{ 461 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); 462 struct drm_i915_private *i915 = to_i915(crtc->base.dev); 463 intel_wakeref_t wakeref; 464 struct intel_dsb *dsb; 465 unsigned int size; 466 467 if (!HAS_DSB(i915)) 468 return NULL; 469 470 /* TODO: DSB is broken in Xe KMD, so disabling it until fixed */ 471 if (!IS_ENABLED(I915)) 472 return NULL; 473 474 dsb = kzalloc(sizeof(*dsb), GFP_KERNEL); 475 if (!dsb) 476 goto out; 477 478 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 479 480 /* ~1 qword per instruction, full cachelines */ 481 size = ALIGN(max_cmds * 8, CACHELINE_BYTES); 482 483 if (!intel_dsb_buffer_create(crtc, &dsb->dsb_buf, size)) 484 goto out_put_rpm; 485 486 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 487 488 dsb->id = DSB1; 489 dsb->crtc = crtc; 490 dsb->size = size / 4; /* in dwords */ 491 dsb->free_pos = 0; 492 dsb->ins_start_offset = 0; 493 dsb->dewake_scanline = intel_dsb_dewake_scanline(crtc_state); 494 495 return dsb; 496 497out_put_rpm: 498 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 499 kfree(dsb); 500out: 501 drm_info_once(&i915->drm, 502 "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n", 503 crtc->base.base.id, crtc->base.name, DSB1); 504 505 return NULL; 506} 507 508/** 509 * intel_dsb_cleanup() - To cleanup DSB context. 510 * @dsb: DSB context 511 * 512 * This function cleanup the DSB context by unpinning and releasing 513 * the VMA object associated with it. 514 */ 515void intel_dsb_cleanup(struct intel_dsb *dsb) 516{ 517 intel_dsb_buffer_cleanup(&dsb->dsb_buf); 518 kfree(dsb); 519} 520