1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2022 Intel Corporation 4 */ 5 6#include "intel_gt.h" 7#include "intel_gt_mcr.h" 8#include "intel_gt_print.h" 9#include "intel_gt_regs.h" 10 11/** 12 * DOC: GT Multicast/Replicated (MCR) Register Support 13 * 14 * Some GT registers are designed as "multicast" or "replicated" registers: 15 * multiple instances of the same register share a single MMIO offset. MCR 16 * registers are generally used when the hardware needs to potentially track 17 * independent values of a register per hardware unit (e.g., per-subslice, 18 * per-L3bank, etc.). The specific types of replication that exist vary 19 * per-platform. 20 * 21 * MMIO accesses to MCR registers are controlled according to the settings 22 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR 23 * registers can be done in either a (i.e., a single write updates all 24 * instances of the register to the same value) or unicast (a write updates only 25 * one specific instance). Reads of MCR registers always operate in a unicast 26 * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR. 27 * Selection of a specific MCR instance for unicast operations is referred to 28 * as "steering." 29 * 30 * If MCR register operations are steered toward a hardware unit that is 31 * fused off or currently powered down due to power gating, the MMIO operation 32 * is "terminated" by the hardware. Terminated read operations will return a 33 * value of zero and terminated unicast write operations will be silently 34 * ignored. 35 */ 36 37#define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering) 38 39static const char * const intel_steering_types[] = { 40 "L3BANK", 41 "MSLICE", 42 "LNCF", 43 "GAM", 44 "DSS", 45 "OADDRM", 46 "INSTANCE 0", 47}; 48 49static const struct intel_mmio_range icl_l3bank_steering_table[] = { 50 { 0x00B100, 0x00B3FF }, 51 {}, 52}; 53 54/* 55 * Although the bspec lists more "MSLICE" ranges than shown here, some of those 56 * are of a "GAM" subclass that has special rules. Thus we use a separate 57 * GAM table farther down for those. 58 */ 59static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = { 60 { 0x00DD00, 0x00DDFF }, 61 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ 62 {}, 63}; 64 65static const struct intel_mmio_range xehpsdv_gam_steering_table[] = { 66 { 0x004000, 0x004AFF }, 67 { 0x00C800, 0x00CFFF }, 68 {}, 69}; 70 71static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = { 72 { 0x00B000, 0x00B0FF }, 73 { 0x00D800, 0x00D8FF }, 74 {}, 75}; 76 77static const struct intel_mmio_range dg2_lncf_steering_table[] = { 78 { 0x00B000, 0x00B0FF }, 79 { 0x00D880, 0x00D8FF }, 80 {}, 81}; 82 83/* 84 * We have several types of MCR registers on PVC where steering to (0,0) 85 * will always provide us with a non-terminated value. We'll stick them 86 * all in the same table for simplicity. 87 */ 88static const struct intel_mmio_range pvc_instance0_steering_table[] = { 89 { 0x004000, 0x004AFF }, /* HALF-BSLICE */ 90 { 0x008800, 0x00887F }, /* CC */ 91 { 0x008A80, 0x008AFF }, /* TILEPSMI */ 92 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ 93 { 0x00B100, 0x00B3FF }, /* L3BANK */ 94 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ 95 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ 96 { 0x00DD00, 0x00DDFF }, /* BSLICE */ 97 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ 98 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ 99 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ 100 { 0x024180, 0x0241FF }, /* HALF-BSLICE */ 101 {}, 102}; 103 104static const struct intel_mmio_range xelpg_instance0_steering_table[] = { 105 { 0x000B00, 0x000BFF }, /* SQIDI */ 106 { 0x001000, 0x001FFF }, /* SQIDI */ 107 { 0x004000, 0x0048FF }, /* GAM */ 108 { 0x008700, 0x0087FF }, /* SQIDI */ 109 { 0x00B000, 0x00B0FF }, /* NODE */ 110 { 0x00C800, 0x00CFFF }, /* GAM */ 111 { 0x00D880, 0x00D8FF }, /* NODE */ 112 { 0x00DD00, 0x00DDFF }, /* OAAL2 */ 113 {}, 114}; 115 116static const struct intel_mmio_range xelpg_l3bank_steering_table[] = { 117 { 0x00B100, 0x00B3FF }, 118 {}, 119}; 120 121/* DSS steering is used for SLICE ranges as well */ 122static const struct intel_mmio_range xelpg_dss_steering_table[] = { 123 { 0x005200, 0x0052FF }, /* SLICE */ 124 { 0x005500, 0x007FFF }, /* SLICE */ 125 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ 126 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ 127 { 0x009680, 0x0096FF }, /* DSS */ 128 { 0x00D800, 0x00D87F }, /* SLICE */ 129 { 0x00DC00, 0x00DCFF }, /* SLICE */ 130 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ 131 {}, 132}; 133 134static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = { 135 { 0x393200, 0x39323F }, 136 { 0x393400, 0x3934FF }, 137 {}, 138}; 139 140void intel_gt_mcr_init(struct intel_gt *gt) 141{ 142 struct drm_i915_private *i915 = gt->i915; 143 unsigned long fuse; 144 int i; 145 146 mtx_init(>->mcr_lock, IPL_TTY); 147 148 /* 149 * An mslice is unavailable only if both the meml3 for the slice is 150 * disabled *and* all of the DSS in the slice (quadrant) are disabled. 151 */ 152 if (HAS_MSLICE_STEERING(i915)) { 153 gt->info.mslice_mask = 154 intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask, 155 GEN_DSS_PER_MSLICE); 156 gt->info.mslice_mask |= 157 (intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & 158 GEN12_MEML3_EN_MASK); 159 160 if (!gt->info.mslice_mask) /* should be impossible! */ 161 gt_warn(gt, "mslice mask all zero!\n"); 162 } 163 164 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) { 165 gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table; 166 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { 167 /* Wa_14016747170 */ 168 if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) || 169 IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0)) 170 fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK, 171 intel_uncore_read(gt->uncore, 172 MTL_GT_ACTIVITY_FACTOR)); 173 else 174 fuse = REG_FIELD_GET(GT_L3_EXC_MASK, 175 intel_uncore_read(gt->uncore, XEHP_FUSE4)); 176 177 /* 178 * Despite the register field being named "exclude mask" the 179 * bits actually represent enabled banks (two banks per bit). 180 */ 181 for_each_set_bit(i, &fuse, 3) 182 gt->info.l3bank_mask |= 0x3 << 2 * i; 183 184 gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table; 185 gt->steering_table[L3BANK] = xelpg_l3bank_steering_table; 186 gt->steering_table[DSS] = xelpg_dss_steering_table; 187 } else if (IS_PONTEVECCHIO(i915)) { 188 gt->steering_table[INSTANCE0] = pvc_instance0_steering_table; 189 } else if (IS_DG2(i915)) { 190 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; 191 gt->steering_table[LNCF] = dg2_lncf_steering_table; 192 /* 193 * No need to hook up the GAM table since it has a dedicated 194 * steering control register on DG2 and can use implicit 195 * steering. 196 */ 197 } else if (IS_XEHPSDV(i915)) { 198 gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; 199 gt->steering_table[LNCF] = xehpsdv_lncf_steering_table; 200 gt->steering_table[GAM] = xehpsdv_gam_steering_table; 201 } else if (GRAPHICS_VER(i915) >= 11 && 202 GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) { 203 gt->steering_table[L3BANK] = icl_l3bank_steering_table; 204 gt->info.l3bank_mask = 205 ~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) & 206 GEN10_L3BANK_MASK; 207 if (!gt->info.l3bank_mask) /* should be impossible! */ 208 gt_warn(gt, "L3 bank mask is all zero!\n"); 209 } else if (GRAPHICS_VER(i915) >= 11) { 210 /* 211 * We expect all modern platforms to have at least some 212 * type of steering that needs to be initialized. 213 */ 214 MISSING_CASE(INTEL_INFO(i915)->platform); 215 } 216} 217 218/* 219 * Although the rest of the driver should use MCR-specific functions to 220 * read/write MCR registers, we still use the regular intel_uncore_* functions 221 * internally to implement those, so we need a way for the functions in this 222 * file to "cast" an i915_mcr_reg_t into an i915_reg_t. 223 */ 224static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr) 225{ 226 i915_reg_t r = { .reg = mcr.reg }; 227 228 return r; 229} 230 231/* 232 * rw_with_mcr_steering_fw - Access a register with specific MCR steering 233 * @gt: GT to read register from 234 * @reg: register being accessed 235 * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access 236 * @group: group number (documented as "sliceid" on older platforms) 237 * @instance: instance number (documented as "subsliceid" on older platforms) 238 * @value: register value to be written (ignored for read) 239 * 240 * Context: The caller must hold the MCR lock 241 * Return: 0 for write access. register value for read access. 242 * 243 * Caller needs to make sure the relevant forcewake wells are up. 244 */ 245static u32 rw_with_mcr_steering_fw(struct intel_gt *gt, 246 i915_mcr_reg_t reg, u8 rw_flag, 247 int group, int instance, u32 value) 248{ 249 struct intel_uncore *uncore = gt->uncore; 250 u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0; 251 252 lockdep_assert_held(>->mcr_lock); 253 254 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) { 255 /* 256 * Always leave the hardware in multicast mode when doing reads 257 * (see comment about Wa_22013088509 below) and only change it 258 * to unicast mode when doing writes of a specific instance. 259 * 260 * No need to save old steering reg value. 261 */ 262 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, 263 REG_FIELD_PREP(MTL_MCR_GROUPID, group) | 264 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) | 265 (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0)); 266 } else if (GRAPHICS_VER(uncore->i915) >= 11) { 267 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 268 mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance); 269 270 /* 271 * Wa_22013088509 272 * 273 * The setting of the multicast/unicast bit usually wouldn't 274 * matter for read operations (which always return the value 275 * from a single register instance regardless of how that bit 276 * is set), but some platforms have a workaround requiring us 277 * to remain in multicast mode for reads. There's no real 278 * downside to this, so we'll just go ahead and do so on all 279 * platforms; we'll only clear the multicast bit from the mask 280 * when exlicitly doing a write operation. 281 */ 282 if (rw_flag == FW_REG_WRITE) 283 mcr_mask |= GEN11_MCR_MULTICAST; 284 285 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 286 old_mcr = mcr; 287 288 mcr &= ~mcr_mask; 289 mcr |= mcr_ss; 290 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 291 } else { 292 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; 293 mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance); 294 295 mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); 296 old_mcr = mcr; 297 298 mcr &= ~mcr_mask; 299 mcr |= mcr_ss; 300 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); 301 } 302 303 if (rw_flag == FW_REG_READ) 304 val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg)); 305 else 306 intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value); 307 308 /* 309 * For pre-MTL platforms, we need to restore the old value of the 310 * steering control register to ensure that implicit steering continues 311 * to behave as expected. For MTL and beyond, we need only reinstate 312 * the 'multicast' bit (and only if we did a write that cleared it). 313 */ 314 if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE) 315 intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 316 else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70)) 317 intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr); 318 319 return val; 320} 321 322static u32 rw_with_mcr_steering(struct intel_gt *gt, 323 i915_mcr_reg_t reg, u8 rw_flag, 324 int group, int instance, 325 u32 value) 326{ 327 struct intel_uncore *uncore = gt->uncore; 328 enum forcewake_domains fw_domains; 329 unsigned long flags; 330 u32 val; 331 332 fw_domains = intel_uncore_forcewake_for_reg(uncore, mcr_reg_cast(reg), 333 rw_flag); 334 fw_domains |= intel_uncore_forcewake_for_reg(uncore, 335 GEN8_MCR_SELECTOR, 336 FW_REG_READ | FW_REG_WRITE); 337 338 intel_gt_mcr_lock(gt, &flags); 339 spin_lock(&uncore->lock); 340 intel_uncore_forcewake_get__locked(uncore, fw_domains); 341 342 val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value); 343 344 intel_uncore_forcewake_put__locked(uncore, fw_domains); 345 spin_unlock(&uncore->lock); 346 intel_gt_mcr_unlock(gt, flags); 347 348 return val; 349} 350 351/** 352 * intel_gt_mcr_lock - Acquire MCR steering lock 353 * @gt: GT structure 354 * @flags: storage to save IRQ flags to 355 * 356 * Performs locking to protect the steering for the duration of an MCR 357 * operation. On MTL and beyond, a hardware lock will also be taken to 358 * serialize access not only for the driver, but also for external hardware and 359 * firmware agents. 360 * 361 * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this 362 * function is called, although it may be acquired after this 363 * function call. 364 */ 365void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags) 366 __acquires(>->mcr_lock) 367{ 368 unsigned long __flags; 369 int err = 0; 370 371 lockdep_assert_not_held(>->uncore->lock); 372 373 /* 374 * Starting with MTL, we need to coordinate not only with other 375 * driver threads, but also with hardware/firmware agents. A dedicated 376 * locking register is used. 377 */ 378 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { 379 /* 380 * The steering control and semaphore registers are inside an 381 * "always on" power domain with respect to RC6. However there 382 * are some issues if higher-level platform sleep states are 383 * entering/exiting at the same time these registers are 384 * accessed. Grabbing GT forcewake and holding it over the 385 * entire lock/steer/unlock cycle ensures that those sleep 386 * states have been fully exited before we access these 387 * registers. This wakeref will be released in the unlock 388 * routine. 389 * 390 * This is expected to become a formally documented/numbered 391 * workaround soon. 392 */ 393 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_GT); 394 395 err = wait_for(intel_uncore_read_fw(gt->uncore, 396 MTL_STEER_SEMAPHORE) == 0x1, 100); 397 } 398 399 /* 400 * Even on platforms with a hardware lock, we'll continue to grab 401 * a software spinlock too for lockdep purposes. If the hardware lock 402 * was already acquired, there should never be contention on the 403 * software lock. 404 */ 405 spin_lock_irqsave(>->mcr_lock, __flags); 406 407 *flags = __flags; 408 409 /* 410 * In theory we should never fail to acquire the HW semaphore; this 411 * would indicate some hardware/firmware is misbehaving and not 412 * releasing it properly. 413 */ 414 if (err == -ETIMEDOUT) { 415 gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out"); 416 add_taint_for_CI(gt->i915, TAINT_WARN); /* CI is now unreliable */ 417 } 418} 419 420/** 421 * intel_gt_mcr_unlock - Release MCR steering lock 422 * @gt: GT structure 423 * @flags: IRQ flags to restore 424 * 425 * Releases the lock acquired by intel_gt_mcr_lock(). 426 * 427 * Context: Releases gt->mcr_lock 428 */ 429void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags) 430 __releases(>->mcr_lock) 431{ 432 spin_unlock_irqrestore(>->mcr_lock, flags); 433 434 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { 435 intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1); 436 437 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_GT); 438 } 439} 440 441/** 442 * intel_gt_mcr_read - read a specific instance of an MCR register 443 * @gt: GT structure 444 * @reg: the MCR register to read 445 * @group: the MCR group 446 * @instance: the MCR instance 447 * 448 * Context: Takes and releases gt->mcr_lock 449 * 450 * Returns the value read from an MCR register after steering toward a specific 451 * group/instance. 452 */ 453u32 intel_gt_mcr_read(struct intel_gt *gt, 454 i915_mcr_reg_t reg, 455 int group, int instance) 456{ 457 return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, 0); 458} 459 460/** 461 * intel_gt_mcr_unicast_write - write a specific instance of an MCR register 462 * @gt: GT structure 463 * @reg: the MCR register to write 464 * @value: value to write 465 * @group: the MCR group 466 * @instance: the MCR instance 467 * 468 * Write an MCR register in unicast mode after steering toward a specific 469 * group/instance. 470 * 471 * Context: Calls a function that takes and releases gt->mcr_lock 472 */ 473void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value, 474 int group, int instance) 475{ 476 rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value); 477} 478 479/** 480 * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register 481 * @gt: GT structure 482 * @reg: the MCR register to write 483 * @value: value to write 484 * 485 * Write an MCR register in multicast mode to update all instances. 486 * 487 * Context: Takes and releases gt->mcr_lock 488 */ 489void intel_gt_mcr_multicast_write(struct intel_gt *gt, 490 i915_mcr_reg_t reg, u32 value) 491{ 492 unsigned long flags; 493 494 intel_gt_mcr_lock(gt, &flags); 495 496 /* 497 * Ensure we have multicast behavior, just in case some non-i915 agent 498 * left the hardware in unicast mode. 499 */ 500 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 501 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 502 503 intel_uncore_write(gt->uncore, mcr_reg_cast(reg), value); 504 505 intel_gt_mcr_unlock(gt, flags); 506} 507 508/** 509 * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register 510 * @gt: GT structure 511 * @reg: the MCR register to write 512 * @value: value to write 513 * 514 * Write an MCR register in multicast mode to update all instances. This 515 * function assumes the caller is already holding any necessary forcewake 516 * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should 517 * be obtained automatically. 518 * 519 * Context: The caller must hold gt->mcr_lock. 520 */ 521void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value) 522{ 523 lockdep_assert_held(>->mcr_lock); 524 525 /* 526 * Ensure we have multicast behavior, just in case some non-i915 agent 527 * left the hardware in unicast mode. 528 */ 529 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) 530 intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); 531 532 intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value); 533} 534 535/** 536 * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations 537 * @gt: GT structure 538 * @reg: the MCR register to read and write 539 * @clear: bits to clear during RMW 540 * @set: bits to set during RMW 541 * 542 * Performs a read-modify-write on an MCR register in a multicast manner. 543 * This operation only makes sense on MCR registers where all instances are 544 * expected to have the same value. The read will target any non-terminated 545 * instance and the write will be applied to all instances. 546 * 547 * This function assumes the caller is already holding any necessary forcewake 548 * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should 549 * be obtained automatically. 550 * 551 * Context: Calls functions that take and release gt->mcr_lock 552 * 553 * Returns the old (unmodified) value read. 554 */ 555u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg, 556 u32 clear, u32 set) 557{ 558 u32 val = intel_gt_mcr_read_any(gt, reg); 559 560 intel_gt_mcr_multicast_write(gt, reg, (val & ~clear) | set); 561 562 return val; 563} 564 565/* 566 * reg_needs_read_steering - determine whether a register read requires 567 * explicit steering 568 * @gt: GT structure 569 * @reg: the register to check steering requirements for 570 * @type: type of multicast steering to check 571 * 572 * Determines whether @reg needs explicit steering of a specific type for 573 * reads. 574 * 575 * Returns false if @reg does not belong to a register range of the given 576 * steering type, or if the default (subslice-based) steering IDs are suitable 577 * for @type steering too. 578 */ 579static bool reg_needs_read_steering(struct intel_gt *gt, 580 i915_mcr_reg_t reg, 581 enum intel_steering_type type) 582{ 583 u32 offset = i915_mmio_reg_offset(reg); 584 const struct intel_mmio_range *entry; 585 586 if (likely(!gt->steering_table[type])) 587 return false; 588 589 if (IS_GSI_REG(offset)) 590 offset += gt->uncore->gsi_offset; 591 592 for (entry = gt->steering_table[type]; entry->end; entry++) { 593 if (offset >= entry->start && offset <= entry->end) 594 return true; 595 } 596 597 return false; 598} 599 600/* 601 * get_nonterminated_steering - determines valid IDs for a class of MCR steering 602 * @gt: GT structure 603 * @type: multicast register type 604 * @group: Group ID returned 605 * @instance: Instance ID returned 606 * 607 * Determines group and instance values that will steer reads of the specified 608 * MCR class to a non-terminated instance. 609 */ 610static void get_nonterminated_steering(struct intel_gt *gt, 611 enum intel_steering_type type, 612 u8 *group, u8 *instance) 613{ 614 u32 dss; 615 616 switch (type) { 617 case L3BANK: 618 *group = 0; /* unused */ 619 *instance = __ffs(gt->info.l3bank_mask); 620 break; 621 case MSLICE: 622 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 623 *group = __ffs(gt->info.mslice_mask); 624 *instance = 0; /* unused */ 625 break; 626 case LNCF: 627 /* 628 * An LNCF is always present if its mslice is present, so we 629 * can safely just steer to LNCF 0 in all cases. 630 */ 631 GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); 632 *group = __ffs(gt->info.mslice_mask) << 1; 633 *instance = 0; /* unused */ 634 break; 635 case GAM: 636 *group = IS_DG2(gt->i915) ? 1 : 0; 637 *instance = 0; 638 break; 639 case DSS: 640 dss = intel_sseu_find_first_xehp_dss(>->info.sseu, 0, 0); 641 *group = dss / GEN_DSS_PER_GSLICE; 642 *instance = dss % GEN_DSS_PER_GSLICE; 643 break; 644 case INSTANCE0: 645 /* 646 * There are a lot of MCR types for which instance (0, 0) 647 * will always provide a non-terminated value. 648 */ 649 *group = 0; 650 *instance = 0; 651 break; 652 case OADDRM: 653 if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0)) 654 *group = 0; 655 else 656 *group = 1; 657 *instance = 0; 658 break; 659 default: 660 MISSING_CASE(type); 661 *group = 0; 662 *instance = 0; 663 } 664} 665 666/** 667 * intel_gt_mcr_get_nonterminated_steering - find group/instance values that 668 * will steer a register to a non-terminated instance 669 * @gt: GT structure 670 * @reg: register for which the steering is required 671 * @group: return variable for group steering 672 * @instance: return variable for instance steering 673 * 674 * This function returns a group/instance pair that is guaranteed to work for 675 * read steering of the given register. Note that a value will be returned even 676 * if the register is not replicated and therefore does not actually require 677 * steering. 678 */ 679void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt, 680 i915_mcr_reg_t reg, 681 u8 *group, u8 *instance) 682{ 683 int type; 684 685 for (type = 0; type < NUM_STEERING_TYPES; type++) { 686 if (reg_needs_read_steering(gt, reg, type)) { 687 get_nonterminated_steering(gt, type, group, instance); 688 return; 689 } 690 } 691 692 *group = gt->default_steering.groupid; 693 *instance = gt->default_steering.instanceid; 694} 695 696/** 697 * intel_gt_mcr_read_any_fw - reads one instance of an MCR register 698 * @gt: GT structure 699 * @reg: register to read 700 * 701 * Reads a GT MCR register. The read will be steered to a non-terminated 702 * instance (i.e., one that isn't fused off or powered down by power gating). 703 * This function assumes the caller is already holding any necessary forcewake 704 * domains; use intel_gt_mcr_read_any() in cases where forcewake should be 705 * obtained automatically. 706 * 707 * Context: The caller must hold gt->mcr_lock. 708 * 709 * Returns the value from a non-terminated instance of @reg. 710 */ 711u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg) 712{ 713 int type; 714 u8 group, instance; 715 716 lockdep_assert_held(>->mcr_lock); 717 718 for (type = 0; type < NUM_STEERING_TYPES; type++) { 719 if (reg_needs_read_steering(gt, reg, type)) { 720 get_nonterminated_steering(gt, type, &group, &instance); 721 return rw_with_mcr_steering_fw(gt, reg, 722 FW_REG_READ, 723 group, instance, 0); 724 } 725 } 726 727 return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg)); 728} 729 730/** 731 * intel_gt_mcr_read_any - reads one instance of an MCR register 732 * @gt: GT structure 733 * @reg: register to read 734 * 735 * Reads a GT MCR register. The read will be steered to a non-terminated 736 * instance (i.e., one that isn't fused off or powered down by power gating). 737 * 738 * Context: Calls a function that takes and releases gt->mcr_lock. 739 * 740 * Returns the value from a non-terminated instance of @reg. 741 */ 742u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg) 743{ 744 int type; 745 u8 group, instance; 746 747 for (type = 0; type < NUM_STEERING_TYPES; type++) { 748 if (reg_needs_read_steering(gt, reg, type)) { 749 get_nonterminated_steering(gt, type, &group, &instance); 750 return rw_with_mcr_steering(gt, reg, 751 FW_REG_READ, 752 group, instance, 0); 753 } 754 } 755 756 return intel_uncore_read(gt->uncore, mcr_reg_cast(reg)); 757} 758 759static void report_steering_type(struct drm_printer *p, 760 struct intel_gt *gt, 761 enum intel_steering_type type, 762 bool dump_table) 763{ 764 const struct intel_mmio_range *entry; 765 u8 group, instance; 766 767 BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES); 768 769 if (!gt->steering_table[type]) { 770 drm_printf(p, "%s steering: uses default steering\n", 771 intel_steering_types[type]); 772 return; 773 } 774 775 get_nonterminated_steering(gt, type, &group, &instance); 776 drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n", 777 intel_steering_types[type], group, instance); 778 779 if (!dump_table) 780 return; 781 782 for (entry = gt->steering_table[type]; entry->end; entry++) 783 drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end); 784} 785 786void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt, 787 bool dump_table) 788{ 789 /* 790 * Starting with MTL we no longer have default steering; 791 * all ranges are explicitly steered. 792 */ 793 if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70)) 794 drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n", 795 gt->default_steering.groupid, 796 gt->default_steering.instanceid); 797 798 if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { 799 for (int i = 0; i < NUM_STEERING_TYPES; i++) 800 if (gt->steering_table[i]) 801 report_steering_type(p, gt, i, dump_table); 802 } else if (IS_PONTEVECCHIO(gt->i915)) { 803 report_steering_type(p, gt, INSTANCE0, dump_table); 804 } else if (HAS_MSLICE_STEERING(gt->i915)) { 805 report_steering_type(p, gt, MSLICE, dump_table); 806 report_steering_type(p, gt, LNCF, dump_table); 807 } 808} 809 810/** 811 * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS 812 * @gt: GT structure 813 * @dss: DSS ID to obtain steering for 814 * @group: pointer to storage for steering group ID 815 * @instance: pointer to storage for steering instance ID 816 * 817 * Returns the steering IDs (via the @group and @instance parameters) that 818 * correspond to a specific subslice/DSS ID. 819 */ 820void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss, 821 unsigned int *group, unsigned int *instance) 822{ 823 if (IS_PONTEVECCHIO(gt->i915)) { 824 *group = dss / GEN_DSS_PER_CSLICE; 825 *instance = dss % GEN_DSS_PER_CSLICE; 826 } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) { 827 *group = dss / GEN_DSS_PER_GSLICE; 828 *instance = dss % GEN_DSS_PER_GSLICE; 829 } else { 830 *group = dss / GEN_MAX_SS_PER_HSW_SLICE; 831 *instance = dss % GEN_MAX_SS_PER_HSW_SLICE; 832 return; 833 } 834} 835 836/** 837 * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state 838 * @gt: GT structure 839 * @reg: the register to read 840 * @mask: mask to apply to register value 841 * @value: value to wait for 842 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait 843 * @slow_timeout_ms: slow timeout in millisecond 844 * 845 * This routine waits until the target register @reg contains the expected 846 * @value after applying the @mask, i.e. it waits until :: 847 * 848 * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value 849 * 850 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. 851 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us 852 * must be not larger than 20,0000 microseconds. 853 * 854 * This function is basically an MCR-friendly version of 855 * __intel_wait_for_register_fw(). Generally this function will only be used 856 * on GAM registers which are a bit special --- although they're MCR registers, 857 * reads (e.g., waiting for status updates) are always directed to the primary 858 * instance. 859 * 860 * Note that this routine assumes the caller holds forcewake asserted, it is 861 * not suitable for very long waits. 862 * 863 * Context: Calls a function that takes and releases gt->mcr_lock 864 * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. 865 */ 866int intel_gt_mcr_wait_for_reg(struct intel_gt *gt, 867 i915_mcr_reg_t reg, 868 u32 mask, 869 u32 value, 870 unsigned int fast_timeout_us, 871 unsigned int slow_timeout_ms) 872{ 873 int ret; 874 875 lockdep_assert_not_held(>->mcr_lock); 876 877#define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value) 878 879 /* Catch any overuse of this function */ 880 might_sleep_if(slow_timeout_ms); 881 GEM_BUG_ON(fast_timeout_us > 20000); 882 GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms); 883 884 ret = -ETIMEDOUT; 885 if (fast_timeout_us && fast_timeout_us <= 20000) 886 ret = _wait_for_atomic(done, fast_timeout_us, 0); 887 if (ret && slow_timeout_ms) 888 ret = wait_for(done, slow_timeout_ms); 889 890 return ret; 891#undef done 892} 893