1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2019 Intel Corporation 4 */ 5 6#include <linux/string_helpers.h> 7#include <linux/suspend.h> 8 9#include "i915_drv.h" 10#include "i915_irq.h" 11#include "i915_params.h" 12#include "intel_context.h" 13#include "intel_engine_pm.h" 14#include "intel_gt.h" 15#include "intel_gt_clock_utils.h" 16#include "intel_gt_mcr.h" 17#include "intel_gt_pm.h" 18#include "intel_gt_print.h" 19#include "intel_gt_requests.h" 20#include "intel_llc.h" 21#include "intel_rc6.h" 22#include "intel_rps.h" 23#include "intel_wakeref.h" 24#include "pxp/intel_pxp_pm.h" 25 26#define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2) 27 28static void user_forcewake(struct intel_gt *gt, bool suspend) 29{ 30 int count = atomic_read(>->user_wakeref); 31 intel_wakeref_t wakeref; 32 33 /* Inside suspend/resume so single threaded, no races to worry about. */ 34 if (likely(!count)) 35 return; 36 37 wakeref = intel_gt_pm_get(gt); 38 if (suspend) { 39 GEM_BUG_ON(count > atomic_read(>->wakeref.count)); 40 atomic_sub(count, >->wakeref.count); 41 } else { 42 atomic_add(count, >->wakeref.count); 43 } 44 intel_gt_pm_put(gt, wakeref); 45} 46 47static void runtime_begin(struct intel_gt *gt) 48{ 49 local_irq_disable(); 50 write_seqcount_begin(>->stats.lock); 51 gt->stats.start = ktime_get(); 52 gt->stats.active = true; 53 write_seqcount_end(>->stats.lock); 54 local_irq_enable(); 55} 56 57static void runtime_end(struct intel_gt *gt) 58{ 59 local_irq_disable(); 60 write_seqcount_begin(>->stats.lock); 61 gt->stats.active = false; 62 gt->stats.total = 63 ktime_add(gt->stats.total, 64 ktime_sub(ktime_get(), gt->stats.start)); 65 write_seqcount_end(>->stats.lock); 66 local_irq_enable(); 67} 68 69static int __gt_unpark(struct intel_wakeref *wf) 70{ 71 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 72 struct drm_i915_private *i915 = gt->i915; 73 74 GT_TRACE(gt, "\n"); 75 76 /* 77 * It seems that the DMC likes to transition between the DC states a lot 78 * when there are no connected displays (no active power domains) during 79 * command submission. 80 * 81 * This activity has negative impact on the performance of the chip with 82 * huge latencies observed in the interrupt handler and elsewhere. 83 * 84 * Work around it by grabbing a GT IRQ power domain whilst there is any 85 * GT activity, preventing any DC state transitions. 86 */ 87 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); 88 GEM_BUG_ON(!gt->awake); 89 90 intel_rc6_unpark(>->rc6); 91 intel_rps_unpark(>->rps); 92 i915_pmu_gt_unparked(gt); 93 intel_guc_busyness_unpark(gt); 94 95 intel_gt_unpark_requests(gt); 96 runtime_begin(gt); 97 98 return 0; 99} 100 101static int __gt_park(struct intel_wakeref *wf) 102{ 103 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 104 intel_wakeref_t wakeref = fetch_and_zero(>->awake); 105 struct drm_i915_private *i915 = gt->i915; 106 107 GT_TRACE(gt, "\n"); 108 109 runtime_end(gt); 110 intel_gt_park_requests(gt); 111 112 intel_guc_busyness_park(gt); 113 i915_vma_parked(gt); 114 i915_pmu_gt_parked(gt); 115 intel_rps_park(>->rps); 116 intel_rc6_park(>->rc6); 117 118 /* Everything switched off, flush any residual interrupt just in case */ 119 intel_synchronize_irq(i915); 120 121 /* Defer dropping the display power well for 100ms, it's slow! */ 122 GEM_BUG_ON(!wakeref); 123 intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref); 124 125 return 0; 126} 127 128static const struct intel_wakeref_ops wf_ops = { 129 .get = __gt_unpark, 130 .put = __gt_park, 131}; 132 133void intel_gt_pm_init_early(struct intel_gt *gt) 134{ 135 /* 136 * We access the runtime_pm structure via gt->i915 here rather than 137 * gt->uncore as we do elsewhere in the file because gt->uncore is not 138 * yet initialized for all tiles at this point in the driver startup. 139 * runtime_pm is per-device rather than per-tile, so this is still the 140 * correct structure. 141 */ 142 intel_wakeref_init(>->wakeref, gt->i915, &wf_ops, "GT"); 143 seqcount_mutex_init(>->stats.lock, >->wakeref.mutex); 144} 145 146void intel_gt_pm_init(struct intel_gt *gt) 147{ 148 /* 149 * Enabling power-management should be "self-healing". If we cannot 150 * enable a feature, simply leave it disabled with a notice to the 151 * user. 152 */ 153 intel_rc6_init(>->rc6); 154 intel_rps_init(>->rps); 155} 156 157static bool reset_engines(struct intel_gt *gt) 158{ 159 if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) 160 return false; 161 162 return __intel_gt_reset(gt, ALL_ENGINES) == 0; 163} 164 165static void gt_sanitize(struct intel_gt *gt, bool force) 166{ 167 struct intel_engine_cs *engine; 168 enum intel_engine_id id; 169 intel_wakeref_t wakeref; 170 171 GT_TRACE(gt, "force:%s\n", str_yes_no(force)); 172 173 /* Use a raw wakeref to avoid calling intel_display_power_get early */ 174 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 175 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 176 177 intel_gt_check_clock_frequency(gt); 178 179 /* 180 * As we have just resumed the machine and woken the device up from 181 * deep PCI sleep (presumably D3_cold), assume the HW has been reset 182 * back to defaults, recovering from whatever wedged state we left it 183 * in and so worth trying to use the device once more. 184 */ 185 if (intel_gt_is_wedged(gt)) 186 intel_gt_unset_wedged(gt); 187 188 /* For GuC mode, ensure submission is disabled before stopping ring */ 189 intel_uc_reset_prepare(>->uc); 190 191 for_each_engine(engine, gt, id) { 192 if (engine->reset.prepare) 193 engine->reset.prepare(engine); 194 195 if (engine->sanitize) 196 engine->sanitize(engine); 197 } 198 199 if (reset_engines(gt) || force) { 200 for_each_engine(engine, gt, id) 201 __intel_engine_reset(engine, false); 202 } 203 204 intel_uc_reset(>->uc, false); 205 206 for_each_engine(engine, gt, id) 207 if (engine->reset.finish) 208 engine->reset.finish(engine); 209 210 intel_rps_sanitize(>->rps); 211 212 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 213 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 214} 215 216void intel_gt_pm_fini(struct intel_gt *gt) 217{ 218 intel_rc6_fini(>->rc6); 219} 220 221void intel_gt_resume_early(struct intel_gt *gt) 222{ 223 /* 224 * Sanitize steer semaphores during driver resume. This is necessary 225 * to address observed cases of steer semaphores being 226 * held after a suspend operation. Confirmation from the hardware team 227 * assures the safety of this operation, as no lock acquisitions 228 * by other agents occur during driver load/resume process. 229 */ 230 intel_gt_mcr_lock_sanitize(gt); 231 232 intel_uncore_resume_early(gt->uncore); 233 intel_gt_check_and_clear_faults(gt); 234} 235 236int intel_gt_resume(struct intel_gt *gt) 237{ 238 struct intel_engine_cs *engine; 239 enum intel_engine_id id; 240 intel_wakeref_t wakeref; 241 int err; 242 243 err = intel_gt_has_unrecoverable_error(gt); 244 if (err) 245 return err; 246 247 GT_TRACE(gt, "\n"); 248 249 /* 250 * After resume, we may need to poke into the pinned kernel 251 * contexts to paper over any damage caused by the sudden suspend. 252 * Only the kernel contexts should remain pinned over suspend, 253 * allowing us to fixup the user contexts on their first pin. 254 */ 255 gt_sanitize(gt, true); 256 257 wakeref = intel_gt_pm_get(gt); 258 259 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 260 intel_rc6_sanitize(>->rc6); 261 if (intel_gt_is_wedged(gt)) { 262 err = -EIO; 263 goto out_fw; 264 } 265 266 /* Only when the HW is re-initialised, can we replay the requests */ 267 err = intel_gt_init_hw(gt); 268 if (err) { 269 gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n"); 270 goto err_wedged; 271 } 272 273 intel_uc_reset_finish(>->uc); 274 275 intel_rps_enable(>->rps); 276 intel_llc_enable(>->llc); 277 278 for_each_engine(engine, gt, id) { 279 intel_engine_pm_get(engine); 280 281 engine->serial++; /* kernel context lost */ 282 err = intel_engine_resume(engine); 283 284 intel_engine_pm_put(engine); 285 if (err) { 286 gt_err(gt, "Failed to restart %s (%d)\n", 287 engine->name, err); 288 goto err_wedged; 289 } 290 } 291 292 intel_rc6_enable(>->rc6); 293 294 intel_uc_resume(>->uc); 295 296 user_forcewake(gt, false); 297 298out_fw: 299 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 300 intel_gt_pm_put(gt, wakeref); 301 intel_gt_bind_context_set_ready(gt); 302 return err; 303 304err_wedged: 305 intel_gt_set_wedged(gt); 306 goto out_fw; 307} 308 309static void wait_for_suspend(struct intel_gt *gt) 310{ 311 if (!intel_gt_pm_is_awake(gt)) 312 return; 313 314 if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) { 315 /* 316 * Forcibly cancel outstanding work and leave 317 * the gpu quiet. 318 */ 319 intel_gt_set_wedged(gt); 320 intel_gt_retire_requests(gt); 321 } 322 323 intel_gt_pm_wait_for_idle(gt); 324} 325 326void intel_gt_suspend_prepare(struct intel_gt *gt) 327{ 328 intel_gt_bind_context_set_unready(gt); 329 user_forcewake(gt, true); 330 wait_for_suspend(gt); 331} 332 333static suspend_state_t pm_suspend_target(void) 334{ 335#if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP) 336 return pm_suspend_target_state; 337#else 338 return PM_SUSPEND_TO_IDLE; 339#endif 340} 341 342void intel_gt_suspend_late(struct intel_gt *gt) 343{ 344 intel_wakeref_t wakeref; 345 346 /* We expect to be idle already; but also want to be independent */ 347 wait_for_suspend(gt); 348 349 if (is_mock_gt(gt)) 350 return; 351 352 GEM_BUG_ON(gt->awake); 353 354 intel_uc_suspend(>->uc); 355 356 /* 357 * On disabling the device, we want to turn off HW access to memory 358 * that we no longer own. 359 * 360 * However, not all suspend-states disable the device. S0 (s2idle) 361 * is effectively runtime-suspend, the device is left powered on 362 * but needs to be put into a low power state. We need to keep 363 * powermanagement enabled, but we also retain system state and so 364 * it remains safe to keep on using our allocated memory. 365 */ 366 if (pm_suspend_target() == PM_SUSPEND_TO_IDLE) 367 return; 368 369 with_intel_runtime_pm(gt->uncore->rpm, wakeref) { 370 intel_rps_disable(>->rps); 371 intel_rc6_disable(>->rc6); 372 intel_llc_disable(>->llc); 373 } 374 375 gt_sanitize(gt, false); 376 377 GT_TRACE(gt, "\n"); 378} 379 380void intel_gt_runtime_suspend(struct intel_gt *gt) 381{ 382 intel_gt_bind_context_set_unready(gt); 383 intel_uc_runtime_suspend(>->uc); 384 385 GT_TRACE(gt, "\n"); 386} 387 388int intel_gt_runtime_resume(struct intel_gt *gt) 389{ 390 int ret; 391 392 GT_TRACE(gt, "\n"); 393 intel_gt_init_swizzling(gt); 394 intel_ggtt_restore_fences(gt->ggtt); 395 396 ret = intel_uc_runtime_resume(>->uc); 397 if (ret) 398 return ret; 399 400 intel_gt_bind_context_set_ready(gt); 401 return 0; 402} 403 404static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt) 405{ 406 ktime_t total = gt->stats.total; 407 408 if (gt->stats.active) 409 total = ktime_add(total, 410 ktime_sub(ktime_get(), gt->stats.start)); 411 412 return total; 413} 414 415ktime_t intel_gt_get_awake_time(const struct intel_gt *gt) 416{ 417 unsigned int seq; 418 ktime_t total; 419 420 do { 421 seq = read_seqcount_begin(>->stats.lock); 422 total = __intel_gt_get_awake_time(gt); 423 } while (read_seqcount_retry(>->stats.lock, seq)); 424 425 return total; 426} 427 428#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 429#include "selftest_gt_pm.c" 430#endif 431