1/* $NetBSD: intel_runtime_pm.c,v 1.12 2021/12/19 12:32:15 riastradh Exp $ */ 2 3/* 4 * Copyright �� 2012-2014 Intel Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * Eugeni Dodonov <eugeni.dodonov@intel.com> 27 * Daniel Vetter <daniel.vetter@ffwll.ch> 28 * 29 */ 30 31#include <sys/cdefs.h> 32__KERNEL_RCSID(0, "$NetBSD: intel_runtime_pm.c,v 1.12 2021/12/19 12:32:15 riastradh Exp $"); 33 34#include <linux/pm_runtime.h> 35 36#include <drm/drm_print.h> 37 38#include "i915_drv.h" 39#include "i915_trace.h" 40 41#include <linux/nbsd-namespace.h> 42 43/** 44 * DOC: runtime pm 45 * 46 * The i915 driver supports dynamic enabling and disabling of entire hardware 47 * blocks at runtime. This is especially important on the display side where 48 * software is supposed to control many power gates manually on recent hardware, 49 * since on the GT side a lot of the power management is done by the hardware. 50 * But even there some manual control at the device level is required. 51 * 52 * Since i915 supports a diverse set of platforms with a unified codebase and 53 * hardware engineers just love to shuffle functionality around between power 54 * domains there's a sizeable amount of indirection required. This file provides 55 * generic functions to the driver for grabbing and releasing references for 56 * abstract power domains. It then maps those to the actual power wells 57 * present for a given platform. 58 */ 59 60#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) 61 62#include <linux/sort.h> 63 64#define STACKDEPTH 8 65 66static noinline depot_stack_handle_t __save_depot_stack(void) 67{ 68 unsigned long entries[STACKDEPTH]; 69 unsigned int n; 70 71 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 72 return stack_depot_save(entries, n, GFP_NOWAIT | __GFP_NOWARN); 73} 74 75static void __print_depot_stack(depot_stack_handle_t stack, 76 char *buf, int sz, int indent) 77{ 78 unsigned long *entries; 79 unsigned int nr_entries; 80 81 nr_entries = stack_depot_fetch(stack, &entries); 82 stack_trace_snprint(buf, sz, entries, nr_entries, indent); 83} 84 85static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 86{ 87 spin_lock_init(&rpm->debug.lock); 88} 89 90static void fini_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 91{ 92 spin_lock_fini(&rpm->debug.lock); 93} 94 95static noinline depot_stack_handle_t 96track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 97{ 98 depot_stack_handle_t stack, *stacks; 99 unsigned long flags; 100 101 if (!rpm->available) 102 return -1; 103 104 stack = __save_depot_stack(); 105 if (!stack) 106 return -1; 107 108 spin_lock_irqsave(&rpm->debug.lock, flags); 109 110 if (!rpm->debug.count) 111 rpm->debug.last_acquire = stack; 112 113 stacks = krealloc(rpm->debug.owners, 114 (rpm->debug.count + 1) * sizeof(*stacks), 115 GFP_NOWAIT | __GFP_NOWARN); 116 if (stacks) { 117 stacks[rpm->debug.count++] = stack; 118 rpm->debug.owners = stacks; 119 } else { 120 stack = -1; 121 } 122 123 spin_unlock_irqrestore(&rpm->debug.lock, flags); 124 125 return stack; 126} 127 128static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, 129 depot_stack_handle_t stack) 130{ 131 unsigned long flags, n; 132 bool found = false; 133 134 if (unlikely(stack == -1)) 135 return; 136 137 spin_lock_irqsave(&rpm->debug.lock, flags); 138 for (n = rpm->debug.count; n--; ) { 139 if (rpm->debug.owners[n] == stack) { 140 memmove(rpm->debug.owners + n, 141 rpm->debug.owners + n + 1, 142 (--rpm->debug.count - n) * sizeof(stack)); 143 found = true; 144 break; 145 } 146 } 147 spin_unlock_irqrestore(&rpm->debug.lock, flags); 148 149 if (WARN(!found, 150 "Unmatched wakeref (tracking %lu), count %u\n", 151 rpm->debug.count, atomic_read(&rpm->wakeref_count))) { 152 char *buf; 153 154 buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN); 155 if (!buf) 156 return; 157 158 __print_depot_stack(stack, buf, PAGE_SIZE, 2); 159 DRM_DEBUG_DRIVER("wakeref %x from\n%s", stack, buf); 160 161 stack = READ_ONCE(rpm->debug.last_release); 162 if (stack) { 163 __print_depot_stack(stack, buf, PAGE_SIZE, 2); 164 DRM_DEBUG_DRIVER("wakeref last released at\n%s", buf); 165 } 166 167 kfree(buf); 168 } 169} 170 171static int cmphandle(const void *_a, const void *_b) 172{ 173 const depot_stack_handle_t * const a = _a, * const b = _b; 174 175 if (*a < *b) 176 return -1; 177 else if (*a > *b) 178 return 1; 179 else 180 return 0; 181} 182 183static void 184__print_intel_runtime_pm_wakeref(struct drm_printer *p, 185 const struct intel_runtime_pm_debug *dbg) 186{ 187 unsigned long i; 188 char *buf; 189 190 buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN); 191 if (!buf) 192 return; 193 194 if (dbg->last_acquire) { 195 __print_depot_stack(dbg->last_acquire, buf, PAGE_SIZE, 2); 196 drm_printf(p, "Wakeref last acquired:\n%s", buf); 197 } 198 199 if (dbg->last_release) { 200 __print_depot_stack(dbg->last_release, buf, PAGE_SIZE, 2); 201 drm_printf(p, "Wakeref last released:\n%s", buf); 202 } 203 204 drm_printf(p, "Wakeref count: %lu\n", dbg->count); 205 206 sort(dbg->owners, dbg->count, sizeof(*dbg->owners), cmphandle, NULL); 207 208 for (i = 0; i < dbg->count; i++) { 209 depot_stack_handle_t stack = dbg->owners[i]; 210 unsigned long rep; 211 212 rep = 1; 213 while (i + 1 < dbg->count && dbg->owners[i + 1] == stack) 214 rep++, i++; 215 __print_depot_stack(stack, buf, PAGE_SIZE, 2); 216 drm_printf(p, "Wakeref x%lu taken at:\n%s", rep, buf); 217 } 218 219 kfree(buf); 220} 221 222static noinline void 223__untrack_all_wakerefs(struct intel_runtime_pm_debug *debug, 224 struct intel_runtime_pm_debug *saved) 225{ 226 *saved = *debug; 227 228 debug->owners = NULL; 229 debug->count = 0; 230 debug->last_release = __save_depot_stack(); 231} 232 233static void 234dump_and_free_wakeref_tracking(struct intel_runtime_pm_debug *debug) 235{ 236 if (debug->count) { 237 struct drm_printer p = drm_debug_printer("i915"); 238 239 __print_intel_runtime_pm_wakeref(&p, debug); 240 } 241 242 kfree(debug->owners); 243} 244 245static noinline void 246__intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm) 247{ 248 struct intel_runtime_pm_debug dbg = {}; 249 unsigned long flags; 250 251 if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count, 252 &rpm->debug.lock, 253 flags)) 254 return; 255 256 __untrack_all_wakerefs(&rpm->debug, &dbg); 257 spin_unlock_irqrestore(&rpm->debug.lock, flags); 258 259 dump_and_free_wakeref_tracking(&dbg); 260} 261 262static noinline void 263untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm) 264{ 265 struct intel_runtime_pm_debug dbg = {}; 266 unsigned long flags; 267 268 spin_lock_irqsave(&rpm->debug.lock, flags); 269 __untrack_all_wakerefs(&rpm->debug, &dbg); 270 spin_unlock_irqrestore(&rpm->debug.lock, flags); 271 272 dump_and_free_wakeref_tracking(&dbg); 273} 274 275void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, 276 struct drm_printer *p) 277{ 278 struct intel_runtime_pm_debug dbg = {}; 279 280 do { 281 unsigned long alloc = dbg.count; 282 depot_stack_handle_t *s; 283 284 spin_lock_irq(&rpm->debug.lock); 285 dbg.count = rpm->debug.count; 286 if (dbg.count <= alloc) { 287 memcpy(dbg.owners, 288 rpm->debug.owners, 289 dbg.count * sizeof(*s)); 290 } 291 dbg.last_acquire = rpm->debug.last_acquire; 292 dbg.last_release = rpm->debug.last_release; 293 spin_unlock_irq(&rpm->debug.lock); 294 if (dbg.count <= alloc) 295 break; 296 297 s = krealloc(dbg.owners, 298 dbg.count * sizeof(*s), 299 GFP_NOWAIT | __GFP_NOWARN); 300 if (!s) 301 goto out; 302 303 dbg.owners = s; 304 } while (1); 305 306 __print_intel_runtime_pm_wakeref(p, &dbg); 307 308out: 309 kfree(dbg.owners); 310} 311 312#else 313 314static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 315{ 316} 317 318static void fini_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 319{ 320} 321 322static depot_stack_handle_t 323track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 324{ 325 return -1; 326} 327 328static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, 329 intel_wakeref_t wref) 330{ 331} 332 333static void 334__intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm) 335{ 336 atomic_dec(&rpm->wakeref_count); 337} 338 339static void 340untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm) 341{ 342} 343 344#endif 345 346static void 347intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock) 348{ 349 if (wakelock) { 350 atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count); 351 assert_rpm_wakelock_held(rpm); 352 } else { 353 atomic_inc(&rpm->wakeref_count); 354 assert_rpm_raw_wakeref_held(rpm); 355 } 356} 357 358static void 359intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock) 360{ 361 if (wakelock) { 362 assert_rpm_wakelock_held(rpm); 363 atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count); 364 } else { 365 assert_rpm_raw_wakeref_held(rpm); 366 } 367 368 __intel_wakeref_dec_and_check_tracking(rpm); 369} 370 371static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm, 372 bool wakelock) 373{ 374 int ret; 375 376 ret = pm_runtime_get_sync(rpm->kdev); 377 WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret); 378 379 intel_runtime_pm_acquire(rpm, wakelock); 380 381 return track_intel_runtime_pm_wakeref(rpm); 382} 383 384/** 385 * intel_runtime_pm_get_raw - grab a raw runtime pm reference 386 * @rpm: the intel_runtime_pm structure 387 * 388 * This is the unlocked version of intel_display_power_is_enabled() and should 389 * only be used from error capture and recovery code where deadlocks are 390 * possible. 391 * This function grabs a device-level runtime pm reference (mostly used for 392 * asynchronous PM management from display code) and ensures that it is powered 393 * up. Raw references are not considered during wakelock assert checks. 394 * 395 * Any runtime pm reference obtained by this function must have a symmetric 396 * call to intel_runtime_pm_put_raw() to release the reference again. 397 * 398 * Returns: the wakeref cookie to pass to intel_runtime_pm_put_raw(), evaluates 399 * as True if the wakeref was acquired, or False otherwise. 400 */ 401intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm) 402{ 403 return __intel_runtime_pm_get(rpm, false); 404} 405 406/** 407 * intel_runtime_pm_get - grab a runtime pm reference 408 * @rpm: the intel_runtime_pm structure 409 * 410 * This function grabs a device-level runtime pm reference (mostly used for GEM 411 * code to ensure the GTT or GT is on) and ensures that it is powered up. 412 * 413 * Any runtime pm reference obtained by this function must have a symmetric 414 * call to intel_runtime_pm_put() to release the reference again. 415 * 416 * Returns: the wakeref cookie to pass to intel_runtime_pm_put() 417 */ 418intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm) 419{ 420 return __intel_runtime_pm_get(rpm, true); 421} 422 423/** 424 * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use 425 * @rpm: the intel_runtime_pm structure 426 * 427 * This function grabs a device-level runtime pm reference if the device is 428 * already in use and ensures that it is powered up. It is illegal to try 429 * and access the HW should intel_runtime_pm_get_if_in_use() report failure. 430 * 431 * Any runtime pm reference obtained by this function must have a symmetric 432 * call to intel_runtime_pm_put() to release the reference again. 433 * 434 * Returns: the wakeref cookie to pass to intel_runtime_pm_put(), evaluates 435 * as True if the wakeref was acquired, or False otherwise. 436 */ 437intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm) 438{ 439 if (IS_ENABLED(CONFIG_PM)) { 440 /* 441 * In cases runtime PM is disabled by the RPM core and we get 442 * an -EINVAL return value we are not supposed to call this 443 * function, since the power state is undefined. This applies 444 * atm to the late/early system suspend/resume handlers. 445 */ 446 if (pm_runtime_get_if_in_use(rpm->kdev) <= 0) 447 return 0; 448 } 449 450 intel_runtime_pm_acquire(rpm, true); 451 452 return track_intel_runtime_pm_wakeref(rpm); 453} 454 455/** 456 * intel_runtime_pm_get_noresume - grab a runtime pm reference 457 * @rpm: the intel_runtime_pm structure 458 * 459 * This function grabs a device-level runtime pm reference (mostly used for GEM 460 * code to ensure the GTT or GT is on). 461 * 462 * It will _not_ power up the device but instead only check that it's powered 463 * on. Therefore it is only valid to call this functions from contexts where 464 * the device is known to be powered up and where trying to power it up would 465 * result in hilarity and deadlocks. That pretty much means only the system 466 * suspend/resume code where this is used to grab runtime pm references for 467 * delayed setup down in work items. 468 * 469 * Any runtime pm reference obtained by this function must have a symmetric 470 * call to intel_runtime_pm_put() to release the reference again. 471 * 472 * Returns: the wakeref cookie to pass to intel_runtime_pm_put() 473 */ 474intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm) 475{ 476 assert_rpm_wakelock_held(rpm); 477 pm_runtime_get_noresume(rpm->kdev); 478 479 intel_runtime_pm_acquire(rpm, true); 480 481 return track_intel_runtime_pm_wakeref(rpm); 482} 483 484static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm, 485 intel_wakeref_t wref, 486 bool wakelock) 487{ 488 struct device *kdev = rpm->kdev; 489 490 untrack_intel_runtime_pm_wakeref(rpm, wref); 491 492 intel_runtime_pm_release(rpm, wakelock); 493 494 pm_runtime_mark_last_busy(kdev); 495 pm_runtime_put_autosuspend(kdev); 496} 497 498/** 499 * intel_runtime_pm_put_raw - release a raw runtime pm reference 500 * @rpm: the intel_runtime_pm structure 501 * @wref: wakeref acquired for the reference that is being released 502 * 503 * This function drops the device-level runtime pm reference obtained by 504 * intel_runtime_pm_get_raw() and might power down the corresponding 505 * hardware block right away if this is the last reference. 506 */ 507void 508intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref) 509{ 510 __intel_runtime_pm_put(rpm, wref, false); 511} 512 513/** 514 * intel_runtime_pm_put_unchecked - release an unchecked runtime pm reference 515 * @rpm: the intel_runtime_pm structure 516 * 517 * This function drops the device-level runtime pm reference obtained by 518 * intel_runtime_pm_get() and might power down the corresponding 519 * hardware block right away if this is the last reference. 520 * 521 * This function exists only for historical reasons and should be avoided in 522 * new code, as the correctness of its use cannot be checked. Always use 523 * intel_runtime_pm_put() instead. 524 */ 525void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm) 526{ 527 __intel_runtime_pm_put(rpm, -1, true); 528} 529 530#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) 531/** 532 * intel_runtime_pm_put - release a runtime pm reference 533 * @rpm: the intel_runtime_pm structure 534 * @wref: wakeref acquired for the reference that is being released 535 * 536 * This function drops the device-level runtime pm reference obtained by 537 * intel_runtime_pm_get() and might power down the corresponding 538 * hardware block right away if this is the last reference. 539 */ 540void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref) 541{ 542 __intel_runtime_pm_put(rpm, wref, true); 543} 544#endif 545 546/** 547 * intel_runtime_pm_enable - enable runtime pm 548 * @rpm: the intel_runtime_pm structure 549 * 550 * This function enables runtime pm at the end of the driver load sequence. 551 * 552 * Note that this function does currently not enable runtime pm for the 553 * subordinate display power domains. That is done by 554 * intel_power_domains_enable(). 555 */ 556void intel_runtime_pm_enable(struct intel_runtime_pm *rpm) 557{ 558 struct device *kdev = rpm->kdev; 559 560 /* 561 * Disable the system suspend direct complete optimization, which can 562 * leave the device suspended skipping the driver's suspend handlers 563 * if the device was already runtime suspended. This is needed due to 564 * the difference in our runtime and system suspend sequence and 565 * becaue the HDA driver may require us to enable the audio power 566 * domain during system suspend. 567 */ 568 dev_pm_set_driver_flags(kdev, DPM_FLAG_NEVER_SKIP); 569 570 pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */ 571 pm_runtime_mark_last_busy(kdev); 572 573 /* 574 * Take a permanent reference to disable the RPM functionality and drop 575 * it only when unloading the driver. Use the low level get/put helpers, 576 * so the driver's own RPM reference tracking asserts also work on 577 * platforms without RPM support. 578 */ 579 if (!rpm->available) { 580 int ret; 581 582 pm_runtime_dont_use_autosuspend(kdev); 583 ret = pm_runtime_get_sync(kdev); 584 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret); 585 } else { 586 pm_runtime_use_autosuspend(kdev); 587 } 588 589 /* 590 * The core calls the driver load handler with an RPM reference held. 591 * We drop that here and will reacquire it during unloading in 592 * intel_power_domains_fini(). 593 */ 594 pm_runtime_put_autosuspend(kdev); 595} 596 597void intel_runtime_pm_disable(struct intel_runtime_pm *rpm) 598{ 599 struct device *kdev = rpm->kdev; 600 601 /* Transfer rpm ownership back to core */ 602 WARN(pm_runtime_get_sync(kdev) < 0, 603 "Failed to pass rpm ownership back to core\n"); 604 605 pm_runtime_dont_use_autosuspend(kdev); 606 607 if (!rpm->available) 608 pm_runtime_put(kdev); 609} 610 611void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm) 612{ 613 int count = atomic_read(&rpm->wakeref_count); 614 615 WARN(count, 616 "i915 raw-wakerefs=%d wakelocks=%d on cleanup\n", 617 intel_rpm_raw_wakeref_count(count), 618 intel_rpm_wakelock_count(count)); 619 620 untrack_all_intel_runtime_pm_wakerefs(rpm); 621 fini_intel_runtime_pm_wakeref(rpm); 622} 623 624void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm) 625{ 626 struct drm_i915_private *i915 = 627 container_of(rpm, struct drm_i915_private, runtime_pm); 628 struct pci_dev *pdev = i915->drm.pdev; 629 struct device *kdev = pci_dev_dev(pdev); 630 631 rpm->kdev = kdev; 632 rpm->available = HAS_RUNTIME_PM(i915); 633 634 init_intel_runtime_pm_wakeref(rpm); 635} 636