11590Srgrimes/* 21590Srgrimes * SPDX-License-Identifier: MIT 31590Srgrimes * 41590Srgrimes * Copyright �� 2019 Intel Corporation 51590Srgrimes */ 61590Srgrimes 71590Srgrimes#include "gem/i915_gem_pm.h" 81590Srgrimes#include "gem/i915_gem_ttm_pm.h" 91590Srgrimes#include "gt/intel_gt.h" 101590Srgrimes#include "gt/intel_gt_pm.h" 111590Srgrimes#include "gt/intel_gt_requests.h" 121590Srgrimes 131590Srgrimes#include "i915_driver.h" 141590Srgrimes#include "i915_drv.h" 151590Srgrimes 161590Srgrimes#if defined(CONFIG_X86) 171590Srgrimes#include <asm/smp.h> 181590Srgrimes#else 191590Srgrimes#define wbinvd_on_all_cpus() \ 201590Srgrimes pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__) 211590Srgrimes#endif 221590Srgrimes 231590Srgrimesvoid i915_gem_suspend(struct drm_i915_private *i915) 241590Srgrimes{ 251590Srgrimes struct intel_gt *gt; 261590Srgrimes unsigned int i; 271590Srgrimes 281590Srgrimes GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 291590Srgrimes 301590Srgrimes intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0); 3174769Smikeh /* 321590Srgrimes * On rare occasions, we've observed the fence completion triggers 3374769Smikeh * free_engines asynchronously via rcu_call. Ensure those are done. 341590Srgrimes * This path is only called on suspend, so it's an acceptable cost. 3599112Sobrien */ 3699112Sobrien rcu_barrier(); 371590Srgrimes 381590Srgrimes flush_workqueue(i915->wq); 391590Srgrimes 401590Srgrimes /* 411590Srgrimes * We have to flush all the executing contexts to main memory so 421590Srgrimes * that they can saved in the hibernation image. To ensure the last 431590Srgrimes * context image is coherent, we have to switch away from it. That 441590Srgrimes * leaves the i915->kernel_context still active when 451590Srgrimes * we actually suspend, and its image in memory may not match the GPU 4688428Smikeh * state. Fortunately, the kernel_context is disposable and we do 471590Srgrimes * not rely on its state. 481590Srgrimes */ 491590Srgrimes for_each_gt(gt, i915, i) 501590Srgrimes intel_gt_suspend_prepare(gt); 511590Srgrimes 521590Srgrimes i915_gem_drain_freed_objects(i915); 531590Srgrimes} 541590Srgrimes 551590Srgrimesstatic int lmem_restore(struct drm_i915_private *i915, u32 flags) 561590Srgrimes{ 571590Srgrimes struct intel_memory_region *mr; 581590Srgrimes int ret = 0, id; 591590Srgrimes 601590Srgrimes for_each_memory_region(mr, i915, id) { 611590Srgrimes if (mr->type == INTEL_MEMORY_LOCAL) { 621590Srgrimes ret = i915_ttm_restore_region(mr, flags); 631590Srgrimes if (ret) 641590Srgrimes break; 651590Srgrimes } 661590Srgrimes } 671590Srgrimes 681590Srgrimes return ret; 691590Srgrimes} 701590Srgrimes 711590Srgrimesstatic int lmem_suspend(struct drm_i915_private *i915, u32 flags) 721590Srgrimes{ 73216564Scharnier struct intel_memory_region *mr; 741590Srgrimes int ret = 0, id; 751590Srgrimes 7677274Smikeh for_each_memory_region(mr, i915, id) { 7777274Smikeh if (mr->type == INTEL_MEMORY_LOCAL) { 7888150Smikeh ret = i915_ttm_backup_region(mr, flags); 7988150Smikeh if (ret) 8088150Smikeh break; 811590Srgrimes } 821590Srgrimes } 831590Srgrimes 841590Srgrimes return ret; 851590Srgrimes} 861590Srgrimes 8788150Smikehstatic void lmem_recover(struct drm_i915_private *i915) 8888150Smikeh{ 8988150Smikeh struct intel_memory_region *mr; 9088150Smikeh int id; 911590Srgrimes 9277274Smikeh for_each_memory_region(mr, i915, id) 931590Srgrimes if (mr->type == INTEL_MEMORY_LOCAL) 9477274Smikeh i915_ttm_recover_region(mr); 951590Srgrimes} 961590Srgrimes 971590Srgrimesint i915_gem_backup_suspend(struct drm_i915_private *i915) 981590Srgrimes{ 9977274Smikeh int ret; 1001590Srgrimes 1011590Srgrimes /* Opportunistically try to evict unpinned objects */ 10288150Smikeh ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU); 1031590Srgrimes if (ret) 1041590Srgrimes goto out_recover; 10577274Smikeh 10677274Smikeh i915_gem_suspend(i915); 10774769Smikeh 10874769Smikeh /* 10974769Smikeh * More objects may have become unpinned as requests were 1101590Srgrimes * retired. Now try to evict again. The gt may be wedged here 1111590Srgrimes * in which case we automatically fall back to memcpy. 11277274Smikeh * We allow also backing up pinned objects that have not been 1131590Srgrimes * marked for early recover, and that may contain, for example, 1141590Srgrimes * page-tables for the migrate context. 1151590Srgrimes */ 1161590Srgrimes ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU | 1171590Srgrimes I915_TTM_BACKUP_PINNED); 1181590Srgrimes if (ret) 1191590Srgrimes goto out_recover; 1201590Srgrimes 12177274Smikeh /* 12277274Smikeh * Remaining objects are backed up using memcpy once we've stopped 1231590Srgrimes * using the migrate context. 1241590Srgrimes */ 1251590Srgrimes ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED); 12677274Smikeh if (ret) 1271590Srgrimes goto out_recover; 12877274Smikeh 1291590Srgrimes return 0; 1301590Srgrimes 1311590Srgrimesout_recover: 1321590Srgrimes lmem_recover(i915); 1331590Srgrimes 13488150Smikeh return ret; 13588150Smikeh} 1361590Srgrimes 1371590Srgrimesvoid i915_gem_suspend_late(struct drm_i915_private *i915) 1381590Srgrimes{ 1391590Srgrimes struct drm_i915_gem_object *obj; 1401590Srgrimes struct list_head *phases[] = { 1411590Srgrimes &i915->mm.shrink_list, 1421590Srgrimes &i915->mm.purge_list, 1431590Srgrimes NULL 1441590Srgrimes }, **phase; 1451590Srgrimes struct intel_gt *gt; 1461590Srgrimes unsigned long flags; 1471590Srgrimes unsigned int i; 14877274Smikeh bool flush = false; 1491590Srgrimes 1501590Srgrimes /* 1511590Srgrimes * Neither the BIOS, ourselves or any other kernel 1521590Srgrimes * expects the system to be in execlists mode on startup, 15377274Smikeh * so we need to reset the GPU back to legacy mode. And the only 1541590Srgrimes * known way to disable logical contexts is through a GPU reset. 1551590Srgrimes * 1561590Srgrimes * So in order to leave the system in a known default configuration, 1571590Srgrimes * always reset the GPU upon unload and suspend. Afterwards we then 1581590Srgrimes * clean up the GEM state tracking, flushing off the requests and 1591590Srgrimes * leaving the system in a known idle state. 1601590Srgrimes * 16177274Smikeh * Note that is of the upmost importance that the GPU is idle and 16277274Smikeh * all stray writes are flushed *before* we dismantle the backing 1631590Srgrimes * storage for the pinned objects. 1641590Srgrimes * 1651590Srgrimes * However, since we are uncertain that resetting the GPU on older 1661590Srgrimes * machines is a good idea, we don't - just in case it leaves the 1671590Srgrimes * machine in an unusable condition. 16888150Smikeh */ 16988150Smikeh 1701590Srgrimes /* Like i915_gem_suspend, flush tasks staged from fence triggers */ 1711590Srgrimes rcu_barrier(); 1721590Srgrimes 17388150Smikeh for_each_gt(gt, i915, i) 17477274Smikeh intel_gt_suspend_late(gt); 1751590Srgrimes 17688150Smikeh spin_lock_irqsave(&i915->mm.obj_lock, flags); 17788150Smikeh for (phase = phases; *phase; phase++) { 17888150Smikeh list_for_each_entry(obj, *phase, mm.link) { 1791590Srgrimes if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) 1801590Srgrimes flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0; 1811590Srgrimes __start_cpu_write(obj); /* presume auto-hibernate */ 1821590Srgrimes } 1831590Srgrimes } 1841590Srgrimes spin_unlock_irqrestore(&i915->mm.obj_lock, flags); 1851590Srgrimes if (flush) 1861590Srgrimes wbinvd_on_all_cpus(); 1871590Srgrimes} 1881590Srgrimes 1891590Srgrimesint i915_gem_freeze(struct drm_i915_private *i915) 19088150Smikeh{ 1911590Srgrimes /* Discard all purgeable objects, let userspace recover those as 1921590Srgrimes * required after resuming. 1931590Srgrimes */ 1941590Srgrimes i915_gem_shrink_all(i915); 1951590Srgrimes 1961590Srgrimes return 0; 1971590Srgrimes} 1981590Srgrimes 1991590Srgrimesint i915_gem_freeze_late(struct drm_i915_private *i915) 2001590Srgrimes{ 2011590Srgrimes struct drm_i915_gem_object *obj; 2021590Srgrimes intel_wakeref_t wakeref; 2031590Srgrimes 2041590Srgrimes /* 2051590Srgrimes * Called just before we write the hibernation image. 2061590Srgrimes * 2071590Srgrimes * We need to update the domain tracking to reflect that the CPU 2081590Srgrimes * will be accessing all the pages to create and restore from the 2091590Srgrimes * hibernation, and so upon restoration those pages will be in the 2101590Srgrimes * CPU domain. 21188428Smikeh * 2121590Srgrimes * To make sure the hibernation image contains the latest state, 2131590Srgrimes * we update that state just before writing out the image. 2141590Srgrimes * 2151590Srgrimes * To try and reduce the hibernation image, we manually shrink 2161590Srgrimes * the objects as well, see i915_gem_freeze() 2171590Srgrimes */ 2181590Srgrimes 2191590Srgrimes with_intel_runtime_pm(&i915->runtime_pm, wakeref) 2201590Srgrimes i915_gem_shrink(NULL, i915, -1UL, NULL, ~0); 2211590Srgrimes i915_gem_drain_freed_objects(i915); 2221590Srgrimes 2231590Srgrimes wbinvd_on_all_cpus(); 2241590Srgrimes list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) 2251590Srgrimes __start_cpu_write(obj); 2261590Srgrimes 2271590Srgrimes return 0; 2281590Srgrimes} 2291590Srgrimes 23088428Smikehvoid i915_gem_resume(struct drm_i915_private *i915) 23188428Smikeh{ 23288428Smikeh struct intel_gt *gt; 23388428Smikeh int ret, i, j; 23488428Smikeh 2351590Srgrimes GEM_TRACE("%s\n", dev_name(i915->drm.dev)); 2361590Srgrimes 2371590Srgrimes ret = lmem_restore(i915, 0); 2381590Srgrimes GEM_WARN_ON(ret); 2391590Srgrimes 2401590Srgrimes /* 2411590Srgrimes * As we didn't flush the kernel context before suspend, we cannot 2421590Srgrimes * guarantee that the context image is complete. So let's just reset 2431590Srgrimes * it and start again. 2441590Srgrimes */ 2451590Srgrimes for_each_gt(gt, i915, i) 2461590Srgrimes if (intel_gt_resume(gt)) 2471590Srgrimes goto err_wedged; 2481590Srgrimes 24932189Sjoerg ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU); 2501590Srgrimes GEM_WARN_ON(ret); 2511590Srgrimes 25288227Sache return; 2531590Srgrimes 2541590Srgrimeserr_wedged: 2551590Srgrimes for_each_gt(gt, i915, j) { 25632189Sjoerg if (!intel_gt_is_wedged(gt)) { 25732189Sjoerg dev_err(i915->drm.dev, 25832189Sjoerg "Failed to re-initialize GPU[%u], declaring it wedged!\n", 25932189Sjoerg j); 26032189Sjoerg intel_gt_set_wedged(gt); 26188227Sache } 26232189Sjoerg 26332189Sjoerg if (j == i) 26432189Sjoerg break; 2651590Srgrimes } 2661590Srgrimes} 2671590Srgrimes