1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2019 Intel Corporation
4 */
5
6#include <linux/string_helpers.h>
7#include <linux/suspend.h>
8
9#include "i915_drv.h"
10#include "i915_irq.h"
11#include "i915_params.h"
12#include "intel_context.h"
13#include "intel_engine_pm.h"
14#include "intel_gt.h"
15#include "intel_gt_clock_utils.h"
16#include "intel_gt_mcr.h"
17#include "intel_gt_pm.h"
18#include "intel_gt_print.h"
19#include "intel_gt_requests.h"
20#include "intel_llc.h"
21#include "intel_rc6.h"
22#include "intel_rps.h"
23#include "intel_wakeref.h"
24#include "pxp/intel_pxp_pm.h"
25
26#define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
27
28static void user_forcewake(struct intel_gt *gt, bool suspend)
29{
30	int count = atomic_read(&gt->user_wakeref);
31	intel_wakeref_t wakeref;
32
33	/* Inside suspend/resume so single threaded, no races to worry about. */
34	if (likely(!count))
35		return;
36
37	wakeref = intel_gt_pm_get(gt);
38	if (suspend) {
39		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
40		atomic_sub(count, &gt->wakeref.count);
41	} else {
42		atomic_add(count, &gt->wakeref.count);
43	}
44	intel_gt_pm_put(gt, wakeref);
45}
46
47static void runtime_begin(struct intel_gt *gt)
48{
49	local_irq_disable();
50	write_seqcount_begin(&gt->stats.lock);
51	gt->stats.start = ktime_get();
52	gt->stats.active = true;
53	write_seqcount_end(&gt->stats.lock);
54	local_irq_enable();
55}
56
57static void runtime_end(struct intel_gt *gt)
58{
59	local_irq_disable();
60	write_seqcount_begin(&gt->stats.lock);
61	gt->stats.active = false;
62	gt->stats.total =
63		ktime_add(gt->stats.total,
64			  ktime_sub(ktime_get(), gt->stats.start));
65	write_seqcount_end(&gt->stats.lock);
66	local_irq_enable();
67}
68
69static int __gt_unpark(struct intel_wakeref *wf)
70{
71	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
72	struct drm_i915_private *i915 = gt->i915;
73
74	GT_TRACE(gt, "\n");
75
76	/*
77	 * It seems that the DMC likes to transition between the DC states a lot
78	 * when there are no connected displays (no active power domains) during
79	 * command submission.
80	 *
81	 * This activity has negative impact on the performance of the chip with
82	 * huge latencies observed in the interrupt handler and elsewhere.
83	 *
84	 * Work around it by grabbing a GT IRQ power domain whilst there is any
85	 * GT activity, preventing any DC state transitions.
86	 */
87	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
88	GEM_BUG_ON(!gt->awake);
89
90	intel_rc6_unpark(&gt->rc6);
91	intel_rps_unpark(&gt->rps);
92	i915_pmu_gt_unparked(gt);
93	intel_guc_busyness_unpark(gt);
94
95	intel_gt_unpark_requests(gt);
96	runtime_begin(gt);
97
98	return 0;
99}
100
101static int __gt_park(struct intel_wakeref *wf)
102{
103	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
104	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
105	struct drm_i915_private *i915 = gt->i915;
106
107	GT_TRACE(gt, "\n");
108
109	runtime_end(gt);
110	intel_gt_park_requests(gt);
111
112	intel_guc_busyness_park(gt);
113	i915_vma_parked(gt);
114	i915_pmu_gt_parked(gt);
115	intel_rps_park(&gt->rps);
116	intel_rc6_park(&gt->rc6);
117
118	/* Everything switched off, flush any residual interrupt just in case */
119	intel_synchronize_irq(i915);
120
121	/* Defer dropping the display power well for 100ms, it's slow! */
122	GEM_BUG_ON(!wakeref);
123	intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
124
125	return 0;
126}
127
128static const struct intel_wakeref_ops wf_ops = {
129	.get = __gt_unpark,
130	.put = __gt_park,
131};
132
133void intel_gt_pm_init_early(struct intel_gt *gt)
134{
135	/*
136	 * We access the runtime_pm structure via gt->i915 here rather than
137	 * gt->uncore as we do elsewhere in the file because gt->uncore is not
138	 * yet initialized for all tiles at this point in the driver startup.
139	 * runtime_pm is per-device rather than per-tile, so this is still the
140	 * correct structure.
141	 */
142	intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops, "GT");
143	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
144}
145
146void intel_gt_pm_init(struct intel_gt *gt)
147{
148	/*
149	 * Enabling power-management should be "self-healing". If we cannot
150	 * enable a feature, simply leave it disabled with a notice to the
151	 * user.
152	 */
153	intel_rc6_init(&gt->rc6);
154	intel_rps_init(&gt->rps);
155}
156
157static bool reset_engines(struct intel_gt *gt)
158{
159	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
160		return false;
161
162	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
163}
164
165static void gt_sanitize(struct intel_gt *gt, bool force)
166{
167	struct intel_engine_cs *engine;
168	enum intel_engine_id id;
169	intel_wakeref_t wakeref;
170
171	GT_TRACE(gt, "force:%s\n", str_yes_no(force));
172
173	/* Use a raw wakeref to avoid calling intel_display_power_get early */
174	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
175	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
176
177	intel_gt_check_clock_frequency(gt);
178
179	/*
180	 * As we have just resumed the machine and woken the device up from
181	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
182	 * back to defaults, recovering from whatever wedged state we left it
183	 * in and so worth trying to use the device once more.
184	 */
185	if (intel_gt_is_wedged(gt))
186		intel_gt_unset_wedged(gt);
187
188	/* For GuC mode, ensure submission is disabled before stopping ring */
189	intel_uc_reset_prepare(&gt->uc);
190
191	for_each_engine(engine, gt, id) {
192		if (engine->reset.prepare)
193			engine->reset.prepare(engine);
194
195		if (engine->sanitize)
196			engine->sanitize(engine);
197	}
198
199	if (reset_engines(gt) || force) {
200		for_each_engine(engine, gt, id)
201			__intel_engine_reset(engine, false);
202	}
203
204	intel_uc_reset(&gt->uc, false);
205
206	for_each_engine(engine, gt, id)
207		if (engine->reset.finish)
208			engine->reset.finish(engine);
209
210	intel_rps_sanitize(&gt->rps);
211
212	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
213	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
214}
215
216void intel_gt_pm_fini(struct intel_gt *gt)
217{
218	intel_rc6_fini(&gt->rc6);
219}
220
221void intel_gt_resume_early(struct intel_gt *gt)
222{
223	/*
224	 * Sanitize steer semaphores during driver resume. This is necessary
225	 * to address observed cases of steer semaphores being
226	 * held after a suspend operation. Confirmation from the hardware team
227	 * assures the safety of this operation, as no lock acquisitions
228	 * by other agents occur during driver load/resume process.
229	 */
230	intel_gt_mcr_lock_sanitize(gt);
231
232	intel_uncore_resume_early(gt->uncore);
233	intel_gt_check_and_clear_faults(gt);
234}
235
236int intel_gt_resume(struct intel_gt *gt)
237{
238	struct intel_engine_cs *engine;
239	enum intel_engine_id id;
240	intel_wakeref_t wakeref;
241	int err;
242
243	err = intel_gt_has_unrecoverable_error(gt);
244	if (err)
245		return err;
246
247	GT_TRACE(gt, "\n");
248
249	/*
250	 * After resume, we may need to poke into the pinned kernel
251	 * contexts to paper over any damage caused by the sudden suspend.
252	 * Only the kernel contexts should remain pinned over suspend,
253	 * allowing us to fixup the user contexts on their first pin.
254	 */
255	gt_sanitize(gt, true);
256
257	wakeref = intel_gt_pm_get(gt);
258
259	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
260	intel_rc6_sanitize(&gt->rc6);
261	if (intel_gt_is_wedged(gt)) {
262		err = -EIO;
263		goto out_fw;
264	}
265
266	/* Only when the HW is re-initialised, can we replay the requests */
267	err = intel_gt_init_hw(gt);
268	if (err) {
269		gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");
270		goto err_wedged;
271	}
272
273	intel_uc_reset_finish(&gt->uc);
274
275	intel_rps_enable(&gt->rps);
276	intel_llc_enable(&gt->llc);
277
278	for_each_engine(engine, gt, id) {
279		intel_engine_pm_get(engine);
280
281		engine->serial++; /* kernel context lost */
282		err = intel_engine_resume(engine);
283
284		intel_engine_pm_put(engine);
285		if (err) {
286			gt_err(gt, "Failed to restart %s (%d)\n",
287			       engine->name, err);
288			goto err_wedged;
289		}
290	}
291
292	intel_rc6_enable(&gt->rc6);
293
294	intel_uc_resume(&gt->uc);
295
296	user_forcewake(gt, false);
297
298out_fw:
299	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
300	intel_gt_pm_put(gt, wakeref);
301	intel_gt_bind_context_set_ready(gt);
302	return err;
303
304err_wedged:
305	intel_gt_set_wedged(gt);
306	goto out_fw;
307}
308
309static void wait_for_suspend(struct intel_gt *gt)
310{
311	if (!intel_gt_pm_is_awake(gt))
312		return;
313
314	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
315		/*
316		 * Forcibly cancel outstanding work and leave
317		 * the gpu quiet.
318		 */
319		intel_gt_set_wedged(gt);
320		intel_gt_retire_requests(gt);
321	}
322
323	intel_gt_pm_wait_for_idle(gt);
324}
325
326void intel_gt_suspend_prepare(struct intel_gt *gt)
327{
328	intel_gt_bind_context_set_unready(gt);
329	user_forcewake(gt, true);
330	wait_for_suspend(gt);
331}
332
333static suspend_state_t pm_suspend_target(void)
334{
335#if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
336	return pm_suspend_target_state;
337#else
338	return PM_SUSPEND_TO_IDLE;
339#endif
340}
341
342void intel_gt_suspend_late(struct intel_gt *gt)
343{
344	intel_wakeref_t wakeref;
345
346	/* We expect to be idle already; but also want to be independent */
347	wait_for_suspend(gt);
348
349	if (is_mock_gt(gt))
350		return;
351
352	GEM_BUG_ON(gt->awake);
353
354	intel_uc_suspend(&gt->uc);
355
356	/*
357	 * On disabling the device, we want to turn off HW access to memory
358	 * that we no longer own.
359	 *
360	 * However, not all suspend-states disable the device. S0 (s2idle)
361	 * is effectively runtime-suspend, the device is left powered on
362	 * but needs to be put into a low power state. We need to keep
363	 * powermanagement enabled, but we also retain system state and so
364	 * it remains safe to keep on using our allocated memory.
365	 */
366	if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
367		return;
368
369	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
370		intel_rps_disable(&gt->rps);
371		intel_rc6_disable(&gt->rc6);
372		intel_llc_disable(&gt->llc);
373	}
374
375	gt_sanitize(gt, false);
376
377	GT_TRACE(gt, "\n");
378}
379
380void intel_gt_runtime_suspend(struct intel_gt *gt)
381{
382	intel_gt_bind_context_set_unready(gt);
383	intel_uc_runtime_suspend(&gt->uc);
384
385	GT_TRACE(gt, "\n");
386}
387
388int intel_gt_runtime_resume(struct intel_gt *gt)
389{
390	int ret;
391
392	GT_TRACE(gt, "\n");
393	intel_gt_init_swizzling(gt);
394	intel_ggtt_restore_fences(gt->ggtt);
395
396	ret = intel_uc_runtime_resume(&gt->uc);
397	if (ret)
398		return ret;
399
400	intel_gt_bind_context_set_ready(gt);
401	return 0;
402}
403
404static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
405{
406	ktime_t total = gt->stats.total;
407
408	if (gt->stats.active)
409		total = ktime_add(total,
410				  ktime_sub(ktime_get(), gt->stats.start));
411
412	return total;
413}
414
415ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
416{
417	unsigned int seq;
418	ktime_t total;
419
420	do {
421		seq = read_seqcount_begin(&gt->stats.lock);
422		total = __intel_gt_get_awake_time(gt);
423	} while (read_seqcount_retry(&gt->stats.lock, seq));
424
425	return total;
426}
427
428#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
429#include "selftest_gt_pm.c"
430#endif
431