1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright �� 2019 Intel Corporation
5 */
6
7#include <linux/wait_bit.h>
8
9#include "intel_runtime_pm.h"
10#include "intel_wakeref.h"
11#include "i915_drv.h"
12
13static void rpm_get(struct intel_wakeref *wf)
14{
15	wf->wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm);
16}
17
18static void rpm_put(struct intel_wakeref *wf)
19{
20	intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref);
21
22	intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
23	INTEL_WAKEREF_BUG_ON(!wakeref);
24}
25
26int __intel_wakeref_get_first(struct intel_wakeref *wf)
27{
28	/*
29	 * Treat get/put as different subclasses, as we may need to run
30	 * the put callback from under the shrinker and do not want to
31	 * cross-contanimate that callback with any extra work performed
32	 * upon acquiring the wakeref.
33	 */
34	mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
35	if (!atomic_read(&wf->count)) {
36		int err;
37
38		rpm_get(wf);
39
40		err = wf->ops->get(wf);
41		if (unlikely(err)) {
42			rpm_put(wf);
43			mutex_unlock(&wf->mutex);
44			return err;
45		}
46
47		smp_mb__before_atomic(); /* release wf->count */
48	}
49	atomic_inc(&wf->count);
50	mutex_unlock(&wf->mutex);
51
52	INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
53	return 0;
54}
55
56static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
57{
58	INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
59	if (unlikely(!atomic_dec_and_test(&wf->count)))
60		goto unlock;
61
62	/* ops->put() must reschedule its own release on error/deferral */
63	if (likely(!wf->ops->put(wf))) {
64		rpm_put(wf);
65		wake_up_var(&wf->wakeref);
66	}
67
68unlock:
69	mutex_unlock(&wf->mutex);
70}
71
72void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
73{
74	INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work));
75
76	/* Assume we are not in process context and so cannot sleep. */
77	if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
78		mod_delayed_work(wf->i915->unordered_wq, &wf->work,
79				 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));
80		return;
81	}
82
83	____intel_wakeref_put_last(wf);
84}
85
86static void __intel_wakeref_put_work(struct work_struct *wrk)
87{
88	struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work);
89
90	if (atomic_add_unless(&wf->count, -1, 1))
91		return;
92
93	mutex_lock(&wf->mutex);
94	____intel_wakeref_put_last(wf);
95}
96
97void __intel_wakeref_init(struct intel_wakeref *wf,
98			  struct drm_i915_private *i915,
99			  const struct intel_wakeref_ops *ops,
100			  struct intel_wakeref_lockclass *key)
101{
102	wf->i915 = i915;
103	wf->ops = ops;
104
105#ifdef __linux__
106	__mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);
107#else
108	rw_init(&wf->mutex, "wakeref.mutex");
109#endif
110	atomic_set(&wf->count, 0);
111	wf->wakeref = 0;
112
113	INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work);
114	lockdep_init_map(&wf->work.work.lockdep_map,
115			 "wakeref.work", &key->work, 0);
116}
117
118int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
119{
120	int err;
121
122	might_sleep();
123
124	err = wait_var_event_killable(&wf->wakeref,
125				      !intel_wakeref_is_active(wf));
126	if (err)
127		return err;
128
129	intel_wakeref_unlock_wait(wf);
130	return 0;
131}
132
133static void wakeref_auto_timeout(void *arg)
134{
135	struct intel_wakeref_auto *wf = arg;
136	intel_wakeref_t wakeref;
137	unsigned long flags;
138
139	if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
140		return;
141
142	wakeref = fetch_and_zero(&wf->wakeref);
143	spin_unlock_irqrestore(&wf->lock, flags);
144
145	intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
146}
147
148void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
149			     struct drm_i915_private *i915)
150{
151	mtx_init(&wf->lock, IPL_TTY);
152#ifdef __linux__
153	timer_setup(&wf->timer, wakeref_auto_timeout, 0);
154#else
155	timeout_set(&wf->timer, wakeref_auto_timeout, wf);
156#endif
157	refcount_set(&wf->count, 0);
158	wf->wakeref = 0;
159	wf->i915 = i915;
160}
161
162void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
163{
164	unsigned long flags;
165
166	if (!timeout) {
167		if (del_timer_sync(&wf->timer))
168			wakeref_auto_timeout(&wf->timer);
169		return;
170	}
171
172	/* Our mission is that we only extend an already active wakeref */
173	assert_rpm_wakelock_held(&wf->i915->runtime_pm);
174
175	if (!refcount_inc_not_zero(&wf->count)) {
176		spin_lock_irqsave(&wf->lock, flags);
177		if (!refcount_inc_not_zero(&wf->count)) {
178			INTEL_WAKEREF_BUG_ON(wf->wakeref);
179			wf->wakeref =
180				intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm);
181			refcount_set(&wf->count, 1);
182		}
183		spin_unlock_irqrestore(&wf->lock, flags);
184	}
185
186	/*
187	 * If we extend a pending timer, we will only get a single timer
188	 * callback and so need to cancel the local inc by running the
189	 * elided callback to keep the wf->count balanced.
190	 */
191	if (mod_timer(&wf->timer, jiffies + timeout))
192		wakeref_auto_timeout(&wf->timer);
193}
194
195void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
196{
197	intel_wakeref_auto(wf, 0);
198	INTEL_WAKEREF_BUG_ON(wf->wakeref);
199}
200