1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright �� 2018 Intel Corporation
5 */
6
7#include <linux/kref.h>
8#include <linux/string_helpers.h>
9
10#include "gem/i915_gem_pm.h"
11#include "gt/intel_gt.h"
12
13#include "i915_selftest.h"
14
15#include "igt_flush_test.h"
16#include "lib_sw_fence.h"
17
18struct live_active {
19	struct i915_active base;
20	struct kref ref;
21	bool retired;
22};
23
24static void __live_get(struct live_active *active)
25{
26	kref_get(&active->ref);
27}
28
29static void __live_free(struct live_active *active)
30{
31	i915_active_fini(&active->base);
32	kfree(active);
33}
34
35static void __live_release(struct kref *ref)
36{
37	struct live_active *active = container_of(ref, typeof(*active), ref);
38
39	__live_free(active);
40}
41
42static void __live_put(struct live_active *active)
43{
44	kref_put(&active->ref, __live_release);
45}
46
47static int __live_active(struct i915_active *base)
48{
49	struct live_active *active = container_of(base, typeof(*active), base);
50
51	__live_get(active);
52	return 0;
53}
54
55static void __live_retire(struct i915_active *base)
56{
57	struct live_active *active = container_of(base, typeof(*active), base);
58
59	active->retired = true;
60	__live_put(active);
61}
62
63static struct live_active *__live_alloc(struct drm_i915_private *i915)
64{
65	struct live_active *active;
66
67	active = kzalloc(sizeof(*active), GFP_KERNEL);
68	if (!active)
69		return NULL;
70
71	kref_init(&active->ref);
72	i915_active_init(&active->base, __live_active, __live_retire, 0);
73
74	return active;
75}
76
77static struct live_active *
78__live_active_setup(struct drm_i915_private *i915)
79{
80	struct intel_engine_cs *engine;
81	struct i915_sw_fence *submit;
82	struct live_active *active;
83	unsigned int count = 0;
84	int err = 0;
85
86	active = __live_alloc(i915);
87	if (!active)
88		return ERR_PTR(-ENOMEM);
89
90	submit = heap_fence_create(GFP_KERNEL);
91	if (!submit) {
92		kfree(active);
93		return ERR_PTR(-ENOMEM);
94	}
95
96	err = i915_active_acquire(&active->base);
97	if (err)
98		goto out;
99
100	for_each_uabi_engine(engine, i915) {
101		struct i915_request *rq;
102
103		rq = intel_engine_create_kernel_request(engine);
104		if (IS_ERR(rq)) {
105			err = PTR_ERR(rq);
106			break;
107		}
108
109		err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
110						       submit,
111						       GFP_KERNEL);
112		if (err >= 0)
113			err = i915_active_add_request(&active->base, rq);
114		i915_request_add(rq);
115		if (err) {
116			pr_err("Failed to track active ref!\n");
117			break;
118		}
119
120		count++;
121	}
122
123	i915_active_release(&active->base);
124	if (READ_ONCE(active->retired) && count) {
125		pr_err("i915_active retired before submission!\n");
126		err = -EINVAL;
127	}
128	if (atomic_read(&active->base.count) != count) {
129		pr_err("i915_active not tracking all requests, found %d, expected %d\n",
130		       atomic_read(&active->base.count), count);
131		err = -EINVAL;
132	}
133
134out:
135	i915_sw_fence_commit(submit);
136	heap_fence_put(submit);
137	if (err) {
138		__live_put(active);
139		active = ERR_PTR(err);
140	}
141
142	return active;
143}
144
145static int live_active_wait(void *arg)
146{
147	struct drm_i915_private *i915 = arg;
148	struct live_active *active;
149	int err = 0;
150
151	/* Check that we get a callback when requests retire upon waiting */
152
153	active = __live_active_setup(i915);
154	if (IS_ERR(active))
155		return PTR_ERR(active);
156
157	__i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
158	if (!READ_ONCE(active->retired)) {
159		struct drm_printer p = drm_err_printer(&i915->drm, __func__);
160
161		drm_printf(&p, "i915_active not retired after waiting!\n");
162		i915_active_print(&active->base, &p);
163
164		err = -EINVAL;
165	}
166
167	__live_put(active);
168
169	if (igt_flush_test(i915))
170		err = -EIO;
171
172	return err;
173}
174
175static int live_active_retire(void *arg)
176{
177	struct drm_i915_private *i915 = arg;
178	struct live_active *active;
179	int err = 0;
180
181	/* Check that we get a callback when requests are indirectly retired */
182
183	active = __live_active_setup(i915);
184	if (IS_ERR(active))
185		return PTR_ERR(active);
186
187	/* waits for & retires all requests */
188	if (igt_flush_test(i915))
189		err = -EIO;
190
191	if (!READ_ONCE(active->retired)) {
192		struct drm_printer p = drm_err_printer(&i915->drm, __func__);
193
194		drm_printf(&p, "i915_active not retired after flushing!\n");
195		i915_active_print(&active->base, &p);
196
197		err = -EINVAL;
198	}
199
200	__live_put(active);
201
202	return err;
203}
204
205static int live_active_barrier(void *arg)
206{
207	struct drm_i915_private *i915 = arg;
208	struct intel_engine_cs *engine;
209	struct live_active *active;
210	int err = 0;
211
212	/* Check that we get a callback when requests retire upon waiting */
213
214	active = __live_alloc(i915);
215	if (!active)
216		return -ENOMEM;
217
218	err = i915_active_acquire(&active->base);
219	if (err)
220		goto out;
221
222	for_each_uabi_engine(engine, i915) {
223		err = i915_active_acquire_preallocate_barrier(&active->base,
224							      engine);
225		if (err)
226			break;
227
228		i915_active_acquire_barrier(&active->base);
229	}
230
231	i915_active_release(&active->base);
232	if (err)
233		goto out;
234
235	__i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE);
236	if (!READ_ONCE(active->retired)) {
237		pr_err("i915_active not retired after flushing barriers!\n");
238		err = -EINVAL;
239	}
240
241out:
242	__live_put(active);
243
244	if (igt_flush_test(i915))
245		err = -EIO;
246
247	return err;
248}
249
250int i915_active_live_selftests(struct drm_i915_private *i915)
251{
252	static const struct i915_subtest tests[] = {
253		SUBTEST(live_active_wait),
254		SUBTEST(live_active_retire),
255		SUBTEST(live_active_barrier),
256	};
257
258	if (intel_gt_is_wedged(to_gt(i915)))
259		return 0;
260
261	return i915_subtests(tests, i915);
262}
263
264static struct intel_engine_cs *node_to_barrier(struct active_node *it)
265{
266	struct intel_engine_cs *engine;
267
268	if (!is_barrier(&it->base))
269		return NULL;
270
271	engine = __barrier_to_engine(it);
272	smp_rmb(); /* serialise with add_active_barriers */
273	if (!is_barrier(&it->base))
274		return NULL;
275
276	return engine;
277}
278
279void i915_active_print(struct i915_active *ref, struct drm_printer *m)
280{
281	drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire);
282	drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count));
283	drm_printf(m, "\tpreallocated barriers? %s\n",
284		   str_yes_no(!llist_empty(&ref->preallocated_barriers)));
285
286	if (i915_active_acquire_if_busy(ref)) {
287		struct active_node *it, *n;
288
289		rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
290			struct intel_engine_cs *engine;
291
292			engine = node_to_barrier(it);
293			if (engine) {
294				drm_printf(m, "\tbarrier: %s\n", engine->name);
295				continue;
296			}
297
298			if (i915_active_fence_isset(&it->base)) {
299				drm_printf(m,
300					   "\ttimeline: %llx\n", it->timeline);
301				continue;
302			}
303		}
304
305		i915_active_release(ref);
306	}
307}
308
309static void spin_unlock_wait(spinlock_t *lock)
310{
311	spin_lock_irq(lock);
312	spin_unlock_irq(lock);
313}
314
315static void active_flush(struct i915_active *ref,
316			 struct i915_active_fence *active)
317{
318	struct dma_fence *fence;
319
320	fence = xchg(__active_fence_slot(active), NULL);
321	if (!fence)
322		return;
323
324	spin_lock_irq(fence->lock);
325	__list_del_entry(&active->cb.node);
326	spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */
327	atomic_dec(&ref->count);
328
329	GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
330}
331
332void i915_active_unlock_wait(struct i915_active *ref)
333{
334	if (i915_active_acquire_if_busy(ref)) {
335		struct active_node *it, *n;
336
337		/* Wait for all active callbacks */
338		rcu_read_lock();
339		active_flush(ref, &ref->excl);
340		rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node)
341			active_flush(ref, &it->base);
342		rcu_read_unlock();
343
344		i915_active_release(ref);
345	}
346
347	/* And wait for the retire callback */
348	spin_unlock_wait(&ref->tree_lock);
349
350	/* ... which may have been on a thread instead */
351	flush_work(&ref->work);
352}
353