1// SPDX-License-Identifier: GPL-2.0 OR MIT
2
3#include <drm/drm_exec.h>
4#include <drm/drm_gem.h>
5#include <linux/dma-resv.h>
6
7/**
8 * DOC: Overview
9 *
10 * This component mainly abstracts the retry loop necessary for locking
11 * multiple GEM objects while preparing hardware operations (e.g. command
12 * submissions, page table updates etc..).
13 *
14 * If a contention is detected while locking a GEM object the cleanup procedure
15 * unlocks all previously locked GEM objects and locks the contended one first
16 * before locking any further objects.
17 *
18 * After an object is locked fences slots can optionally be reserved on the
19 * dma_resv object inside the GEM object.
20 *
21 * A typical usage pattern should look like this::
22 *
23 *	struct drm_gem_object *obj;
24 *	struct drm_exec exec;
25 *	unsigned long index;
26 *	int ret;
27 *
28 *	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
29 *	drm_exec_until_all_locked(&exec) {
30 *		ret = drm_exec_prepare_obj(&exec, boA, 1);
31 *		drm_exec_retry_on_contention(&exec);
32 *		if (ret)
33 *			goto error;
34 *
35 *		ret = drm_exec_prepare_obj(&exec, boB, 1);
36 *		drm_exec_retry_on_contention(&exec);
37 *		if (ret)
38 *			goto error;
39 *	}
40 *
41 *	drm_exec_for_each_locked_object(&exec, index, obj) {
42 *		dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ);
43 *		...
44 *	}
45 *	drm_exec_fini(&exec);
46 *
47 * See struct dma_exec for more details.
48 */
49
50/* Dummy value used to initially enter the retry loop */
51#define DRM_EXEC_DUMMY ((void *)~0)
52
53/* Unlock all objects and drop references */
54static void drm_exec_unlock_all(struct drm_exec *exec)
55{
56	struct drm_gem_object *obj;
57	unsigned long index;
58
59	drm_exec_for_each_locked_object_reverse(exec, index, obj) {
60		dma_resv_unlock(obj->resv);
61		drm_gem_object_put(obj);
62	}
63
64	drm_gem_object_put(exec->prelocked);
65	exec->prelocked = NULL;
66}
67
68/**
69 * drm_exec_init - initialize a drm_exec object
70 * @exec: the drm_exec object to initialize
71 * @flags: controls locking behavior, see DRM_EXEC_* defines
72 * @nr: the initial # of objects
73 *
74 * Initialize the object and make sure that we can track locked objects.
75 *
76 * If nr is non-zero then it is used as the initial objects table size.
77 * In either case, the table will grow (be re-allocated) on demand.
78 */
79void drm_exec_init(struct drm_exec *exec, u32 flags, unsigned nr)
80{
81	if (!nr)
82		nr = PAGE_SIZE / sizeof(void *);
83
84	exec->flags = flags;
85	exec->objects = kvmalloc_array(nr, sizeof(void *), GFP_KERNEL);
86
87	/* If allocation here fails, just delay that till the first use */
88	exec->max_objects = exec->objects ? nr : 0;
89	exec->num_objects = 0;
90	exec->contended = DRM_EXEC_DUMMY;
91	exec->prelocked = NULL;
92}
93EXPORT_SYMBOL(drm_exec_init);
94
95/**
96 * drm_exec_fini - finalize a drm_exec object
97 * @exec: the drm_exec object to finalize
98 *
99 * Unlock all locked objects, drop the references to objects and free all memory
100 * used for tracking the state.
101 */
102void drm_exec_fini(struct drm_exec *exec)
103{
104	drm_exec_unlock_all(exec);
105	kvfree(exec->objects);
106	if (exec->contended != DRM_EXEC_DUMMY) {
107		drm_gem_object_put(exec->contended);
108		ww_acquire_fini(&exec->ticket);
109	}
110}
111EXPORT_SYMBOL(drm_exec_fini);
112
113/**
114 * drm_exec_cleanup - cleanup when contention is detected
115 * @exec: the drm_exec object to cleanup
116 *
117 * Cleanup the current state and return true if we should stay inside the retry
118 * loop, false if there wasn't any contention detected and we can keep the
119 * objects locked.
120 */
121bool drm_exec_cleanup(struct drm_exec *exec)
122{
123	if (likely(!exec->contended)) {
124		ww_acquire_done(&exec->ticket);
125		return false;
126	}
127
128	if (likely(exec->contended == DRM_EXEC_DUMMY)) {
129		exec->contended = NULL;
130		ww_acquire_init(&exec->ticket, &reservation_ww_class);
131		return true;
132	}
133
134	drm_exec_unlock_all(exec);
135	exec->num_objects = 0;
136	return true;
137}
138EXPORT_SYMBOL(drm_exec_cleanup);
139
140/* Track the locked object in the array */
141static int drm_exec_obj_locked(struct drm_exec *exec,
142			       struct drm_gem_object *obj)
143{
144	if (unlikely(exec->num_objects == exec->max_objects)) {
145		size_t size = exec->max_objects * sizeof(void *);
146		void *tmp;
147
148		tmp = kvrealloc(exec->objects, size, size + PAGE_SIZE,
149				GFP_KERNEL);
150		if (!tmp)
151			return -ENOMEM;
152
153		exec->objects = tmp;
154		exec->max_objects += PAGE_SIZE / sizeof(void *);
155	}
156	drm_gem_object_get(obj);
157	exec->objects[exec->num_objects++] = obj;
158
159	return 0;
160}
161
162/* Make sure the contended object is locked first */
163static int drm_exec_lock_contended(struct drm_exec *exec)
164{
165	struct drm_gem_object *obj = exec->contended;
166	int ret;
167
168	if (likely(!obj))
169		return 0;
170
171	/* Always cleanup the contention so that error handling can kick in */
172	exec->contended = NULL;
173	if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) {
174		ret = dma_resv_lock_slow_interruptible(obj->resv,
175						       &exec->ticket);
176		if (unlikely(ret))
177			goto error_dropref;
178	} else {
179		dma_resv_lock_slow(obj->resv, &exec->ticket);
180	}
181
182	ret = drm_exec_obj_locked(exec, obj);
183	if (unlikely(ret))
184		goto error_unlock;
185
186	exec->prelocked = obj;
187	return 0;
188
189error_unlock:
190	dma_resv_unlock(obj->resv);
191
192error_dropref:
193	drm_gem_object_put(obj);
194	return ret;
195}
196
197/**
198 * drm_exec_lock_obj - lock a GEM object for use
199 * @exec: the drm_exec object with the state
200 * @obj: the GEM object to lock
201 *
202 * Lock a GEM object for use and grab a reference to it.
203 *
204 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
205 * already locked (can be suppressed by setting the DRM_EXEC_IGNORE_DUPLICATES
206 * flag), -ENOMEM when memory allocation failed and zero for success.
207 */
208int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
209{
210	int ret;
211
212	ret = drm_exec_lock_contended(exec);
213	if (unlikely(ret))
214		return ret;
215
216	if (exec->prelocked == obj) {
217		drm_gem_object_put(exec->prelocked);
218		exec->prelocked = NULL;
219		return 0;
220	}
221
222	if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT)
223		ret = dma_resv_lock_interruptible(obj->resv, &exec->ticket);
224	else
225		ret = dma_resv_lock(obj->resv, &exec->ticket);
226
227	if (unlikely(ret == -EDEADLK)) {
228		drm_gem_object_get(obj);
229		exec->contended = obj;
230		return -EDEADLK;
231	}
232
233	if (unlikely(ret == -EALREADY) &&
234	    exec->flags & DRM_EXEC_IGNORE_DUPLICATES)
235		return 0;
236
237	if (unlikely(ret))
238		return ret;
239
240	ret = drm_exec_obj_locked(exec, obj);
241	if (ret)
242		goto error_unlock;
243
244	return 0;
245
246error_unlock:
247	dma_resv_unlock(obj->resv);
248	return ret;
249}
250EXPORT_SYMBOL(drm_exec_lock_obj);
251
252/**
253 * drm_exec_unlock_obj - unlock a GEM object in this exec context
254 * @exec: the drm_exec object with the state
255 * @obj: the GEM object to unlock
256 *
257 * Unlock the GEM object and remove it from the collection of locked objects.
258 * Should only be used to unlock the most recently locked objects. It's not time
259 * efficient to unlock objects locked long ago.
260 */
261void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
262{
263	unsigned int i;
264
265	for (i = exec->num_objects; i--;) {
266		if (exec->objects[i] == obj) {
267			dma_resv_unlock(obj->resv);
268			for (++i; i < exec->num_objects; ++i)
269				exec->objects[i - 1] = exec->objects[i];
270			--exec->num_objects;
271			drm_gem_object_put(obj);
272			return;
273		}
274
275	}
276}
277EXPORT_SYMBOL(drm_exec_unlock_obj);
278
279/**
280 * drm_exec_prepare_obj - prepare a GEM object for use
281 * @exec: the drm_exec object with the state
282 * @obj: the GEM object to prepare
283 * @num_fences: how many fences to reserve
284 *
285 * Prepare a GEM object for use by locking it and reserving fence slots.
286 *
287 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
288 * already locked, -ENOMEM when memory allocation failed and zero for success.
289 */
290int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
291			 unsigned int num_fences)
292{
293	int ret;
294
295	ret = drm_exec_lock_obj(exec, obj);
296	if (ret)
297		return ret;
298
299	ret = dma_resv_reserve_fences(obj->resv, num_fences);
300	if (ret) {
301		drm_exec_unlock_obj(exec, obj);
302		return ret;
303	}
304
305	return 0;
306}
307EXPORT_SYMBOL(drm_exec_prepare_obj);
308
309/**
310 * drm_exec_prepare_array - helper to prepare an array of objects
311 * @exec: the drm_exec object with the state
312 * @objects: array of GEM object to prepare
313 * @num_objects: number of GEM objects in the array
314 * @num_fences: number of fences to reserve on each GEM object
315 *
316 * Prepares all GEM objects in an array, aborts on first error.
317 * Reserves @num_fences on each GEM object after locking it.
318 *
319 * Returns: -EDEADLOCK on contention, -EALREADY when object is already locked,
320 * -ENOMEM when memory allocation failed and zero for success.
321 */
322int drm_exec_prepare_array(struct drm_exec *exec,
323			   struct drm_gem_object **objects,
324			   unsigned int num_objects,
325			   unsigned int num_fences)
326{
327	int ret;
328
329	for (unsigned int i = 0; i < num_objects; ++i) {
330		ret = drm_exec_prepare_obj(exec, objects[i], num_fences);
331		if (unlikely(ret))
332			return ret;
333	}
334
335	return 0;
336}
337EXPORT_SYMBOL(drm_exec_prepare_array);
338
339MODULE_DESCRIPTION("DRM execution context");
340MODULE_LICENSE("Dual MIT/GPL");
341