1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2021 Intel Corporation
4 */
5
6#include "xe_sync.h"
7
8#include <linux/dma-fence-array.h>
9#include <linux/kthread.h>
10#include <linux/sched/mm.h>
11#include <linux/uaccess.h>
12
13#include <drm/drm_print.h>
14#include <drm/drm_syncobj.h>
15#include <drm/xe_drm.h>
16
17#include "xe_device_types.h"
18#include "xe_exec_queue.h"
19#include "xe_macros.h"
20#include "xe_sched_job_types.h"
21
22struct xe_user_fence {
23	struct xe_device *xe;
24	struct kref refcount;
25	struct dma_fence_cb cb;
26	struct work_struct worker;
27	struct mm_struct *mm;
28	u64 __user *addr;
29	u64 value;
30	int signalled;
31};
32
33static void user_fence_destroy(struct kref *kref)
34{
35	struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
36						 refcount);
37
38	mmdrop(ufence->mm);
39	kfree(ufence);
40}
41
42static void user_fence_get(struct xe_user_fence *ufence)
43{
44	kref_get(&ufence->refcount);
45}
46
47static void user_fence_put(struct xe_user_fence *ufence)
48{
49	kref_put(&ufence->refcount, user_fence_destroy);
50}
51
52static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,
53					       u64 value)
54{
55	struct xe_user_fence *ufence;
56
57	ufence = kmalloc(sizeof(*ufence), GFP_KERNEL);
58	if (!ufence)
59		return NULL;
60
61	ufence->xe = xe;
62	kref_init(&ufence->refcount);
63	ufence->addr = u64_to_user_ptr(addr);
64	ufence->value = value;
65	ufence->mm = current->mm;
66	mmgrab(ufence->mm);
67
68	return ufence;
69}
70
71static void user_fence_worker(struct work_struct *w)
72{
73	struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);
74
75	if (mmget_not_zero(ufence->mm)) {
76		kthread_use_mm(ufence->mm);
77		if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
78			XE_WARN_ON("Copy to user failed");
79		kthread_unuse_mm(ufence->mm);
80		mmput(ufence->mm);
81	}
82
83	wake_up_all(&ufence->xe->ufence_wq);
84	WRITE_ONCE(ufence->signalled, 1);
85	user_fence_put(ufence);
86}
87
88static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)
89{
90	INIT_WORK(&ufence->worker, user_fence_worker);
91	queue_work(ufence->xe->ordered_wq, &ufence->worker);
92	dma_fence_put(fence);
93}
94
95static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
96{
97	struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);
98
99	kick_ufence(ufence, fence);
100}
101
102int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
103			struct xe_sync_entry *sync,
104			struct drm_xe_sync __user *sync_user,
105			unsigned int flags)
106{
107	struct drm_xe_sync sync_in;
108	int err;
109	bool exec = flags & SYNC_PARSE_FLAG_EXEC;
110	bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
111	bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
112	bool signal;
113
114	if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
115		return -EFAULT;
116
117	if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||
118	    XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))
119		return -EINVAL;
120
121	signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
122	switch (sync_in.type) {
123	case DRM_XE_SYNC_TYPE_SYNCOBJ:
124		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
125			return -EOPNOTSUPP;
126
127		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
128			return -EINVAL;
129
130		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
131		if (XE_IOCTL_DBG(xe, !sync->syncobj))
132			return -ENOENT;
133
134		if (!signal) {
135			sync->fence = drm_syncobj_fence_get(sync->syncobj);
136			if (XE_IOCTL_DBG(xe, !sync->fence))
137				return -EINVAL;
138		}
139		break;
140
141	case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
142		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
143			return -EOPNOTSUPP;
144
145		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
146			return -EINVAL;
147
148		if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
149			return -EINVAL;
150
151		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
152		if (XE_IOCTL_DBG(xe, !sync->syncobj))
153			return -ENOENT;
154
155		if (signal) {
156			sync->chain_fence = dma_fence_chain_alloc();
157			if (!sync->chain_fence)
158				return -ENOMEM;
159		} else {
160			sync->fence = drm_syncobj_fence_get(sync->syncobj);
161			if (XE_IOCTL_DBG(xe, !sync->fence))
162				return -EINVAL;
163
164			err = dma_fence_chain_find_seqno(&sync->fence,
165							 sync_in.timeline_value);
166			if (err)
167				return err;
168		}
169		break;
170
171	case DRM_XE_SYNC_TYPE_USER_FENCE:
172		if (XE_IOCTL_DBG(xe, disallow_user_fence))
173			return -EOPNOTSUPP;
174
175		if (XE_IOCTL_DBG(xe, !signal))
176			return -EOPNOTSUPP;
177
178		if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
179			return -EINVAL;
180
181		if (exec) {
182			sync->addr = sync_in.addr;
183		} else {
184			sync->ufence = user_fence_create(xe, sync_in.addr,
185							 sync_in.timeline_value);
186			if (XE_IOCTL_DBG(xe, !sync->ufence))
187				return -ENOMEM;
188		}
189
190		break;
191
192	default:
193		return -EINVAL;
194	}
195
196	sync->type = sync_in.type;
197	sync->flags = sync_in.flags;
198	sync->timeline_value = sync_in.timeline_value;
199
200	return 0;
201}
202
203int xe_sync_entry_wait(struct xe_sync_entry *sync)
204{
205	if (sync->fence)
206		dma_fence_wait(sync->fence, true);
207
208	return 0;
209}
210
211int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
212{
213	int err;
214
215	if (sync->fence) {
216		err = drm_sched_job_add_dependency(&job->drm,
217						   dma_fence_get(sync->fence));
218		if (err) {
219			dma_fence_put(sync->fence);
220			return err;
221		}
222	}
223
224	return 0;
225}
226
227void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job,
228			  struct dma_fence *fence)
229{
230	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
231		return;
232
233	if (sync->chain_fence) {
234		drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
235				      fence, sync->timeline_value);
236		/*
237		 * The chain's ownership is transferred to the
238		 * timeline.
239		 */
240		sync->chain_fence = NULL;
241	} else if (sync->syncobj) {
242		drm_syncobj_replace_fence(sync->syncobj, fence);
243	} else if (sync->ufence) {
244		int err;
245
246		dma_fence_get(fence);
247		user_fence_get(sync->ufence);
248		err = dma_fence_add_callback(fence, &sync->ufence->cb,
249					     user_fence_cb);
250		if (err == -ENOENT) {
251			kick_ufence(sync->ufence, fence);
252		} else if (err) {
253			XE_WARN_ON("failed to add user fence");
254			user_fence_put(sync->ufence);
255			dma_fence_put(fence);
256		}
257	} else if (sync->type == DRM_XE_SYNC_TYPE_USER_FENCE) {
258		job->user_fence.used = true;
259		job->user_fence.addr = sync->addr;
260		job->user_fence.value = sync->timeline_value;
261	}
262}
263
264void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
265{
266	if (sync->syncobj)
267		drm_syncobj_put(sync->syncobj);
268	if (sync->fence)
269		dma_fence_put(sync->fence);
270	if (sync->chain_fence)
271		dma_fence_put(&sync->chain_fence->base);
272	if (sync->ufence)
273		user_fence_put(sync->ufence);
274}
275
276/**
277 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
278 * @sync: input syncs
279 * @num_sync: number of syncs
280 * @q: exec queue
281 * @vm: VM
282 *
283 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
284 * and return a composite fence of all in-fences + last fence. If no in-fences
285 * return last fence on  input exec queue. Caller must drop reference to
286 * returned fence.
287 *
288 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
289 */
290struct dma_fence *
291xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
292		     struct xe_exec_queue *q, struct xe_vm *vm)
293{
294	struct dma_fence **fences = NULL;
295	struct dma_fence_array *cf = NULL;
296	struct dma_fence *fence;
297	int i, num_in_fence = 0, current_fence = 0;
298
299	lockdep_assert_held(&vm->lock);
300
301	/* Count in-fences */
302	for (i = 0; i < num_sync; ++i) {
303		if (sync[i].fence) {
304			++num_in_fence;
305			fence = sync[i].fence;
306		}
307	}
308
309	/* Easy case... */
310	if (!num_in_fence) {
311		fence = xe_exec_queue_last_fence_get(q, vm);
312		return fence;
313	}
314
315	/* Create composite fence */
316	fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL);
317	if (!fences)
318		return ERR_PTR(-ENOMEM);
319	for (i = 0; i < num_sync; ++i) {
320		if (sync[i].fence) {
321			dma_fence_get(sync[i].fence);
322			fences[current_fence++] = sync[i].fence;
323		}
324	}
325	fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm);
326	cf = dma_fence_array_create(num_in_fence, fences,
327				    vm->composite_fence_ctx,
328				    vm->composite_fence_seqno++,
329				    false);
330	if (!cf) {
331		--vm->composite_fence_seqno;
332		goto err_out;
333	}
334
335	return &cf->base;
336
337err_out:
338	while (current_fence)
339		dma_fence_put(fences[--current_fence]);
340	kfree(fences);
341	kfree(cf);
342
343	return ERR_PTR(-ENOMEM);
344}
345
346/**
347 * xe_sync_ufence_get() - Get user fence from sync
348 * @sync: input sync
349 *
350 * Get a user fence reference from sync.
351 *
352 * Return: xe_user_fence pointer with reference
353 */
354struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)
355{
356	user_fence_get(sync->ufence);
357
358	return sync->ufence;
359}
360
361/**
362 * xe_sync_ufence_put() - Put user fence reference
363 * @ufence: user fence reference
364 *
365 */
366void xe_sync_ufence_put(struct xe_user_fence *ufence)
367{
368	user_fence_put(ufence);
369}
370
371/**
372 * xe_sync_ufence_get_status() - Get user fence status
373 * @ufence: user fence
374 *
375 * Return: 1 if signalled, 0 not signalled, <0 on error
376 */
377int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
378{
379	return READ_ONCE(ufence->signalled);
380}
381