1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <drm/drm_file.h>
7
8#include <linux/bitfield.h>
9#include <linux/highmem.h>
10#include <linux/pci.h>
11#include <linux/module.h>
12#include <uapi/drm/ivpu_accel.h>
13
14#include "ivpu_drv.h"
15#include "ivpu_hw.h"
16#include "ivpu_ipc.h"
17#include "ivpu_job.h"
18#include "ivpu_jsm_msg.h"
19#include "ivpu_pm.h"
20
21#define CMD_BUF_IDX	     0
22#define JOB_ID_JOB_MASK	     GENMASK(7, 0)
23#define JOB_ID_CONTEXT_MASK  GENMASK(31, 8)
24#define JOB_MAX_BUFFER_COUNT 65535
25
26static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
27{
28	ivpu_hw_reg_db_set(vdev, cmdq->db_id);
29}
30
31static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine)
32{
33	struct xa_limit db_xa_limit = {.max = IVPU_MAX_DB, .min = IVPU_MIN_DB};
34	struct ivpu_device *vdev = file_priv->vdev;
35	struct vpu_job_queue_header *jobq_header;
36	struct ivpu_cmdq *cmdq;
37	int ret;
38
39	cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);
40	if (!cmdq)
41		return NULL;
42
43	ret = xa_alloc(&vdev->db_xa, &cmdq->db_id, NULL, db_xa_limit, GFP_KERNEL);
44	if (ret) {
45		ivpu_err(vdev, "Failed to allocate doorbell id: %d\n", ret);
46		goto err_free_cmdq;
47	}
48
49	cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
50	if (!cmdq->mem)
51		goto err_erase_xa;
52
53	cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
54				  sizeof(struct vpu_job_queue_entry));
55
56	cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
57	jobq_header = &cmdq->jobq->header;
58	jobq_header->engine_idx = engine;
59	jobq_header->head = 0;
60	jobq_header->tail = 0;
61	wmb(); /* Flush WC buffer for jobq->header */
62
63	return cmdq;
64
65err_erase_xa:
66	xa_erase(&vdev->db_xa, cmdq->db_id);
67err_free_cmdq:
68	kfree(cmdq);
69	return NULL;
70}
71
72static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
73{
74	if (!cmdq)
75		return;
76
77	ivpu_bo_free(cmdq->mem);
78	xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
79	kfree(cmdq);
80}
81
82static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine)
83{
84	struct ivpu_device *vdev = file_priv->vdev;
85	struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
86	int ret;
87
88	lockdep_assert_held(&file_priv->lock);
89
90	if (!cmdq) {
91		cmdq = ivpu_cmdq_alloc(file_priv, engine);
92		if (!cmdq)
93			return NULL;
94		file_priv->cmdq[engine] = cmdq;
95	}
96
97	if (cmdq->db_registered)
98		return cmdq;
99
100	ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
101				   cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
102	if (ret)
103		return NULL;
104
105	cmdq->db_registered = true;
106
107	return cmdq;
108}
109
110static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine)
111{
112	struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
113
114	lockdep_assert_held(&file_priv->lock);
115
116	if (cmdq) {
117		file_priv->cmdq[engine] = NULL;
118		if (cmdq->db_registered)
119			ivpu_jsm_unregister_db(file_priv->vdev, cmdq->db_id);
120
121		ivpu_cmdq_free(file_priv, cmdq);
122	}
123}
124
125void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
126{
127	int i;
128
129	lockdep_assert_held(&file_priv->lock);
130
131	for (i = 0; i < IVPU_NUM_ENGINES; i++)
132		ivpu_cmdq_release_locked(file_priv, i);
133}
134
135/*
136 * Mark the doorbell as unregistered and reset job queue pointers.
137 * This function needs to be called when the VPU hardware is restarted
138 * and FW loses job queue state. The next time job queue is used it
139 * will be registered again.
140 */
141static void ivpu_cmdq_reset_locked(struct ivpu_file_priv *file_priv, u16 engine)
142{
143	struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
144
145	lockdep_assert_held(&file_priv->lock);
146
147	if (cmdq) {
148		cmdq->db_registered = false;
149		cmdq->jobq->header.head = 0;
150		cmdq->jobq->header.tail = 0;
151		wmb(); /* Flush WC buffer for jobq header */
152	}
153}
154
155static void ivpu_cmdq_reset_all(struct ivpu_file_priv *file_priv)
156{
157	int i;
158
159	mutex_lock(&file_priv->lock);
160
161	for (i = 0; i < IVPU_NUM_ENGINES; i++)
162		ivpu_cmdq_reset_locked(file_priv, i);
163
164	mutex_unlock(&file_priv->lock);
165}
166
167void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
168{
169	struct ivpu_file_priv *file_priv;
170	unsigned long ctx_id;
171
172	mutex_lock(&vdev->context_list_lock);
173
174	xa_for_each(&vdev->context_xa, ctx_id, file_priv)
175		ivpu_cmdq_reset_all(file_priv);
176
177	mutex_unlock(&vdev->context_list_lock);
178
179}
180
181static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
182{
183	struct ivpu_device *vdev = job->vdev;
184	struct vpu_job_queue_header *header = &cmdq->jobq->header;
185	struct vpu_job_queue_entry *entry;
186	u32 tail = READ_ONCE(header->tail);
187	u32 next_entry = (tail + 1) % cmdq->entry_count;
188
189	/* Check if there is space left in job queue */
190	if (next_entry == header->head) {
191		ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",
192			 job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);
193		return -EBUSY;
194	}
195
196	entry = &cmdq->jobq->job[tail];
197	entry->batch_buf_addr = job->cmd_buf_vpu_addr;
198	entry->job_id = job->job_id;
199	entry->flags = 0;
200	if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
201		entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
202	wmb(); /* Ensure that tail is updated after filling entry */
203	header->tail = next_entry;
204	wmb(); /* Flush WC buffer for jobq header */
205
206	return 0;
207}
208
209struct ivpu_fence {
210	struct dma_fence base;
211	spinlock_t lock; /* protects base */
212	struct ivpu_device *vdev;
213};
214
215static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)
216{
217	return container_of(fence, struct ivpu_fence, base);
218}
219
220static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)
221{
222	return DRIVER_NAME;
223}
224
225static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)
226{
227	struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);
228
229	return dev_name(ivpu_fence->vdev->drm.dev);
230}
231
232static const struct dma_fence_ops ivpu_fence_ops = {
233	.get_driver_name = ivpu_fence_get_driver_name,
234	.get_timeline_name = ivpu_fence_get_timeline_name,
235};
236
237static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)
238{
239	struct ivpu_fence *fence;
240
241	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
242	if (!fence)
243		return NULL;
244
245	fence->vdev = vdev;
246	spin_lock_init(&fence->lock);
247	dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);
248
249	return &fence->base;
250}
251
252static void ivpu_job_destroy(struct ivpu_job *job)
253{
254	struct ivpu_device *vdev = job->vdev;
255	u32 i;
256
257	ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",
258		 job->job_id, job->file_priv->ctx.id, job->engine_idx);
259
260	for (i = 0; i < job->bo_count; i++)
261		if (job->bos[i])
262			drm_gem_object_put(&job->bos[i]->base.base);
263
264	dma_fence_put(job->done_fence);
265	ivpu_file_priv_put(&job->file_priv);
266	kfree(job);
267}
268
269static struct ivpu_job *
270ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
271{
272	struct ivpu_device *vdev = file_priv->vdev;
273	struct ivpu_job *job;
274
275	job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);
276	if (!job)
277		return NULL;
278
279	job->vdev = vdev;
280	job->engine_idx = engine_idx;
281	job->bo_count = bo_count;
282	job->done_fence = ivpu_fence_create(vdev);
283	if (!job->done_fence) {
284		ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");
285		goto err_free_job;
286	}
287
288	job->file_priv = ivpu_file_priv_get(file_priv);
289
290	ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
291	return job;
292
293err_free_job:
294	kfree(job);
295	return NULL;
296}
297
298static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
299{
300	struct ivpu_job *job;
301
302	job = xa_erase(&vdev->submitted_jobs_xa, job_id);
303	if (!job)
304		return -ENOENT;
305
306	if (job->file_priv->has_mmu_faults)
307		job_status = DRM_IVPU_JOB_STATUS_ABORTED;
308
309	job->bos[CMD_BUF_IDX]->job_status = job_status;
310	dma_fence_signal(job->done_fence);
311
312	ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",
313		 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
314
315	ivpu_job_destroy(job);
316	ivpu_stop_job_timeout_detection(vdev);
317
318	ivpu_rpm_put(vdev);
319	return 0;
320}
321
322void ivpu_jobs_abort_all(struct ivpu_device *vdev)
323{
324	struct ivpu_job *job;
325	unsigned long id;
326
327	xa_for_each(&vdev->submitted_jobs_xa, id, job)
328		ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);
329}
330
331static int ivpu_job_submit(struct ivpu_job *job)
332{
333	struct ivpu_file_priv *file_priv = job->file_priv;
334	struct ivpu_device *vdev = job->vdev;
335	struct xa_limit job_id_range;
336	struct ivpu_cmdq *cmdq;
337	int ret;
338
339	ret = ivpu_rpm_get(vdev);
340	if (ret < 0)
341		return ret;
342
343	mutex_lock(&file_priv->lock);
344
345	cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx);
346	if (!cmdq) {
347		ivpu_warn_ratelimited(vdev, "Failed get job queue, ctx %d engine %d\n",
348				      file_priv->ctx.id, job->engine_idx);
349		ret = -EINVAL;
350		goto err_unlock_file_priv;
351	}
352
353	job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));
354	job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK;
355
356	xa_lock(&vdev->submitted_jobs_xa);
357	ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);
358	if (ret) {
359		ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",
360			 file_priv->ctx.id);
361		ret = -EBUSY;
362		goto err_unlock_submitted_jobs_xa;
363	}
364
365	ret = ivpu_cmdq_push_job(cmdq, job);
366	if (ret)
367		goto err_erase_xa;
368
369	ivpu_start_job_timeout_detection(vdev);
370
371	if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {
372		cmdq->jobq->header.head = cmdq->jobq->header.tail;
373		wmb(); /* Flush WC buffer for jobq header */
374	} else {
375		ivpu_cmdq_ring_db(vdev, cmdq);
376	}
377
378	ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d addr 0x%llx next %d\n",
379		 job->job_id, file_priv->ctx.id, job->engine_idx,
380		 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);
381
382	xa_unlock(&vdev->submitted_jobs_xa);
383
384	mutex_unlock(&file_priv->lock);
385
386	if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW))
387		ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
388
389	return 0;
390
391err_erase_xa:
392	__xa_erase(&vdev->submitted_jobs_xa, job->job_id);
393err_unlock_submitted_jobs_xa:
394	xa_unlock(&vdev->submitted_jobs_xa);
395err_unlock_file_priv:
396	mutex_unlock(&file_priv->lock);
397	ivpu_rpm_put(vdev);
398	return ret;
399}
400
401static int
402ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,
403				u32 buf_count, u32 commands_offset)
404{
405	struct ivpu_file_priv *file_priv = file->driver_priv;
406	struct ivpu_device *vdev = file_priv->vdev;
407	struct ww_acquire_ctx acquire_ctx;
408	enum dma_resv_usage usage;
409	struct ivpu_bo *bo;
410	int ret;
411	u32 i;
412
413	for (i = 0; i < buf_count; i++) {
414		struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);
415
416		if (!obj)
417			return -ENOENT;
418
419		job->bos[i] = to_ivpu_bo(obj);
420
421		ret = ivpu_bo_pin(job->bos[i]);
422		if (ret)
423			return ret;
424	}
425
426	bo = job->bos[CMD_BUF_IDX];
427	if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
428		ivpu_warn(vdev, "Buffer is already in use\n");
429		return -EBUSY;
430	}
431
432	if (commands_offset >= ivpu_bo_size(bo)) {
433		ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);
434		return -EINVAL;
435	}
436
437	job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
438
439	ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
440					&acquire_ctx);
441	if (ret) {
442		ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
443		return ret;
444	}
445
446	for (i = 0; i < buf_count; i++) {
447		ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
448		if (ret) {
449			ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
450			goto unlock_reservations;
451		}
452	}
453
454	for (i = 0; i < buf_count; i++) {
455		usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
456		dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
457	}
458
459unlock_reservations:
460	drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
461
462	wmb(); /* Flush write combining buffers */
463
464	return ret;
465}
466
467int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
468{
469	struct ivpu_file_priv *file_priv = file->driver_priv;
470	struct ivpu_device *vdev = file_priv->vdev;
471	struct drm_ivpu_submit *params = data;
472	struct ivpu_job *job;
473	u32 *buf_handles;
474	int idx, ret;
475
476	if (params->engine > DRM_IVPU_ENGINE_COPY)
477		return -EINVAL;
478
479	if (params->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
480		return -EINVAL;
481
482	if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT)
483		return -EINVAL;
484
485	if (!IS_ALIGNED(params->commands_offset, 8))
486		return -EINVAL;
487
488	if (!file_priv->ctx.id)
489		return -EINVAL;
490
491	if (file_priv->has_mmu_faults)
492		return -EBADFD;
493
494	buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL);
495	if (!buf_handles)
496		return -ENOMEM;
497
498	ret = copy_from_user(buf_handles,
499			     (void __user *)params->buffers_ptr,
500			     params->buffer_count * sizeof(u32));
501	if (ret) {
502		ret = -EFAULT;
503		goto err_free_handles;
504	}
505
506	if (!drm_dev_enter(&vdev->drm, &idx)) {
507		ret = -ENODEV;
508		goto err_free_handles;
509	}
510
511	ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",
512		 file_priv->ctx.id, params->buffer_count);
513
514	job = ivpu_job_create(file_priv, params->engine, params->buffer_count);
515	if (!job) {
516		ivpu_err(vdev, "Failed to create job\n");
517		ret = -ENOMEM;
518		goto err_exit_dev;
519	}
520
521	ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,
522					      params->commands_offset);
523	if (ret) {
524		ivpu_err(vdev, "Failed to prepare job: %d\n", ret);
525		goto err_destroy_job;
526	}
527
528	down_read(&vdev->pm->reset_lock);
529	ret = ivpu_job_submit(job);
530	up_read(&vdev->pm->reset_lock);
531	if (ret)
532		goto err_signal_fence;
533
534	drm_dev_exit(idx);
535	kfree(buf_handles);
536	return ret;
537
538err_signal_fence:
539	dma_fence_signal(job->done_fence);
540err_destroy_job:
541	ivpu_job_destroy(job);
542err_exit_dev:
543	drm_dev_exit(idx);
544err_free_handles:
545	kfree(buf_handles);
546	return ret;
547}
548
549static void
550ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
551		       struct vpu_jsm_msg *jsm_msg)
552{
553	struct vpu_ipc_msg_payload_job_done *payload;
554	int ret;
555
556	if (!jsm_msg) {
557		ivpu_err(vdev, "IPC message has no JSM payload\n");
558		return;
559	}
560
561	if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
562		ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
563		return;
564	}
565
566	payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
567	ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);
568	if (!ret && !xa_empty(&vdev->submitted_jobs_xa))
569		ivpu_start_job_timeout_detection(vdev);
570}
571
572void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
573{
574	ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
575			      VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
576}
577
578void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
579{
580	ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
581}
582