1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2021 Intel Corporation
4 */
5
6#include "xe_hw_fence.h"
7
8#include <linux/device.h>
9#include <linux/slab.h>
10
11#include "xe_bo.h"
12#include "xe_device.h"
13#include "xe_gt.h"
14#include "xe_hw_engine.h"
15#include "xe_macros.h"
16#include "xe_map.h"
17#include "xe_trace.h"
18
19static struct kmem_cache *xe_hw_fence_slab;
20
21int __init xe_hw_fence_module_init(void)
22{
23	xe_hw_fence_slab = kmem_cache_create("xe_hw_fence",
24					     sizeof(struct xe_hw_fence), 0,
25					     SLAB_HWCACHE_ALIGN, NULL);
26	if (!xe_hw_fence_slab)
27		return -ENOMEM;
28
29	return 0;
30}
31
32void xe_hw_fence_module_exit(void)
33{
34	rcu_barrier();
35	kmem_cache_destroy(xe_hw_fence_slab);
36}
37
38static struct xe_hw_fence *fence_alloc(void)
39{
40	return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL);
41}
42
43static void fence_free(struct rcu_head *rcu)
44{
45	struct xe_hw_fence *fence =
46		container_of(rcu, struct xe_hw_fence, dma.rcu);
47
48	if (!WARN_ON_ONCE(!fence))
49		kmem_cache_free(xe_hw_fence_slab, fence);
50}
51
52static void hw_fence_irq_run_cb(struct irq_work *work)
53{
54	struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work);
55	struct xe_hw_fence *fence, *next;
56	bool tmp;
57
58	tmp = dma_fence_begin_signalling();
59	spin_lock(&irq->lock);
60	if (irq->enabled) {
61		list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
62			struct dma_fence *dma_fence = &fence->dma;
63
64			trace_xe_hw_fence_try_signal(fence);
65			if (dma_fence_is_signaled_locked(dma_fence)) {
66				trace_xe_hw_fence_signal(fence);
67				list_del_init(&fence->irq_link);
68				dma_fence_put(dma_fence);
69			}
70		}
71	}
72	spin_unlock(&irq->lock);
73	dma_fence_end_signalling(tmp);
74}
75
76void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq)
77{
78	spin_lock_init(&irq->lock);
79	init_irq_work(&irq->work, hw_fence_irq_run_cb);
80	INIT_LIST_HEAD(&irq->pending);
81	irq->enabled = true;
82}
83
84void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq)
85{
86	struct xe_hw_fence *fence, *next;
87	unsigned long flags;
88	int err;
89	bool tmp;
90
91	if (XE_WARN_ON(!list_empty(&irq->pending))) {
92		tmp = dma_fence_begin_signalling();
93		spin_lock_irqsave(&irq->lock, flags);
94		list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
95			list_del_init(&fence->irq_link);
96			err = dma_fence_signal_locked(&fence->dma);
97			dma_fence_put(&fence->dma);
98			XE_WARN_ON(err);
99		}
100		spin_unlock_irqrestore(&irq->lock, flags);
101		dma_fence_end_signalling(tmp);
102	}
103}
104
105void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq)
106{
107	irq_work_queue(&irq->work);
108}
109
110void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq)
111{
112	spin_lock_irq(&irq->lock);
113	irq->enabled = false;
114	spin_unlock_irq(&irq->lock);
115}
116
117void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq)
118{
119	spin_lock_irq(&irq->lock);
120	irq->enabled = true;
121	spin_unlock_irq(&irq->lock);
122
123	irq_work_queue(&irq->work);
124}
125
126void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt,
127			  struct xe_hw_fence_irq *irq, const char *name)
128{
129	ctx->gt = gt;
130	ctx->irq = irq;
131	ctx->dma_fence_ctx = dma_fence_context_alloc(1);
132	ctx->next_seqno = XE_FENCE_INITIAL_SEQNO;
133	sprintf(ctx->name, "%s", name);
134}
135
136void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx)
137{
138}
139
140static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence);
141
142static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence)
143{
144	return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock);
145}
146
147static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence)
148{
149	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
150
151	return dev_name(gt_to_xe(fence->ctx->gt)->drm.dev);
152}
153
154static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence)
155{
156	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
157
158	return fence->ctx->name;
159}
160
161static bool xe_hw_fence_signaled(struct dma_fence *dma_fence)
162{
163	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
164	struct xe_device *xe = gt_to_xe(fence->ctx->gt);
165	u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32);
166
167	return dma_fence->error ||
168		!__dma_fence_is_later(dma_fence->seqno, seqno, dma_fence->ops);
169}
170
171static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence)
172{
173	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
174	struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence);
175
176	dma_fence_get(dma_fence);
177	list_add_tail(&fence->irq_link, &irq->pending);
178
179	/* SW completed (no HW IRQ) so kick handler to signal fence */
180	if (xe_hw_fence_signaled(dma_fence))
181		xe_hw_fence_irq_run(irq);
182
183	return true;
184}
185
186static void xe_hw_fence_release(struct dma_fence *dma_fence)
187{
188	struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
189
190	trace_xe_hw_fence_free(fence);
191	XE_WARN_ON(!list_empty(&fence->irq_link));
192	call_rcu(&dma_fence->rcu, fence_free);
193}
194
195static const struct dma_fence_ops xe_hw_fence_ops = {
196	.get_driver_name = xe_hw_fence_get_driver_name,
197	.get_timeline_name = xe_hw_fence_get_timeline_name,
198	.enable_signaling = xe_hw_fence_enable_signaling,
199	.signaled = xe_hw_fence_signaled,
200	.release = xe_hw_fence_release,
201};
202
203static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence)
204{
205	if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops))
206		return NULL;
207
208	return container_of(fence, struct xe_hw_fence, dma);
209}
210
211struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx,
212				       struct iosys_map seqno_map)
213{
214	struct xe_hw_fence *fence;
215
216	fence = fence_alloc();
217	if (!fence)
218		return ERR_PTR(-ENOMEM);
219
220	fence->ctx = ctx;
221	fence->seqno_map = seqno_map;
222	INIT_LIST_HEAD(&fence->irq_link);
223
224	dma_fence_init(&fence->dma, &xe_hw_fence_ops, &ctx->irq->lock,
225		       ctx->dma_fence_ctx, ctx->next_seqno++);
226
227	trace_xe_hw_fence_create(fence);
228
229	return fence;
230}
231