1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2021 Intel Corporation 4 */ 5 6#include "xe_hw_fence.h" 7 8#include <linux/device.h> 9#include <linux/slab.h> 10 11#include "xe_bo.h" 12#include "xe_device.h" 13#include "xe_gt.h" 14#include "xe_hw_engine.h" 15#include "xe_macros.h" 16#include "xe_map.h" 17#include "xe_trace.h" 18 19static struct kmem_cache *xe_hw_fence_slab; 20 21int __init xe_hw_fence_module_init(void) 22{ 23 xe_hw_fence_slab = kmem_cache_create("xe_hw_fence", 24 sizeof(struct xe_hw_fence), 0, 25 SLAB_HWCACHE_ALIGN, NULL); 26 if (!xe_hw_fence_slab) 27 return -ENOMEM; 28 29 return 0; 30} 31 32void xe_hw_fence_module_exit(void) 33{ 34 rcu_barrier(); 35 kmem_cache_destroy(xe_hw_fence_slab); 36} 37 38static struct xe_hw_fence *fence_alloc(void) 39{ 40 return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL); 41} 42 43static void fence_free(struct rcu_head *rcu) 44{ 45 struct xe_hw_fence *fence = 46 container_of(rcu, struct xe_hw_fence, dma.rcu); 47 48 if (!WARN_ON_ONCE(!fence)) 49 kmem_cache_free(xe_hw_fence_slab, fence); 50} 51 52static void hw_fence_irq_run_cb(struct irq_work *work) 53{ 54 struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work); 55 struct xe_hw_fence *fence, *next; 56 bool tmp; 57 58 tmp = dma_fence_begin_signalling(); 59 spin_lock(&irq->lock); 60 if (irq->enabled) { 61 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 62 struct dma_fence *dma_fence = &fence->dma; 63 64 trace_xe_hw_fence_try_signal(fence); 65 if (dma_fence_is_signaled_locked(dma_fence)) { 66 trace_xe_hw_fence_signal(fence); 67 list_del_init(&fence->irq_link); 68 dma_fence_put(dma_fence); 69 } 70 } 71 } 72 spin_unlock(&irq->lock); 73 dma_fence_end_signalling(tmp); 74} 75 76void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq) 77{ 78 spin_lock_init(&irq->lock); 79 init_irq_work(&irq->work, hw_fence_irq_run_cb); 80 INIT_LIST_HEAD(&irq->pending); 81 irq->enabled = true; 82} 83 84void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq) 85{ 86 struct xe_hw_fence *fence, *next; 87 unsigned long flags; 88 int err; 89 bool tmp; 90 91 if (XE_WARN_ON(!list_empty(&irq->pending))) { 92 tmp = dma_fence_begin_signalling(); 93 spin_lock_irqsave(&irq->lock, flags); 94 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 95 list_del_init(&fence->irq_link); 96 err = dma_fence_signal_locked(&fence->dma); 97 dma_fence_put(&fence->dma); 98 XE_WARN_ON(err); 99 } 100 spin_unlock_irqrestore(&irq->lock, flags); 101 dma_fence_end_signalling(tmp); 102 } 103} 104 105void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq) 106{ 107 irq_work_queue(&irq->work); 108} 109 110void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq) 111{ 112 spin_lock_irq(&irq->lock); 113 irq->enabled = false; 114 spin_unlock_irq(&irq->lock); 115} 116 117void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq) 118{ 119 spin_lock_irq(&irq->lock); 120 irq->enabled = true; 121 spin_unlock_irq(&irq->lock); 122 123 irq_work_queue(&irq->work); 124} 125 126void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, 127 struct xe_hw_fence_irq *irq, const char *name) 128{ 129 ctx->gt = gt; 130 ctx->irq = irq; 131 ctx->dma_fence_ctx = dma_fence_context_alloc(1); 132 ctx->next_seqno = XE_FENCE_INITIAL_SEQNO; 133 sprintf(ctx->name, "%s", name); 134} 135 136void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx) 137{ 138} 139 140static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence); 141 142static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence) 143{ 144 return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock); 145} 146 147static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence) 148{ 149 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 150 151 return dev_name(gt_to_xe(fence->ctx->gt)->drm.dev); 152} 153 154static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence) 155{ 156 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 157 158 return fence->ctx->name; 159} 160 161static bool xe_hw_fence_signaled(struct dma_fence *dma_fence) 162{ 163 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 164 struct xe_device *xe = gt_to_xe(fence->ctx->gt); 165 u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32); 166 167 return dma_fence->error || 168 !__dma_fence_is_later(dma_fence->seqno, seqno, dma_fence->ops); 169} 170 171static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence) 172{ 173 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 174 struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence); 175 176 dma_fence_get(dma_fence); 177 list_add_tail(&fence->irq_link, &irq->pending); 178 179 /* SW completed (no HW IRQ) so kick handler to signal fence */ 180 if (xe_hw_fence_signaled(dma_fence)) 181 xe_hw_fence_irq_run(irq); 182 183 return true; 184} 185 186static void xe_hw_fence_release(struct dma_fence *dma_fence) 187{ 188 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 189 190 trace_xe_hw_fence_free(fence); 191 XE_WARN_ON(!list_empty(&fence->irq_link)); 192 call_rcu(&dma_fence->rcu, fence_free); 193} 194 195static const struct dma_fence_ops xe_hw_fence_ops = { 196 .get_driver_name = xe_hw_fence_get_driver_name, 197 .get_timeline_name = xe_hw_fence_get_timeline_name, 198 .enable_signaling = xe_hw_fence_enable_signaling, 199 .signaled = xe_hw_fence_signaled, 200 .release = xe_hw_fence_release, 201}; 202 203static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence) 204{ 205 if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops)) 206 return NULL; 207 208 return container_of(fence, struct xe_hw_fence, dma); 209} 210 211struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx, 212 struct iosys_map seqno_map) 213{ 214 struct xe_hw_fence *fence; 215 216 fence = fence_alloc(); 217 if (!fence) 218 return ERR_PTR(-ENOMEM); 219 220 fence->ctx = ctx; 221 fence->seqno_map = seqno_map; 222 INIT_LIST_HEAD(&fence->irq_link); 223 224 dma_fence_init(&fence->dma, &xe_hw_fence_ops, &ctx->irq->lock, 225 ctx->dma_fence_ctx, ctx->next_seqno++); 226 227 trace_xe_hw_fence_create(fence); 228 229 return fence; 230} 231