intel_breadcrumbs.c revision 1.4
1/* $NetBSD: intel_breadcrumbs.c,v 1.4 2021/12/19 11:38:04 riastradh Exp $ */ 2 3/* 4 * Copyright �� 2015 Intel Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 */ 26 27#include <sys/cdefs.h> 28__KERNEL_RCSID(0, "$NetBSD: intel_breadcrumbs.c,v 1.4 2021/12/19 11:38:04 riastradh Exp $"); 29 30#include <linux/kthread.h> 31#include <trace/events/dma_fence.h> 32#include <uapi/linux/sched/types.h> 33 34#include "i915_drv.h" 35#include "i915_trace.h" 36#include "intel_gt_pm.h" 37#include "intel_gt_requests.h" 38 39#include <linux/nbsd-namespace.h> 40 41static void irq_enable(struct intel_engine_cs *engine) 42{ 43 if (!engine->irq_enable) 44 return; 45 46 /* Caller disables interrupts */ 47 spin_lock(&engine->gt->irq_lock); 48 engine->irq_enable(engine); 49 spin_unlock(&engine->gt->irq_lock); 50} 51 52static void irq_disable(struct intel_engine_cs *engine) 53{ 54 if (!engine->irq_disable) 55 return; 56 57 /* Caller disables interrupts */ 58 spin_lock(&engine->gt->irq_lock); 59 engine->irq_disable(engine); 60 spin_unlock(&engine->gt->irq_lock); 61} 62 63static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) 64{ 65 struct intel_engine_cs *engine = 66 container_of(b, struct intel_engine_cs, breadcrumbs); 67 68 lockdep_assert_held(&b->irq_lock); 69 70 GEM_BUG_ON(!b->irq_enabled); 71 if (!--b->irq_enabled) 72 irq_disable(engine); 73 74 b->irq_armed = false; 75 intel_gt_pm_put_async(engine->gt); 76} 77 78void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) 79{ 80 struct intel_breadcrumbs *b = &engine->breadcrumbs; 81 unsigned long flags; 82 83 if (!b->irq_armed) 84 return; 85 86 spin_lock_irqsave(&b->irq_lock, flags); 87 if (b->irq_armed) 88 __intel_breadcrumbs_disarm_irq(b); 89 spin_unlock_irqrestore(&b->irq_lock, flags); 90} 91 92static inline bool __request_completed(const struct i915_request *rq) 93{ 94 return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno); 95} 96 97__maybe_unused static bool 98check_signal_order(struct intel_context *ce, struct i915_request *rq) 99{ 100 if (!list_is_last(&rq->signal_link, &ce->signals) && 101 i915_seqno_passed(rq->fence.seqno, 102 list_next_entry(rq, signal_link)->fence.seqno)) 103 return false; 104 105 if (!list_is_first(&rq->signal_link, &ce->signals) && 106 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno, 107 rq->fence.seqno)) 108 return false; 109 110 return true; 111} 112 113#ifndef __NetBSD__ 114 115static bool 116__dma_fence_signal(struct dma_fence *fence) 117{ 118 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); 119} 120 121static void 122__dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp) 123{ 124 fence->timestamp = timestamp; 125 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 126 trace_dma_fence_signaled(fence); 127} 128 129static void 130__dma_fence_signal__notify(struct dma_fence *fence, 131 const struct list_head *list) 132{ 133 struct dma_fence_cb *cur, *tmp; 134 135 lockdep_assert_held(fence->lock); 136 137 list_for_each_entry_safe(cur, tmp, list, node) { 138 INIT_LIST_HEAD(&cur->node); 139 cur->func(fence, cur); 140 } 141} 142 143#endif 144 145static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl) 146{ 147 struct intel_engine_cs *engine = 148 container_of(b, struct intel_engine_cs, breadcrumbs); 149 150 if (unlikely(intel_engine_is_virtual(engine))) 151 engine = intel_virtual_engine_get_sibling(engine, 0); 152 153 intel_engine_add_retire(engine, tl); 154} 155 156static void signal_irq_work(struct irq_work *work) 157{ 158 struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work); 159 const ktime_t timestamp = ktime_get(); 160 struct intel_context *ce, *cn; 161 struct list_head *pos, *next; 162 LIST_HEAD(signal); 163 164 spin_lock(&b->irq_lock); 165 166 if (b->irq_armed && list_empty(&b->signalers)) 167 __intel_breadcrumbs_disarm_irq(b); 168 169 list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { 170 GEM_BUG_ON(list_empty(&ce->signals)); 171 172 list_for_each_safe(pos, next, &ce->signals) { 173 struct i915_request *rq = 174 list_entry(pos, typeof(*rq), signal_link); 175 176 GEM_BUG_ON(!check_signal_order(ce, rq)); 177 178 if (!__request_completed(rq)) 179 break; 180 181 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, 182 &rq->fence.flags)); 183 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 184 185 if (!__dma_fence_signal(&rq->fence)) 186 continue; 187 188 /* 189 * Queue for execution after dropping the signaling 190 * spinlock as the callback chain may end up adding 191 * more signalers to the same context or engine. 192 */ 193 i915_request_get(rq); 194 list_add_tail(&rq->signal_link, &signal); 195 } 196 197 /* 198 * We process the list deletion in bulk, only using a list_add 199 * (not list_move) above but keeping the status of 200 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. 201 */ 202 if (!list_is_first(pos, &ce->signals)) { 203 /* Advance the list to the first incomplete request */ 204 __list_del_many(&ce->signals, pos); 205 if (&ce->signals == pos) { /* now empty */ 206 list_del_init(&ce->signal_link); 207 add_retire(b, ce->timeline); 208 } 209 } 210 } 211 212 spin_unlock(&b->irq_lock); 213 214 list_for_each_safe(pos, next, &signal) { 215 struct i915_request *rq = 216 list_entry(pos, typeof(*rq), signal_link); 217#ifdef __NetBSD__ 218 __dma_fence_signal_wake(&rq->fence, timestamp); 219#else 220 struct list_head cb_list; 221 222 spin_lock(&rq->lock); 223 list_replace(&rq->fence.cb_list, &cb_list); 224 __dma_fence_signal__timestamp(&rq->fence, timestamp); 225 __dma_fence_signal__notify(&rq->fence, &cb_list); 226 spin_unlock(&rq->lock); 227#endif 228 229 i915_request_put(rq); 230 } 231} 232 233static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) 234{ 235 struct intel_engine_cs *engine = 236 container_of(b, struct intel_engine_cs, breadcrumbs); 237 238 lockdep_assert_held(&b->irq_lock); 239 if (b->irq_armed) 240 return true; 241 242 if (!intel_gt_pm_get_if_awake(engine->gt)) 243 return false; 244 245 /* 246 * The breadcrumb irq will be disarmed on the interrupt after the 247 * waiters are signaled. This gives us a single interrupt window in 248 * which we can add a new waiter and avoid the cost of re-enabling 249 * the irq. 250 */ 251 b->irq_armed = true; 252 253 /* 254 * Since we are waiting on a request, the GPU should be busy 255 * and should have its own rpm reference. This is tracked 256 * by i915->gt.awake, we can forgo holding our own wakref 257 * for the interrupt as before i915->gt.awake is released (when 258 * the driver is idle) we disarm the breadcrumbs. 259 */ 260 261 if (!b->irq_enabled++) 262 irq_enable(engine); 263 264 return true; 265} 266 267void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) 268{ 269 struct intel_breadcrumbs *b = &engine->breadcrumbs; 270 271 spin_lock_init(&b->irq_lock); 272 INIT_LIST_HEAD(&b->signalers); 273 274 init_irq_work(&b->irq_work, signal_irq_work); 275} 276 277void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) 278{ 279 struct intel_breadcrumbs *b = &engine->breadcrumbs; 280 unsigned long flags; 281 282 spin_lock_irqsave(&b->irq_lock, flags); 283 284 if (b->irq_enabled) 285 irq_enable(engine); 286 else 287 irq_disable(engine); 288 289 spin_unlock_irqrestore(&b->irq_lock, flags); 290} 291 292void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) 293{ 294} 295 296bool i915_request_enable_breadcrumb(struct i915_request *rq) 297{ 298 lockdep_assert_held(&rq->lock); 299 300 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { 301 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 302 struct intel_context *ce = rq->context; 303 struct list_head *pos; 304 305 spin_lock(&b->irq_lock); 306 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); 307 308 if (!__intel_breadcrumbs_arm_irq(b)) 309 goto unlock; 310 311 /* 312 * We keep the seqno in retirement order, so we can break 313 * inside intel_engine_signal_breadcrumbs as soon as we've 314 * passed the last completed request (or seen a request that 315 * hasn't event started). We could walk the timeline->requests, 316 * but keeping a separate signalers_list has the advantage of 317 * hopefully being much smaller than the full list and so 318 * provides faster iteration and detection when there are no 319 * more interrupts required for this context. 320 * 321 * We typically expect to add new signalers in order, so we 322 * start looking for our insertion point from the tail of 323 * the list. 324 */ 325 list_for_each_prev(pos, &ce->signals) { 326 struct i915_request *it = 327 list_entry(pos, typeof(*it), signal_link); 328 329 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) 330 break; 331 } 332 list_add(&rq->signal_link, pos); 333 if (pos == &ce->signals) /* catch transitions from empty list */ 334 list_move_tail(&ce->signal_link, &b->signalers); 335 GEM_BUG_ON(!check_signal_order(ce, rq)); 336 337 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 338unlock: 339 spin_unlock(&b->irq_lock); 340 } 341 342 return !__request_completed(rq); 343} 344 345void i915_request_cancel_breadcrumb(struct i915_request *rq) 346{ 347 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 348 349 lockdep_assert_held(&rq->lock); 350 351 /* 352 * We must wait for b->irq_lock so that we know the interrupt handler 353 * has released its reference to the intel_context and has completed 354 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if 355 * required). 356 */ 357 spin_lock(&b->irq_lock); 358 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { 359 struct intel_context *ce = rq->context; 360 361 list_del(&rq->signal_link); 362 if (list_empty(&ce->signals)) 363 list_del_init(&ce->signal_link); 364 365 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 366 } 367 spin_unlock(&b->irq_lock); 368} 369 370void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, 371 struct drm_printer *p) 372{ 373 struct intel_breadcrumbs *b = &engine->breadcrumbs; 374 struct intel_context *ce; 375 struct i915_request *rq; 376 377 if (list_empty(&b->signalers)) 378 return; 379 380 drm_printf(p, "Signals:\n"); 381 382 spin_lock_irq(&b->irq_lock); 383 list_for_each_entry(ce, &b->signalers, signal_link) { 384 list_for_each_entry(rq, &ce->signals, signal_link) { 385 drm_printf(p, "\t[%"PRIx64":%"PRIx64"%s] @ %dms\n", 386 (uint64_t)rq->fence.context, 387 (uint64_t)rq->fence.seqno, 388 i915_request_completed(rq) ? "!" : 389 i915_request_started(rq) ? "*" : 390 "", 391 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 392 } 393 } 394 spin_unlock_irq(&b->irq_lock); 395} 396