1/* $NetBSD: linux_dma_fence.c,v 1.42 2022/09/01 09:37:06 riastradh Exp $ */ 2 3/*- 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__KERNEL_RCSID(0, "$NetBSD: linux_dma_fence.c,v 1.42 2022/09/01 09:37:06 riastradh Exp $"); 34 35#include <sys/atomic.h> 36#include <sys/condvar.h> 37#include <sys/lock.h> 38#include <sys/queue.h> 39#include <sys/sdt.h> 40 41#include <linux/atomic.h> 42#include <linux/dma-fence.h> 43#include <linux/errno.h> 44#include <linux/kref.h> 45#include <linux/sched.h> 46#include <linux/spinlock.h> 47 48#define FENCE_MAGIC_GOOD 0x607ba424048c37e5ULL 49#define FENCE_MAGIC_BAD 0x7641ca721344505fULL 50 51SDT_PROBE_DEFINE1(sdt, drm, fence, init, 52 "struct dma_fence *"/*fence*/); 53SDT_PROBE_DEFINE1(sdt, drm, fence, reset, 54 "struct dma_fence *"/*fence*/); 55SDT_PROBE_DEFINE1(sdt, drm, fence, release, 56 "struct dma_fence *"/*fence*/); 57SDT_PROBE_DEFINE1(sdt, drm, fence, free, 58 "struct dma_fence *"/*fence*/); 59SDT_PROBE_DEFINE1(sdt, drm, fence, destroy, 60 "struct dma_fence *"/*fence*/); 61 62SDT_PROBE_DEFINE1(sdt, drm, fence, enable_signaling, 63 "struct dma_fence *"/*fence*/); 64SDT_PROBE_DEFINE2(sdt, drm, fence, add_callback, 65 "struct dma_fence *"/*fence*/, 66 "struct dma_fence_callback *"/*callback*/); 67SDT_PROBE_DEFINE2(sdt, drm, fence, remove_callback, 68 "struct dma_fence *"/*fence*/, 69 "struct dma_fence_callback *"/*callback*/); 70SDT_PROBE_DEFINE2(sdt, drm, fence, callback, 71 "struct dma_fence *"/*fence*/, 72 "struct dma_fence_callback *"/*callback*/); 73SDT_PROBE_DEFINE1(sdt, drm, fence, test, 74 "struct dma_fence *"/*fence*/); 75SDT_PROBE_DEFINE2(sdt, drm, fence, set_error, 76 "struct dma_fence *"/*fence*/, 77 "int"/*error*/); 78SDT_PROBE_DEFINE1(sdt, drm, fence, signal, 79 "struct dma_fence *"/*fence*/); 80 81SDT_PROBE_DEFINE3(sdt, drm, fence, wait_start, 82 "struct dma_fence *"/*fence*/, 83 "bool"/*intr*/, 84 "long"/*timeout*/); 85SDT_PROBE_DEFINE2(sdt, drm, fence, wait_done, 86 "struct dma_fence *"/*fence*/, 87 "long"/*ret*/); 88 89/* 90 * linux_dma_fence_trace 91 * 92 * True if we print DMA_FENCE_TRACE messages, false if not. These 93 * are extremely noisy, too much even for AB_VERBOSE and AB_DEBUG 94 * in boothowto. 95 */ 96int linux_dma_fence_trace = 0; 97 98static struct { 99 spinlock_t lock; 100 struct dma_fence fence; 101} dma_fence_stub __cacheline_aligned; 102 103static const char *dma_fence_stub_name(struct dma_fence *f) 104{ 105 106 KASSERT(f == &dma_fence_stub.fence); 107 return "stub"; 108} 109 110static void 111dma_fence_stub_release(struct dma_fence *f) 112{ 113 114 KASSERT(f == &dma_fence_stub.fence); 115 dma_fence_destroy(f); 116} 117 118static const struct dma_fence_ops dma_fence_stub_ops = { 119 .get_driver_name = dma_fence_stub_name, 120 .get_timeline_name = dma_fence_stub_name, 121 .release = dma_fence_stub_release, 122}; 123 124/* 125 * linux_dma_fences_init(), linux_dma_fences_fini() 126 * 127 * Set up and tear down module state. 128 */ 129void 130linux_dma_fences_init(void) 131{ 132 int error __diagused; 133 134 spin_lock_init(&dma_fence_stub.lock); 135 dma_fence_init(&dma_fence_stub.fence, &dma_fence_stub_ops, 136 &dma_fence_stub.lock, /*context*/0, /*seqno*/0); 137 error = dma_fence_signal(&dma_fence_stub.fence); 138 KASSERTMSG(error == 0, "error=%d", error); 139} 140 141void 142linux_dma_fences_fini(void) 143{ 144 145 dma_fence_put(&dma_fence_stub.fence); 146 spin_lock_destroy(&dma_fence_stub.lock); 147} 148 149/* 150 * dma_fence_referenced_p(fence) 151 * 152 * True if fence has a positive reference count. True after 153 * dma_fence_init; after the last dma_fence_put, this becomes 154 * false. The fence must have been initialized and must not have 155 * been destroyed. 156 */ 157static inline bool __diagused 158dma_fence_referenced_p(struct dma_fence *fence) 159{ 160 161 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 162 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 163 164 return kref_referenced_p(&fence->refcount); 165} 166 167/* 168 * dma_fence_init(fence, ops, lock, context, seqno) 169 * 170 * Initialize fence. Caller should call dma_fence_destroy when 171 * done, after all references have been released. 172 */ 173void 174dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, 175 spinlock_t *lock, uint64_t context, uint64_t seqno) 176{ 177 178 kref_init(&fence->refcount); 179 fence->lock = lock; 180 fence->flags = 0; 181 fence->context = context; 182 fence->seqno = seqno; 183 fence->ops = ops; 184 fence->error = 0; 185 TAILQ_INIT(&fence->f_callbacks); 186 cv_init(&fence->f_cv, "dmafence"); 187 188#ifdef DIAGNOSTIC 189 fence->f_magic = FENCE_MAGIC_GOOD; 190#endif 191 192 SDT_PROBE1(sdt, drm, fence, init, fence); 193} 194 195/* 196 * dma_fence_reset(fence) 197 * 198 * Ensure fence is in a quiescent state. Allowed either for newly 199 * initialized or freed fences, but not fences with more than one 200 * reference. 201 * 202 * XXX extension to Linux API 203 */ 204void 205dma_fence_reset(struct dma_fence *fence, const struct dma_fence_ops *ops, 206 spinlock_t *lock, uint64_t context, uint64_t seqno) 207{ 208 209 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 210 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 211 KASSERT(kref_read(&fence->refcount) == 0 || 212 kref_read(&fence->refcount) == 1); 213 KASSERT(TAILQ_EMPTY(&fence->f_callbacks)); 214 KASSERT(fence->lock == lock); 215 KASSERT(fence->ops == ops); 216 217 kref_init(&fence->refcount); 218 fence->flags = 0; 219 fence->context = context; 220 fence->seqno = seqno; 221 fence->error = 0; 222 223 SDT_PROBE1(sdt, drm, fence, reset, fence); 224} 225 226/* 227 * dma_fence_destroy(fence) 228 * 229 * Clean up memory initialized with dma_fence_init. This is meant 230 * to be used after a fence release callback. 231 * 232 * XXX extension to Linux API 233 */ 234void 235dma_fence_destroy(struct dma_fence *fence) 236{ 237 238 KASSERT(!dma_fence_referenced_p(fence)); 239 240 SDT_PROBE1(sdt, drm, fence, destroy, fence); 241 242#ifdef DIAGNOSTIC 243 fence->f_magic = FENCE_MAGIC_BAD; 244#endif 245 246 KASSERT(TAILQ_EMPTY(&fence->f_callbacks)); 247 cv_destroy(&fence->f_cv); 248} 249 250static void 251dma_fence_free_cb(struct rcu_head *rcu) 252{ 253 struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu); 254 255 KASSERT(!dma_fence_referenced_p(fence)); 256 257 dma_fence_destroy(fence); 258 kfree(fence); 259} 260 261/* 262 * dma_fence_free(fence) 263 * 264 * Schedule fence to be destroyed and then freed with kfree after 265 * any pending RCU read sections on all CPUs have completed. 266 * Caller must guarantee all references have been released. This 267 * is meant to be used after a fence release callback. 268 * 269 * NOTE: Callers assume kfree will be used. We don't even use 270 * kmalloc to allocate these -- caller is expected to allocate 271 * memory with kmalloc to be initialized with dma_fence_init. 272 */ 273void 274dma_fence_free(struct dma_fence *fence) 275{ 276 277 KASSERT(!dma_fence_referenced_p(fence)); 278 279 SDT_PROBE1(sdt, drm, fence, free, fence); 280 281 call_rcu(&fence->rcu, &dma_fence_free_cb); 282} 283 284/* 285 * dma_fence_context_alloc(n) 286 * 287 * Return the first of a contiguous sequence of unique 288 * identifiers, at least until the system wraps around. 289 */ 290uint64_t 291dma_fence_context_alloc(unsigned n) 292{ 293 static struct { 294 volatile unsigned lock; 295 uint64_t context; 296 } S; 297 uint64_t c; 298 299 while (__predict_false(atomic_swap_uint(&S.lock, 1))) 300 SPINLOCK_BACKOFF_HOOK; 301 membar_acquire(); 302 c = S.context; 303 S.context += n; 304 atomic_store_release(&S.lock, 0); 305 306 return c; 307} 308 309/* 310 * __dma_fence_is_later(a, b, ops) 311 * 312 * True if sequence number a is later than sequence number b, 313 * according to the given fence ops. 314 * 315 * - For fence ops with 64-bit sequence numbers, this is simply 316 * defined to be a > b as unsigned 64-bit integers. 317 * 318 * - For fence ops with 32-bit sequence numbers, this is defined 319 * to mean that the 32-bit unsigned difference a - b is less 320 * than INT_MAX. 321 */ 322bool 323__dma_fence_is_later(uint64_t a, uint64_t b, const struct dma_fence_ops *ops) 324{ 325 326 if (ops->use_64bit_seqno) 327 return a > b; 328 else 329 return (unsigned)a - (unsigned)b < INT_MAX; 330} 331 332/* 333 * dma_fence_is_later(a, b) 334 * 335 * True if the sequence number of fence a is later than the 336 * sequence number of fence b. Since sequence numbers wrap 337 * around, we define this to mean that the sequence number of 338 * fence a is no more than INT_MAX past the sequence number of 339 * fence b. 340 * 341 * The two fences must have the context. Whether sequence numbers 342 * are 32-bit is determined by a. 343 */ 344bool 345dma_fence_is_later(struct dma_fence *a, struct dma_fence *b) 346{ 347 348 KASSERTMSG(a->f_magic != FENCE_MAGIC_BAD, "fence %p", a); 349 KASSERTMSG(a->f_magic == FENCE_MAGIC_GOOD, "fence %p", a); 350 KASSERTMSG(b->f_magic != FENCE_MAGIC_BAD, "fence %p", b); 351 KASSERTMSG(b->f_magic == FENCE_MAGIC_GOOD, "fence %p", b); 352 KASSERTMSG(a->context == b->context, "incommensurate fences" 353 ": %"PRIu64" @ %p =/= %"PRIu64" @ %p", 354 a->context, a, b->context, b); 355 356 return __dma_fence_is_later(a->seqno, b->seqno, a->ops); 357} 358 359/* 360 * dma_fence_get_stub() 361 * 362 * Return a dma fence that is always already signalled. 363 */ 364struct dma_fence * 365dma_fence_get_stub(void) 366{ 367 368 return dma_fence_get(&dma_fence_stub.fence); 369} 370 371/* 372 * dma_fence_get(fence) 373 * 374 * Acquire a reference to fence and return it, or return NULL if 375 * fence is NULL. The fence, if nonnull, must not be being 376 * destroyed. 377 */ 378struct dma_fence * 379dma_fence_get(struct dma_fence *fence) 380{ 381 382 if (fence == NULL) 383 return NULL; 384 385 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 386 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 387 388 kref_get(&fence->refcount); 389 return fence; 390} 391 392/* 393 * dma_fence_get_rcu(fence) 394 * 395 * Attempt to acquire a reference to a fence that may be about to 396 * be destroyed, during a read section. Return the fence on 397 * success, or NULL on failure. The fence must be nonnull. 398 */ 399struct dma_fence * 400dma_fence_get_rcu(struct dma_fence *fence) 401{ 402 403 __insn_barrier(); 404 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 405 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 406 if (!kref_get_unless_zero(&fence->refcount)) 407 return NULL; 408 return fence; 409} 410 411/* 412 * dma_fence_get_rcu_safe(fencep) 413 * 414 * Attempt to acquire a reference to the fence *fencep, which may 415 * be about to be destroyed, during a read section. If the value 416 * of *fencep changes after we read *fencep but before we 417 * increment its reference count, retry. Return *fencep on 418 * success, or NULL on failure. 419 */ 420struct dma_fence * 421dma_fence_get_rcu_safe(struct dma_fence *volatile const *fencep) 422{ 423 struct dma_fence *fence; 424 425retry: 426 /* 427 * Load the fence, ensuring we observe the fully initialized 428 * content. 429 */ 430 if ((fence = atomic_load_consume(fencep)) == NULL) 431 return NULL; 432 433 /* Try to acquire a reference. If we can't, try again. */ 434 if (!dma_fence_get_rcu(fence)) 435 goto retry; 436 437 /* 438 * Confirm that it's still the same fence. If not, release it 439 * and retry. 440 */ 441 if (fence != atomic_load_relaxed(fencep)) { 442 dma_fence_put(fence); 443 goto retry; 444 } 445 446 /* Success! */ 447 KASSERT(dma_fence_referenced_p(fence)); 448 return fence; 449} 450 451static void 452dma_fence_release(struct kref *refcount) 453{ 454 struct dma_fence *fence = container_of(refcount, struct dma_fence, 455 refcount); 456 457 KASSERTMSG(TAILQ_EMPTY(&fence->f_callbacks), 458 "fence %p has pending callbacks", fence); 459 KASSERT(!dma_fence_referenced_p(fence)); 460 461 SDT_PROBE1(sdt, drm, fence, release, fence); 462 463 if (fence->ops->release) 464 (*fence->ops->release)(fence); 465 else 466 dma_fence_free(fence); 467} 468 469/* 470 * dma_fence_put(fence) 471 * 472 * Release a reference to fence. If this was the last one, call 473 * the fence's release callback. 474 */ 475void 476dma_fence_put(struct dma_fence *fence) 477{ 478 479 if (fence == NULL) 480 return; 481 KASSERT(dma_fence_referenced_p(fence)); 482 kref_put(&fence->refcount, &dma_fence_release); 483} 484 485/* 486 * dma_fence_ensure_signal_enabled(fence) 487 * 488 * Internal subroutine. If the fence was already signalled, 489 * return -ENOENT. Otherwise, if the enable signalling callback 490 * has not been called yet, call it. If fails, signal the fence 491 * and return -ENOENT. If it succeeds, or if it had already been 492 * called, return zero to indicate success. 493 * 494 * Caller must hold the fence's lock. 495 */ 496static int 497dma_fence_ensure_signal_enabled(struct dma_fence *fence) 498{ 499 bool already_enabled; 500 501 KASSERT(dma_fence_referenced_p(fence)); 502 KASSERT(spin_is_locked(fence->lock)); 503 504 /* Determine whether signalling was enabled, and enable it. */ 505 already_enabled = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 506 &fence->flags); 507 508 /* If the fence was already signalled, fail with -ENOENT. */ 509 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) 510 return -ENOENT; 511 512 /* 513 * Otherwise, if it wasn't enabled yet, try to enable 514 * signalling. 515 */ 516 if (!already_enabled && fence->ops->enable_signaling) { 517 SDT_PROBE1(sdt, drm, fence, enable_signaling, fence); 518 if (!(*fence->ops->enable_signaling)(fence)) { 519 /* If it failed, signal and return -ENOENT. */ 520 dma_fence_signal_locked(fence); 521 return -ENOENT; 522 } 523 } 524 525 /* Success! */ 526 return 0; 527} 528 529/* 530 * dma_fence_add_callback(fence, fcb, fn) 531 * 532 * If fence has been signalled, return -ENOENT. If the enable 533 * signalling callback hasn't been called yet, call it; if it 534 * fails, return -ENOENT. Otherwise, arrange to call fn(fence, 535 * fcb) when it is signalled, and return 0. 536 * 537 * The fence uses memory allocated by the caller in fcb from the 538 * time of dma_fence_add_callback either to the time of 539 * dma_fence_remove_callback, or just before calling fn. 540 */ 541int 542dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *fcb, 543 dma_fence_func_t fn) 544{ 545 int ret; 546 547 KASSERT(dma_fence_referenced_p(fence)); 548 549 /* Optimistically try to skip the lock if it's already signalled. */ 550 if (atomic_load_relaxed(&fence->flags) & 551 (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) { 552 ret = -ENOENT; 553 goto out0; 554 } 555 556 /* Acquire the lock. */ 557 spin_lock(fence->lock); 558 559 /* Ensure signalling is enabled, or fail if we can't. */ 560 ret = dma_fence_ensure_signal_enabled(fence); 561 if (ret) 562 goto out1; 563 564 /* Insert the callback. */ 565 SDT_PROBE2(sdt, drm, fence, add_callback, fence, fcb); 566 fcb->func = fn; 567 TAILQ_INSERT_TAIL(&fence->f_callbacks, fcb, fcb_entry); 568 fcb->fcb_onqueue = true; 569 ret = 0; 570 571 /* Release the lock and we're done. */ 572out1: spin_unlock(fence->lock); 573out0: if (ret) { 574 fcb->func = NULL; 575 fcb->fcb_onqueue = false; 576 } 577 return ret; 578} 579 580/* 581 * dma_fence_remove_callback(fence, fcb) 582 * 583 * Remove the callback fcb from fence. Return true if it was 584 * removed from the list, or false if it had already run and so 585 * was no longer queued anyway. Caller must have already called 586 * dma_fence_add_callback(fence, fcb). 587 */ 588bool 589dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *fcb) 590{ 591 bool onqueue; 592 593 KASSERT(dma_fence_referenced_p(fence)); 594 595 spin_lock(fence->lock); 596 onqueue = fcb->fcb_onqueue; 597 if (onqueue) { 598 SDT_PROBE2(sdt, drm, fence, remove_callback, fence, fcb); 599 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry); 600 fcb->fcb_onqueue = false; 601 } 602 spin_unlock(fence->lock); 603 604 return onqueue; 605} 606 607/* 608 * dma_fence_enable_sw_signaling(fence) 609 * 610 * If it hasn't been called yet and the fence hasn't been 611 * signalled yet, call the fence's enable_sw_signaling callback. 612 * If when that happens, the callback indicates failure by 613 * returning false, signal the fence. 614 */ 615void 616dma_fence_enable_sw_signaling(struct dma_fence *fence) 617{ 618 619 KASSERT(dma_fence_referenced_p(fence)); 620 621 spin_lock(fence->lock); 622 if ((fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) == 0) 623 (void)dma_fence_ensure_signal_enabled(fence); 624 spin_unlock(fence->lock); 625} 626 627/* 628 * dma_fence_is_signaled(fence) 629 * 630 * Test whether the fence has been signalled. If it has been 631 * signalled by dma_fence_signal(_locked), return true. If the 632 * signalled callback returns true indicating that some implicit 633 * external condition has changed, call the callbacks as if with 634 * dma_fence_signal. 635 */ 636bool 637dma_fence_is_signaled(struct dma_fence *fence) 638{ 639 bool signaled; 640 641 KASSERT(dma_fence_referenced_p(fence)); 642 643 spin_lock(fence->lock); 644 signaled = dma_fence_is_signaled_locked(fence); 645 spin_unlock(fence->lock); 646 647 return signaled; 648} 649 650/* 651 * dma_fence_is_signaled_locked(fence) 652 * 653 * Test whether the fence has been signalled. Like 654 * dma_fence_is_signaleed, but caller already holds the fence's lock. 655 */ 656bool 657dma_fence_is_signaled_locked(struct dma_fence *fence) 658{ 659 660 KASSERT(dma_fence_referenced_p(fence)); 661 KASSERT(spin_is_locked(fence->lock)); 662 663 /* Check whether we already set the signalled bit. */ 664 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) 665 return true; 666 667 /* If there's a signalled callback, test it. */ 668 if (fence->ops->signaled) { 669 SDT_PROBE1(sdt, drm, fence, test, fence); 670 if ((*fence->ops->signaled)(fence)) { 671 /* 672 * It's been signalled implicitly by some 673 * external phenomonen. Act as though someone 674 * has called dma_fence_signal. 675 */ 676 dma_fence_signal_locked(fence); 677 return true; 678 } 679 } 680 681 return false; 682} 683 684/* 685 * dma_fence_set_error(fence, error) 686 * 687 * Set an error code prior to dma_fence_signal for use by a 688 * waiter to learn about success or failure of the fence. 689 */ 690void 691dma_fence_set_error(struct dma_fence *fence, int error) 692{ 693 694 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 695 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 696 KASSERT((atomic_load_relaxed(&fence->flags) & 697 (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) == 0); 698 KASSERTMSG(error >= -ELAST, "%d", error); 699 KASSERTMSG(error < 0, "%d", error); 700 701 SDT_PROBE2(sdt, drm, fence, set_error, fence, error); 702 fence->error = error; 703} 704 705/* 706 * dma_fence_get_status(fence) 707 * 708 * Return 0 if fence has yet to be signalled, 1 if it has been 709 * signalled without error, or negative error code if 710 * dma_fence_set_error was used. 711 */ 712int 713dma_fence_get_status(struct dma_fence *fence) 714{ 715 int ret; 716 717 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 718 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 719 720 spin_lock(fence->lock); 721 if (!dma_fence_is_signaled_locked(fence)) { 722 ret = 0; 723 } else if (fence->error) { 724 ret = fence->error; 725 KASSERTMSG(ret < 0, "%d", ret); 726 } else { 727 ret = 1; 728 } 729 spin_unlock(fence->lock); 730 731 return ret; 732} 733 734/* 735 * dma_fence_signal(fence) 736 * 737 * Signal the fence. If it has already been signalled, return 738 * -EINVAL. If it has not been signalled, call the enable 739 * signalling callback if it hasn't been called yet, and remove 740 * each registered callback from the queue and call it; then 741 * return 0. 742 */ 743int 744dma_fence_signal(struct dma_fence *fence) 745{ 746 int ret; 747 748 KASSERT(dma_fence_referenced_p(fence)); 749 750 spin_lock(fence->lock); 751 ret = dma_fence_signal_locked(fence); 752 spin_unlock(fence->lock); 753 754 return ret; 755} 756 757/* 758 * dma_fence_signal_locked(fence) 759 * 760 * Signal the fence. Like dma_fence_signal, but caller already 761 * holds the fence's lock. 762 */ 763int 764dma_fence_signal_locked(struct dma_fence *fence) 765{ 766 struct dma_fence_cb *fcb, *next; 767 768 KASSERT(dma_fence_referenced_p(fence)); 769 KASSERT(spin_is_locked(fence->lock)); 770 771 /* If it's been signalled, fail; otherwise set the signalled bit. */ 772 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 773 return -EINVAL; 774 775 SDT_PROBE1(sdt, drm, fence, signal, fence); 776 777 /* Set the timestamp. */ 778 fence->timestamp = ktime_get(); 779 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 780 781 /* Wake waiters. */ 782 cv_broadcast(&fence->f_cv); 783 784 /* Remove and call the callbacks. */ 785 TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) { 786 SDT_PROBE2(sdt, drm, fence, callback, fence, fcb); 787 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry); 788 fcb->fcb_onqueue = false; 789 (*fcb->func)(fence, fcb); 790 } 791 792 /* Success! */ 793 return 0; 794} 795 796struct wait_any { 797 struct dma_fence_cb fcb; 798 struct wait_any1 { 799 kmutex_t lock; 800 kcondvar_t cv; 801 struct wait_any *cb; 802 bool done; 803 } *common; 804}; 805 806static void 807wait_any_cb(struct dma_fence *fence, struct dma_fence_cb *fcb) 808{ 809 struct wait_any *cb = container_of(fcb, struct wait_any, fcb); 810 811 KASSERT(dma_fence_referenced_p(fence)); 812 813 mutex_enter(&cb->common->lock); 814 cb->common->done = true; 815 cv_broadcast(&cb->common->cv); 816 mutex_exit(&cb->common->lock); 817} 818 819/* 820 * dma_fence_wait_any_timeout(fence, nfences, intr, timeout, ip) 821 * 822 * Wait for any of fences[0], fences[1], fences[2], ..., 823 * fences[nfences-1] to be signalled. If ip is nonnull, set *ip 824 * to the index of the first one. 825 * 826 * Return -ERESTARTSYS if interrupted, 0 on timeout, or time 827 * remaining (at least 1) on success. 828 */ 829long 830dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t nfences, 831 bool intr, long timeout, uint32_t *ip) 832{ 833 struct wait_any1 common; 834 struct wait_any *cb; 835 uint32_t i, j; 836 int start, end; 837 long ret = 0; 838 839 KASSERTMSG(timeout >= 0, "timeout %ld", timeout); 840 KASSERTMSG(timeout <= MAX_SCHEDULE_TIMEOUT, "timeout %ld", timeout); 841 842 /* Optimistically check whether any are signalled. */ 843 for (i = 0; i < nfences; i++) { 844 KASSERT(dma_fence_referenced_p(fences[i])); 845 if (dma_fence_is_signaled(fences[i])) { 846 if (ip) 847 *ip = i; 848 return MAX(1, timeout); 849 } 850 } 851 852 /* 853 * If timeout is zero, we're just polling, so stop here as if 854 * we timed out instantly. 855 */ 856 if (timeout == 0) 857 return 0; 858 859 /* Allocate an array of callback records. */ 860 cb = kcalloc(nfences, sizeof(cb[0]), GFP_KERNEL); 861 if (cb == NULL) 862 return -ENOMEM; 863 864 /* Initialize a mutex and condvar for the common wait. */ 865 mutex_init(&common.lock, MUTEX_DEFAULT, IPL_VM); 866 cv_init(&common.cv, "fence"); 867 common.cb = cb; 868 common.done = false; 869 870 /* 871 * Add a callback to each of the fences, or stop if already 872 * signalled. 873 */ 874 for (i = 0; i < nfences; i++) { 875 cb[i].common = &common; 876 KASSERT(dma_fence_referenced_p(fences[i])); 877 ret = dma_fence_add_callback(fences[i], &cb[i].fcb, 878 &wait_any_cb); 879 if (ret) { 880 KASSERT(ret == -ENOENT); 881 if (ip) 882 *ip = i; 883 ret = MAX(1, timeout); 884 goto out; 885 } 886 } 887 888 /* 889 * None of them was ready immediately. Wait for one of the 890 * callbacks to notify us when it is done. 891 */ 892 mutex_enter(&common.lock); 893 while (!common.done) { 894 /* Wait for the time remaining. */ 895 start = getticks(); 896 if (intr) { 897 if (timeout != MAX_SCHEDULE_TIMEOUT) { 898 ret = -cv_timedwait_sig(&common.cv, 899 &common.lock, MIN(timeout, /* paranoia */ 900 MAX_SCHEDULE_TIMEOUT)); 901 } else { 902 ret = -cv_wait_sig(&common.cv, &common.lock); 903 } 904 } else { 905 if (timeout != MAX_SCHEDULE_TIMEOUT) { 906 ret = -cv_timedwait(&common.cv, 907 &common.lock, MIN(timeout, /* paranoia */ 908 MAX_SCHEDULE_TIMEOUT)); 909 } else { 910 cv_wait(&common.cv, &common.lock); 911 ret = 0; 912 } 913 } 914 end = getticks(); 915 916 /* Deduct from time remaining. If none left, time out. */ 917 if (timeout != MAX_SCHEDULE_TIMEOUT) { 918 timeout -= MIN(timeout, 919 (unsigned)end - (unsigned)start); 920 if (timeout == 0) 921 ret = -EWOULDBLOCK; 922 } 923 924 /* If the wait failed, give up. */ 925 if (ret) 926 break; 927 } 928 mutex_exit(&common.lock); 929 930 /* 931 * Massage the return code if nonzero: 932 * - if we were interrupted, return -ERESTARTSYS; 933 * - if we timed out, return 0. 934 * No other failure is possible. On success, ret=0 but we 935 * check again below to verify anyway. 936 */ 937 if (ret) { 938 KASSERTMSG((ret == -EINTR || ret == -ERESTART || 939 ret == -EWOULDBLOCK), "ret=%ld", ret); 940 if (ret == -EINTR || ret == -ERESTART) { 941 ret = -ERESTARTSYS; 942 } else if (ret == -EWOULDBLOCK) { 943 KASSERT(timeout != MAX_SCHEDULE_TIMEOUT); 944 ret = 0; /* timed out */ 945 } 946 } 947 948 KASSERT(ret != -ERESTART); /* would be confused with time left */ 949 950 /* 951 * Test whether any of the fences has been signalled. If they 952 * have, return success. 953 */ 954 for (j = 0; j < nfences; j++) { 955 if (dma_fence_is_signaled(fences[i])) { 956 if (ip) 957 *ip = j; 958 ret = MAX(1, timeout); 959 goto out; 960 } 961 } 962 963 /* 964 * If user passed MAX_SCHEDULE_TIMEOUT, we can't return 0 965 * meaning timed out because we're supposed to wait forever. 966 */ 967 KASSERT(timeout == MAX_SCHEDULE_TIMEOUT ? ret != 0 : 1); 968 969out: while (i --> 0) 970 (void)dma_fence_remove_callback(fences[i], &cb[i].fcb); 971 cv_destroy(&common.cv); 972 mutex_destroy(&common.lock); 973 kfree(cb); 974 return ret; 975} 976 977/* 978 * dma_fence_wait_timeout(fence, intr, timeout) 979 * 980 * Wait until fence is signalled; or until interrupt, if intr is 981 * true; or until timeout, if positive. Return -ERESTARTSYS if 982 * interrupted, negative error code on any other error, zero on 983 * timeout, or positive number of ticks remaining if the fence is 984 * signalled before the timeout. Works by calling the fence wait 985 * callback. 986 * 987 * The timeout must be nonnegative and at most 988 * MAX_SCHEDULE_TIMEOUT, which means wait indefinitely. 989 */ 990long 991dma_fence_wait_timeout(struct dma_fence *fence, bool intr, long timeout) 992{ 993 long ret; 994 995 KASSERT(dma_fence_referenced_p(fence)); 996 KASSERTMSG(timeout >= 0, "timeout %ld", timeout); 997 KASSERTMSG(timeout <= MAX_SCHEDULE_TIMEOUT, "timeout %ld", timeout); 998 999 SDT_PROBE3(sdt, drm, fence, wait_start, fence, intr, timeout); 1000 if (fence->ops->wait) 1001 ret = (*fence->ops->wait)(fence, intr, timeout); 1002 else 1003 ret = dma_fence_default_wait(fence, intr, timeout); 1004 SDT_PROBE2(sdt, drm, fence, wait_done, fence, ret); 1005 1006 return ret; 1007} 1008 1009/* 1010 * dma_fence_wait(fence, intr) 1011 * 1012 * Wait until fence is signalled; or until interrupt, if intr is 1013 * true. Return -ERESTARTSYS if interrupted, negative error code 1014 * on any other error, zero on sucess. Works by calling the fence 1015 * wait callback with MAX_SCHEDULE_TIMEOUT. 1016 */ 1017long 1018dma_fence_wait(struct dma_fence *fence, bool intr) 1019{ 1020 long ret; 1021 1022 KASSERT(dma_fence_referenced_p(fence)); 1023 1024 ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT); 1025 KASSERT(ret != 0); 1026 KASSERTMSG(ret == -ERESTARTSYS || ret == MAX_SCHEDULE_TIMEOUT, 1027 "ret=%ld", ret); 1028 1029 return (ret < 0 ? ret : 0); 1030} 1031 1032/* 1033 * dma_fence_default_wait(fence, intr, timeout) 1034 * 1035 * Default implementation of fence wait callback using a condition 1036 * variable. If the fence is already signalled, return timeout, 1037 * or 1 if timeout is zero meaning poll. If the enable signalling 1038 * callback hasn't been called, call it, and if it fails, act as 1039 * if the fence had been signalled. Otherwise, wait on the 1040 * internal condvar. If timeout is MAX_SCHEDULE_TIMEOUT, wait 1041 * indefinitely. 1042 */ 1043long 1044dma_fence_default_wait(struct dma_fence *fence, bool intr, long timeout) 1045{ 1046 int starttime = 0, now = 0, deadline = 0; /* XXXGCC */ 1047 kmutex_t *lock = &fence->lock->sl_lock; 1048 long ret = 0; 1049 1050 KASSERT(dma_fence_referenced_p(fence)); 1051 KASSERTMSG(timeout >= 0, "timeout %ld", timeout); 1052 KASSERTMSG(timeout <= MAX_SCHEDULE_TIMEOUT, "timeout %ld", timeout); 1053 1054 /* Optimistically try to skip the lock if it's already signalled. */ 1055 if (atomic_load_relaxed(&fence->flags) & 1056 (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) 1057 return MAX(1, timeout); 1058 1059 /* Acquire the lock. */ 1060 spin_lock(fence->lock); 1061 1062 /* Ensure signalling is enabled, or stop if already completed. */ 1063 if (dma_fence_ensure_signal_enabled(fence) != 0) { 1064 ret = MAX(1, timeout); 1065 goto out; 1066 } 1067 1068 /* If merely polling, stop here. */ 1069 if (timeout == 0) { 1070 ret = 0; 1071 goto out; 1072 } 1073 1074 /* Find out what our deadline is so we can handle spurious wakeup. */ 1075 if (timeout < MAX_SCHEDULE_TIMEOUT) { 1076 now = getticks(); 1077 starttime = now; 1078 deadline = starttime + timeout; 1079 } 1080 1081 /* Wait until the signalled bit is set. */ 1082 while (!(fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT))) { 1083 /* 1084 * If there's a timeout and we've passed the deadline, 1085 * give up. 1086 */ 1087 if (timeout < MAX_SCHEDULE_TIMEOUT) { 1088 now = getticks(); 1089 if (deadline <= now) { 1090 ret = -EWOULDBLOCK; 1091 break; 1092 } 1093 } 1094 1095 /* Wait for the time remaining. */ 1096 if (intr) { 1097 if (timeout < MAX_SCHEDULE_TIMEOUT) { 1098 ret = -cv_timedwait_sig(&fence->f_cv, lock, 1099 deadline - now); 1100 } else { 1101 ret = -cv_wait_sig(&fence->f_cv, lock); 1102 } 1103 } else { 1104 if (timeout < MAX_SCHEDULE_TIMEOUT) { 1105 ret = -cv_timedwait(&fence->f_cv, lock, 1106 deadline - now); 1107 } else { 1108 cv_wait(&fence->f_cv, lock); 1109 ret = 0; 1110 } 1111 } 1112 1113 /* If the wait failed, give up. */ 1114 if (ret) 1115 break; 1116 } 1117 1118 /* 1119 * Massage the return code if nonzero: 1120 * - if we were interrupted, return -ERESTARTSYS; 1121 * - if we timed out, return 0. 1122 * No other failure is possible. On success, ret=0 but we 1123 * check again below to verify anyway. 1124 */ 1125 if (ret) { 1126 KASSERTMSG((ret == -EINTR || ret == -ERESTART || 1127 ret == -EWOULDBLOCK), "ret=%ld", ret); 1128 if (ret == -EINTR || ret == -ERESTART) { 1129 ret = -ERESTARTSYS; 1130 } else if (ret == -EWOULDBLOCK) { 1131 KASSERT(timeout < MAX_SCHEDULE_TIMEOUT); 1132 ret = 0; /* timed out */ 1133 } 1134 } 1135 1136 KASSERT(ret != -ERESTART); /* would be confused with time left */ 1137 1138 /* Check again in case it was signalled after a wait. */ 1139 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) { 1140 if (timeout < MAX_SCHEDULE_TIMEOUT) 1141 ret = MAX(1, deadline - now); 1142 else 1143 ret = MAX_SCHEDULE_TIMEOUT; 1144 } 1145 1146out: /* All done. Release the lock. */ 1147 spin_unlock(fence->lock); 1148 return ret; 1149} 1150 1151/* 1152 * __dma_fence_signal(fence) 1153 * 1154 * Set fence's signalled bit, without waking waiters yet. Return 1155 * true if it was newly set, false if it was already set. 1156 */ 1157bool 1158__dma_fence_signal(struct dma_fence *fence) 1159{ 1160 1161 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 1162 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 1163 1164 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 1165 return false; 1166 1167 return true; 1168} 1169 1170/* 1171 * __dma_fence_signal_wake(fence) 1172 * 1173 * Set fence's timestamp and wake fence's waiters. Caller must 1174 * have previously called __dma_fence_signal and it must have 1175 * previously returned true. 1176 */ 1177void 1178__dma_fence_signal_wake(struct dma_fence *fence, ktime_t timestamp) 1179{ 1180 struct dma_fence_cb *fcb, *next; 1181 1182 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence); 1183 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence); 1184 1185 spin_lock(fence->lock); 1186 1187 KASSERT(fence->flags & DMA_FENCE_FLAG_SIGNALED_BIT); 1188 1189 SDT_PROBE1(sdt, drm, fence, signal, fence); 1190 1191 /* Set the timestamp. */ 1192 fence->timestamp = timestamp; 1193 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 1194 1195 /* Wake waiters. */ 1196 cv_broadcast(&fence->f_cv); 1197 1198 /* Remove and call the callbacks. */ 1199 TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) { 1200 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry); 1201 fcb->fcb_onqueue = false; 1202 (*fcb->func)(fence, fcb); 1203 } 1204 1205 spin_unlock(fence->lock); 1206} 1207