drivers/dma-buf/dma-fence.c

16 #include <linux/dma-fence.h>
31  * fence context counter: each execution context should have its own
32  * fence context, this allows checking if fences belong to the same
45  * A fence is initialized using dma_fence_init() and completed using
60  *   fence to be updated.
69  * DOC: fence cross-driver contract
95  *   This means any code required for fence completion cannot acquire a
100  *   callbacks. This means any code required for fence completion cannot
105  *   for fence completion cannot allocate memory with GFP_NOFS or GFP_NOIO.
114 static const char *dma_fence_stub_get_name(struct dma_fence *fence)
125  * dma_fence_get_stub - return a signaled fence
127  * Return a stub fence which is already signaled. The fence's
152  * dma_fence_allocate_private_stub - return a private, signaled fence
153  * @timestamp: timestamp when the fence was signaled
155  * Return a newly allocated and signaled stub fence.
159 	struct dma_fence *fence;
161 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
162 	if (fence == NULL)
165 	dma_fence_init(fence,
171 		&fence->flags);
173 	dma_fence_signal_timestamp(fence, timestamp);
175 	return fence;
180  * dma_fence_context_alloc - allocate an array of fence contexts
183  * This function will return the first index of the number of fence contexts
184  * allocated.  The fence context is used for setting &dma_fence.context to a
195  * DOC: fence signalling annotation
235  * prevents it from signalling the fence the previous thread is stuck waiting
256  *   point where a fence is accessible to other threads, to the point where
289  * dma_fence_begin_signalling - begin a critical DMA fence signalling section
320  * dma_fence_end_signalling - end a critical DMA fence signalling section
350  * dma_fence_signal_timestamp_locked - signal completion of a fence
351  * @fence: the fence to signal
352  * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
354  * Signal completion for software callbacks on a fence, this will unblock
356  * dma_fence_add_callback(). Can be called multiple times, but since a fence
358  * only be effective the first time. Set the timestamp provided as the fence
364  * Returns 0 on success and a negative error value when @fence has been
367 int dma_fence_signal_timestamp_locked(struct dma_fence *fence,
373 	lockdep_assert_held(fence->lock);
376 				      &fence->flags)))
380 	list_replace(&fence->cb_list, &cb_list);
382 	fence->timestamp = timestamp;
383 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
384 	trace_dma_fence_signaled(fence);
388 		cur->func(fence, cur);
396  * dma_fence_signal_timestamp - signal completion of a fence
397  * @fence: the fence to signal
398  * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
400  * Signal completion for software callbacks on a fence, this will unblock
402  * dma_fence_add_callback(). Can be called multiple times, but since a fence
404  * only be effective the first time. Set the timestamp provided as the fence
407  * Returns 0 on success and a negative error value when @fence has been
410 int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)
415 	if (!fence)
418 	spin_lock_irqsave(fence->lock, flags);
419 	ret = dma_fence_signal_timestamp_locked(fence, timestamp);
420 	spin_unlock_irqrestore(fence->lock, flags);
427  * dma_fence_signal_locked - signal completion of a fence
428  * @fence: the fence to signal
430  * Signal completion for software callbacks on a fence, this will unblock
432  * dma_fence_add_callback(). Can be called multiple times, but since a fence
439  * Returns 0 on success and a negative error value when @fence has been
442 int dma_fence_signal_locked(struct dma_fence *fence)
444 	return dma_fence_signal_timestamp_locked(fence, ktime_get());
449  * dma_fence_signal - signal completion of a fence
450  * @fence: the fence to signal
452  * Signal completion for software callbacks on a fence, this will unblock
454  * dma_fence_add_callback(). Can be called multiple times, but since a fence
458  * Returns 0 on success and a negative error value when @fence has been
461 int dma_fence_signal(struct dma_fence *fence)
467 	if (!fence)
472 	spin_lock_irqsave(fence->lock, flags);
473 	ret = dma_fence_signal_timestamp_locked(fence, ktime_get());
474 	spin_unlock_irqrestore(fence->lock, flags);
483  * dma_fence_wait_timeout - sleep until the fence gets signaled
485  * @fence: the fence to wait on
493  * Performs a synchronous wait on this fence. It is assumed the caller
495  * holds a reference to the fence, otherwise the fence might be
501 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
512 	dma_fence_enable_sw_signaling(fence);
514 	trace_dma_fence_wait_start(fence);
515 	if (fence->ops->wait)
516 		ret = fence->ops->wait(fence, intr, timeout);
518 		ret = dma_fence_default_wait(fence, intr, timeout);
519 	trace_dma_fence_wait_end(fence);
533 	struct dma_fence *fence =
536 	trace_dma_fence_destroy(fence);
538 	if (WARN(!list_empty(&fence->cb_list) &&
539 		 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),
541 		 fence->ops->get_driver_name(fence),
542 		 fence->ops->get_timeline_name(fence),
543 		 fence->context, fence->seqno)) {
553 		spin_lock_irqsave(fence->lock, flags);
554 		fence->error = -EDEADLK;
555 		dma_fence_signal_locked(fence);
556 		spin_unlock_irqrestore(fence->lock, flags);
559 	if (fence->ops->release)
560 		fence->ops->release(fence);
562 		dma_fence_free(fence);
568  * @fence: fence to release
571  * kfree_rcu() on @fence.
573 void dma_fence_free(struct dma_fence *fence)
575 	kfree_rcu(fence, rcu);
579 static bool __dma_fence_enable_signaling(struct dma_fence *fence)
583 	lockdep_assert_held(fence->lock);
586 				   &fence->flags);
588 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
591 	if (!was_set && fence->ops->enable_signaling) {
592 		trace_dma_fence_enable_signal(fence);
594 		if (!fence->ops->enable_signaling(fence)) {
595 			dma_fence_signal_locked(fence);
604  * dma_fence_enable_sw_signaling - enable signaling on fence
605  * @fence: the fence to enable
607  * This will request for sw signaling to be enabled, to make the fence
611 void dma_fence_enable_sw_signaling(struct dma_fence *fence)
615 	spin_lock_irqsave(fence->lock, flags);
616 	__dma_fence_enable_signaling(fence);
617 	spin_unlock_irqrestore(fence->lock, flags);
622  * dma_fence_add_callback - add a callback to be called when the fence
624  * @fence: the fence to wait on
628  * Add a software callback to the fence. The caller should keep a reference to
629  * the fence.
633  * to a fence, but a callback can only be registered to one fence at a time.
635  * If fence is already signaled, this function will return -ENOENT (and
640  * Returns 0 in case of success, -ENOENT if the fence is already signaled
643 int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
649 	if (WARN_ON(!fence || !func))
652 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
657 	spin_lock_irqsave(fence->lock, flags);
659 	if (__dma_fence_enable_signaling(fence)) {
661 		list_add_tail(&cb->node, &fence->cb_list);
667 	spin_unlock_irqrestore(fence->lock, flags);
675  * @fence: the dma_fence to query
678  * condition on a signaled fence. See dma_fence_get_status_locked() for more
681  * Returns 0 if the fence has not yet been signaled, 1 if the fence has
683  * if the fence has been completed in err.
685 int dma_fence_get_status(struct dma_fence *fence)
690 	spin_lock_irqsave(fence->lock, flags);
691 	status = dma_fence_get_status_locked(fence);
692 	spin_unlock_irqrestore(fence->lock, flags);
700  * @fence: the fence to wait on
703  * Remove a previously queued callback from the fence. This function returns
704  * true if the callback is successfully removed, or false if the fence has
711  * with a reference held to the fence.
713  * Behaviour is undefined if @cb has not been added to @fence using
717 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
722 	spin_lock_irqsave(fence->lock, flags);
728 	spin_unlock_irqrestore(fence->lock, flags);
740 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
749  * dma_fence_default_wait - default sleep until the fence gets signaled
751  * @fence: the fence to wait on
757  * returned if the fence is already signaled for consistency with other
761 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
767 	spin_lock_irqsave(fence->lock, flags);
769 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
784 	list_add(&cb.base.node, &fence->cb_list);
786 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
791 		spin_unlock_irqrestore(fence->lock, flags);
795 		spin_lock_irqsave(fence->lock, flags);
805 	spin_unlock_irqrestore(fence->lock, flags);
817 		struct dma_fence *fence = fences[i];
818 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
828  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
834  * @idx: used to store the first signaled fence index, meaningful only on
837  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
841  * Synchronous waits for the first fence in the array to be signaled. The
843  * fence might be freed before return, resulting in undefined behavior.
876 		struct dma_fence *fence = fences[i];
879 		if (dma_fence_add_callback(fence, &cb[i].base,
881 			/* This fence is already signaled */
946  * hint (Ie. forward progress cannot be made until this fence is signaled).
948  * Multiple deadlines may be set on a given fence, even in parallel.  See the
951  * The deadline hint is just that, a hint.  The driver that created the fence
957  * dma_fence_set_deadline - set desired fence-wait deadline hint
958  * @fence:    the fence that is to be waited on
959  * @deadline: the time by which the waiter hopes for the fence to be
962  * Give the fence signaler a hint about an upcoming deadline, such as
963  * vblank, by which point the waiter would prefer the fence to be
964  * signaled by.  This is intended to give feedback to the fence signaler
966  * if a periodic vblank deadline is approaching but the fence is not
969 void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
971 	if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))
972 		fence->ops->set_deadline(fence, deadline);
977  * dma_fence_describe - Dump fence description into seq_file
978  * @fence: the fence to describe
981  * Dump a textual description of the fence and it's state into the seq_file.
983 void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq)
986 		   fence->ops->get_driver_name(fence),
987 		   fence->ops->get_timeline_name(fence), fence->seqno,
988 		   dma_fence_is_signaled(fence) ? "" : "un");
993  * dma_fence_init - Initialize a custom fence.
994  * @fence: the fence to initialize
995  * @ops: the dma_fence_ops for operations on this fence
996  * @lock: the irqsafe spinlock to use for locking this fence
997  * @context: the execution context this fence is run on
1000  * Initializes an allocated fence, the caller doesn't have to keep its
1001  * refcount after committing with this fence, but it will need to hold a
1005  * to check which fence is later by simply using dma_fence_later().
1008 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
1014 	kref_init(&fence->refcount);
1015 	fence->ops = ops;
1016 	INIT_LIST_HEAD(&fence->cb_list);
1017 	fence->lock = lock;
1018 	fence->context = context;
1019 	fence->seqno = seqno;
1020 	fence->flags = 0UL;
1021 	fence->error = 0;
1023 	trace_dma_fence_init(fence);