drm/i915/i915_active.c

29 	struct i915_active *ref;
77 	struct i915_active *ref = addr;
79 	return (void *)ref->active ?: (void *)ref->retire ?: (void *)ref;
87 static void debug_active_init(struct i915_active *ref)
89 	debug_object_init(ref, &active_debug_desc);
92 static void debug_active_activate(struct i915_active *ref)
94 	lockdep_assert_held(&ref->tree_lock);
95 	debug_object_activate(ref, &active_debug_desc);
98 static void debug_active_deactivate(struct i915_active *ref)
100 	lockdep_assert_held(&ref->tree_lock);
101 	if (!atomic_read(&ref->count)) /* after the last dec */
102 		debug_object_deactivate(ref, &active_debug_desc);
105 static void debug_active_fini(struct i915_active *ref)
107 	debug_object_free(ref, &active_debug_desc);
110 static void debug_active_assert(struct i915_active *ref)
112 	debug_object_assert_init(ref, &active_debug_desc);
117 static inline void debug_active_init(struct i915_active *ref) { }
118 static inline void debug_active_activate(struct i915_active *ref) { }
119 static inline void debug_active_deactivate(struct i915_active *ref) { }
120 static inline void debug_active_fini(struct i915_active *ref) { }
121 static inline void debug_active_assert(struct i915_active *ref) { }
126 __active_retire(struct i915_active *ref)
132 	GEM_BUG_ON(i915_active_is_idle(ref));
135 	if (!atomic_dec_and_lock_irqsave(&ref->count, &ref->tree_lock, flags))
138 	GEM_BUG_ON(rcu_access_pointer(ref->excl.fence));
139 	debug_active_deactivate(ref);
142 	if (!ref->cache)
143 		ref->cache = fetch_node(ref->tree.rb_node);
146 	if (ref->cache) {
148 		rb_erase(&ref->cache->node, &ref->tree);
149 		root = ref->tree;
152 		rb_link_node(&ref->cache->node, NULL, &ref->tree.rb_node);
153 		rb_insert_color(&ref->cache->node, &ref->tree);
154 		GEM_BUG_ON(ref->tree.rb_node != &ref->cache->node);
157 		ref->cache->timeline = 0; /* needs cmpxchg(u64) */
160 	spin_unlock_irqrestore(&ref->tree_lock, flags);
163 	if (ref->retire)
164 		ref->retire(ref);
167 	wake_up_var(ref);
179 	struct i915_active *ref = container_of(wrk, typeof(*ref), work);
181 	GEM_BUG_ON(!atomic_read(&ref->count));
182 	if (atomic_add_unless(&ref->count, -1, 1))
185 	__active_retire(ref);
189 active_retire(struct i915_active *ref)
191 	GEM_BUG_ON(!atomic_read(&ref->count));
192 	if (atomic_add_unless(&ref->count, -1, 1))
195 	if (ref->flags & I915_ACTIVE_RETIRE_SLEEPS) {
196 		queue_work(system_unbound_wq, &ref->work);
200 	__active_retire(ref);
222 		active_retire(container_of(cb, struct active_node, base.cb)->ref);
232 static struct active_node *__active_lookup(struct i915_active *ref, u64 idx)
245 	it = READ_ONCE(ref->cache);
270 	GEM_BUG_ON(i915_active_is_idle(ref));
272 	it = fetch_node(ref->tree.rb_node);
279 			WRITE_ONCE(ref->cache, it);
289 active_instance(struct i915_active *ref, u64 idx)
294 	node = __active_lookup(ref, idx);
298 	spin_lock_irq(&ref->tree_lock);
299 	GEM_BUG_ON(i915_active_is_idle(ref));
302 	p = &ref->tree.rb_node;
325 	node->ref = ref;
329 	rb_insert_color(&node->node, &ref->tree);
332 	WRITE_ONCE(ref->cache, node);
333 	spin_unlock_irq(&ref->tree_lock);
338 void __i915_active_init(struct i915_active *ref,
339 			int (*active)(struct i915_active *ref),
340 			void (*retire)(struct i915_active *ref),
345 	debug_active_init(ref);
347 	ref->flags = flags;
348 	ref->active = active;
349 	ref->retire = retire;
351 	spin_lock_init(&ref->tree_lock);
352 	ref->tree = RB_ROOT;
353 	ref->cache = NULL;
355 	init_llist_head(&ref->preallocated_barriers);
356 	atomic_set(&ref->count, 0);
357 	__mutex_init(&ref->mutex, "i915_active", mkey);
358 	__i915_active_fence_init(&ref->excl, NULL, excl_retire);
359 	INIT_WORK(&ref->work, active_work);
361 	lockdep_init_map(&ref->work.lockdep_map, "i915_active.work", wkey, 0);
365 static bool ____active_del_barrier(struct i915_active *ref,
408 __active_del_barrier(struct i915_active *ref, struct active_node *node)
410 	return ____active_del_barrier(ref, node, barrier_to_engine(node));
414 replace_barrier(struct i915_active *ref, struct i915_active_fence *active)
424 	return __active_del_barrier(ref, node_from_active(active));
427 int i915_active_add_request(struct i915_active *ref, struct i915_request *rq)
435 	err = i915_active_acquire(ref);
440 		active = active_instance(ref, idx);
446 		if (replace_barrier(ref, active)) {
448 			atomic_dec(&ref->count);
454 		__i915_active_acquire(ref);
459 	i915_active_release(ref);
464 __i915_active_set_fence(struct i915_active *ref,
470 	if (replace_barrier(ref, active)) {
477 		__i915_active_acquire(ref);
483 i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f)
486 	return __i915_active_set_fence(ref, &ref->excl, f);
489 bool i915_active_acquire_if_busy(struct i915_active *ref)
491 	debug_active_assert(ref);
492 	return atomic_add_unless(&ref->count, 1, 0);
495 static void __i915_active_activate(struct i915_active *ref)
497 	spin_lock_irq(&ref->tree_lock); /* __active_retire() */
498 	if (!atomic_fetch_inc(&ref->count))
499 		debug_active_activate(ref);
500 	spin_unlock_irq(&ref->tree_lock);
503 int i915_active_acquire(struct i915_active *ref)
507 	if (i915_active_acquire_if_busy(ref))
510 	if (!ref->active) {
511 		__i915_active_activate(ref);
515 	err = mutex_lock_interruptible(&ref->mutex);
519 	if (likely(!i915_active_acquire_if_busy(ref))) {
520 		err = ref->active(ref);
522 			__i915_active_activate(ref);
525 	mutex_unlock(&ref->mutex);
530 int i915_active_acquire_for_context(struct i915_active *ref, u64 idx)
535 	err = i915_active_acquire(ref);
539 	active = active_instance(ref, idx);
541 		i915_active_release(ref);
545 	return 0; /* return with active ref */
548 void i915_active_release(struct i915_active *ref)
550 	debug_active_assert(ref);
551 	active_retire(ref);
584 static int flush_lazy_signals(struct i915_active *ref)
589 	enable_signaling(&ref->excl);
590 	rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
601 int __i915_active_wait(struct i915_active *ref, int state)
606 	if (i915_active_acquire_if_busy(ref)) {
609 		err = flush_lazy_signals(ref);
610 		i915_active_release(ref);
614 		if (___wait_var_event(ref, i915_active_is_idle(ref),
623 	flush_work(&ref->work);
651 	struct i915_active *ref;
659 	if (i915_active_is_idle(wb->ref)) {
668 static int __await_barrier(struct i915_active *ref, struct i915_sw_fence *fence)
676 	GEM_BUG_ON(i915_active_is_idle(ref));
685 	wb->ref = ref;
687 	add_wait_queue(__var_waitqueue(ref), &wb->base);
691 static int await_active(struct i915_active *ref,
698 	if (!i915_active_acquire_if_busy(ref))
702 	    rcu_access_pointer(ref->excl.fence)) {
703 		err = __await_active(&ref->excl, fn, arg);
711 		rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
719 		err = flush_lazy_signals(ref);
723 		err = __await_barrier(ref, barrier);
729 	i915_active_release(ref);
739 			      struct i915_active *ref,
742 	return await_active(ref, flags, rq_await_fence, rq, &rq->submit);
752 			       struct i915_active *ref,
755 	return await_active(ref, flags, sw_await_fence, fence, fence);
758 void i915_active_fini(struct i915_active *ref)
760 	debug_active_fini(ref);
761 	GEM_BUG_ON(atomic_read(&ref->count));
762 	GEM_BUG_ON(work_pending(&ref->work));
763 	mutex_destroy(&ref->mutex);
765 	if (ref->cache)
766 		kmem_cache_free(slab_cache, ref->cache);
774 static struct active_node *reuse_idle_barrier(struct i915_active *ref, u64 idx)
778 	if (RB_EMPTY_ROOT(&ref->tree))
781 	GEM_BUG_ON(i915_active_is_idle(ref));
790 	if (ref->cache && is_idle_barrier(ref->cache, idx)) {
791 		p = &ref->cache->node;
796 	p = ref->tree.rb_node;
841 		    ____active_del_barrier(ref, node, engine))
848 	spin_lock_irq(&ref->tree_lock);
849 	rb_erase(p, &ref->tree); /* Hide from waits and sibling allocations */
850 	if (p == &ref->cache->node)
851 		WRITE_ONCE(ref->cache, NULL);
852 	spin_unlock_irq(&ref->tree_lock);
857 int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
864 	GEM_BUG_ON(i915_active_is_idle(ref));
867 	while (!llist_empty(&ref->preallocated_barriers))
883 		node = reuse_idle_barrier(ref, idx);
893 			node->ref = ref;
908 			__i915_active_acquire(ref);
920 	GEM_BUG_ON(!llist_empty(&ref->preallocated_barriers));
921 	llist_add_batch(first, last, &ref->preallocated_barriers);
931 		atomic_dec(&ref->count);
939 void i915_active_acquire_barrier(struct i915_active *ref)
944 	GEM_BUG_ON(i915_active_is_idle(ref));
952 	llist_for_each_safe(pos, next, take_preallocated_barriers(ref)) {
957 		spin_lock_irqsave_nested(&ref->tree_lock, flags,
960 		p = &ref->tree.rb_node;
973 		rb_insert_color(&node->node, &ref->tree);
974 		spin_unlock_irqrestore(&ref->tree_lock, flags);
1141 	struct kref ref;
1144 struct i915_active *i915_active_get(struct i915_active *ref)
1146 	struct auto_active *aa = container_of(ref, typeof(*aa), base);
1148 	kref_get(&aa->ref);
1152 static void auto_release(struct kref *ref)
1154 	struct auto_active *aa = container_of(ref, typeof(*aa), ref);
1160 void i915_active_put(struct i915_active *ref)
1162 	struct auto_active *aa = container_of(ref, typeof(*aa), base);
1164 	kref_put(&aa->ref, auto_release);
1167 static int auto_active(struct i915_active *ref)
1169 	i915_active_get(ref);
1173 static void auto_retire(struct i915_active *ref)
1175 	i915_active_put(ref);
1186 	kref_init(&aa->ref);