kernel/rcu/sync.c

19  * @rsp: Pointer to rcu_sync structure to be initialized
21 void rcu_sync_init(struct rcu_sync *rsp)
23 	memset(rsp, 0, sizeof(*rsp));
24 	init_waitqueue_head(&rsp->gp_wait);
29 static void rcu_sync_call(struct rcu_sync *rsp)
31 	call_rcu_hurry(&rsp->cb_head, rcu_sync_func);
59 	struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head);
62 	WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE);
63 	WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED);
65 	spin_lock_irqsave(&rsp->rss_lock, flags);
66 	if (rsp->gp_count) {
70 		WRITE_ONCE(rsp->gp_state, GP_PASSED);
71 		wake_up_locked(&rsp->gp_wait);
72 	} else if (rsp->gp_state == GP_REPLAY) {
77 		WRITE_ONCE(rsp->gp_state, GP_EXIT);
78 		rcu_sync_call(rsp);
85 		WRITE_ONCE(rsp->gp_state, GP_IDLE);
87 	spin_unlock_irqrestore(&rsp->rss_lock, flags);
92  * @rsp: Pointer to rcu_sync structure to use for synchronization
105 void rcu_sync_enter(struct rcu_sync *rsp)
109 	spin_lock_irq(&rsp->rss_lock);
110 	gp_state = rsp->gp_state;
112 		WRITE_ONCE(rsp->gp_state, GP_ENTER);
113 		WARN_ON_ONCE(rsp->gp_count);
115 		 * Note that we could simply do rcu_sync_call(rsp) here and
125 	rsp->gp_count++;
126 	spin_unlock_irq(&rsp->rss_lock);
134 		rcu_sync_func(&rsp->cb_head);
139 	wait_event(rsp->gp_wait, READ_ONCE(rsp->gp_state) >= GP_PASSED);
144  * @rsp: Pointer to rcu_sync structure to use for synchronization
152 void rcu_sync_exit(struct rcu_sync *rsp)
154 	WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE);
155 	WARN_ON_ONCE(READ_ONCE(rsp->gp_count) == 0);
157 	spin_lock_irq(&rsp->rss_lock);
158 	if (!--rsp->gp_count) {
159 		if (rsp->gp_state == GP_PASSED) {
160 			WRITE_ONCE(rsp->gp_state, GP_EXIT);
161 			rcu_sync_call(rsp);
162 		} else if (rsp->gp_state == GP_EXIT) {
163 			WRITE_ONCE(rsp->gp_state, GP_REPLAY);
166 	spin_unlock_irq(&rsp->rss_lock);
171  * @rsp: Pointer to rcu_sync structure to be cleaned up
173 void rcu_sync_dtor(struct rcu_sync *rsp)
177 	WARN_ON_ONCE(READ_ONCE(rsp->gp_count));
178 	WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED);
180 	spin_lock_irq(&rsp->rss_lock);
181 	if (rsp->gp_state == GP_REPLAY)
182 		WRITE_ONCE(rsp->gp_state, GP_EXIT);
183 	gp_state = rsp->gp_state;
184 	spin_unlock_irq(&rsp->rss_lock);
188 		WARN_ON_ONCE(rsp->gp_state != GP_IDLE);