kernel/rcu/tree_nocb.h

215 static bool __wake_nocb_gp(struct rcu_data *rdp_gp,
218 	__releases(rdp_gp->nocb_gp_lock)
222 	if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {
223 		raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
229 	if (rdp_gp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {
230 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
231 		del_timer(&rdp_gp->nocb_timer);
234 	if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {
235 		WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);
238 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
241 		wake_up_process(rdp_gp->nocb_gp_kthread);
253 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
255 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
256 	return __wake_nocb_gp(rdp_gp, rdp, force, flags);
292 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
294 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
302 		mod_timer(&rdp_gp->nocb_timer, jiffies + rcu_get_jiffies_lazy_flush());
303 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
305 		mod_timer(&rdp_gp->nocb_timer, jiffies + 2);
306 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
308 		if (rdp_gp->nocb_defer_wakeup < RCU_NOCB_WAKE)
309 			mod_timer(&rdp_gp->nocb_timer, jiffies + 1);
310 		if (rdp_gp->nocb_defer_wakeup < waketype)
311 			WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
314 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
567 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
612 		    !timer_pending(&rdp_gp->nocb_timer)) {
1031 static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp_gp,
1034 	__releases(rdp_gp->nocb_gp_lock)
1039 	if (!rcu_nocb_need_deferred_wakeup(rdp_gp, level)) {
1040 		raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
1044 	ndw = rdp_gp->nocb_defer_wakeup;
1045 	ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);
1073 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1075 	if (!rdp_gp || !rcu_nocb_need_deferred_wakeup(rdp_gp, RCU_NOCB_WAKE))
1078 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
1079 	return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags);
1093 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1108 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
1110 	WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp);
1111 	if (rdp_gp->nocb_gp_sleep) {
1112 		rdp_gp->nocb_gp_sleep = false;
1115 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
1126 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1159 	mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);
1160 	if (rdp_gp->nocb_gp_kthread) {
1162 			wake_up_process(rdp_gp->nocb_gp_kthread);
1190 	mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1245 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1255 	if (WARN_ON_ONCE(!rdp_gp->nocb_gp_kthread))
1284 		wake_up_process(rdp_gp->nocb_gp_kthread);
1500 	struct rcu_data *rdp_gp;
1513 	rdp_gp = rdp->nocb_gp_rdp;
1514 	mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);
1515 	if (!rdp_gp->nocb_gp_kthread) {
1516 		t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,
1517 				"rcuog/%d", rdp_gp->cpu);
1519 			mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1522 		WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);
1526 	mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1538 	WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
1565 	struct rcu_data *rdp_gp = NULL;  /* Suppress misguided gcc warn. */
1585 			rdp_gp = rdp;
1602 		rdp->nocb_gp_rdp = rdp_gp;
1604 			list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);