kernel/rcu/srcutiny.c

27 static int init_srcu_struct_fields(struct srcu_struct *ssp)
29 	ssp->srcu_lock_nesting[0] = 0;
30 	ssp->srcu_lock_nesting[1] = 0;
31 	init_swait_queue_head(&ssp->srcu_wq);
32 	ssp->srcu_cb_head = NULL;
33 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;
34 	ssp->srcu_gp_running = false;
35 	ssp->srcu_gp_waiting = false;
36 	ssp->srcu_idx = 0;
37 	ssp->srcu_idx_max = 0;
38 	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
39 	INIT_LIST_HEAD(&ssp->srcu_work.entry);
45 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
49 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
50 	lockdep_init_map(&ssp->dep_map, name, key, 0);
51 	return init_srcu_struct_fields(ssp);
59  * @ssp: structure to initialize.
65 int init_srcu_struct(struct srcu_struct *ssp)
67 	return init_srcu_struct_fields(ssp);
75  * @ssp: structure to clean up.
80 void cleanup_srcu_struct(struct srcu_struct *ssp)
82 	WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);
83 	flush_work(&ssp->srcu_work);
84 	WARN_ON(ssp->srcu_gp_running);
85 	WARN_ON(ssp->srcu_gp_waiting);
86 	WARN_ON(ssp->srcu_cb_head);
87 	WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);
88 	WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);
89 	WARN_ON(ssp->srcu_idx & 0x1);
97 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
99 	int newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) - 1;
101 	WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);
102 	if (!newval && READ_ONCE(ssp->srcu_gp_waiting) && in_task())
103 		swake_up_one(&ssp->srcu_wq);
117 	struct srcu_struct *ssp;
119 	ssp = container_of(wp, struct srcu_struct, srcu_work);
120 	if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
124 	WRITE_ONCE(ssp->srcu_gp_running, true);
126 	lh = ssp->srcu_cb_head;
127 	ssp->srcu_cb_head = NULL;
128 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;
130 	idx = (ssp->srcu_idx & 0x2) / 2;
131 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
132 	WRITE_ONCE(ssp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */
133 	swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));
134 	WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
135 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
153 	WRITE_ONCE(ssp->srcu_gp_running, false);
154 	if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
155 		schedule_work(&ssp->srcu_work);
159 static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
163 	cookie = get_state_synchronize_srcu(ssp);
164 	if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
166 	WRITE_ONCE(ssp->srcu_idx_max, cookie);
167 	if (!READ_ONCE(ssp->srcu_gp_running)) {
169 			schedule_work(&ssp->srcu_work);
170 		else if (list_empty(&ssp->srcu_work.entry))
171 			list_add(&ssp->srcu_work.entry, &srcu_boot_list);
179 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
187 	*ssp->srcu_cb_tail = rhp;
188 	ssp->srcu_cb_tail = &rhp->next;
190 	srcu_gp_start_if_needed(ssp);
197 void synchronize_srcu(struct srcu_struct *ssp)
201 	srcu_lock_sync(&ssp->dep_map);
203 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||
215 	call_srcu(ssp, &rs.head, wakeme_after_rcu);
224 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
229 	ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
242 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)
244 	unsigned long ret = get_state_synchronize_srcu(ssp);
246 	srcu_gp_start_if_needed(ssp);
254 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
256 	unsigned long cur_s = READ_ONCE(ssp->srcu_idx);
276 	struct srcu_struct *ssp;
280 		ssp = list_first_entry(&srcu_boot_list,
282 		list_del_init(&ssp->srcu_work.entry);
283 		schedule_work(&ssp->srcu_work);