kernel/rcu/tree_stall.h

222 	struct rcu_node *rnp;
225 	rnp = rdp->mynode;
226 	raw_spin_lock_rcu_node(rnp);
228 		rdp->rcu_iw_gp_seq = rnp->gp_seq;
231 	raw_spin_unlock_rcu_node(rnp);
244 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
249 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
250 	if (!rcu_preempt_blocked_readers_cgp(rnp)) {
251 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
254 	t = list_entry(rnp->gp_tasks->prev,
256 	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
264 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
294 static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
295 	__releases(rnp->lock)
304 	if (!rcu_preempt_blocked_readers_cgp(rnp)) {
305 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
309 	       rnp->level, rnp->grplo, rnp->grphi);
310 	t = list_entry(rnp->gp_tasks->prev,
312 	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
318 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
344 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
352 static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
353 	__releases(rnp->lock)
355 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
370 	struct rcu_node *rnp;
372 	rcu_for_each_leaf_node(rnp) {
373 		raw_spin_lock_irqsave_rcu_node(rnp, flags);
374 		for_each_leaf_node_possible_cpu(rnp, cpu)
375 			if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
381 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
598 	struct rcu_node *rnp;
615 	rcu_for_each_leaf_node(rnp) {
616 		raw_spin_lock_irqsave_rcu_node(rnp, flags);
617 		if (rnp->qsmask != 0) {
618 			for_each_leaf_node_possible_cpu(rnp, cpu)
619 				if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
624 		ndetected += rcu_print_task_stall(rnp, flags); // Releases rnp->lock.
638 		rcu_for_each_leaf_node(rnp)
639 			rcu_print_detail_task_stall_rnp(rnp);
670 	struct rcu_node *rnp = rcu_get_root();
702 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
707 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
731 	struct rcu_node *rnp;
779 	rnp = rdp->mynode;
781 	self_detected = READ_ONCE(rnp->qsmask) & rdp->grpmask;
838 	struct rcu_node *rnp;
840 	rcu_for_each_leaf_node(rnp) {
842 			if (data_race(READ_ONCE(rnp->qsmask))) {
845 				if (READ_ONCE(rnp->gp_tasks))
851 		raw_spin_lock_irqsave_rcu_node(rnp, flags);
852 		if (rnp->gp_tasks)
854 		if (!rnp->qsmask) {
855 			// No CPUs without quiescent states for this rnp.
856 			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
860 		for_each_leaf_node_possible_cpu(rnp, cpu) {
861 			if (rnp->qsmask & (1UL << (cpu - rnp->grplo))) {
862 				raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
867 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
887 	struct rcu_node *rnp;
904 	rcu_for_each_node_breadth_first(rnp) {
905 		if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) &&
906 		    !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) &&
907 		    !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks)))
910 			rnp->grplo, rnp->grphi,
911 			(long)data_race(READ_ONCE(rnp->gp_seq)),
912 			(long)data_race(READ_ONCE(rnp->gp_seq_needed)),
913 			data_race(READ_ONCE(rnp->qsmask)),
914 			".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))],
915 			".B"[!!data_race(READ_ONCE(rnp->boost_tasks))],
916 			".E"[!!data_race(READ_ONCE(rnp->exp_tasks))],
917 			".G"[!!data_race(READ_ONCE(rnp->gp_tasks))],
918 			data_race(READ_ONCE(rnp->n_boosts)));
919 		if (!rcu_is_leaf_node(rnp))
921 		for_each_leaf_node_possible_cpu(rnp, cpu) {
946 static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp,
965 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
973 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
978 	if (rnp_root != rnp)
987 		if (rnp_root != rnp)
990 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
994 	if (rnp_root != rnp)
996 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);