| /linux-master/kernel/rcu/ |
| H A D | tree_exp.h | 13 static int rcu_print_task_exp_stall(struct rcu_node *rnp);
14 static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp);
83 struct rcu_node *rnp; local
95 rcu_for_each_leaf_node(rnp) {
96 raw_spin_lock_irqsave_rcu_node(rnp, flags);
97 if (rnp->expmaskinit == rnp->expmaskinitnext) {
98 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
103 oldmask = rnp->expmaskinit;
104 rnp
136 struct rcu_node *rnp; local
153 sync_rcu_exp_done(struct rcu_node *rnp) argument
164 sync_rcu_exp_done_unlocked(struct rcu_node *rnp) argument
190 raw_lockdep_assert_held_rcu_node(rnp); variable
207 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled */ variable
209 raw_spin_lock_rcu_node(rnp); /* irqs already disabled */ variable
219 rcu_report_exp_rnp(struct rcu_node *rnp, bool wake) argument
231 rcu_report_exp_cpu_mult(struct rcu_node *rnp, unsigned long mask, bool wake) argument
284 struct rcu_node *rnp = rdp->mynode; local
345 struct rcu_node *rnp = container_of(rewp, struct rcu_node, rew); local
434 rcu_exp_par_worker_started(struct rcu_node *rnp) argument
439 sync_rcu_exp_select_cpus_queue_work(struct rcu_node *rnp) argument
450 sync_rcu_exp_select_cpus_flush_work(struct rcu_node *rnp) argument
478 struct rcu_node *rnp; local
[all...] |
| H A D | tree_plugin.h | 105 static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake);
151 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
152 __releases(rnp->lock) /* But leaves rrupts disabled. */
154 int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
155 (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
156 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
157 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
160 raw_lockdep_assert_held_rcu_node(rnp); variable
161 WARN_ON_ONCE(rdp->mynode != rnp);
162 WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
253 raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */ variable
316 struct rcu_node *rnp; local
369 rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) argument
440 rcu_next_node_entry(struct task_struct *t, struct rcu_node *rnp) argument
455 rcu_preempt_has_tasks(struct rcu_node *rnp) argument
474 struct rcu_node *rnp; local
642 struct rcu_node *rnp = rdp->mynode; local
693 rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) argument
778 dump_blkd_tasks(struct rcu_node *rnp, int ncheck) argument
913 rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) argument
921 rcu_preempt_has_tasks(struct rcu_node *rnp) argument
955 rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) argument
995 dump_blkd_tasks(struct rcu_node *rnp, int ncheck) argument
1047 rcu_boost(struct rcu_node *rnp) argument
1112 struct rcu_node *rnp = (struct rcu_node *)arg; local
1155 raw_lockdep_assert_held_rcu_node(rnp); variable
1182 rcu_preempt_boost_start_gp(struct rcu_node *rnp) argument
1191 rcu_spawn_one_boost_kthread(struct rcu_node *rnp) argument
1214 rcu_boost_task(struct rcu_node *rnp) argument
1227 rcu_preempt_boost_start_gp(struct rcu_node *rnp) argument
1231 rcu_spawn_one_boost_kthread(struct rcu_node *rnp) argument
1235 rcu_boost_task(struct rcu_node *rnp) argument
[all...] |
| H A D | tree.c | 146 static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
148 static struct task_struct *rcu_boost_task(struct rcu_node *rnp);
152 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
731 static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) argument
733 raw_lockdep_assert_held_rcu_node(rnp);
735 rnp->gp_seq))
737 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))
738 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;
771 struct rcu_node *rnp = rdp->mynode; local
783 rcu_gpnum_ovf(rnp, rd
904 trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, unsigned long gp_seq_req, const char *s) argument
932 struct rcu_node *rnp; local
1003 rcu_future_gp_cleanup(struct rcu_node *rnp) argument
1087 rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) argument
1133 rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, struct rcu_data *rdp) argument
1163 rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) argument
1186 rcu_advance_cbs_nowake(struct rcu_node *rnp, struct rcu_data *rdp) argument
1217 __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) argument
1270 struct rcu_node *rnp; local
1358 struct rcu_node *rnp = rcu_get_root(); local
1374 struct rcu_node *rnp = rcu_get_root(); local
1396 struct rcu_node *rnp = rcu_get_root(); local
1413 struct rcu_node *rnp = rcu_get_root(); local
1434 struct rcu_node *rnp = rcu_get_root(); local
1574 struct rcu_node *rnp = rcu_get_root(); local
1598 struct rcu_node *rnp = rcu_get_root(); local
1638 struct rcu_node *rnp = rcu_get_root(); local
1726 struct rcu_node *rnp = rcu_get_root(); local
1912 raw_lockdep_assert_held_rcu_node(rnp); variable
1977 raw_lockdep_assert_held_rcu_node(rnp); variable
1999 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ variable
2014 struct rcu_node *rnp; local
2317 struct rcu_node *rnp; local
2376 struct rcu_node *rnp; local
2421 struct rcu_node *rnp = rdp->mynode; local
2672 check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp) argument
2697 struct rcu_node *const rnp = rdp->mynode; local
3603 struct rcu_node *rnp; local
3690 struct rcu_node *rnp = rcu_get_root(); local
3711 struct rcu_node *rnp; local
3852 struct rcu_node *rnp = rcu_get_root(); local
3929 struct rcu_node *rnp = rdp->mynode; local
4240 rcu_rnp_online_cpus(struct rcu_node *rnp) argument
4329 struct rcu_node *rnp = rnp_leaf; local
4364 struct rcu_node *rnp = rnp_leaf; local
4408 rcu_spawn_exp_par_gp_kworker(struct rcu_node *rnp) argument
4430 rcu_exp_par_gp_task(struct rcu_node *rnp) argument
4456 rcu_spawn_rnp_kthreads(struct rcu_node *rnp) argument
4481 struct rcu_node *rnp = rcu_get_root(); local
4539 struct rcu_node *rnp; local
4600 struct rcu_node *rnp; local
4635 struct rcu_node *rnp; local
4693 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ local
4798 struct rcu_node *rnp = rdp->mynode; local
4814 struct rcu_node *rnp; local
4860 struct rcu_node *rnp; local
4909 struct rcu_node *rnp; local
4940 struct rcu_node *rnp; local
5135 struct rcu_node *rnp; local
[all...] |
| H A D | tree_stall.h | 222 struct rcu_node *rnp; local
225 rnp = rdp->mynode;
226 raw_spin_lock_rcu_node(rnp);
228 rdp->rcu_iw_gp_seq = rnp->gp_seq;
231 raw_spin_unlock_rcu_node(rnp);
244 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp) argument
249 raw_spin_lock_irqsave_rcu_node(rnp, flags);
250 if (!rcu_preempt_blocked_readers_cgp(rnp)) {
251 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
254 t = list_entry(rnp
344 rcu_print_detail_task_stall_rnp(struct rcu_node *rnp) argument
370 struct rcu_node *rnp; local
597 struct rcu_node *rnp; local
668 struct rcu_node *rnp = rcu_get_root(); local
728 struct rcu_node *rnp; local
835 struct rcu_node *rnp; local
884 struct rcu_node *rnp; local
943 rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, const unsigned long gpssdelay) argument
[all...] |
| H A D | rcu.h | 380 #define rcu_is_leaf_node(rnp) ((rnp)->level == rcu_num_lvls - 1)
383 #define rcu_is_last_leaf_node(rnp) ((rnp) == &rcu_state.node[rcu_num_nodes - 1])
390 #define _rcu_for_each_node_breadth_first(sp, rnp) \
391 for ((rnp) = &(sp)->node[0]; \
392 (rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++)
393 #define rcu_for_each_node_breadth_first(rnp) \
394 _rcu_for_each_node_breadth_first(&rcu_state, rnp)
[all...] |
| H A D | tree.h | 146 #define leaf_node_cpu_bit(rnp, cpu) (BIT((cpu) - (rnp)->grplo))
447 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);
449 static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
451 static int rcu_print_task_exp_stall(struct rcu_node *rnp);
452 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
454 static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck);
455 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
456 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
459 static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp);
[all...] |
| H A D | tree_nocb.h | 204 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) argument
206 return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1];
209 static void rcu_init_one_nocb(struct rcu_node *rnp) argument
211 init_swait_queue_head(&rnp->nocb_gp_wq[0]);
212 init_swait_queue_head(&rnp->nocb_gp_wq[1]);
701 struct rcu_node *rnp; local
764 rnp = rdp->mynode;
771 rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
772 raw_spin_lock_rcu_node(rnp); /* irq
940 struct rcu_node *rnp = rdp->mynode; local
1640 struct rcu_node *rnp = rdp->mynode; local
1760 rcu_nocb_gp_get(struct rcu_node *rnp) argument
1765 rcu_init_one_nocb(struct rcu_node *rnp) argument
[all...] |