Lines Matching defs:rcu_state
85 static struct rcu_state rcu_state = {
86 .level = { &rcu_state.node[0] },
89 .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
90 .barrier_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.barrier_lock),
93 .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
94 .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
227 return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
484 return READ_ONCE(rcu_state.gp_seq);
496 return rcu_state.expedited_sequence;
501 * Return the root node of the rcu_state structure.
505 return &rcu_state.node[0];
516 *flags = READ_ONCE(rcu_state.gp_flags);
517 *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
750 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
782 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
834 (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||
835 time_after(jiffies, rcu_state.jiffies_resched) ||
836 rcu_state.cbovld)) {
840 } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
854 rcu_state.cbovld)) {
867 if (time_after(jiffies, rcu_state.jiffies_resched)) {
907 trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
980 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);
981 WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
982 if (!READ_ONCE(rcu_state.gp_kthread)) {
986 trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));
1065 struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
1068 !READ_ONCE(rcu_state.gp_flags) || !t)
1070 WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1071 WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
1072 swake_up_one_online(&rcu_state.gp_wq);
1111 gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
1117 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));
1119 trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));
1140 c = rcu_seq_snap(&rcu_state.gp_seq);
1234 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1250 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
1317 !(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
1364 if (!rcu_seq_state(rcu_state.gp_seq_polled))
1365 rcu_seq_start(&rcu_state.gp_seq_polled);
1368 *snap = rcu_state.gp_seq_polled;
1382 if (*snap && *snap == rcu_state.gp_seq_polled) {
1383 rcu_seq_end(&rcu_state.gp_seq_polled);
1384 rcu_state.gp_seq_polled_snap = 0;
1385 rcu_state.gp_seq_polled_exp_snap = 0;
1436 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1438 if (!READ_ONCE(rcu_state.gp_flags)) {
1443 WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */
1457 rcu_seq_start(&rcu_state.gp_seq);
1458 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
1459 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
1460 rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap);
1472 WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF);
1476 arch_spin_lock(&rcu_state.ofl_lock);
1482 arch_spin_unlock(&rcu_state.ofl_lock);
1519 arch_spin_unlock(&rcu_state.ofl_lock);
1528 * layout of the tree within the rcu_state.node[] array. Note that
1536 WRITE_ONCE(rcu_state.gp_state, RCU_GP_INIT);
1543 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
1547 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
1558 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1581 *gfp = READ_ONCE(rcu_state.gp_flags);
1597 int nr_fqs = READ_ONCE(rcu_state.nr_fqs_jiffies_stall);
1600 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1601 WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);
1607 WRITE_ONCE(rcu_state.jiffies_stall,
1610 WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, --nr_fqs);
1621 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
1623 WRITE_ONCE(rcu_state.gp_flags,
1624 READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
1641 if (rcu_state.cbovld)
1645 if (rcu_state.cbovld) {
1650 if (!ret || time_before(jiffies + j, rcu_state.jiffies_force_qs)) {
1651 WRITE_ONCE(rcu_state.jiffies_force_qs, jiffies + j);
1657 WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
1660 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1662 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_FQS);
1663 (void)swait_event_idle_timeout_exclusive(rcu_state.gp_wq,
1666 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DOING_FQS);
1681 if (!time_after(rcu_state.jiffies_force_qs, jiffies) ||
1683 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1689 gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
1691 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1694 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1700 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1702 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1706 if (time_after(jiffies, rcu_state.jiffies_force_qs))
1709 j = rcu_state.jiffies_force_qs - j;
1729 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1731 rcu_state.gp_end = jiffies;
1732 gp_duration = rcu_state.gp_end - rcu_state.gp_start;
1733 if (gp_duration > rcu_state.gp_max)
1734 rcu_state.gp_max = gp_duration;
1744 rcu_poll_gp_seq_end(&rcu_state.gp_seq_polled_snap);
1756 new_gp_seq = rcu_state.gp_seq;
1781 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1788 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
1789 rcu_seq_end(&rcu_state.gp_seq);
1790 ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);
1791 WRITE_ONCE(rcu_state.gp_state, RCU_GP_IDLE);
1813 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
1814 WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
1815 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));
1824 WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & RCU_GP_FLAG_INIT);
1843 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1845 WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_GPS);
1846 swait_event_idle_exclusive(rcu_state.gp_wq,
1847 READ_ONCE(rcu_state.gp_flags) &
1850 WRITE_ONCE(rcu_state.gp_state, RCU_GP_DONE_GPS);
1855 WRITE_ONCE(rcu_state.gp_activity, jiffies);
1857 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1865 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANUP);
1867 WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANED);
1872 * Report a full set of quiescent states to the rcu_state data structure.
1885 WRITE_ONCE(rcu_state.gp_flags,
1886 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
1929 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
2135 trace_rcu_batch_start(rcu_state.name,
2137 trace_rcu_batch_end(rcu_state.name, 0,
2171 trace_rcu_batch_start(rcu_state.name,
2191 trace_rcu_invoke_callback(rcu_state.name, rhp);
2232 trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
2247 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
2319 rcu_state.cbovld = rcu_state.cbovldnext;
2320 rcu_state.cbovldnext = false;
2327 rcu_state.cbovldnext |= !!rnp->cbovldmask;
2384 ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||
2397 if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
2401 WRITE_ONCE(rcu_state.gp_flags,
2402 READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
2604 trace_rcu_kvfree_callback(rcu_state.name, head,
2608 trace_rcu_callback(rcu_state.name, head,
2650 if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&
2653 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
3030 rcu_state.name, bnode->nr_records,
3037 rcu_state.name, bnode->records[i], 0);
3068 trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset);
3623 rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap);
3624 rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap);
3632 rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT);
3634 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;
3668 return rcu_seq_snap(&rcu_state.gp_seq_polled);
3698 rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence);
3724 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq));
3809 rcu_seq_done_exact(&rcu_state.gp_seq_polled, oldstate)) {
3842 * ->gp_seq field be checked instead of that of the rcu_state structure.
3845 * field is updated and the time that the rcu_state structure's ->gp_seq
3858 rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) {
3975 trace_rcu_barrier(rcu_state.name, s, cpu,
3976 atomic_read(&rcu_state.barrier_cpu_count), done);
3983 * Note that the value of rcu_state.barrier_sequence must be captured
3991 unsigned long __maybe_unused s = rcu_state.barrier_sequence;
3993 if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
3995 complete(&rcu_state.barrier_completion);
4006 unsigned long gseq = READ_ONCE(rcu_state.barrier_sequence);
4011 lockdep_assert_held(&rcu_state.barrier_lock);
4014 rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
4027 atomic_inc(&rcu_state.barrier_cpu_count);
4030 rcu_barrier_trace(TPS("IRQNQ"), -1, rcu_state.barrier_sequence);
4049 raw_spin_lock(&rcu_state.barrier_lock);
4051 raw_spin_unlock(&rcu_state.barrier_lock);
4068 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
4073 mutex_lock(&rcu_state.barrier_mutex);
4076 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
4077 rcu_barrier_trace(TPS("EarlyExit"), -1, rcu_state.barrier_sequence);
4079 mutex_unlock(&rcu_state.barrier_mutex);
4084 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);
4085 rcu_seq_start(&rcu_state.barrier_sequence);
4086 gseq = rcu_state.barrier_sequence;
4087 rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
4096 init_completion(&rcu_state.barrier_completion);
4097 atomic_set(&rcu_state.barrier_cpu_count, 2);
4098 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);
4110 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);
4113 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);
4114 rcu_barrier_trace(TPS("NQ"), cpu, rcu_state.barrier_sequence);
4120 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);
4121 rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu, rcu_state.barrier_sequence);
4124 raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);
4130 rcu_barrier_trace(TPS("OnlineQ"), cpu, rcu_state.barrier_sequence);
4137 if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
4138 complete(&rcu_state.barrier_completion);
4141 wait_for_completion(&rcu_state.barrier_completion);
4144 rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
4145 rcu_seq_end(&rcu_state.barrier_sequence);
4146 gseq = rcu_state.barrier_sequence;
4154 mutex_unlock(&rcu_state.barrier_mutex);
4183 unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
4188 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
4293 if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock))
4306 return !!rcu_state.n_online_cpus;
4396 rdp->barrier_seq_snap = rcu_state.barrier_sequence;
4397 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
4399 rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
4486 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
4512 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
4516 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1);
4646 arch_spin_lock(&rcu_state.ofl_lock);
4648 raw_spin_lock(&rcu_state.barrier_lock);
4651 raw_spin_unlock(&rcu_state.barrier_lock);
4655 smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */
4656 ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);
4658 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
4659 rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
4673 arch_spin_unlock(&rcu_state.ofl_lock);
4707 arch_spin_lock(&rcu_state.ofl_lock);
4709 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
4710 rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
4719 arch_spin_unlock(&rcu_state.ofl_lock);
4741 raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);
4753 raw_spin_unlock(&rcu_state.barrier_lock); /* irqs remain disabled. */
4784 WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1);
4801 trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
4866 t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
4875 WRITE_ONCE(rcu_state.gp_activity, jiffies);
4876 WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
4878 smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
4918 rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;
4927 * Helper function for rcu_init() that initializes the rcu_state structure.
4951 rcu_state.level[i] =
4952 rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
4959 rnp = rcu_state.level[i];
4967 rnp->gp_seq = rcu_state.gp_seq;
4968 rnp->gp_seq_needed = rcu_state.gp_seq;
4969 rnp->completedqs = rcu_state.gp_seq;
4983 rnp->parent = rcu_state.level[i - 1] +
5001 init_swait_queue_head(&rcu_state.gp_wq);
5002 init_swait_queue_head(&rcu_state.expedited_wq);
5037 * the ->node array in the rcu_state structure.
5130 * with the rcu_state structure.