Lines Matching refs:rdp
80 static void force_quiescent_state(struct rcu_data *rdp,
90 * rdp->cpu is the current cpu.
93 cpu_clear(rdp->cpu, cpumask);
99 static inline void force_quiescent_state(struct rcu_data *rdp,
121 struct rcu_data *rdp;
126 rdp = &__get_cpu_var(rcu_data);
127 *rdp->nxttail = head;
128 rdp->nxttail = &head->next;
129 if (unlikely(++rdp->qlen > qhimark)) {
130 rdp->blimit = INT_MAX;
131 force_quiescent_state(rdp, &rcu_ctrlblk);
156 struct rcu_data *rdp;
161 rdp = &__get_cpu_var(rcu_bh_data);
162 *rdp->nxttail = head;
163 rdp->nxttail = &head->next;
165 if (unlikely(++rdp->qlen > qhimark)) {
166 rdp->blimit = INT_MAX;
167 force_quiescent_state(rdp, &rcu_bh_ctrlblk);
203 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
206 head = &rdp->barrier;
231 static void rcu_do_batch(struct rcu_data *rdp)
236 list = rdp->donelist;
242 if (++count >= rdp->blimit)
245 rdp->donelist = list;
248 rdp->qlen -= count;
250 if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark)
251 rdp->blimit = blimit;
253 if (!rdp->donelist)
254 rdp->donetail = &rdp->donelist;
256 tasklet_schedule(&per_cpu(rcu_tasklet, rdp->cpu));
265 * rdp->quiescbatch. All cpus are recorded in the
328 struct rcu_data *rdp)
330 if (rdp->quiescbatch != rcp->cur) {
332 rdp->qs_pending = 1;
333 rdp->passed_quiesc = 0;
334 rdp->quiescbatch = rcp->cur;
342 if (!rdp->qs_pending)
349 if (!rdp->passed_quiesc)
351 rdp->qs_pending = 0;
355 * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync
358 if (likely(rdp->quiescbatch == rcp->cur))
359 cpu_quiet(rdp->cpu, rcp);
382 struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
390 cpu_quiet(rdp->cpu, rcp);
392 rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail);
393 rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail);
394 rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail);
423 struct rcu_data *rdp)
425 if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) {
426 *rdp->donetail = rdp->curlist;
427 rdp->donetail = rdp->curtail;
428 rdp->curlist = NULL;
429 rdp->curtail = &rdp->curlist;
432 if (rdp->nxtlist && !rdp->curlist) {
434 rdp->curlist = rdp->nxtlist;
435 rdp->curtail = rdp->nxttail;
436 rdp->nxtlist = NULL;
437 rdp->nxttail = &rdp->nxtlist;
445 rdp->batch = rcp->cur + 1;
460 rcu_check_quiescent_state(rcp, rdp);
461 if (rdp->donelist)
462 rcu_do_batch(rdp);
471 static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp)
476 if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch))
480 if (!rdp->curlist && rdp->nxtlist)
484 if (rdp->donelist)
488 if (rdp->quiescbatch != rcp->cur || rdp->qs_pending)
514 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
517 return (!!rdp->curlist || !!rdp_bh->curlist || rcu_pending(cpu));
533 struct rcu_data *rdp)
535 memset(rdp, 0, sizeof(*rdp));
536 rdp->curtail = &rdp->curlist;
537 rdp->nxttail = &rdp->nxtlist;
538 rdp->donetail = &rdp->donelist;
539 rdp->quiescbatch = rcp->completed;
540 rdp->qs_pending = 0;
541 rdp->cpu = cpu;
542 rdp->blimit = blimit;
547 struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
550 rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp);