Lines Matching defs:cur_ops
396 static struct rcu_torture_ops *cur_ops;
428 started = cur_ops->get_gp_seq();
434 completed = cur_ops->get_gp_seq();
435 do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
513 cur_ops->deferred_free(rp);
640 if (cur_ops == &srcud_ops)
667 if (cur_ops == &srcud_ops)
1000 if (!cur_ops->gp_diff)
1002 return cur_ops->gp_diff(new, old);
1052 if (cur_ops->poll_gp_state(gp_state))
1054 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) {
1064 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) {
1067 cur_ops->gp_kthread_dbg();
1069 gp_done = cur_ops->poll_gp_state(gp_state);
1076 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) {
1117 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1120 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) {
1121 gp_state = cur_ops->start_gp_poll();
1130 if (cur_ops->poll_gp_state(gp_state))
1138 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1197 cur_ops->fqs();
1246 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
1249 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {
1252 if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) {
1255 } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {
1258 if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) {
1261 } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {
1264 if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {
1268 (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {
1271 if (gp_exp1 && cur_ops->exp_sync) {
1274 } else if (gp_exp && !cur_ops->exp_sync) {
1277 if (gp_normal1 && cur_ops->deferred_free) {
1280 } else if (gp_normal && !cur_ops->deferred_free) {
1283 if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state &&
1284 cur_ops->start_gp_poll && cur_ops->poll_gp_state) {
1287 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
1290 if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full
1291 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
1294 } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {
1297 if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {
1300 } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {
1303 if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {
1307 (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {
1310 if (gp_sync1 && cur_ops->sync) {
1313 } else if (gp_sync && !cur_ops->sync) {
1331 dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300);
1332 dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00);
1336 cookie = cur_ops->get_gp_state();
1338 cur_ops->get_gp_state_full(&cookie_full);
1339 if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full))
1342 WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),
1345 WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full),
1387 torture_type, cur_ops->name);
1426 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {
1427 idx = cur_ops->readlock();
1428 cookie = cur_ops->get_gp_state();
1429 WARN_ONCE(cur_ops->poll_gp_state(cookie),
1434 cookie, cur_ops->get_gp_state());
1435 if (cur_ops->get_gp_completed) {
1436 cookie = cur_ops->get_gp_completed();
1437 WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));
1439 cur_ops->readunlock(idx);
1441 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) {
1442 idx = cur_ops->readlock();
1443 cur_ops->get_gp_state_full(&cookie_full);
1444 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
1450 if (cur_ops->get_gp_completed_full) {
1451 cur_ops->get_gp_completed_full(&cookie_full);
1452 WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));
1454 cur_ops->readunlock(idx);
1459 cur_ops->deferred_free(old_rp);
1463 do_rtws_sync(&rand, cur_ops->exp_sync);
1468 gp_snap = cur_ops->get_gp_state();
1471 cur_ops->cond_sync(gp_snap);
1476 gp_snap = cur_ops->get_gp_state_exp();
1479 cur_ops->cond_sync_exp(gp_snap);
1484 cur_ops->get_gp_state_full(&gp_snap_full);
1487 cur_ops->cond_sync_full(&gp_snap_full);
1492 cur_ops->get_gp_state_full(&gp_snap_full);
1495 cur_ops->cond_sync_exp_full(&gp_snap_full);
1501 ulo[i] = cur_ops->get_comp_state();
1502 gp_snap = cur_ops->start_gp_poll();
1504 while (!cur_ops->poll_gp_state(gp_snap)) {
1505 gp_snap1 = cur_ops->get_gp_state();
1507 if (cur_ops->poll_gp_state(ulo[i]) ||
1508 cur_ops->same_gp_state(ulo[i], gp_snap1)) {
1521 cur_ops->get_comp_state_full(&rgo[i]);
1522 cur_ops->start_gp_poll_full(&gp_snap_full);
1524 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1525 cur_ops->get_gp_state_full(&gp_snap1_full);
1527 if (cur_ops->poll_gp_state_full(&rgo[i]) ||
1528 cur_ops->same_gp_state_full(&rgo[i],
1541 gp_snap = cur_ops->start_gp_poll_exp();
1543 while (!cur_ops->poll_gp_state_exp(gp_snap))
1550 cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1552 while (!cur_ops->poll_gp_state_full(&gp_snap_full))
1559 do_rtws_sync(&rand, cur_ops->sync);
1588 !cur_ops->slow_gps &&
1596 if (cur_ops->gp_kthread_dbg)
1597 cur_ops->gp_kthread_dbg();
1647 if (cur_ops->cb_barrier != NULL &&
1649 cur_ops->cb_barrier();
1655 cur_ops->exp_sync();
1658 gp_snap = cur_ops->get_gp_state();
1660 cur_ops->cond_sync(gp_snap);
1663 gp_snap = cur_ops->get_gp_state_exp();
1665 cur_ops->cond_sync_exp(gp_snap);
1668 cur_ops->get_gp_state_full(&gp_snap_full);
1670 cur_ops->cond_sync_full(&gp_snap_full);
1673 cur_ops->get_gp_state_full(&gp_snap_full);
1675 cur_ops->cond_sync_exp_full(&gp_snap_full);
1678 gp_snap = cur_ops->start_gp_poll();
1679 while (!cur_ops->poll_gp_state(gp_snap)) {
1685 cur_ops->start_gp_poll_full(&gp_snap_full);
1686 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1692 gp_snap = cur_ops->start_gp_poll_exp();
1693 while (!cur_ops->poll_gp_state_exp(gp_snap)) {
1699 cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1700 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1706 cur_ops->sync();
1818 idxnew1 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_1;
1820 idxnew2 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_2;
1840 cur_ops->readunlock((idxold2 >> RCUTORTURE_RDR_SHIFT_2) & 0x1);
1847 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff);
1850 cur_ops->readunlock((idxold1 >> RCUTORTURE_RDR_SHIFT_1) & 0x1);
1859 cur_ops->read_delay(trsp, rtrsp);
1883 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;
1987 if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
1988 cookie = cur_ops->get_gp_state();
1989 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
1990 cur_ops->get_gp_state_full(&cookie_full);
1992 started = cur_ops->get_gp_seq();
1995 !cur_ops->readlock_held || cur_ops->readlock_held());
2012 completed = cur_ops->get_gp_seq();
2014 do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
2027 if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
2028 WARN_ONCE(cur_ops->poll_gp_state(cookie),
2033 cookie, cur_ops->get_gp_state());
2034 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
2035 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
2073 if (cur_ops->call) {
2077 cur_ops->call(rhp, rcu_torture_timer_cb);
2098 if (irqreader && cur_ops->irq_capable)
2102 if (irqreader && cur_ops->irq_capable) {
2116 if (irqreader && cur_ops->irq_capable) {
2263 if (cur_ops->stats)
2264 cur_ops->stats();
2271 if (cur_ops->get_gp_data)
2272 cur_ops->get_gp_data(&flags, &gp_seq);
2283 if (cur_ops->gp_kthread_dbg)
2284 cur_ops->gp_kthread_dbg();
2358 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
2377 test_boost, cur_ops->can_boost,
2489 idx = cur_ops->readlock();
2511 cur_ops->readunlock(idx);
2552 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb);
2635 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();
2701 if (!cur_ops->sync)
2703 if (cur_ops->call && cur_ops->cb_barrier) {
2710 cur_ops->sync(); /* Later readers see above write. */
2713 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
2716 gps = cur_ops->get_gp_seq();
2717 sd = cur_ops->stall_dur() + 1;
2725 idx = cur_ops->readlock();
2727 cur_ops->readunlock(idx);
2737 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
2744 cur_ops->sync(); /* Wait for running CB to complete. */
2745 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
2746 cur_ops->cb_barrier(); /* Wait for queued callbacks. */
2777 if (!cur_ops->call)
2782 cur_ops->sync(); /* Later readers see above write. */
2793 gps = cur_ops->get_gp_seq();
2810 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) {
2824 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
2835 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
2836 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
2837 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
2842 if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg)
2843 cur_ops->gp_kthread_dbg();
2890 cur_ops->cb_barrier();
2895 cur_ops->cb_barrier();
2944 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) &&
2978 if ((!cur_ops->sync && !cur_ops->call) ||
2979 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) ||
2980 cur_ops == &rcu_busted_ops) {
3055 cur_ops->call(rhp, rcu_torture_barrier_cbf);
3083 cur_ops->call(&rcu, rcu_torture_barrier_cbf);
3088 if (cur_ops->cb_barrier != NULL)
3089 cur_ops->cb_barrier();
3114 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
3127 cur_ops->cb_barrier();
3152 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
3155 torture_type, cur_ops->name);
3205 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2)
3207 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)
3346 if (cur_ops->cb_barrier != NULL) {
3347 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
3348 cur_ops->cb_barrier();
3350 if (cur_ops->gp_slow_unregister)
3351 cur_ops->gp_slow_unregister(NULL);
3354 if (!cur_ops) {
3361 if (cur_ops->gp_kthread_dbg)
3362 cur_ops->gp_kthread_dbg();
3394 if (cur_ops->get_gp_data)
3395 cur_ops->get_gp_data(&flags, &gp_seq);
3397 cur_ops->name, (long)gp_seq, flags,
3408 if (cur_ops->cb_barrier != NULL) {
3409 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
3410 cur_ops->cb_barrier();
3412 if (cur_ops->cleanup != NULL)
3413 cur_ops->cleanup();
3447 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
3449 rcu_torture_print_module_parms(cur_ops,
3452 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
3454 if (cur_ops->gp_slow_unregister)
3455 cur_ops->gp_slow_unregister(NULL);
3524 if (cur_ops->sync && !(++n & 0xfff))
3525 cur_ops->sync();
3729 cur_ops = torture_ops[i];
3730 if (strcmp(torture_type, cur_ops->name) == 0)
3741 cur_ops = NULL;
3744 if (cur_ops->fqs == NULL && fqs_duration != 0) {
3748 if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops ||
3751 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU));
3754 if (cur_ops->init)
3755 cur_ops->init();
3766 rcu_torture_print_module_parms(cur_ops, "Start of test");
3767 if (cur_ops->get_gp_data)
3768 cur_ops->get_gp_data(&flags, &gp_seq);
3771 cur_ops->name, (long)gp_seq, flags);
3883 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;
3937 if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister))
3938 cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay);