1/* 2 * Read-Copy Update module-based torture test facility 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright (C) IBM Corporation, 2005, 2006 19 * 20 * Authors: Paul E. McKenney <paulmck@us.ibm.com> 21 * Josh Triplett <josh@freedesktop.org> 22 * 23 * See also: Documentation/RCU/torture.txt 24 */ 25#include <linux/types.h> 26#include <linux/kernel.h> 27#include <linux/init.h> 28#include <linux/module.h> 29#include <linux/kthread.h> 30#include <linux/err.h> 31#include <linux/spinlock.h> 32#include <linux/smp.h> 33#include <linux/rcupdate.h> 34#include <linux/interrupt.h> 35#include <linux/sched.h> 36#include <asm/atomic.h> 37#include <linux/bitops.h> 38#include <linux/completion.h> 39#include <linux/moduleparam.h> 40#include <linux/percpu.h> 41#include <linux/notifier.h> 42#include <linux/reboot.h> 43#include <linux/freezer.h> 44#include <linux/cpu.h> 45#include <linux/delay.h> 46#include <linux/stat.h> 47#include <linux/srcu.h> 48#include <linux/slab.h> 49#include <asm/byteorder.h> 50 51MODULE_LICENSE("GPL"); 52MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and " 53 "Josh Triplett <josh@freedesktop.org>"); 54 55static int nreaders = -1; /* # reader threads, defaults to 2*ncpus */ 56static int nfakewriters = 4; /* # fake writer threads */ 57static int stat_interval; /* Interval between stats, in seconds. */ 58 /* Defaults to "only at end of test". */ 59static int verbose; /* Print more debug info. */ 60static int test_no_idle_hz; /* Test RCU's support for tickless idle CPUs. */ 61static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/ 62static int stutter = 5; /* Start/stop testing interval (in sec) */ 63static int irqreader = 1; /* RCU readers from irq (timers). */ 64static int fqs_duration = 0; /* Duration of bursts (us), 0 to disable. */ 65static int fqs_holdoff = 0; /* Hold time within burst (us). */ 66static int fqs_stutter = 3; /* Wait time between bursts (s). */ 67static char *torture_type = "rcu"; /* What RCU implementation to torture. */ 68 69module_param(nreaders, int, 0444); 70MODULE_PARM_DESC(nreaders, "Number of RCU reader threads"); 71module_param(nfakewriters, int, 0444); 72MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads"); 73module_param(stat_interval, int, 0444); 74MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s"); 75module_param(verbose, bool, 0444); 76MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s"); 77module_param(test_no_idle_hz, bool, 0444); 78MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs"); 79module_param(shuffle_interval, int, 0444); 80MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles"); 81module_param(stutter, int, 0444); 82MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test"); 83module_param(irqreader, int, 0444); 84MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers"); 85module_param(fqs_duration, int, 0444); 86MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)"); 87module_param(fqs_holdoff, int, 0444); 88MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)"); 89module_param(fqs_stutter, int, 0444); 90MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)"); 91module_param(torture_type, charp, 0444); 92MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)"); 93 94#define TORTURE_FLAG "-torture:" 95#define PRINTK_STRING(s) \ 96 do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0) 97#define VERBOSE_PRINTK_STRING(s) \ 98 do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0) 99#define VERBOSE_PRINTK_ERRSTRING(s) \ 100 do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0) 101 102static char printk_buf[4096]; 103 104static int nrealreaders; 105static struct task_struct *writer_task; 106static struct task_struct **fakewriter_tasks; 107static struct task_struct **reader_tasks; 108static struct task_struct *stats_task; 109static struct task_struct *shuffler_task; 110static struct task_struct *stutter_task; 111static struct task_struct *fqs_task; 112 113#define RCU_TORTURE_PIPE_LEN 10 114 115struct rcu_torture { 116 struct rcu_head rtort_rcu; 117 int rtort_pipe_count; 118 struct list_head rtort_free; 119 int rtort_mbtest; 120}; 121 122static LIST_HEAD(rcu_torture_freelist); 123static struct rcu_torture *rcu_torture_current; 124static long rcu_torture_current_version; 125static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 126static DEFINE_SPINLOCK(rcu_torture_lock); 127static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) = 128 { 0 }; 129static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) = 130 { 0 }; 131static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 132static atomic_t n_rcu_torture_alloc; 133static atomic_t n_rcu_torture_alloc_fail; 134static atomic_t n_rcu_torture_free; 135static atomic_t n_rcu_torture_mberror; 136static atomic_t n_rcu_torture_error; 137static long n_rcu_torture_timers; 138static struct list_head rcu_torture_removed; 139static cpumask_var_t shuffle_tmp_mask; 140 141static int stutter_pause_test; 142 143#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE) 144#define RCUTORTURE_RUNNABLE_INIT 1 145#else 146#define RCUTORTURE_RUNNABLE_INIT 0 147#endif 148int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT; 149 150/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */ 151 152#define FULLSTOP_DONTSTOP 0 /* Normal operation. */ 153#define FULLSTOP_SHUTDOWN 1 /* System shutdown with rcutorture running. */ 154#define FULLSTOP_RMMOD 2 /* Normal rmmod of rcutorture. */ 155static int fullstop = FULLSTOP_RMMOD; 156DEFINE_MUTEX(fullstop_mutex); /* Protect fullstop transitions and spawning */ 157 /* of kthreads. */ 158 159/* 160 * Detect and respond to a system shutdown. 161 */ 162static int 163rcutorture_shutdown_notify(struct notifier_block *unused1, 164 unsigned long unused2, void *unused3) 165{ 166 mutex_lock(&fullstop_mutex); 167 if (fullstop == FULLSTOP_DONTSTOP) 168 fullstop = FULLSTOP_SHUTDOWN; 169 else 170 printk(KERN_WARNING /* but going down anyway, so... */ 171 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n"); 172 mutex_unlock(&fullstop_mutex); 173 return NOTIFY_DONE; 174} 175 176/* 177 * Absorb kthreads into a kernel function that won't return, so that 178 * they won't ever access module text or data again. 179 */ 180static void rcutorture_shutdown_absorb(char *title) 181{ 182 if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) { 183 printk(KERN_NOTICE 184 "rcutorture thread %s parking due to system shutdown\n", 185 title); 186 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT); 187 } 188} 189 190/* 191 * Allocate an element from the rcu_tortures pool. 192 */ 193static struct rcu_torture * 194rcu_torture_alloc(void) 195{ 196 struct list_head *p; 197 198 spin_lock_bh(&rcu_torture_lock); 199 if (list_empty(&rcu_torture_freelist)) { 200 atomic_inc(&n_rcu_torture_alloc_fail); 201 spin_unlock_bh(&rcu_torture_lock); 202 return NULL; 203 } 204 atomic_inc(&n_rcu_torture_alloc); 205 p = rcu_torture_freelist.next; 206 list_del_init(p); 207 spin_unlock_bh(&rcu_torture_lock); 208 return container_of(p, struct rcu_torture, rtort_free); 209} 210 211/* 212 * Free an element to the rcu_tortures pool. 213 */ 214static void 215rcu_torture_free(struct rcu_torture *p) 216{ 217 atomic_inc(&n_rcu_torture_free); 218 spin_lock_bh(&rcu_torture_lock); 219 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 220 spin_unlock_bh(&rcu_torture_lock); 221} 222 223struct rcu_random_state { 224 unsigned long rrs_state; 225 long rrs_count; 226}; 227 228#define RCU_RANDOM_MULT 39916801 /* prime */ 229#define RCU_RANDOM_ADD 479001701 /* prime */ 230#define RCU_RANDOM_REFRESH 10000 231 232#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 } 233 234/* 235 * Crude but fast random-number generator. Uses a linear congruential 236 * generator, with occasional help from cpu_clock(). 237 */ 238static unsigned long 239rcu_random(struct rcu_random_state *rrsp) 240{ 241 if (--rrsp->rrs_count < 0) { 242 rrsp->rrs_state += (unsigned long)local_clock(); 243 rrsp->rrs_count = RCU_RANDOM_REFRESH; 244 } 245 rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD; 246 return swahw32(rrsp->rrs_state); 247} 248 249static void 250rcu_stutter_wait(char *title) 251{ 252 while (stutter_pause_test || !rcutorture_runnable) { 253 if (rcutorture_runnable) 254 schedule_timeout_interruptible(1); 255 else 256 schedule_timeout_interruptible(round_jiffies_relative(HZ)); 257 rcutorture_shutdown_absorb(title); 258 } 259} 260 261/* 262 * Operations vector for selecting different types of tests. 263 */ 264 265struct rcu_torture_ops { 266 void (*init)(void); 267 void (*cleanup)(void); 268 int (*readlock)(void); 269 void (*read_delay)(struct rcu_random_state *rrsp); 270 void (*readunlock)(int idx); 271 int (*completed)(void); 272 void (*deferred_free)(struct rcu_torture *p); 273 void (*sync)(void); 274 void (*cb_barrier)(void); 275 void (*fqs)(void); 276 int (*stats)(char *page); 277 int irq_capable; 278 char *name; 279}; 280 281static struct rcu_torture_ops *cur_ops; 282 283/* 284 * Definitions for rcu torture testing. 285 */ 286 287static int rcu_torture_read_lock(void) __acquires(RCU) 288{ 289 rcu_read_lock(); 290 return 0; 291} 292 293static void rcu_read_delay(struct rcu_random_state *rrsp) 294{ 295 const unsigned long shortdelay_us = 200; 296 const unsigned long longdelay_ms = 50; 297 298 /* We want a short delay sometimes to make a reader delay the grace 299 * period, and we want a long delay occasionally to trigger 300 * force_quiescent_state. */ 301 302 if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) 303 mdelay(longdelay_ms); 304 if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) 305 udelay(shortdelay_us); 306} 307 308static void rcu_torture_read_unlock(int idx) __releases(RCU) 309{ 310 rcu_read_unlock(); 311} 312 313static int rcu_torture_completed(void) 314{ 315 return rcu_batches_completed(); 316} 317 318static void 319rcu_torture_cb(struct rcu_head *p) 320{ 321 int i; 322 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 323 324 if (fullstop != FULLSTOP_DONTSTOP) { 325 /* Test is ending, just drop callbacks on the floor. */ 326 /* The next initialization will pick up the pieces. */ 327 return; 328 } 329 i = rp->rtort_pipe_count; 330 if (i > RCU_TORTURE_PIPE_LEN) 331 i = RCU_TORTURE_PIPE_LEN; 332 atomic_inc(&rcu_torture_wcount[i]); 333 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 334 rp->rtort_mbtest = 0; 335 rcu_torture_free(rp); 336 } else 337 cur_ops->deferred_free(rp); 338} 339 340static int rcu_no_completed(void) 341{ 342 return 0; 343} 344 345static void rcu_torture_deferred_free(struct rcu_torture *p) 346{ 347 call_rcu(&p->rtort_rcu, rcu_torture_cb); 348} 349 350static struct rcu_torture_ops rcu_ops = { 351 .init = NULL, 352 .cleanup = NULL, 353 .readlock = rcu_torture_read_lock, 354 .read_delay = rcu_read_delay, 355 .readunlock = rcu_torture_read_unlock, 356 .completed = rcu_torture_completed, 357 .deferred_free = rcu_torture_deferred_free, 358 .sync = synchronize_rcu, 359 .cb_barrier = rcu_barrier, 360 .fqs = rcu_force_quiescent_state, 361 .stats = NULL, 362 .irq_capable = 1, 363 .name = "rcu" 364}; 365 366static void rcu_sync_torture_deferred_free(struct rcu_torture *p) 367{ 368 int i; 369 struct rcu_torture *rp; 370 struct rcu_torture *rp1; 371 372 cur_ops->sync(); 373 list_add(&p->rtort_free, &rcu_torture_removed); 374 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 375 i = rp->rtort_pipe_count; 376 if (i > RCU_TORTURE_PIPE_LEN) 377 i = RCU_TORTURE_PIPE_LEN; 378 atomic_inc(&rcu_torture_wcount[i]); 379 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { 380 rp->rtort_mbtest = 0; 381 list_del(&rp->rtort_free); 382 rcu_torture_free(rp); 383 } 384 } 385} 386 387static void rcu_sync_torture_init(void) 388{ 389 INIT_LIST_HEAD(&rcu_torture_removed); 390} 391 392static struct rcu_torture_ops rcu_sync_ops = { 393 .init = rcu_sync_torture_init, 394 .cleanup = NULL, 395 .readlock = rcu_torture_read_lock, 396 .read_delay = rcu_read_delay, 397 .readunlock = rcu_torture_read_unlock, 398 .completed = rcu_torture_completed, 399 .deferred_free = rcu_sync_torture_deferred_free, 400 .sync = synchronize_rcu, 401 .cb_barrier = NULL, 402 .fqs = rcu_force_quiescent_state, 403 .stats = NULL, 404 .irq_capable = 1, 405 .name = "rcu_sync" 406}; 407 408static struct rcu_torture_ops rcu_expedited_ops = { 409 .init = rcu_sync_torture_init, 410 .cleanup = NULL, 411 .readlock = rcu_torture_read_lock, 412 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 413 .readunlock = rcu_torture_read_unlock, 414 .completed = rcu_no_completed, 415 .deferred_free = rcu_sync_torture_deferred_free, 416 .sync = synchronize_rcu_expedited, 417 .cb_barrier = NULL, 418 .fqs = rcu_force_quiescent_state, 419 .stats = NULL, 420 .irq_capable = 1, 421 .name = "rcu_expedited" 422}; 423 424/* 425 * Definitions for rcu_bh torture testing. 426 */ 427 428static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH) 429{ 430 rcu_read_lock_bh(); 431 return 0; 432} 433 434static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH) 435{ 436 rcu_read_unlock_bh(); 437} 438 439static int rcu_bh_torture_completed(void) 440{ 441 return rcu_batches_completed_bh(); 442} 443 444static void rcu_bh_torture_deferred_free(struct rcu_torture *p) 445{ 446 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb); 447} 448 449struct rcu_bh_torture_synchronize { 450 struct rcu_head head; 451 struct completion completion; 452}; 453 454static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head) 455{ 456 struct rcu_bh_torture_synchronize *rcu; 457 458 rcu = container_of(head, struct rcu_bh_torture_synchronize, head); 459 complete(&rcu->completion); 460} 461 462static void rcu_bh_torture_synchronize(void) 463{ 464 struct rcu_bh_torture_synchronize rcu; 465 466 init_rcu_head_on_stack(&rcu.head); 467 init_completion(&rcu.completion); 468 call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb); 469 wait_for_completion(&rcu.completion); 470 destroy_rcu_head_on_stack(&rcu.head); 471} 472 473static struct rcu_torture_ops rcu_bh_ops = { 474 .init = NULL, 475 .cleanup = NULL, 476 .readlock = rcu_bh_torture_read_lock, 477 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 478 .readunlock = rcu_bh_torture_read_unlock, 479 .completed = rcu_bh_torture_completed, 480 .deferred_free = rcu_bh_torture_deferred_free, 481 .sync = rcu_bh_torture_synchronize, 482 .cb_barrier = rcu_barrier_bh, 483 .fqs = rcu_bh_force_quiescent_state, 484 .stats = NULL, 485 .irq_capable = 1, 486 .name = "rcu_bh" 487}; 488 489static struct rcu_torture_ops rcu_bh_sync_ops = { 490 .init = rcu_sync_torture_init, 491 .cleanup = NULL, 492 .readlock = rcu_bh_torture_read_lock, 493 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 494 .readunlock = rcu_bh_torture_read_unlock, 495 .completed = rcu_bh_torture_completed, 496 .deferred_free = rcu_sync_torture_deferred_free, 497 .sync = rcu_bh_torture_synchronize, 498 .cb_barrier = NULL, 499 .fqs = rcu_bh_force_quiescent_state, 500 .stats = NULL, 501 .irq_capable = 1, 502 .name = "rcu_bh_sync" 503}; 504 505/* 506 * Definitions for srcu torture testing. 507 */ 508 509static struct srcu_struct srcu_ctl; 510 511static void srcu_torture_init(void) 512{ 513 init_srcu_struct(&srcu_ctl); 514 rcu_sync_torture_init(); 515} 516 517static void srcu_torture_cleanup(void) 518{ 519 synchronize_srcu(&srcu_ctl); 520 cleanup_srcu_struct(&srcu_ctl); 521} 522 523static int srcu_torture_read_lock(void) __acquires(&srcu_ctl) 524{ 525 return srcu_read_lock(&srcu_ctl); 526} 527 528static void srcu_read_delay(struct rcu_random_state *rrsp) 529{ 530 long delay; 531 const long uspertick = 1000000 / HZ; 532 const long longdelay = 10; 533 534 /* We want there to be long-running readers, but not all the time. */ 535 536 delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); 537 if (!delay) 538 schedule_timeout_interruptible(longdelay); 539} 540 541static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl) 542{ 543 srcu_read_unlock(&srcu_ctl, idx); 544} 545 546static int srcu_torture_completed(void) 547{ 548 return srcu_batches_completed(&srcu_ctl); 549} 550 551static void srcu_torture_synchronize(void) 552{ 553 synchronize_srcu(&srcu_ctl); 554} 555 556static int srcu_torture_stats(char *page) 557{ 558 int cnt = 0; 559 int cpu; 560 int idx = srcu_ctl.completed & 0x1; 561 562 cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):", 563 torture_type, TORTURE_FLAG, idx); 564 for_each_possible_cpu(cpu) { 565 cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu, 566 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx], 567 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]); 568 } 569 cnt += sprintf(&page[cnt], "\n"); 570 return cnt; 571} 572 573static struct rcu_torture_ops srcu_ops = { 574 .init = srcu_torture_init, 575 .cleanup = srcu_torture_cleanup, 576 .readlock = srcu_torture_read_lock, 577 .read_delay = srcu_read_delay, 578 .readunlock = srcu_torture_read_unlock, 579 .completed = srcu_torture_completed, 580 .deferred_free = rcu_sync_torture_deferred_free, 581 .sync = srcu_torture_synchronize, 582 .cb_barrier = NULL, 583 .stats = srcu_torture_stats, 584 .name = "srcu" 585}; 586 587static void srcu_torture_synchronize_expedited(void) 588{ 589 synchronize_srcu_expedited(&srcu_ctl); 590} 591 592static struct rcu_torture_ops srcu_expedited_ops = { 593 .init = srcu_torture_init, 594 .cleanup = srcu_torture_cleanup, 595 .readlock = srcu_torture_read_lock, 596 .read_delay = srcu_read_delay, 597 .readunlock = srcu_torture_read_unlock, 598 .completed = srcu_torture_completed, 599 .deferred_free = rcu_sync_torture_deferred_free, 600 .sync = srcu_torture_synchronize_expedited, 601 .cb_barrier = NULL, 602 .stats = srcu_torture_stats, 603 .name = "srcu_expedited" 604}; 605 606/* 607 * Definitions for sched torture testing. 608 */ 609 610static int sched_torture_read_lock(void) 611{ 612 preempt_disable(); 613 return 0; 614} 615 616static void sched_torture_read_unlock(int idx) 617{ 618 preempt_enable(); 619} 620 621static void rcu_sched_torture_deferred_free(struct rcu_torture *p) 622{ 623 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); 624} 625 626static void sched_torture_synchronize(void) 627{ 628 synchronize_sched(); 629} 630 631static struct rcu_torture_ops sched_ops = { 632 .init = rcu_sync_torture_init, 633 .cleanup = NULL, 634 .readlock = sched_torture_read_lock, 635 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 636 .readunlock = sched_torture_read_unlock, 637 .completed = rcu_no_completed, 638 .deferred_free = rcu_sched_torture_deferred_free, 639 .sync = sched_torture_synchronize, 640 .cb_barrier = rcu_barrier_sched, 641 .fqs = rcu_sched_force_quiescent_state, 642 .stats = NULL, 643 .irq_capable = 1, 644 .name = "sched" 645}; 646 647static struct rcu_torture_ops sched_sync_ops = { 648 .init = rcu_sync_torture_init, 649 .cleanup = NULL, 650 .readlock = sched_torture_read_lock, 651 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 652 .readunlock = sched_torture_read_unlock, 653 .completed = rcu_no_completed, 654 .deferred_free = rcu_sync_torture_deferred_free, 655 .sync = sched_torture_synchronize, 656 .cb_barrier = NULL, 657 .fqs = rcu_sched_force_quiescent_state, 658 .stats = NULL, 659 .name = "sched_sync" 660}; 661 662static struct rcu_torture_ops sched_expedited_ops = { 663 .init = rcu_sync_torture_init, 664 .cleanup = NULL, 665 .readlock = sched_torture_read_lock, 666 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 667 .readunlock = sched_torture_read_unlock, 668 .completed = rcu_no_completed, 669 .deferred_free = rcu_sync_torture_deferred_free, 670 .sync = synchronize_sched_expedited, 671 .cb_barrier = NULL, 672 .fqs = rcu_sched_force_quiescent_state, 673 .stats = NULL, 674 .irq_capable = 1, 675 .name = "sched_expedited" 676}; 677 678/* 679 * RCU torture force-quiescent-state kthread. Repeatedly induces 680 * bursts of calls to force_quiescent_state(), increasing the probability 681 * of occurrence of some important types of race conditions. 682 */ 683static int 684rcu_torture_fqs(void *arg) 685{ 686 unsigned long fqs_resume_time; 687 int fqs_burst_remaining; 688 689 VERBOSE_PRINTK_STRING("rcu_torture_fqs task started"); 690 do { 691 fqs_resume_time = jiffies + fqs_stutter * HZ; 692 while (jiffies - fqs_resume_time > LONG_MAX) { 693 schedule_timeout_interruptible(1); 694 } 695 fqs_burst_remaining = fqs_duration; 696 while (fqs_burst_remaining > 0) { 697 cur_ops->fqs(); 698 udelay(fqs_holdoff); 699 fqs_burst_remaining -= fqs_holdoff; 700 } 701 rcu_stutter_wait("rcu_torture_fqs"); 702 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 703 VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping"); 704 rcutorture_shutdown_absorb("rcu_torture_fqs"); 705 while (!kthread_should_stop()) 706 schedule_timeout_uninterruptible(1); 707 return 0; 708} 709 710/* 711 * RCU torture writer kthread. Repeatedly substitutes a new structure 712 * for that pointed to by rcu_torture_current, freeing the old structure 713 * after a series of grace periods (the "pipeline"). 714 */ 715static int 716rcu_torture_writer(void *arg) 717{ 718 int i; 719 long oldbatch = rcu_batches_completed(); 720 struct rcu_torture *rp; 721 struct rcu_torture *old_rp; 722 static DEFINE_RCU_RANDOM(rand); 723 724 VERBOSE_PRINTK_STRING("rcu_torture_writer task started"); 725 set_user_nice(current, 19); 726 727 do { 728 schedule_timeout_uninterruptible(1); 729 rp = rcu_torture_alloc(); 730 if (rp == NULL) 731 continue; 732 rp->rtort_pipe_count = 0; 733 udelay(rcu_random(&rand) & 0x3ff); 734 old_rp = rcu_torture_current; 735 rp->rtort_mbtest = 1; 736 rcu_assign_pointer(rcu_torture_current, rp); 737 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 738 if (old_rp) { 739 i = old_rp->rtort_pipe_count; 740 if (i > RCU_TORTURE_PIPE_LEN) 741 i = RCU_TORTURE_PIPE_LEN; 742 atomic_inc(&rcu_torture_wcount[i]); 743 old_rp->rtort_pipe_count++; 744 cur_ops->deferred_free(old_rp); 745 } 746 rcu_torture_current_version++; 747 oldbatch = cur_ops->completed(); 748 rcu_stutter_wait("rcu_torture_writer"); 749 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 750 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping"); 751 rcutorture_shutdown_absorb("rcu_torture_writer"); 752 while (!kthread_should_stop()) 753 schedule_timeout_uninterruptible(1); 754 return 0; 755} 756 757/* 758 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 759 * delay between calls. 760 */ 761static int 762rcu_torture_fakewriter(void *arg) 763{ 764 DEFINE_RCU_RANDOM(rand); 765 766 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started"); 767 set_user_nice(current, 19); 768 769 do { 770 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10); 771 udelay(rcu_random(&rand) & 0x3ff); 772 cur_ops->sync(); 773 rcu_stutter_wait("rcu_torture_fakewriter"); 774 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 775 776 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping"); 777 rcutorture_shutdown_absorb("rcu_torture_fakewriter"); 778 while (!kthread_should_stop()) 779 schedule_timeout_uninterruptible(1); 780 return 0; 781} 782 783/* 784 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 785 * incrementing the corresponding element of the pipeline array. The 786 * counter in the element should never be greater than 1, otherwise, the 787 * RCU implementation is broken. 788 */ 789static void rcu_torture_timer(unsigned long unused) 790{ 791 int idx; 792 int completed; 793 static DEFINE_RCU_RANDOM(rand); 794 static DEFINE_SPINLOCK(rand_lock); 795 struct rcu_torture *p; 796 int pipe_count; 797 798 idx = cur_ops->readlock(); 799 completed = cur_ops->completed(); 800 p = rcu_dereference_check(rcu_torture_current, 801 rcu_read_lock_held() || 802 rcu_read_lock_bh_held() || 803 rcu_read_lock_sched_held() || 804 srcu_read_lock_held(&srcu_ctl)); 805 if (p == NULL) { 806 /* Leave because rcu_torture_writer is not yet underway */ 807 cur_ops->readunlock(idx); 808 return; 809 } 810 if (p->rtort_mbtest == 0) 811 atomic_inc(&n_rcu_torture_mberror); 812 spin_lock(&rand_lock); 813 cur_ops->read_delay(&rand); 814 n_rcu_torture_timers++; 815 spin_unlock(&rand_lock); 816 preempt_disable(); 817 pipe_count = p->rtort_pipe_count; 818 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 819 /* Should not happen, but... */ 820 pipe_count = RCU_TORTURE_PIPE_LEN; 821 } 822 __this_cpu_inc(rcu_torture_count[pipe_count]); 823 completed = cur_ops->completed() - completed; 824 if (completed > RCU_TORTURE_PIPE_LEN) { 825 /* Should not happen, but... */ 826 completed = RCU_TORTURE_PIPE_LEN; 827 } 828 __this_cpu_inc(rcu_torture_batch[completed]); 829 preempt_enable(); 830 cur_ops->readunlock(idx); 831} 832 833/* 834 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 835 * incrementing the corresponding element of the pipeline array. The 836 * counter in the element should never be greater than 1, otherwise, the 837 * RCU implementation is broken. 838 */ 839static int 840rcu_torture_reader(void *arg) 841{ 842 int completed; 843 int idx; 844 DEFINE_RCU_RANDOM(rand); 845 struct rcu_torture *p; 846 int pipe_count; 847 struct timer_list t; 848 849 VERBOSE_PRINTK_STRING("rcu_torture_reader task started"); 850 set_user_nice(current, 19); 851 if (irqreader && cur_ops->irq_capable) 852 setup_timer_on_stack(&t, rcu_torture_timer, 0); 853 854 do { 855 if (irqreader && cur_ops->irq_capable) { 856 if (!timer_pending(&t)) 857 mod_timer(&t, jiffies + 1); 858 } 859 idx = cur_ops->readlock(); 860 completed = cur_ops->completed(); 861 p = rcu_dereference_check(rcu_torture_current, 862 rcu_read_lock_held() || 863 rcu_read_lock_bh_held() || 864 rcu_read_lock_sched_held() || 865 srcu_read_lock_held(&srcu_ctl)); 866 if (p == NULL) { 867 /* Wait for rcu_torture_writer to get underway */ 868 cur_ops->readunlock(idx); 869 schedule_timeout_interruptible(HZ); 870 continue; 871 } 872 if (p->rtort_mbtest == 0) 873 atomic_inc(&n_rcu_torture_mberror); 874 cur_ops->read_delay(&rand); 875 preempt_disable(); 876 pipe_count = p->rtort_pipe_count; 877 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 878 /* Should not happen, but... */ 879 pipe_count = RCU_TORTURE_PIPE_LEN; 880 } 881 __this_cpu_inc(rcu_torture_count[pipe_count]); 882 completed = cur_ops->completed() - completed; 883 if (completed > RCU_TORTURE_PIPE_LEN) { 884 /* Should not happen, but... */ 885 completed = RCU_TORTURE_PIPE_LEN; 886 } 887 __this_cpu_inc(rcu_torture_batch[completed]); 888 preempt_enable(); 889 cur_ops->readunlock(idx); 890 schedule(); 891 rcu_stutter_wait("rcu_torture_reader"); 892 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP); 893 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping"); 894 rcutorture_shutdown_absorb("rcu_torture_reader"); 895 if (irqreader && cur_ops->irq_capable) 896 del_timer_sync(&t); 897 while (!kthread_should_stop()) 898 schedule_timeout_uninterruptible(1); 899 return 0; 900} 901 902/* 903 * Create an RCU-torture statistics message in the specified buffer. 904 */ 905static int 906rcu_torture_printk(char *page) 907{ 908 int cnt = 0; 909 int cpu; 910 int i; 911 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 912 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 913 914 for_each_possible_cpu(cpu) { 915 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 916 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i]; 917 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i]; 918 } 919 } 920 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) { 921 if (pipesummary[i] != 0) 922 break; 923 } 924 cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG); 925 cnt += sprintf(&page[cnt], 926 "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d " 927 "rtmbe: %d nt: %ld", 928 rcu_torture_current, 929 rcu_torture_current_version, 930 list_empty(&rcu_torture_freelist), 931 atomic_read(&n_rcu_torture_alloc), 932 atomic_read(&n_rcu_torture_alloc_fail), 933 atomic_read(&n_rcu_torture_free), 934 atomic_read(&n_rcu_torture_mberror), 935 n_rcu_torture_timers); 936 if (atomic_read(&n_rcu_torture_mberror) != 0) 937 cnt += sprintf(&page[cnt], " !!!"); 938 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG); 939 if (i > 1) { 940 cnt += sprintf(&page[cnt], "!!! "); 941 atomic_inc(&n_rcu_torture_error); 942 WARN_ON_ONCE(1); 943 } 944 cnt += sprintf(&page[cnt], "Reader Pipe: "); 945 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 946 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]); 947 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG); 948 cnt += sprintf(&page[cnt], "Reader Batch: "); 949 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 950 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]); 951 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG); 952 cnt += sprintf(&page[cnt], "Free-Block Circulation: "); 953 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 954 cnt += sprintf(&page[cnt], " %d", 955 atomic_read(&rcu_torture_wcount[i])); 956 } 957 cnt += sprintf(&page[cnt], "\n"); 958 if (cur_ops->stats) 959 cnt += cur_ops->stats(&page[cnt]); 960 return cnt; 961} 962 963/* 964 * Print torture statistics. Caller must ensure that there is only 965 * one call to this function at a given time!!! This is normally 966 * accomplished by relying on the module system to only have one copy 967 * of the module loaded, and then by giving the rcu_torture_stats 968 * kthread full control (or the init/cleanup functions when rcu_torture_stats 969 * thread is not running). 970 */ 971static void 972rcu_torture_stats_print(void) 973{ 974 int cnt; 975 976 cnt = rcu_torture_printk(printk_buf); 977 printk(KERN_ALERT "%s", printk_buf); 978} 979 980/* 981 * Periodically prints torture statistics, if periodic statistics printing 982 * was specified via the stat_interval module parameter. 983 * 984 * No need to worry about fullstop here, since this one doesn't reference 985 * volatile state or register callbacks. 986 */ 987static int 988rcu_torture_stats(void *arg) 989{ 990 VERBOSE_PRINTK_STRING("rcu_torture_stats task started"); 991 do { 992 schedule_timeout_interruptible(stat_interval * HZ); 993 rcu_torture_stats_print(); 994 rcutorture_shutdown_absorb("rcu_torture_stats"); 995 } while (!kthread_should_stop()); 996 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping"); 997 return 0; 998} 999 1000static int rcu_idle_cpu; /* Force all torture tasks off this CPU */ 1001 1002/* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case 1003 * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs. 1004 */ 1005static void rcu_torture_shuffle_tasks(void) 1006{ 1007 int i; 1008 1009 cpumask_setall(shuffle_tmp_mask); 1010 get_online_cpus(); 1011 1012 /* No point in shuffling if there is only one online CPU (ex: UP) */ 1013 if (num_online_cpus() == 1) { 1014 put_online_cpus(); 1015 return; 1016 } 1017 1018 if (rcu_idle_cpu != -1) 1019 cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask); 1020 1021 set_cpus_allowed_ptr(current, shuffle_tmp_mask); 1022 1023 if (reader_tasks) { 1024 for (i = 0; i < nrealreaders; i++) 1025 if (reader_tasks[i]) 1026 set_cpus_allowed_ptr(reader_tasks[i], 1027 shuffle_tmp_mask); 1028 } 1029 1030 if (fakewriter_tasks) { 1031 for (i = 0; i < nfakewriters; i++) 1032 if (fakewriter_tasks[i]) 1033 set_cpus_allowed_ptr(fakewriter_tasks[i], 1034 shuffle_tmp_mask); 1035 } 1036 1037 if (writer_task) 1038 set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask); 1039 1040 if (stats_task) 1041 set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask); 1042 1043 if (rcu_idle_cpu == -1) 1044 rcu_idle_cpu = num_online_cpus() - 1; 1045 else 1046 rcu_idle_cpu--; 1047 1048 put_online_cpus(); 1049} 1050 1051/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the 1052 * system to become idle at a time and cut off its timer ticks. This is meant 1053 * to test the support for such tickless idle CPU in RCU. 1054 */ 1055static int 1056rcu_torture_shuffle(void *arg) 1057{ 1058 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started"); 1059 do { 1060 schedule_timeout_interruptible(shuffle_interval * HZ); 1061 rcu_torture_shuffle_tasks(); 1062 rcutorture_shutdown_absorb("rcu_torture_shuffle"); 1063 } while (!kthread_should_stop()); 1064 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping"); 1065 return 0; 1066} 1067 1068/* Cause the rcutorture test to "stutter", starting and stopping all 1069 * threads periodically. 1070 */ 1071static int 1072rcu_torture_stutter(void *arg) 1073{ 1074 VERBOSE_PRINTK_STRING("rcu_torture_stutter task started"); 1075 do { 1076 schedule_timeout_interruptible(stutter * HZ); 1077 stutter_pause_test = 1; 1078 if (!kthread_should_stop()) 1079 schedule_timeout_interruptible(stutter * HZ); 1080 stutter_pause_test = 0; 1081 rcutorture_shutdown_absorb("rcu_torture_stutter"); 1082 } while (!kthread_should_stop()); 1083 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping"); 1084 return 0; 1085} 1086 1087static inline void 1088rcu_torture_print_module_parms(char *tag) 1089{ 1090 printk(KERN_ALERT "%s" TORTURE_FLAG 1091 "--- %s: nreaders=%d nfakewriters=%d " 1092 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 1093 "shuffle_interval=%d stutter=%d irqreader=%d " 1094 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d\n", 1095 torture_type, tag, nrealreaders, nfakewriters, 1096 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 1097 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter); 1098} 1099 1100static struct notifier_block rcutorture_nb = { 1101 .notifier_call = rcutorture_shutdown_notify, 1102}; 1103 1104static void 1105rcu_torture_cleanup(void) 1106{ 1107 int i; 1108 1109 mutex_lock(&fullstop_mutex); 1110 if (fullstop == FULLSTOP_SHUTDOWN) { 1111 printk(KERN_WARNING /* but going down anyway, so... */ 1112 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n"); 1113 mutex_unlock(&fullstop_mutex); 1114 schedule_timeout_uninterruptible(10); 1115 if (cur_ops->cb_barrier != NULL) 1116 cur_ops->cb_barrier(); 1117 return; 1118 } 1119 fullstop = FULLSTOP_RMMOD; 1120 mutex_unlock(&fullstop_mutex); 1121 unregister_reboot_notifier(&rcutorture_nb); 1122 if (stutter_task) { 1123 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task"); 1124 kthread_stop(stutter_task); 1125 } 1126 stutter_task = NULL; 1127 if (shuffler_task) { 1128 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task"); 1129 kthread_stop(shuffler_task); 1130 free_cpumask_var(shuffle_tmp_mask); 1131 } 1132 shuffler_task = NULL; 1133 1134 if (writer_task) { 1135 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task"); 1136 kthread_stop(writer_task); 1137 } 1138 writer_task = NULL; 1139 1140 if (reader_tasks) { 1141 for (i = 0; i < nrealreaders; i++) { 1142 if (reader_tasks[i]) { 1143 VERBOSE_PRINTK_STRING( 1144 "Stopping rcu_torture_reader task"); 1145 kthread_stop(reader_tasks[i]); 1146 } 1147 reader_tasks[i] = NULL; 1148 } 1149 kfree(reader_tasks); 1150 reader_tasks = NULL; 1151 } 1152 rcu_torture_current = NULL; 1153 1154 if (fakewriter_tasks) { 1155 for (i = 0; i < nfakewriters; i++) { 1156 if (fakewriter_tasks[i]) { 1157 VERBOSE_PRINTK_STRING( 1158 "Stopping rcu_torture_fakewriter task"); 1159 kthread_stop(fakewriter_tasks[i]); 1160 } 1161 fakewriter_tasks[i] = NULL; 1162 } 1163 kfree(fakewriter_tasks); 1164 fakewriter_tasks = NULL; 1165 } 1166 1167 if (stats_task) { 1168 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task"); 1169 kthread_stop(stats_task); 1170 } 1171 stats_task = NULL; 1172 1173 if (fqs_task) { 1174 VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task"); 1175 kthread_stop(fqs_task); 1176 } 1177 fqs_task = NULL; 1178 1179 /* Wait for all RCU callbacks to fire. */ 1180 1181 if (cur_ops->cb_barrier != NULL) 1182 cur_ops->cb_barrier(); 1183 1184 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 1185 1186 if (cur_ops->cleanup) 1187 cur_ops->cleanup(); 1188 if (atomic_read(&n_rcu_torture_error)) 1189 rcu_torture_print_module_parms("End of test: FAILURE"); 1190 else 1191 rcu_torture_print_module_parms("End of test: SUCCESS"); 1192} 1193 1194static int __init 1195rcu_torture_init(void) 1196{ 1197 int i; 1198 int cpu; 1199 int firsterr = 0; 1200 static struct rcu_torture_ops *torture_ops[] = 1201 { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops, 1202 &rcu_bh_ops, &rcu_bh_sync_ops, 1203 &srcu_ops, &srcu_expedited_ops, 1204 &sched_ops, &sched_sync_ops, &sched_expedited_ops, }; 1205 1206 mutex_lock(&fullstop_mutex); 1207 1208 /* Process args and tell the world that the torturer is on the job. */ 1209 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1210 cur_ops = torture_ops[i]; 1211 if (strcmp(torture_type, cur_ops->name) == 0) 1212 break; 1213 } 1214 if (i == ARRAY_SIZE(torture_ops)) { 1215 printk(KERN_ALERT "rcu-torture: invalid torture type: \"%s\"\n", 1216 torture_type); 1217 printk(KERN_ALERT "rcu-torture types:"); 1218 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 1219 printk(KERN_ALERT " %s", torture_ops[i]->name); 1220 printk(KERN_ALERT "\n"); 1221 mutex_unlock(&fullstop_mutex); 1222 return -EINVAL; 1223 } 1224 if (cur_ops->fqs == NULL && fqs_duration != 0) { 1225 printk(KERN_ALERT "rcu-torture: ->fqs NULL and non-zero " 1226 "fqs_duration, fqs disabled.\n"); 1227 fqs_duration = 0; 1228 } 1229 if (cur_ops->init) 1230 cur_ops->init(); /* no "goto unwind" prior to this point!!! */ 1231 1232 if (nreaders >= 0) 1233 nrealreaders = nreaders; 1234 else 1235 nrealreaders = 2 * num_online_cpus(); 1236 rcu_torture_print_module_parms("Start of test"); 1237 fullstop = FULLSTOP_DONTSTOP; 1238 1239 /* Set up the freelist. */ 1240 1241 INIT_LIST_HEAD(&rcu_torture_freelist); 1242 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 1243 rcu_tortures[i].rtort_mbtest = 0; 1244 list_add_tail(&rcu_tortures[i].rtort_free, 1245 &rcu_torture_freelist); 1246 } 1247 1248 /* Initialize the statistics so that each run gets its own numbers. */ 1249 1250 rcu_torture_current = NULL; 1251 rcu_torture_current_version = 0; 1252 atomic_set(&n_rcu_torture_alloc, 0); 1253 atomic_set(&n_rcu_torture_alloc_fail, 0); 1254 atomic_set(&n_rcu_torture_free, 0); 1255 atomic_set(&n_rcu_torture_mberror, 0); 1256 atomic_set(&n_rcu_torture_error, 0); 1257 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 1258 atomic_set(&rcu_torture_wcount[i], 0); 1259 for_each_possible_cpu(cpu) { 1260 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 1261 per_cpu(rcu_torture_count, cpu)[i] = 0; 1262 per_cpu(rcu_torture_batch, cpu)[i] = 0; 1263 } 1264 } 1265 1266 /* Start up the kthreads. */ 1267 1268 VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task"); 1269 writer_task = kthread_run(rcu_torture_writer, NULL, 1270 "rcu_torture_writer"); 1271 if (IS_ERR(writer_task)) { 1272 firsterr = PTR_ERR(writer_task); 1273 VERBOSE_PRINTK_ERRSTRING("Failed to create writer"); 1274 writer_task = NULL; 1275 goto unwind; 1276 } 1277 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]), 1278 GFP_KERNEL); 1279 if (fakewriter_tasks == NULL) { 1280 VERBOSE_PRINTK_ERRSTRING("out of memory"); 1281 firsterr = -ENOMEM; 1282 goto unwind; 1283 } 1284 for (i = 0; i < nfakewriters; i++) { 1285 VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task"); 1286 fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL, 1287 "rcu_torture_fakewriter"); 1288 if (IS_ERR(fakewriter_tasks[i])) { 1289 firsterr = PTR_ERR(fakewriter_tasks[i]); 1290 VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter"); 1291 fakewriter_tasks[i] = NULL; 1292 goto unwind; 1293 } 1294 } 1295 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]), 1296 GFP_KERNEL); 1297 if (reader_tasks == NULL) { 1298 VERBOSE_PRINTK_ERRSTRING("out of memory"); 1299 firsterr = -ENOMEM; 1300 goto unwind; 1301 } 1302 for (i = 0; i < nrealreaders; i++) { 1303 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task"); 1304 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL, 1305 "rcu_torture_reader"); 1306 if (IS_ERR(reader_tasks[i])) { 1307 firsterr = PTR_ERR(reader_tasks[i]); 1308 VERBOSE_PRINTK_ERRSTRING("Failed to create reader"); 1309 reader_tasks[i] = NULL; 1310 goto unwind; 1311 } 1312 } 1313 if (stat_interval > 0) { 1314 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task"); 1315 stats_task = kthread_run(rcu_torture_stats, NULL, 1316 "rcu_torture_stats"); 1317 if (IS_ERR(stats_task)) { 1318 firsterr = PTR_ERR(stats_task); 1319 VERBOSE_PRINTK_ERRSTRING("Failed to create stats"); 1320 stats_task = NULL; 1321 goto unwind; 1322 } 1323 } 1324 if (test_no_idle_hz) { 1325 rcu_idle_cpu = num_online_cpus() - 1; 1326 1327 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) { 1328 firsterr = -ENOMEM; 1329 VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask"); 1330 goto unwind; 1331 } 1332 1333 /* Create the shuffler thread */ 1334 shuffler_task = kthread_run(rcu_torture_shuffle, NULL, 1335 "rcu_torture_shuffle"); 1336 if (IS_ERR(shuffler_task)) { 1337 free_cpumask_var(shuffle_tmp_mask); 1338 firsterr = PTR_ERR(shuffler_task); 1339 VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler"); 1340 shuffler_task = NULL; 1341 goto unwind; 1342 } 1343 } 1344 if (stutter < 0) 1345 stutter = 0; 1346 if (stutter) { 1347 /* Create the stutter thread */ 1348 stutter_task = kthread_run(rcu_torture_stutter, NULL, 1349 "rcu_torture_stutter"); 1350 if (IS_ERR(stutter_task)) { 1351 firsterr = PTR_ERR(stutter_task); 1352 VERBOSE_PRINTK_ERRSTRING("Failed to create stutter"); 1353 stutter_task = NULL; 1354 goto unwind; 1355 } 1356 } 1357 if (fqs_duration < 0) 1358 fqs_duration = 0; 1359 if (fqs_duration) { 1360 /* Create the stutter thread */ 1361 fqs_task = kthread_run(rcu_torture_fqs, NULL, 1362 "rcu_torture_fqs"); 1363 if (IS_ERR(fqs_task)) { 1364 firsterr = PTR_ERR(fqs_task); 1365 VERBOSE_PRINTK_ERRSTRING("Failed to create fqs"); 1366 fqs_task = NULL; 1367 goto unwind; 1368 } 1369 } 1370 register_reboot_notifier(&rcutorture_nb); 1371 mutex_unlock(&fullstop_mutex); 1372 return 0; 1373 1374unwind: 1375 mutex_unlock(&fullstop_mutex); 1376 rcu_torture_cleanup(); 1377 return firsterr; 1378} 1379 1380module_init(rcu_torture_init); 1381module_exit(rcu_torture_cleanup); 1382