1// SPDX-License-Identifier: GPL-2.0+ 2// 3// Scalability test comparing RCU vs other mechanisms 4// for acquiring references on objects. 5// 6// Copyright (C) Google, 2020. 7// 8// Author: Joel Fernandes <joel@joelfernandes.org> 9 10#define pr_fmt(fmt) fmt 11 12#include <linux/atomic.h> 13#include <linux/bitops.h> 14#include <linux/completion.h> 15#include <linux/cpu.h> 16#include <linux/delay.h> 17#include <linux/err.h> 18#include <linux/init.h> 19#include <linux/interrupt.h> 20#include <linux/kthread.h> 21#include <linux/kernel.h> 22#include <linux/mm.h> 23#include <linux/module.h> 24#include <linux/moduleparam.h> 25#include <linux/notifier.h> 26#include <linux/percpu.h> 27#include <linux/rcupdate.h> 28#include <linux/rcupdate_trace.h> 29#include <linux/reboot.h> 30#include <linux/sched.h> 31#include <linux/spinlock.h> 32#include <linux/smp.h> 33#include <linux/stat.h> 34#include <linux/srcu.h> 35#include <linux/slab.h> 36#include <linux/torture.h> 37#include <linux/types.h> 38 39#include "rcu.h" 40 41#define SCALE_FLAG "-ref-scale: " 42 43#define SCALEOUT(s, x...) \ 44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x) 45 46#define VERBOSE_SCALEOUT(s, x...) \ 47 do { \ 48 if (verbose) \ 49 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ 50 } while (0) 51 52static atomic_t verbose_batch_ctr; 53 54#define VERBOSE_SCALEOUT_BATCH(s, x...) \ 55do { \ 56 if (verbose && \ 57 (verbose_batched <= 0 || \ 58 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \ 59 schedule_timeout_uninterruptible(1); \ 60 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ 61 } \ 62} while (0) 63 64#define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x) 65 66MODULE_LICENSE("GPL"); 67MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); 68 69static char *scale_type = "rcu"; 70module_param(scale_type, charp, 0444); 71MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); 72 73torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); 74torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); 75 76// Wait until there are multiple CPUs before starting test. 77torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, 78 "Holdoff time before test start (s)"); 79// Number of typesafe_lookup structures, that is, the degree of concurrency. 80torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures."); 81// Number of loops per experiment, all readers execute operations concurrently. 82torture_param(long, loops, 10000, "Number of loops per experiment."); 83// Number of readers, with -1 defaulting to about 75% of the CPUs. 84torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs."); 85// Number of runs. 86torture_param(int, nruns, 30, "Number of experiments to run."); 87// Reader delay in nanoseconds, 0 for no delay. 88torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); 89 90#ifdef MODULE 91# define REFSCALE_SHUTDOWN 0 92#else 93# define REFSCALE_SHUTDOWN 1 94#endif 95 96torture_param(bool, shutdown, REFSCALE_SHUTDOWN, 97 "Shutdown at end of scalability tests."); 98 99struct reader_task { 100 struct task_struct *task; 101 int start_reader; 102 wait_queue_head_t wq; 103 u64 last_duration_ns; 104}; 105 106static struct task_struct *shutdown_task; 107static wait_queue_head_t shutdown_wq; 108 109static struct task_struct *main_task; 110static wait_queue_head_t main_wq; 111static int shutdown_start; 112 113static struct reader_task *reader_tasks; 114 115// Number of readers that are part of the current experiment. 116static atomic_t nreaders_exp; 117 118// Use to wait for all threads to start. 119static atomic_t n_init; 120static atomic_t n_started; 121static atomic_t n_warmedup; 122static atomic_t n_cooleddown; 123 124// Track which experiment is currently running. 125static int exp_idx; 126 127// Operations vector for selecting different types of tests. 128struct ref_scale_ops { 129 bool (*init)(void); 130 void (*cleanup)(void); 131 void (*readsection)(const int nloops); 132 void (*delaysection)(const int nloops, const int udl, const int ndl); 133 const char *name; 134}; 135 136static struct ref_scale_ops *cur_ops; 137 138static void un_delay(const int udl, const int ndl) 139{ 140 if (udl) 141 udelay(udl); 142 if (ndl) 143 ndelay(ndl); 144} 145 146static void ref_rcu_read_section(const int nloops) 147{ 148 int i; 149 150 for (i = nloops; i >= 0; i--) { 151 rcu_read_lock(); 152 rcu_read_unlock(); 153 } 154} 155 156static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl) 157{ 158 int i; 159 160 for (i = nloops; i >= 0; i--) { 161 rcu_read_lock(); 162 un_delay(udl, ndl); 163 rcu_read_unlock(); 164 } 165} 166 167static bool rcu_sync_scale_init(void) 168{ 169 return true; 170} 171 172static struct ref_scale_ops rcu_ops = { 173 .init = rcu_sync_scale_init, 174 .readsection = ref_rcu_read_section, 175 .delaysection = ref_rcu_delay_section, 176 .name = "rcu" 177}; 178 179// Definitions for SRCU ref scale testing. 180DEFINE_STATIC_SRCU(srcu_refctl_scale); 181static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; 182 183static void srcu_ref_scale_read_section(const int nloops) 184{ 185 int i; 186 int idx; 187 188 for (i = nloops; i >= 0; i--) { 189 idx = srcu_read_lock(srcu_ctlp); 190 srcu_read_unlock(srcu_ctlp, idx); 191 } 192} 193 194static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 195{ 196 int i; 197 int idx; 198 199 for (i = nloops; i >= 0; i--) { 200 idx = srcu_read_lock(srcu_ctlp); 201 un_delay(udl, ndl); 202 srcu_read_unlock(srcu_ctlp, idx); 203 } 204} 205 206static struct ref_scale_ops srcu_ops = { 207 .init = rcu_sync_scale_init, 208 .readsection = srcu_ref_scale_read_section, 209 .delaysection = srcu_ref_scale_delay_section, 210 .name = "srcu" 211}; 212 213#ifdef CONFIG_TASKS_RCU 214 215// Definitions for RCU Tasks ref scale testing: Empty read markers. 216// These definitions also work for RCU Rude readers. 217static void rcu_tasks_ref_scale_read_section(const int nloops) 218{ 219 int i; 220 221 for (i = nloops; i >= 0; i--) 222 continue; 223} 224 225static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 226{ 227 int i; 228 229 for (i = nloops; i >= 0; i--) 230 un_delay(udl, ndl); 231} 232 233static struct ref_scale_ops rcu_tasks_ops = { 234 .init = rcu_sync_scale_init, 235 .readsection = rcu_tasks_ref_scale_read_section, 236 .delaysection = rcu_tasks_ref_scale_delay_section, 237 .name = "rcu-tasks" 238}; 239 240#define RCU_TASKS_OPS &rcu_tasks_ops, 241 242#else // #ifdef CONFIG_TASKS_RCU 243 244#define RCU_TASKS_OPS 245 246#endif // #else // #ifdef CONFIG_TASKS_RCU 247 248#ifdef CONFIG_TASKS_TRACE_RCU 249 250// Definitions for RCU Tasks Trace ref scale testing. 251static void rcu_trace_ref_scale_read_section(const int nloops) 252{ 253 int i; 254 255 for (i = nloops; i >= 0; i--) { 256 rcu_read_lock_trace(); 257 rcu_read_unlock_trace(); 258 } 259} 260 261static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 262{ 263 int i; 264 265 for (i = nloops; i >= 0; i--) { 266 rcu_read_lock_trace(); 267 un_delay(udl, ndl); 268 rcu_read_unlock_trace(); 269 } 270} 271 272static struct ref_scale_ops rcu_trace_ops = { 273 .init = rcu_sync_scale_init, 274 .readsection = rcu_trace_ref_scale_read_section, 275 .delaysection = rcu_trace_ref_scale_delay_section, 276 .name = "rcu-trace" 277}; 278 279#define RCU_TRACE_OPS &rcu_trace_ops, 280 281#else // #ifdef CONFIG_TASKS_TRACE_RCU 282 283#define RCU_TRACE_OPS 284 285#endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU 286 287// Definitions for reference count 288static atomic_t refcnt; 289 290static void ref_refcnt_section(const int nloops) 291{ 292 int i; 293 294 for (i = nloops; i >= 0; i--) { 295 atomic_inc(&refcnt); 296 atomic_dec(&refcnt); 297 } 298} 299 300static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl) 301{ 302 int i; 303 304 for (i = nloops; i >= 0; i--) { 305 atomic_inc(&refcnt); 306 un_delay(udl, ndl); 307 atomic_dec(&refcnt); 308 } 309} 310 311static struct ref_scale_ops refcnt_ops = { 312 .init = rcu_sync_scale_init, 313 .readsection = ref_refcnt_section, 314 .delaysection = ref_refcnt_delay_section, 315 .name = "refcnt" 316}; 317 318// Definitions for rwlock 319static rwlock_t test_rwlock; 320 321static bool ref_rwlock_init(void) 322{ 323 rwlock_init(&test_rwlock); 324 return true; 325} 326 327static void ref_rwlock_section(const int nloops) 328{ 329 int i; 330 331 for (i = nloops; i >= 0; i--) { 332 read_lock(&test_rwlock); 333 read_unlock(&test_rwlock); 334 } 335} 336 337static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl) 338{ 339 int i; 340 341 for (i = nloops; i >= 0; i--) { 342 read_lock(&test_rwlock); 343 un_delay(udl, ndl); 344 read_unlock(&test_rwlock); 345 } 346} 347 348static struct ref_scale_ops rwlock_ops = { 349 .init = ref_rwlock_init, 350 .readsection = ref_rwlock_section, 351 .delaysection = ref_rwlock_delay_section, 352 .name = "rwlock" 353}; 354 355// Definitions for rwsem 356static struct rw_semaphore test_rwsem; 357 358static bool ref_rwsem_init(void) 359{ 360 init_rwsem(&test_rwsem); 361 return true; 362} 363 364static void ref_rwsem_section(const int nloops) 365{ 366 int i; 367 368 for (i = nloops; i >= 0; i--) { 369 down_read(&test_rwsem); 370 up_read(&test_rwsem); 371 } 372} 373 374static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl) 375{ 376 int i; 377 378 for (i = nloops; i >= 0; i--) { 379 down_read(&test_rwsem); 380 un_delay(udl, ndl); 381 up_read(&test_rwsem); 382 } 383} 384 385static struct ref_scale_ops rwsem_ops = { 386 .init = ref_rwsem_init, 387 .readsection = ref_rwsem_section, 388 .delaysection = ref_rwsem_delay_section, 389 .name = "rwsem" 390}; 391 392// Definitions for global spinlock 393static DEFINE_RAW_SPINLOCK(test_lock); 394 395static void ref_lock_section(const int nloops) 396{ 397 int i; 398 399 preempt_disable(); 400 for (i = nloops; i >= 0; i--) { 401 raw_spin_lock(&test_lock); 402 raw_spin_unlock(&test_lock); 403 } 404 preempt_enable(); 405} 406 407static void ref_lock_delay_section(const int nloops, const int udl, const int ndl) 408{ 409 int i; 410 411 preempt_disable(); 412 for (i = nloops; i >= 0; i--) { 413 raw_spin_lock(&test_lock); 414 un_delay(udl, ndl); 415 raw_spin_unlock(&test_lock); 416 } 417 preempt_enable(); 418} 419 420static struct ref_scale_ops lock_ops = { 421 .readsection = ref_lock_section, 422 .delaysection = ref_lock_delay_section, 423 .name = "lock" 424}; 425 426// Definitions for global irq-save spinlock 427 428static void ref_lock_irq_section(const int nloops) 429{ 430 unsigned long flags; 431 int i; 432 433 preempt_disable(); 434 for (i = nloops; i >= 0; i--) { 435 raw_spin_lock_irqsave(&test_lock, flags); 436 raw_spin_unlock_irqrestore(&test_lock, flags); 437 } 438 preempt_enable(); 439} 440 441static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl) 442{ 443 unsigned long flags; 444 int i; 445 446 preempt_disable(); 447 for (i = nloops; i >= 0; i--) { 448 raw_spin_lock_irqsave(&test_lock, flags); 449 un_delay(udl, ndl); 450 raw_spin_unlock_irqrestore(&test_lock, flags); 451 } 452 preempt_enable(); 453} 454 455static struct ref_scale_ops lock_irq_ops = { 456 .readsection = ref_lock_irq_section, 457 .delaysection = ref_lock_irq_delay_section, 458 .name = "lock-irq" 459}; 460 461// Definitions acquire-release. 462static DEFINE_PER_CPU(unsigned long, test_acqrel); 463 464static void ref_acqrel_section(const int nloops) 465{ 466 unsigned long x; 467 int i; 468 469 preempt_disable(); 470 for (i = nloops; i >= 0; i--) { 471 x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); 472 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); 473 } 474 preempt_enable(); 475} 476 477static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl) 478{ 479 unsigned long x; 480 int i; 481 482 preempt_disable(); 483 for (i = nloops; i >= 0; i--) { 484 x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); 485 un_delay(udl, ndl); 486 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); 487 } 488 preempt_enable(); 489} 490 491static struct ref_scale_ops acqrel_ops = { 492 .readsection = ref_acqrel_section, 493 .delaysection = ref_acqrel_delay_section, 494 .name = "acqrel" 495}; 496 497static volatile u64 stopopts; 498 499static void ref_clock_section(const int nloops) 500{ 501 u64 x = 0; 502 int i; 503 504 preempt_disable(); 505 for (i = nloops; i >= 0; i--) 506 x += ktime_get_real_fast_ns(); 507 preempt_enable(); 508 stopopts = x; 509} 510 511static void ref_clock_delay_section(const int nloops, const int udl, const int ndl) 512{ 513 u64 x = 0; 514 int i; 515 516 preempt_disable(); 517 for (i = nloops; i >= 0; i--) { 518 x += ktime_get_real_fast_ns(); 519 un_delay(udl, ndl); 520 } 521 preempt_enable(); 522 stopopts = x; 523} 524 525static struct ref_scale_ops clock_ops = { 526 .readsection = ref_clock_section, 527 .delaysection = ref_clock_delay_section, 528 .name = "clock" 529}; 530 531static void ref_jiffies_section(const int nloops) 532{ 533 u64 x = 0; 534 int i; 535 536 preempt_disable(); 537 for (i = nloops; i >= 0; i--) 538 x += jiffies; 539 preempt_enable(); 540 stopopts = x; 541} 542 543static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl) 544{ 545 u64 x = 0; 546 int i; 547 548 preempt_disable(); 549 for (i = nloops; i >= 0; i--) { 550 x += jiffies; 551 un_delay(udl, ndl); 552 } 553 preempt_enable(); 554 stopopts = x; 555} 556 557static struct ref_scale_ops jiffies_ops = { 558 .readsection = ref_jiffies_section, 559 .delaysection = ref_jiffies_delay_section, 560 .name = "jiffies" 561}; 562 563//////////////////////////////////////////////////////////////////////// 564// 565// Methods leveraging SLAB_TYPESAFE_BY_RCU. 566// 567 568// Item to look up in a typesafe manner. Array of pointers to these. 569struct refscale_typesafe { 570 atomic_t rts_refctr; // Used by all flavors 571 spinlock_t rts_lock; 572 seqlock_t rts_seqlock; 573 unsigned int a; 574 unsigned int b; 575}; 576 577static struct kmem_cache *typesafe_kmem_cachep; 578static struct refscale_typesafe **rtsarray; 579static long rtsarray_size; 580static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand); 581static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start); 582static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start); 583 584// Conditionally acquire an explicit in-structure reference count. 585static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start) 586{ 587 return atomic_inc_not_zero(&rtsp->rts_refctr); 588} 589 590// Unconditionally release an explicit in-structure reference count. 591static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start) 592{ 593 if (!atomic_dec_return(&rtsp->rts_refctr)) { 594 WRITE_ONCE(rtsp->a, rtsp->a + 1); 595 kmem_cache_free(typesafe_kmem_cachep, rtsp); 596 } 597 return true; 598} 599 600// Unconditionally acquire an explicit in-structure spinlock. 601static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start) 602{ 603 spin_lock(&rtsp->rts_lock); 604 return true; 605} 606 607// Unconditionally release an explicit in-structure spinlock. 608static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start) 609{ 610 spin_unlock(&rtsp->rts_lock); 611 return true; 612} 613 614// Unconditionally acquire an explicit in-structure sequence lock. 615static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start) 616{ 617 *start = read_seqbegin(&rtsp->rts_seqlock); 618 return true; 619} 620 621// Conditionally release an explicit in-structure sequence lock. Return 622// true if this release was successful, that is, if no retry is required. 623static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start) 624{ 625 return !read_seqretry(&rtsp->rts_seqlock, start); 626} 627 628// Do a read-side critical section with the specified delay in 629// microseconds and nanoseconds inserted so as to increase probability 630// of failure. 631static void typesafe_delay_section(const int nloops, const int udl, const int ndl) 632{ 633 unsigned int a; 634 unsigned int b; 635 int i; 636 long idx; 637 struct refscale_typesafe *rtsp; 638 unsigned int start; 639 640 for (i = nloops; i >= 0; i--) { 641 preempt_disable(); 642 idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size; 643 preempt_enable(); 644retry: 645 rcu_read_lock(); 646 rtsp = rcu_dereference(rtsarray[idx]); 647 a = READ_ONCE(rtsp->a); 648 if (!rts_acquire(rtsp, &start)) { 649 rcu_read_unlock(); 650 goto retry; 651 } 652 if (a != READ_ONCE(rtsp->a)) { 653 (void)rts_release(rtsp, start); 654 rcu_read_unlock(); 655 goto retry; 656 } 657 un_delay(udl, ndl); 658 b = READ_ONCE(rtsp->a); 659 // Remember, seqlock read-side release can fail. 660 if (!rts_release(rtsp, start)) { 661 rcu_read_unlock(); 662 goto retry; 663 } 664 WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b); 665 b = rtsp->b; 666 rcu_read_unlock(); 667 WARN_ON_ONCE(a * a != b); 668 } 669} 670 671// Because the acquisition and release methods are expensive, there 672// is no point in optimizing away the un_delay() function's two checks. 673// Thus simply define typesafe_read_section() as a simple wrapper around 674// typesafe_delay_section(). 675static void typesafe_read_section(const int nloops) 676{ 677 typesafe_delay_section(nloops, 0, 0); 678} 679 680// Allocate and initialize one refscale_typesafe structure. 681static struct refscale_typesafe *typesafe_alloc_one(void) 682{ 683 struct refscale_typesafe *rtsp; 684 685 rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL); 686 if (!rtsp) 687 return NULL; 688 atomic_set(&rtsp->rts_refctr, 1); 689 WRITE_ONCE(rtsp->a, rtsp->a + 1); 690 WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a); 691 return rtsp; 692} 693 694// Slab-allocator constructor for refscale_typesafe structures created 695// out of a new slab of system memory. 696static void refscale_typesafe_ctor(void *rtsp_in) 697{ 698 struct refscale_typesafe *rtsp = rtsp_in; 699 700 spin_lock_init(&rtsp->rts_lock); 701 seqlock_init(&rtsp->rts_seqlock); 702 preempt_disable(); 703 rtsp->a = torture_random(this_cpu_ptr(&refscale_rand)); 704 preempt_enable(); 705} 706 707static struct ref_scale_ops typesafe_ref_ops; 708static struct ref_scale_ops typesafe_lock_ops; 709static struct ref_scale_ops typesafe_seqlock_ops; 710 711// Initialize for a typesafe test. 712static bool typesafe_init(void) 713{ 714 long idx; 715 long si = lookup_instances; 716 717 typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe", 718 sizeof(struct refscale_typesafe), sizeof(void *), 719 SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor); 720 if (!typesafe_kmem_cachep) 721 return false; 722 if (si < 0) 723 si = -si * nr_cpu_ids; 724 else if (si == 0) 725 si = nr_cpu_ids; 726 rtsarray_size = si; 727 rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL); 728 if (!rtsarray) 729 return false; 730 for (idx = 0; idx < rtsarray_size; idx++) { 731 rtsarray[idx] = typesafe_alloc_one(); 732 if (!rtsarray[idx]) 733 return false; 734 } 735 if (cur_ops == &typesafe_ref_ops) { 736 rts_acquire = typesafe_ref_acquire; 737 rts_release = typesafe_ref_release; 738 } else if (cur_ops == &typesafe_lock_ops) { 739 rts_acquire = typesafe_lock_acquire; 740 rts_release = typesafe_lock_release; 741 } else if (cur_ops == &typesafe_seqlock_ops) { 742 rts_acquire = typesafe_seqlock_acquire; 743 rts_release = typesafe_seqlock_release; 744 } else { 745 WARN_ON_ONCE(1); 746 return false; 747 } 748 return true; 749} 750 751// Clean up after a typesafe test. 752static void typesafe_cleanup(void) 753{ 754 long idx; 755 756 if (rtsarray) { 757 for (idx = 0; idx < rtsarray_size; idx++) 758 kmem_cache_free(typesafe_kmem_cachep, rtsarray[idx]); 759 kfree(rtsarray); 760 rtsarray = NULL; 761 rtsarray_size = 0; 762 } 763 kmem_cache_destroy(typesafe_kmem_cachep); 764 typesafe_kmem_cachep = NULL; 765 rts_acquire = NULL; 766 rts_release = NULL; 767} 768 769// The typesafe_init() function distinguishes these structures by address. 770static struct ref_scale_ops typesafe_ref_ops = { 771 .init = typesafe_init, 772 .cleanup = typesafe_cleanup, 773 .readsection = typesafe_read_section, 774 .delaysection = typesafe_delay_section, 775 .name = "typesafe_ref" 776}; 777 778static struct ref_scale_ops typesafe_lock_ops = { 779 .init = typesafe_init, 780 .cleanup = typesafe_cleanup, 781 .readsection = typesafe_read_section, 782 .delaysection = typesafe_delay_section, 783 .name = "typesafe_lock" 784}; 785 786static struct ref_scale_ops typesafe_seqlock_ops = { 787 .init = typesafe_init, 788 .cleanup = typesafe_cleanup, 789 .readsection = typesafe_read_section, 790 .delaysection = typesafe_delay_section, 791 .name = "typesafe_seqlock" 792}; 793 794static void rcu_scale_one_reader(void) 795{ 796 if (readdelay <= 0) 797 cur_ops->readsection(loops); 798 else 799 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000); 800} 801 802// Reader kthread. Repeatedly does empty RCU read-side 803// critical section, minimizing update-side interference. 804static int 805ref_scale_reader(void *arg) 806{ 807 unsigned long flags; 808 long me = (long)arg; 809 struct reader_task *rt = &(reader_tasks[me]); 810 u64 start; 811 s64 duration; 812 813 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me); 814 WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids))); 815 set_user_nice(current, MAX_NICE); 816 atomic_inc(&n_init); 817 if (holdoff) 818 schedule_timeout_interruptible(holdoff * HZ); 819repeat: 820 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id()); 821 822 // Wait for signal that this reader can start. 823 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) || 824 torture_must_stop()); 825 826 if (torture_must_stop()) 827 goto end; 828 829 // Make sure that the CPU is affinitized appropriately during testing. 830 WARN_ON_ONCE(raw_smp_processor_id() != me); 831 832 WRITE_ONCE(rt->start_reader, 0); 833 if (!atomic_dec_return(&n_started)) 834 while (atomic_read_acquire(&n_started)) 835 cpu_relax(); 836 837 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx); 838 839 840 // To reduce noise, do an initial cache-warming invocation, check 841 // in, and then keep warming until everyone has checked in. 842 rcu_scale_one_reader(); 843 if (!atomic_dec_return(&n_warmedup)) 844 while (atomic_read_acquire(&n_warmedup)) 845 rcu_scale_one_reader(); 846 // Also keep interrupts disabled. This also has the effect 847 // of preventing entries into slow path for rcu_read_unlock(). 848 local_irq_save(flags); 849 start = ktime_get_mono_fast_ns(); 850 851 rcu_scale_one_reader(); 852 853 duration = ktime_get_mono_fast_ns() - start; 854 local_irq_restore(flags); 855 856 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; 857 // To reduce runtime-skew noise, do maintain-load invocations until 858 // everyone is done. 859 if (!atomic_dec_return(&n_cooleddown)) 860 while (atomic_read_acquire(&n_cooleddown)) 861 rcu_scale_one_reader(); 862 863 if (atomic_dec_and_test(&nreaders_exp)) 864 wake_up(&main_wq); 865 866 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)", 867 me, exp_idx, atomic_read(&nreaders_exp)); 868 869 if (!torture_must_stop()) 870 goto repeat; 871end: 872 torture_kthread_stopping("ref_scale_reader"); 873 return 0; 874} 875 876static void reset_readers(void) 877{ 878 int i; 879 struct reader_task *rt; 880 881 for (i = 0; i < nreaders; i++) { 882 rt = &(reader_tasks[i]); 883 884 rt->last_duration_ns = 0; 885 } 886} 887 888// Print the results of each reader and return the sum of all their durations. 889static u64 process_durations(int n) 890{ 891 int i; 892 struct reader_task *rt; 893 char buf1[64]; 894 char *buf; 895 u64 sum = 0; 896 897 buf = kmalloc(800 + 64, GFP_KERNEL); 898 if (!buf) 899 return 0; 900 buf[0] = 0; 901 sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)", 902 exp_idx); 903 904 for (i = 0; i < n && !torture_must_stop(); i++) { 905 rt = &(reader_tasks[i]); 906 sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns); 907 908 if (i % 5 == 0) 909 strcat(buf, "\n"); 910 if (strlen(buf) >= 800) { 911 pr_alert("%s", buf); 912 buf[0] = 0; 913 } 914 strcat(buf, buf1); 915 916 sum += rt->last_duration_ns; 917 } 918 pr_alert("%s\n", buf); 919 920 kfree(buf); 921 return sum; 922} 923 924// The main_func is the main orchestrator, it performs a bunch of 925// experiments. For every experiment, it orders all the readers 926// involved to start and waits for them to finish the experiment. It 927// then reads their timestamps and starts the next experiment. Each 928// experiment progresses from 1 concurrent reader to N of them at which 929// point all the timestamps are printed. 930static int main_func(void *arg) 931{ 932 int exp, r; 933 char buf1[64]; 934 char *buf; 935 u64 *result_avg; 936 937 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids)); 938 set_user_nice(current, MAX_NICE); 939 940 VERBOSE_SCALEOUT("main_func task started"); 941 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL); 942 buf = kzalloc(800 + 64, GFP_KERNEL); 943 if (!result_avg || !buf) { 944 SCALEOUT_ERRSTRING("out of memory"); 945 goto oom_exit; 946 } 947 if (holdoff) 948 schedule_timeout_interruptible(holdoff * HZ); 949 950 // Wait for all threads to start. 951 atomic_inc(&n_init); 952 while (atomic_read(&n_init) < nreaders + 1) 953 schedule_timeout_uninterruptible(1); 954 955 // Start exp readers up per experiment 956 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) { 957 if (torture_must_stop()) 958 goto end; 959 960 reset_readers(); 961 atomic_set(&nreaders_exp, nreaders); 962 atomic_set(&n_started, nreaders); 963 atomic_set(&n_warmedup, nreaders); 964 atomic_set(&n_cooleddown, nreaders); 965 966 exp_idx = exp; 967 968 for (r = 0; r < nreaders; r++) { 969 smp_store_release(&reader_tasks[r].start_reader, 1); 970 wake_up(&reader_tasks[r].wq); 971 } 972 973 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers", 974 nreaders); 975 976 wait_event(main_wq, 977 !atomic_read(&nreaders_exp) || torture_must_stop()); 978 979 VERBOSE_SCALEOUT("main_func: experiment ended"); 980 981 if (torture_must_stop()) 982 goto end; 983 984 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops); 985 } 986 987 // Print the average of all experiments 988 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n"); 989 990 pr_alert("Runs\tTime(ns)\n"); 991 for (exp = 0; exp < nruns; exp++) { 992 u64 avg; 993 u32 rem; 994 995 avg = div_u64_rem(result_avg[exp], 1000, &rem); 996 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem); 997 strcat(buf, buf1); 998 if (strlen(buf) >= 800) { 999 pr_alert("%s", buf); 1000 buf[0] = 0; 1001 } 1002 } 1003 1004 pr_alert("%s", buf); 1005 1006oom_exit: 1007 // This will shutdown everything including us. 1008 if (shutdown) { 1009 shutdown_start = 1; 1010 wake_up(&shutdown_wq); 1011 } 1012 1013 // Wait for torture to stop us 1014 while (!torture_must_stop()) 1015 schedule_timeout_uninterruptible(1); 1016 1017end: 1018 torture_kthread_stopping("main_func"); 1019 kfree(result_avg); 1020 kfree(buf); 1021 return 0; 1022} 1023 1024static void 1025ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag) 1026{ 1027 pr_alert("%s" SCALE_FLAG 1028 "--- %s: verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag, 1029 verbose, verbose_batched, shutdown, holdoff, lookup_instances, loops, nreaders, nruns, readdelay); 1030} 1031 1032static void 1033ref_scale_cleanup(void) 1034{ 1035 int i; 1036 1037 if (torture_cleanup_begin()) 1038 return; 1039 1040 if (!cur_ops) { 1041 torture_cleanup_end(); 1042 return; 1043 } 1044 1045 if (reader_tasks) { 1046 for (i = 0; i < nreaders; i++) 1047 torture_stop_kthread("ref_scale_reader", 1048 reader_tasks[i].task); 1049 } 1050 kfree(reader_tasks); 1051 1052 torture_stop_kthread("main_task", main_task); 1053 kfree(main_task); 1054 1055 // Do scale-type-specific cleanup operations. 1056 if (cur_ops->cleanup != NULL) 1057 cur_ops->cleanup(); 1058 1059 torture_cleanup_end(); 1060} 1061 1062// Shutdown kthread. Just waits to be awakened, then shuts down system. 1063static int 1064ref_scale_shutdown(void *arg) 1065{ 1066 wait_event_idle(shutdown_wq, shutdown_start); 1067 1068 smp_mb(); // Wake before output. 1069 ref_scale_cleanup(); 1070 kernel_power_off(); 1071 1072 return -EINVAL; 1073} 1074 1075static int __init 1076ref_scale_init(void) 1077{ 1078 long i; 1079 int firsterr = 0; 1080 static struct ref_scale_ops *scale_ops[] = { 1081 &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, 1082 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops, 1083 &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, 1084 }; 1085 1086 if (!torture_init_begin(scale_type, verbose)) 1087 return -EBUSY; 1088 1089 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { 1090 cur_ops = scale_ops[i]; 1091 if (strcmp(scale_type, cur_ops->name) == 0) 1092 break; 1093 } 1094 if (i == ARRAY_SIZE(scale_ops)) { 1095 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); 1096 pr_alert("rcu-scale types:"); 1097 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) 1098 pr_cont(" %s", scale_ops[i]->name); 1099 pr_cont("\n"); 1100 firsterr = -EINVAL; 1101 cur_ops = NULL; 1102 goto unwind; 1103 } 1104 if (cur_ops->init) 1105 if (!cur_ops->init()) { 1106 firsterr = -EUCLEAN; 1107 goto unwind; 1108 } 1109 1110 ref_scale_print_module_parms(cur_ops, "Start of test"); 1111 1112 // Shutdown task 1113 if (shutdown) { 1114 init_waitqueue_head(&shutdown_wq); 1115 firsterr = torture_create_kthread(ref_scale_shutdown, NULL, 1116 shutdown_task); 1117 if (torture_init_error(firsterr)) 1118 goto unwind; 1119 schedule_timeout_uninterruptible(1); 1120 } 1121 1122 // Reader tasks (default to ~75% of online CPUs). 1123 if (nreaders < 0) 1124 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2); 1125 if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops)) 1126 loops = 1; 1127 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders)) 1128 nreaders = 1; 1129 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns)) 1130 nruns = 1; 1131 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]), 1132 GFP_KERNEL); 1133 if (!reader_tasks) { 1134 SCALEOUT_ERRSTRING("out of memory"); 1135 firsterr = -ENOMEM; 1136 goto unwind; 1137 } 1138 1139 VERBOSE_SCALEOUT("Starting %d reader threads", nreaders); 1140 1141 for (i = 0; i < nreaders; i++) { 1142 init_waitqueue_head(&reader_tasks[i].wq); 1143 firsterr = torture_create_kthread(ref_scale_reader, (void *)i, 1144 reader_tasks[i].task); 1145 if (torture_init_error(firsterr)) 1146 goto unwind; 1147 } 1148 1149 // Main Task 1150 init_waitqueue_head(&main_wq); 1151 firsterr = torture_create_kthread(main_func, NULL, main_task); 1152 if (torture_init_error(firsterr)) 1153 goto unwind; 1154 1155 torture_init_end(); 1156 return 0; 1157 1158unwind: 1159 torture_init_end(); 1160 ref_scale_cleanup(); 1161 if (shutdown) { 1162 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST)); 1163 kernel_power_off(); 1164 } 1165 return firsterr; 1166} 1167 1168module_init(ref_scale_init); 1169module_exit(ref_scale_cleanup); 1170