1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2018 Linaro Limited 4 * 5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org> 6 * 7 * The idle injection framework provides a way to force CPUs to enter idle 8 * states for a specified fraction of time over a specified period. 9 * 10 * It relies on the smpboot kthreads feature providing common code for CPU 11 * hotplug and thread [un]parking. 12 * 13 * All of the kthreads used for idle injection are created at init time. 14 * 15 * Next, the users of the idle injection framework provide a cpumask via 16 * its register function. The kthreads will be synchronized with respect to 17 * this cpumask. 18 * 19 * The idle + run duration is specified via separate helpers and that allows 20 * idle injection to be started. 21 * 22 * The idle injection kthreads will call play_idle_precise() with the idle 23 * duration and max allowed latency specified as per the above. 24 * 25 * After all of them have been woken up, a timer is set to start the next idle 26 * injection cycle. 27 * 28 * The timer interrupt handler will wake up the idle injection kthreads for 29 * all of the CPUs in the cpumask provided by the user. 30 * 31 * Idle injection is stopped synchronously and no leftover idle injection 32 * kthread activity after its completion is guaranteed. 33 * 34 * It is up to the user of this framework to provide a lock for higher-level 35 * synchronization to prevent race conditions like starting idle injection 36 * while unregistering from the framework. 37 */ 38#define pr_fmt(fmt) "ii_dev: " fmt 39 40#include <linux/cpu.h> 41#include <linux/hrtimer.h> 42#include <linux/kthread.h> 43#include <linux/sched.h> 44#include <linux/slab.h> 45#include <linux/smpboot.h> 46#include <linux/idle_inject.h> 47 48#include <uapi/linux/sched/types.h> 49 50/** 51 * struct idle_inject_thread - task on/off switch structure 52 * @tsk: task injecting the idle cycles 53 * @should_run: whether or not to run the task (for the smpboot kthread API) 54 */ 55struct idle_inject_thread { 56 struct task_struct *tsk; 57 int should_run; 58}; 59 60/** 61 * struct idle_inject_device - idle injection data 62 * @timer: idle injection period timer 63 * @idle_duration_us: duration of CPU idle time to inject 64 * @run_duration_us: duration of CPU run time to allow 65 * @latency_us: max allowed latency 66 * @update: Optional callback deciding whether or not to skip idle 67 * injection in the given cycle. 68 * @cpumask: mask of CPUs affected by idle injection 69 * 70 * This structure is used to define per instance idle inject device data. Each 71 * instance has an idle duration, a run duration and mask of CPUs to inject 72 * idle. 73 * 74 * Actual CPU idle time is injected by calling kernel scheduler interface 75 * play_idle_precise(). There is one optional callback that can be registered 76 * by calling idle_inject_register_full(): 77 * 78 * update() - This callback is invoked just before waking up CPUs to inject 79 * idle. If it returns false, CPUs are not woken up to inject idle in the given 80 * cycle. It also allows the caller to readjust the idle and run duration by 81 * calling idle_inject_set_duration() for the next cycle. 82 */ 83struct idle_inject_device { 84 struct hrtimer timer; 85 unsigned int idle_duration_us; 86 unsigned int run_duration_us; 87 unsigned int latency_us; 88 bool (*update)(void); 89 unsigned long cpumask[]; 90}; 91 92static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread); 93static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device); 94 95/** 96 * idle_inject_wakeup - Wake up idle injection threads 97 * @ii_dev: target idle injection device 98 * 99 * Every idle injection task associated with the given idle injection device 100 * and running on an online CPU will be woken up. 101 */ 102static void idle_inject_wakeup(struct idle_inject_device *ii_dev) 103{ 104 struct idle_inject_thread *iit; 105 unsigned int cpu; 106 107 for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) { 108 iit = per_cpu_ptr(&idle_inject_thread, cpu); 109 iit->should_run = 1; 110 wake_up_process(iit->tsk); 111 } 112} 113 114/** 115 * idle_inject_timer_fn - idle injection timer function 116 * @timer: idle injection hrtimer 117 * 118 * This function is called when the idle injection timer expires. It wakes up 119 * idle injection tasks associated with the timer and they, in turn, invoke 120 * play_idle_precise() to inject a specified amount of CPU idle time. 121 * 122 * Return: HRTIMER_RESTART. 123 */ 124static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer) 125{ 126 unsigned int duration_us; 127 struct idle_inject_device *ii_dev = 128 container_of(timer, struct idle_inject_device, timer); 129 130 if (!ii_dev->update || (ii_dev->update && ii_dev->update())) 131 idle_inject_wakeup(ii_dev); 132 133 duration_us = READ_ONCE(ii_dev->run_duration_us); 134 duration_us += READ_ONCE(ii_dev->idle_duration_us); 135 136 hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC)); 137 138 return HRTIMER_RESTART; 139} 140 141/** 142 * idle_inject_fn - idle injection work function 143 * @cpu: the CPU owning the task 144 * 145 * This function calls play_idle_precise() to inject a specified amount of CPU 146 * idle time. 147 */ 148static void idle_inject_fn(unsigned int cpu) 149{ 150 struct idle_inject_device *ii_dev; 151 struct idle_inject_thread *iit; 152 153 ii_dev = per_cpu(idle_inject_device, cpu); 154 iit = per_cpu_ptr(&idle_inject_thread, cpu); 155 156 /* 157 * Let the smpboot main loop know that the task should not run again. 158 */ 159 iit->should_run = 0; 160 161 play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC, 162 READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC); 163} 164 165/** 166 * idle_inject_set_duration - idle and run duration update helper 167 * @ii_dev: idle injection control device structure 168 * @run_duration_us: CPU run time to allow in microseconds 169 * @idle_duration_us: CPU idle time to inject in microseconds 170 */ 171void idle_inject_set_duration(struct idle_inject_device *ii_dev, 172 unsigned int run_duration_us, 173 unsigned int idle_duration_us) 174{ 175 if (run_duration_us + idle_duration_us) { 176 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us); 177 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us); 178 } 179 if (!run_duration_us) 180 pr_debug("CPU is forced to 100 percent idle\n"); 181} 182EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT); 183 184/** 185 * idle_inject_get_duration - idle and run duration retrieval helper 186 * @ii_dev: idle injection control device structure 187 * @run_duration_us: memory location to store the current CPU run time 188 * @idle_duration_us: memory location to store the current CPU idle time 189 */ 190void idle_inject_get_duration(struct idle_inject_device *ii_dev, 191 unsigned int *run_duration_us, 192 unsigned int *idle_duration_us) 193{ 194 *run_duration_us = READ_ONCE(ii_dev->run_duration_us); 195 *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us); 196} 197EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT); 198 199/** 200 * idle_inject_set_latency - set the maximum latency allowed 201 * @ii_dev: idle injection control device structure 202 * @latency_us: set the latency requirement for the idle state 203 */ 204void idle_inject_set_latency(struct idle_inject_device *ii_dev, 205 unsigned int latency_us) 206{ 207 WRITE_ONCE(ii_dev->latency_us, latency_us); 208} 209EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT); 210 211/** 212 * idle_inject_start - start idle injections 213 * @ii_dev: idle injection control device structure 214 * 215 * The function starts idle injection by first waking up all of the idle 216 * injection kthreads associated with @ii_dev to let them inject CPU idle time 217 * sets up a timer to start the next idle injection period. 218 * 219 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success. 220 */ 221int idle_inject_start(struct idle_inject_device *ii_dev) 222{ 223 unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us); 224 unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us); 225 226 if (!(idle_duration_us + run_duration_us)) 227 return -EINVAL; 228 229 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n", 230 cpumask_pr_args(to_cpumask(ii_dev->cpumask))); 231 232 idle_inject_wakeup(ii_dev); 233 234 hrtimer_start(&ii_dev->timer, 235 ns_to_ktime((idle_duration_us + run_duration_us) * 236 NSEC_PER_USEC), 237 HRTIMER_MODE_REL); 238 239 return 0; 240} 241EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT); 242 243/** 244 * idle_inject_stop - stops idle injections 245 * @ii_dev: idle injection control device structure 246 * 247 * The function stops idle injection and waits for the threads to finish work. 248 * If CPU idle time is being injected when this function runs, then it will 249 * wait until the end of the cycle. 250 * 251 * When it returns, there is no more idle injection kthread activity. The 252 * kthreads are scheduled out and the periodic timer is off. 253 */ 254void idle_inject_stop(struct idle_inject_device *ii_dev) 255{ 256 struct idle_inject_thread *iit; 257 unsigned int cpu; 258 259 pr_debug("Stopping idle injection on CPUs '%*pbl'\n", 260 cpumask_pr_args(to_cpumask(ii_dev->cpumask))); 261 262 hrtimer_cancel(&ii_dev->timer); 263 264 /* 265 * Stopping idle injection requires all of the idle injection kthreads 266 * associated with the given cpumask to be parked and stay that way, so 267 * prevent CPUs from going online at this point. Any CPUs going online 268 * after the loop below will be covered by clearing the should_run flag 269 * that will cause the smpboot main loop to schedule them out. 270 */ 271 cpu_hotplug_disable(); 272 273 /* 274 * Iterate over all (online + offline) CPUs here in case one of them 275 * goes offline with the should_run flag set so as to prevent its idle 276 * injection kthread from running when the CPU goes online again after 277 * the ii_dev has been freed. 278 */ 279 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { 280 iit = per_cpu_ptr(&idle_inject_thread, cpu); 281 iit->should_run = 0; 282 283 wait_task_inactive(iit->tsk, TASK_ANY); 284 } 285 286 cpu_hotplug_enable(); 287} 288EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT); 289 290/** 291 * idle_inject_setup - prepare the current task for idle injection 292 * @cpu: not used 293 * 294 * Called once, this function is in charge of setting the current task's 295 * scheduler parameters to make it an RT task. 296 */ 297static void idle_inject_setup(unsigned int cpu) 298{ 299 sched_set_fifo(current); 300} 301 302/** 303 * idle_inject_should_run - function helper for the smpboot API 304 * @cpu: CPU the kthread is running on 305 * 306 * Return: whether or not the thread can run. 307 */ 308static int idle_inject_should_run(unsigned int cpu) 309{ 310 struct idle_inject_thread *iit = 311 per_cpu_ptr(&idle_inject_thread, cpu); 312 313 return iit->should_run; 314} 315 316/** 317 * idle_inject_register_full - initialize idle injection on a set of CPUs 318 * @cpumask: CPUs to be affected by idle injection 319 * @update: This callback is called just before waking up CPUs to inject 320 * idle 321 * 322 * This function creates an idle injection control device structure for the 323 * given set of CPUs and initializes the timer associated with it. This 324 * function also allows to register update()callback. 325 * It does not start any injection cycles. 326 * 327 * Return: NULL if memory allocation fails, idle injection control device 328 * pointer on success. 329 */ 330 331struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask, 332 bool (*update)(void)) 333{ 334 struct idle_inject_device *ii_dev; 335 int cpu, cpu_rb; 336 337 ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL); 338 if (!ii_dev) 339 return NULL; 340 341 cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask); 342 hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 343 ii_dev->timer.function = idle_inject_timer_fn; 344 ii_dev->latency_us = UINT_MAX; 345 ii_dev->update = update; 346 347 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { 348 349 if (per_cpu(idle_inject_device, cpu)) { 350 pr_err("cpu%d is already registered\n", cpu); 351 goto out_rollback; 352 } 353 354 per_cpu(idle_inject_device, cpu) = ii_dev; 355 } 356 357 return ii_dev; 358 359out_rollback: 360 for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) { 361 if (cpu == cpu_rb) 362 break; 363 per_cpu(idle_inject_device, cpu_rb) = NULL; 364 } 365 366 kfree(ii_dev); 367 368 return NULL; 369} 370EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT); 371 372/** 373 * idle_inject_register - initialize idle injection on a set of CPUs 374 * @cpumask: CPUs to be affected by idle injection 375 * 376 * This function creates an idle injection control device structure for the 377 * given set of CPUs and initializes the timer associated with it. It does not 378 * start any injection cycles. 379 * 380 * Return: NULL if memory allocation fails, idle injection control device 381 * pointer on success. 382 */ 383struct idle_inject_device *idle_inject_register(struct cpumask *cpumask) 384{ 385 return idle_inject_register_full(cpumask, NULL); 386} 387EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT); 388 389/** 390 * idle_inject_unregister - unregister idle injection control device 391 * @ii_dev: idle injection control device to unregister 392 * 393 * The function stops idle injection for the given control device, 394 * unregisters its kthreads and frees memory allocated when that device was 395 * created. 396 */ 397void idle_inject_unregister(struct idle_inject_device *ii_dev) 398{ 399 unsigned int cpu; 400 401 idle_inject_stop(ii_dev); 402 403 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) 404 per_cpu(idle_inject_device, cpu) = NULL; 405 406 kfree(ii_dev); 407} 408EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT); 409 410static struct smp_hotplug_thread idle_inject_threads = { 411 .store = &idle_inject_thread.tsk, 412 .setup = idle_inject_setup, 413 .thread_fn = idle_inject_fn, 414 .thread_comm = "idle_inject/%u", 415 .thread_should_run = idle_inject_should_run, 416}; 417 418static int __init idle_inject_init(void) 419{ 420 return smpboot_register_percpu_thread(&idle_inject_threads); 421} 422early_initcall(idle_inject_init); 423