1/* 2 * driver.c - driver support 3 * 4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 5 * Shaohua Li <shaohua.li@intel.com> 6 * Adam Belay <abelay@novell.com> 7 * 8 * This code is licenced under the GPL. 9 */ 10 11#include <linux/mutex.h> 12#include <linux/module.h> 13#include <linux/sched.h> 14#include <linux/sched/idle.h> 15#include <linux/cpuidle.h> 16#include <linux/cpumask.h> 17#include <linux/tick.h> 18#include <linux/cpu.h> 19#include <linux/math64.h> 20 21#include "cpuidle.h" 22 23DEFINE_SPINLOCK(cpuidle_driver_lock); 24 25#ifdef CONFIG_CPU_IDLE_MULTIPLE_DRIVERS 26 27static DEFINE_PER_CPU(struct cpuidle_driver *, cpuidle_drivers); 28 29/** 30 * __cpuidle_get_cpu_driver - return the cpuidle driver tied to a CPU. 31 * @cpu: the CPU handled by the driver 32 * 33 * Returns a pointer to struct cpuidle_driver or NULL if no driver has been 34 * registered for @cpu. 35 */ 36static struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu) 37{ 38 return per_cpu(cpuidle_drivers, cpu); 39} 40 41/** 42 * __cpuidle_unset_driver - unset per CPU driver variables. 43 * @drv: a valid pointer to a struct cpuidle_driver 44 * 45 * For each CPU in the driver's CPU mask, unset the registered driver per CPU 46 * variable. If @drv is different from the registered driver, the corresponding 47 * variable is not cleared. 48 */ 49static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv) 50{ 51 int cpu; 52 53 for_each_cpu(cpu, drv->cpumask) { 54 55 if (drv != __cpuidle_get_cpu_driver(cpu)) 56 continue; 57 58 per_cpu(cpuidle_drivers, cpu) = NULL; 59 } 60} 61 62/** 63 * __cpuidle_set_driver - set per CPU driver variables for the given driver. 64 * @drv: a valid pointer to a struct cpuidle_driver 65 * 66 * Returns 0 on success, -EBUSY if any CPU in the cpumask have a driver 67 * different from drv already. 68 */ 69static inline int __cpuidle_set_driver(struct cpuidle_driver *drv) 70{ 71 int cpu; 72 73 for_each_cpu(cpu, drv->cpumask) { 74 struct cpuidle_driver *old_drv; 75 76 old_drv = __cpuidle_get_cpu_driver(cpu); 77 if (old_drv && old_drv != drv) 78 return -EBUSY; 79 } 80 81 for_each_cpu(cpu, drv->cpumask) 82 per_cpu(cpuidle_drivers, cpu) = drv; 83 84 return 0; 85} 86 87#else 88 89static struct cpuidle_driver *cpuidle_curr_driver; 90 91/** 92 * __cpuidle_get_cpu_driver - return the global cpuidle driver pointer. 93 * @cpu: ignored without the multiple driver support 94 * 95 * Return a pointer to a struct cpuidle_driver object or NULL if no driver was 96 * previously registered. 97 */ 98static inline struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu) 99{ 100 return cpuidle_curr_driver; 101} 102 103/** 104 * __cpuidle_set_driver - assign the global cpuidle driver variable. 105 * @drv: pointer to a struct cpuidle_driver object 106 * 107 * Returns 0 on success, -EBUSY if the driver is already registered. 108 */ 109static inline int __cpuidle_set_driver(struct cpuidle_driver *drv) 110{ 111 if (cpuidle_curr_driver) 112 return -EBUSY; 113 114 cpuidle_curr_driver = drv; 115 116 return 0; 117} 118 119/** 120 * __cpuidle_unset_driver - unset the global cpuidle driver variable. 121 * @drv: a pointer to a struct cpuidle_driver 122 * 123 * Reset the global cpuidle variable to NULL. If @drv does not match the 124 * registered driver, do nothing. 125 */ 126static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv) 127{ 128 if (drv == cpuidle_curr_driver) 129 cpuidle_curr_driver = NULL; 130} 131 132#endif 133 134/** 135 * cpuidle_setup_broadcast_timer - enable/disable the broadcast timer on a cpu 136 * @arg: a void pointer used to match the SMP cross call API 137 * 138 * If @arg is NULL broadcast is disabled otherwise enabled 139 * 140 * This function is executed per CPU by an SMP cross call. It's not 141 * supposed to be called directly. 142 */ 143static void cpuidle_setup_broadcast_timer(void *arg) 144{ 145 if (arg) 146 tick_broadcast_enable(); 147 else 148 tick_broadcast_disable(); 149} 150 151/** 152 * __cpuidle_driver_init - initialize the driver's internal data 153 * @drv: a valid pointer to a struct cpuidle_driver 154 */ 155static void __cpuidle_driver_init(struct cpuidle_driver *drv) 156{ 157 int i; 158 159 /* 160 * Use all possible CPUs as the default, because if the kernel boots 161 * with some CPUs offline and then we online one of them, the CPU 162 * notifier has to know which driver to assign. 163 */ 164 if (!drv->cpumask) 165 drv->cpumask = (struct cpumask *)cpu_possible_mask; 166 167 for (i = 0; i < drv->state_count; i++) { 168 struct cpuidle_state *s = &drv->states[i]; 169 170 /* 171 * Look for the timer stop flag in the different states and if 172 * it is found, indicate that the broadcast timer has to be set 173 * up. 174 */ 175 if (s->flags & CPUIDLE_FLAG_TIMER_STOP) 176 drv->bctimer = 1; 177 178 /* 179 * The core will use the target residency and exit latency 180 * values in nanoseconds, but allow drivers to provide them in 181 * microseconds too. 182 */ 183 if (s->target_residency > 0) 184 s->target_residency_ns = s->target_residency * NSEC_PER_USEC; 185 else if (s->target_residency_ns < 0) 186 s->target_residency_ns = 0; 187 else 188 s->target_residency = div_u64(s->target_residency_ns, NSEC_PER_USEC); 189 190 if (s->exit_latency > 0) 191 s->exit_latency_ns = mul_u32_u32(s->exit_latency, NSEC_PER_USEC); 192 else if (s->exit_latency_ns < 0) 193 s->exit_latency_ns = 0; 194 else 195 s->exit_latency = div_u64(s->exit_latency_ns, NSEC_PER_USEC); 196 } 197} 198 199/** 200 * __cpuidle_register_driver: register the driver 201 * @drv: a valid pointer to a struct cpuidle_driver 202 * 203 * Do some sanity checks, initialize the driver, assign the driver to the 204 * global cpuidle driver variable(s) and set up the broadcast timer if the 205 * cpuidle driver has some states that shut down the local timer. 206 * 207 * Returns 0 on success, a negative error code otherwise: 208 * * -EINVAL if the driver pointer is NULL or no idle states are available 209 * * -ENODEV if the cpuidle framework is disabled 210 * * -EBUSY if the driver is already assigned to the global variable(s) 211 */ 212static int __cpuidle_register_driver(struct cpuidle_driver *drv) 213{ 214 int ret; 215 216 if (!drv || !drv->state_count) 217 return -EINVAL; 218 219 ret = cpuidle_coupled_state_verify(drv); 220 if (ret) 221 return ret; 222 223 if (cpuidle_disabled()) 224 return -ENODEV; 225 226 __cpuidle_driver_init(drv); 227 228 ret = __cpuidle_set_driver(drv); 229 if (ret) 230 return ret; 231 232 if (drv->bctimer) 233 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer, 234 (void *)1, 1); 235 236 return 0; 237} 238 239/** 240 * __cpuidle_unregister_driver - unregister the driver 241 * @drv: a valid pointer to a struct cpuidle_driver 242 * 243 * Check if the driver is no longer in use, reset the global cpuidle driver 244 * variable(s) and disable the timer broadcast notification mechanism if it was 245 * in use. 246 * 247 */ 248static void __cpuidle_unregister_driver(struct cpuidle_driver *drv) 249{ 250 if (drv->bctimer) { 251 drv->bctimer = 0; 252 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer, 253 NULL, 1); 254 } 255 256 __cpuidle_unset_driver(drv); 257} 258 259/** 260 * cpuidle_register_driver - registers a driver 261 * @drv: a pointer to a valid struct cpuidle_driver 262 * 263 * Register the driver under a lock to prevent concurrent attempts to 264 * [un]register the driver from occuring at the same time. 265 * 266 * Returns 0 on success, a negative error code (returned by 267 * __cpuidle_register_driver()) otherwise. 268 */ 269int cpuidle_register_driver(struct cpuidle_driver *drv) 270{ 271 struct cpuidle_governor *gov; 272 int ret; 273 274 spin_lock(&cpuidle_driver_lock); 275 ret = __cpuidle_register_driver(drv); 276 spin_unlock(&cpuidle_driver_lock); 277 278 if (!ret && !strlen(param_governor) && drv->governor && 279 (cpuidle_get_driver() == drv)) { 280 mutex_lock(&cpuidle_lock); 281 gov = cpuidle_find_governor(drv->governor); 282 if (gov) { 283 cpuidle_prev_governor = cpuidle_curr_governor; 284 if (cpuidle_switch_governor(gov) < 0) 285 cpuidle_prev_governor = NULL; 286 } 287 mutex_unlock(&cpuidle_lock); 288 } 289 290 return ret; 291} 292EXPORT_SYMBOL_GPL(cpuidle_register_driver); 293 294/** 295 * cpuidle_unregister_driver - unregisters a driver 296 * @drv: a pointer to a valid struct cpuidle_driver 297 * 298 * Unregisters the cpuidle driver under a lock to prevent concurrent attempts 299 * to [un]register the driver from occuring at the same time. @drv has to 300 * match the currently registered driver. 301 */ 302void cpuidle_unregister_driver(struct cpuidle_driver *drv) 303{ 304 bool enabled = (cpuidle_get_driver() == drv); 305 306 spin_lock(&cpuidle_driver_lock); 307 __cpuidle_unregister_driver(drv); 308 spin_unlock(&cpuidle_driver_lock); 309 310 if (!enabled) 311 return; 312 313 mutex_lock(&cpuidle_lock); 314 if (cpuidle_prev_governor) { 315 if (!cpuidle_switch_governor(cpuidle_prev_governor)) 316 cpuidle_prev_governor = NULL; 317 } 318 mutex_unlock(&cpuidle_lock); 319} 320EXPORT_SYMBOL_GPL(cpuidle_unregister_driver); 321 322/** 323 * cpuidle_get_driver - return the driver tied to the current CPU. 324 * 325 * Returns a struct cpuidle_driver pointer, or NULL if no driver is registered. 326 */ 327struct cpuidle_driver *cpuidle_get_driver(void) 328{ 329 struct cpuidle_driver *drv; 330 int cpu; 331 332 cpu = get_cpu(); 333 drv = __cpuidle_get_cpu_driver(cpu); 334 put_cpu(); 335 336 return drv; 337} 338EXPORT_SYMBOL_GPL(cpuidle_get_driver); 339 340/** 341 * cpuidle_get_cpu_driver - return the driver registered for a CPU. 342 * @dev: a valid pointer to a struct cpuidle_device 343 * 344 * Returns a struct cpuidle_driver pointer, or NULL if no driver is registered 345 * for the CPU associated with @dev. 346 */ 347struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev) 348{ 349 if (!dev) 350 return NULL; 351 352 return __cpuidle_get_cpu_driver(dev->cpu); 353} 354EXPORT_SYMBOL_GPL(cpuidle_get_cpu_driver); 355 356/** 357 * cpuidle_driver_state_disabled - Disable or enable an idle state 358 * @drv: cpuidle driver owning the state 359 * @idx: State index 360 * @disable: Whether or not to disable the state 361 */ 362void cpuidle_driver_state_disabled(struct cpuidle_driver *drv, int idx, 363 bool disable) 364{ 365 unsigned int cpu; 366 367 mutex_lock(&cpuidle_lock); 368 369 spin_lock(&cpuidle_driver_lock); 370 371 if (!drv->cpumask) { 372 drv->states[idx].flags |= CPUIDLE_FLAG_UNUSABLE; 373 goto unlock; 374 } 375 376 for_each_cpu(cpu, drv->cpumask) { 377 struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu); 378 379 if (!dev) 380 continue; 381 382 if (disable) 383 dev->states_usage[idx].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER; 384 else 385 dev->states_usage[idx].disable &= ~CPUIDLE_STATE_DISABLED_BY_DRIVER; 386 } 387 388unlock: 389 spin_unlock(&cpuidle_driver_lock); 390 391 mutex_unlock(&cpuidle_lock); 392} 393