1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * intel_idle.c - native hardware idle loop for modern Intel processors 4 * 5 * Copyright (c) 2013 - 2020, Intel Corporation. 6 * Len Brown <len.brown@intel.com> 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com> 8 */ 9 10/* 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to 13 * make Linux more efficient on these processors, as intel_idle knows 14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. 15 */ 16 17/* 18 * Design Assumptions 19 * 20 * All CPUs have same idle states as boot CPU 21 * 22 * Chipset BM_STS (bus master status) bit is a NOP 23 * for preventing entry into deep C-states 24 * 25 * CPU will flush caches as needed when entering a C-state via MWAIT 26 * (in contrast to entering ACPI C3, in which case the WBINVD 27 * instruction needs to be executed to flush the caches) 28 */ 29 30/* 31 * Known limitations 32 * 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend 34 * to avoid complications with the lapic timer workaround. 35 * Have not seen issues with suspend, but may need same workaround here. 36 * 37 */ 38 39/* un-comment DEBUG to enable pr_debug() statements */ 40/* #define DEBUG */ 41 42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 43 44#include <linux/acpi.h> 45#include <linux/kernel.h> 46#include <linux/cpuidle.h> 47#include <linux/tick.h> 48#include <trace/events/power.h> 49#include <linux/sched.h> 50#include <linux/sched/smt.h> 51#include <linux/notifier.h> 52#include <linux/cpu.h> 53#include <linux/moduleparam.h> 54#include <asm/cpu_device_id.h> 55#include <asm/intel-family.h> 56#include <asm/mwait.h> 57#include <asm/spec-ctrl.h> 58#include <asm/fpu/api.h> 59 60#define INTEL_IDLE_VERSION "0.5.1" 61 62static struct cpuidle_driver intel_idle_driver = { 63 .name = "intel_idle", 64 .owner = THIS_MODULE, 65}; 66/* intel_idle.max_cstate=0 disables driver */ 67static int max_cstate = CPUIDLE_STATE_MAX - 1; 68static unsigned int disabled_states_mask __read_mostly; 69static unsigned int preferred_states_mask __read_mostly; 70static bool force_irq_on __read_mostly; 71static bool ibrs_off __read_mostly; 72 73static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; 74 75static unsigned long auto_demotion_disable_flags; 76 77static enum { 78 C1E_PROMOTION_PRESERVE, 79 C1E_PROMOTION_ENABLE, 80 C1E_PROMOTION_DISABLE 81} c1e_promotion = C1E_PROMOTION_PRESERVE; 82 83struct idle_cpu { 84 struct cpuidle_state *state_table; 85 86 /* 87 * Hardware C-state auto-demotion may not always be optimal. 88 * Indicate which enable bits to clear here. 89 */ 90 unsigned long auto_demotion_disable_flags; 91 bool byt_auto_demotion_disable_flag; 92 bool disable_promotion_to_c1e; 93 bool use_acpi; 94}; 95 96static const struct idle_cpu *icpu __initdata; 97static struct cpuidle_state *cpuidle_state_table __initdata; 98 99static unsigned int mwait_substates __initdata; 100 101/* 102 * Enable interrupts before entering the C-state. On some platforms and for 103 * some C-states, this may measurably decrease interrupt latency. 104 */ 105#define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) 106 107/* 108 * Enable this state by default even if the ACPI _CST does not list it. 109 */ 110#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) 111 112/* 113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE 114 * above. 115 */ 116#define CPUIDLE_FLAG_IBRS BIT(16) 117 118/* 119 * Initialize large xstate for the C6-state entrance. 120 */ 121#define CPUIDLE_FLAG_INIT_XSTATE BIT(17) 122 123/* 124 * MWAIT takes an 8-bit "hint" in EAX "suggesting" 125 * the C-state (top nibble) and sub-state (bottom nibble) 126 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. 127 * 128 * We store the hint at the top of our "flags" for each state. 129 */ 130#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) 131#define MWAIT2flg(eax) ((eax & 0xFF) << 24) 132 133static __always_inline int __intel_idle(struct cpuidle_device *dev, 134 struct cpuidle_driver *drv, 135 int index, bool irqoff) 136{ 137 struct cpuidle_state *state = &drv->states[index]; 138 unsigned long eax = flg2MWAIT(state->flags); 139 unsigned long ecx = 1*irqoff; /* break on interrupt flag */ 140 141 mwait_idle_with_hints(eax, ecx); 142 143 return index; 144} 145 146/** 147 * intel_idle - Ask the processor to enter the given idle state. 148 * @dev: cpuidle device of the target CPU. 149 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 150 * @index: Target idle state index. 151 * 152 * Use the MWAIT instruction to notify the processor that the CPU represented by 153 * @dev is idle and it can try to enter the idle state corresponding to @index. 154 * 155 * If the local APIC timer is not known to be reliable in the target idle state, 156 * enable one-shot tick broadcasting for the target CPU before executing MWAIT. 157 * 158 * Must be called under local_irq_disable(). 159 */ 160static __cpuidle int intel_idle(struct cpuidle_device *dev, 161 struct cpuidle_driver *drv, int index) 162{ 163 return __intel_idle(dev, drv, index, true); 164} 165 166static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, 167 struct cpuidle_driver *drv, int index) 168{ 169 return __intel_idle(dev, drv, index, false); 170} 171 172static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, 173 struct cpuidle_driver *drv, int index) 174{ 175 bool smt_active = sched_smt_active(); 176 u64 spec_ctrl = spec_ctrl_current(); 177 int ret; 178 179 if (smt_active) 180 __update_spec_ctrl(0); 181 182 ret = __intel_idle(dev, drv, index, true); 183 184 if (smt_active) 185 __update_spec_ctrl(spec_ctrl); 186 187 return ret; 188} 189 190static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev, 191 struct cpuidle_driver *drv, int index) 192{ 193 fpu_idle_fpregs(); 194 return __intel_idle(dev, drv, index, true); 195} 196 197/** 198 * intel_idle_s2idle - Ask the processor to enter the given idle state. 199 * @dev: cpuidle device of the target CPU. 200 * @drv: cpuidle driver (assumed to point to intel_idle_driver). 201 * @index: Target idle state index. 202 * 203 * Use the MWAIT instruction to notify the processor that the CPU represented by 204 * @dev is idle and it can try to enter the idle state corresponding to @index. 205 * 206 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen 207 * scheduler tick and suspended scheduler clock on the target CPU. 208 */ 209static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, 210 struct cpuidle_driver *drv, int index) 211{ 212 unsigned long ecx = 1; /* break on interrupt flag */ 213 struct cpuidle_state *state = &drv->states[index]; 214 unsigned long eax = flg2MWAIT(state->flags); 215 216 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) 217 fpu_idle_fpregs(); 218 219 mwait_idle_with_hints(eax, ecx); 220 221 return 0; 222} 223 224/* 225 * States are indexed by the cstate number, 226 * which is also the index into the MWAIT hint array. 227 * Thus C0 is a dummy. 228 */ 229static struct cpuidle_state nehalem_cstates[] __initdata = { 230 { 231 .name = "C1", 232 .desc = "MWAIT 0x00", 233 .flags = MWAIT2flg(0x00), 234 .exit_latency = 3, 235 .target_residency = 6, 236 .enter = &intel_idle, 237 .enter_s2idle = intel_idle_s2idle, }, 238 { 239 .name = "C1E", 240 .desc = "MWAIT 0x01", 241 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 242 .exit_latency = 10, 243 .target_residency = 20, 244 .enter = &intel_idle, 245 .enter_s2idle = intel_idle_s2idle, }, 246 { 247 .name = "C3", 248 .desc = "MWAIT 0x10", 249 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 250 .exit_latency = 20, 251 .target_residency = 80, 252 .enter = &intel_idle, 253 .enter_s2idle = intel_idle_s2idle, }, 254 { 255 .name = "C6", 256 .desc = "MWAIT 0x20", 257 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 258 .exit_latency = 200, 259 .target_residency = 800, 260 .enter = &intel_idle, 261 .enter_s2idle = intel_idle_s2idle, }, 262 { 263 .enter = NULL } 264}; 265 266static struct cpuidle_state snb_cstates[] __initdata = { 267 { 268 .name = "C1", 269 .desc = "MWAIT 0x00", 270 .flags = MWAIT2flg(0x00), 271 .exit_latency = 2, 272 .target_residency = 2, 273 .enter = &intel_idle, 274 .enter_s2idle = intel_idle_s2idle, }, 275 { 276 .name = "C1E", 277 .desc = "MWAIT 0x01", 278 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 279 .exit_latency = 10, 280 .target_residency = 20, 281 .enter = &intel_idle, 282 .enter_s2idle = intel_idle_s2idle, }, 283 { 284 .name = "C3", 285 .desc = "MWAIT 0x10", 286 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 287 .exit_latency = 80, 288 .target_residency = 211, 289 .enter = &intel_idle, 290 .enter_s2idle = intel_idle_s2idle, }, 291 { 292 .name = "C6", 293 .desc = "MWAIT 0x20", 294 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 295 .exit_latency = 104, 296 .target_residency = 345, 297 .enter = &intel_idle, 298 .enter_s2idle = intel_idle_s2idle, }, 299 { 300 .name = "C7", 301 .desc = "MWAIT 0x30", 302 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 303 .exit_latency = 109, 304 .target_residency = 345, 305 .enter = &intel_idle, 306 .enter_s2idle = intel_idle_s2idle, }, 307 { 308 .enter = NULL } 309}; 310 311static struct cpuidle_state byt_cstates[] __initdata = { 312 { 313 .name = "C1", 314 .desc = "MWAIT 0x00", 315 .flags = MWAIT2flg(0x00), 316 .exit_latency = 1, 317 .target_residency = 1, 318 .enter = &intel_idle, 319 .enter_s2idle = intel_idle_s2idle, }, 320 { 321 .name = "C6N", 322 .desc = "MWAIT 0x58", 323 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 324 .exit_latency = 300, 325 .target_residency = 275, 326 .enter = &intel_idle, 327 .enter_s2idle = intel_idle_s2idle, }, 328 { 329 .name = "C6S", 330 .desc = "MWAIT 0x52", 331 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 332 .exit_latency = 500, 333 .target_residency = 560, 334 .enter = &intel_idle, 335 .enter_s2idle = intel_idle_s2idle, }, 336 { 337 .name = "C7", 338 .desc = "MWAIT 0x60", 339 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 340 .exit_latency = 1200, 341 .target_residency = 4000, 342 .enter = &intel_idle, 343 .enter_s2idle = intel_idle_s2idle, }, 344 { 345 .name = "C7S", 346 .desc = "MWAIT 0x64", 347 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 348 .exit_latency = 10000, 349 .target_residency = 20000, 350 .enter = &intel_idle, 351 .enter_s2idle = intel_idle_s2idle, }, 352 { 353 .enter = NULL } 354}; 355 356static struct cpuidle_state cht_cstates[] __initdata = { 357 { 358 .name = "C1", 359 .desc = "MWAIT 0x00", 360 .flags = MWAIT2flg(0x00), 361 .exit_latency = 1, 362 .target_residency = 1, 363 .enter = &intel_idle, 364 .enter_s2idle = intel_idle_s2idle, }, 365 { 366 .name = "C6N", 367 .desc = "MWAIT 0x58", 368 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, 369 .exit_latency = 80, 370 .target_residency = 275, 371 .enter = &intel_idle, 372 .enter_s2idle = intel_idle_s2idle, }, 373 { 374 .name = "C6S", 375 .desc = "MWAIT 0x52", 376 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 377 .exit_latency = 200, 378 .target_residency = 560, 379 .enter = &intel_idle, 380 .enter_s2idle = intel_idle_s2idle, }, 381 { 382 .name = "C7", 383 .desc = "MWAIT 0x60", 384 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 385 .exit_latency = 1200, 386 .target_residency = 4000, 387 .enter = &intel_idle, 388 .enter_s2idle = intel_idle_s2idle, }, 389 { 390 .name = "C7S", 391 .desc = "MWAIT 0x64", 392 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 393 .exit_latency = 10000, 394 .target_residency = 20000, 395 .enter = &intel_idle, 396 .enter_s2idle = intel_idle_s2idle, }, 397 { 398 .enter = NULL } 399}; 400 401static struct cpuidle_state ivb_cstates[] __initdata = { 402 { 403 .name = "C1", 404 .desc = "MWAIT 0x00", 405 .flags = MWAIT2flg(0x00), 406 .exit_latency = 1, 407 .target_residency = 1, 408 .enter = &intel_idle, 409 .enter_s2idle = intel_idle_s2idle, }, 410 { 411 .name = "C1E", 412 .desc = "MWAIT 0x01", 413 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 414 .exit_latency = 10, 415 .target_residency = 20, 416 .enter = &intel_idle, 417 .enter_s2idle = intel_idle_s2idle, }, 418 { 419 .name = "C3", 420 .desc = "MWAIT 0x10", 421 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 422 .exit_latency = 59, 423 .target_residency = 156, 424 .enter = &intel_idle, 425 .enter_s2idle = intel_idle_s2idle, }, 426 { 427 .name = "C6", 428 .desc = "MWAIT 0x20", 429 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 430 .exit_latency = 80, 431 .target_residency = 300, 432 .enter = &intel_idle, 433 .enter_s2idle = intel_idle_s2idle, }, 434 { 435 .name = "C7", 436 .desc = "MWAIT 0x30", 437 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 438 .exit_latency = 87, 439 .target_residency = 300, 440 .enter = &intel_idle, 441 .enter_s2idle = intel_idle_s2idle, }, 442 { 443 .enter = NULL } 444}; 445 446static struct cpuidle_state ivt_cstates[] __initdata = { 447 { 448 .name = "C1", 449 .desc = "MWAIT 0x00", 450 .flags = MWAIT2flg(0x00), 451 .exit_latency = 1, 452 .target_residency = 1, 453 .enter = &intel_idle, 454 .enter_s2idle = intel_idle_s2idle, }, 455 { 456 .name = "C1E", 457 .desc = "MWAIT 0x01", 458 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 459 .exit_latency = 10, 460 .target_residency = 80, 461 .enter = &intel_idle, 462 .enter_s2idle = intel_idle_s2idle, }, 463 { 464 .name = "C3", 465 .desc = "MWAIT 0x10", 466 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 467 .exit_latency = 59, 468 .target_residency = 156, 469 .enter = &intel_idle, 470 .enter_s2idle = intel_idle_s2idle, }, 471 { 472 .name = "C6", 473 .desc = "MWAIT 0x20", 474 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 475 .exit_latency = 82, 476 .target_residency = 300, 477 .enter = &intel_idle, 478 .enter_s2idle = intel_idle_s2idle, }, 479 { 480 .enter = NULL } 481}; 482 483static struct cpuidle_state ivt_cstates_4s[] __initdata = { 484 { 485 .name = "C1", 486 .desc = "MWAIT 0x00", 487 .flags = MWAIT2flg(0x00), 488 .exit_latency = 1, 489 .target_residency = 1, 490 .enter = &intel_idle, 491 .enter_s2idle = intel_idle_s2idle, }, 492 { 493 .name = "C1E", 494 .desc = "MWAIT 0x01", 495 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 496 .exit_latency = 10, 497 .target_residency = 250, 498 .enter = &intel_idle, 499 .enter_s2idle = intel_idle_s2idle, }, 500 { 501 .name = "C3", 502 .desc = "MWAIT 0x10", 503 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 504 .exit_latency = 59, 505 .target_residency = 300, 506 .enter = &intel_idle, 507 .enter_s2idle = intel_idle_s2idle, }, 508 { 509 .name = "C6", 510 .desc = "MWAIT 0x20", 511 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 512 .exit_latency = 84, 513 .target_residency = 400, 514 .enter = &intel_idle, 515 .enter_s2idle = intel_idle_s2idle, }, 516 { 517 .enter = NULL } 518}; 519 520static struct cpuidle_state ivt_cstates_8s[] __initdata = { 521 { 522 .name = "C1", 523 .desc = "MWAIT 0x00", 524 .flags = MWAIT2flg(0x00), 525 .exit_latency = 1, 526 .target_residency = 1, 527 .enter = &intel_idle, 528 .enter_s2idle = intel_idle_s2idle, }, 529 { 530 .name = "C1E", 531 .desc = "MWAIT 0x01", 532 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 533 .exit_latency = 10, 534 .target_residency = 500, 535 .enter = &intel_idle, 536 .enter_s2idle = intel_idle_s2idle, }, 537 { 538 .name = "C3", 539 .desc = "MWAIT 0x10", 540 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 541 .exit_latency = 59, 542 .target_residency = 600, 543 .enter = &intel_idle, 544 .enter_s2idle = intel_idle_s2idle, }, 545 { 546 .name = "C6", 547 .desc = "MWAIT 0x20", 548 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 549 .exit_latency = 88, 550 .target_residency = 700, 551 .enter = &intel_idle, 552 .enter_s2idle = intel_idle_s2idle, }, 553 { 554 .enter = NULL } 555}; 556 557static struct cpuidle_state hsw_cstates[] __initdata = { 558 { 559 .name = "C1", 560 .desc = "MWAIT 0x00", 561 .flags = MWAIT2flg(0x00), 562 .exit_latency = 2, 563 .target_residency = 2, 564 .enter = &intel_idle, 565 .enter_s2idle = intel_idle_s2idle, }, 566 { 567 .name = "C1E", 568 .desc = "MWAIT 0x01", 569 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 570 .exit_latency = 10, 571 .target_residency = 20, 572 .enter = &intel_idle, 573 .enter_s2idle = intel_idle_s2idle, }, 574 { 575 .name = "C3", 576 .desc = "MWAIT 0x10", 577 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 578 .exit_latency = 33, 579 .target_residency = 100, 580 .enter = &intel_idle, 581 .enter_s2idle = intel_idle_s2idle, }, 582 { 583 .name = "C6", 584 .desc = "MWAIT 0x20", 585 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 586 .exit_latency = 133, 587 .target_residency = 400, 588 .enter = &intel_idle, 589 .enter_s2idle = intel_idle_s2idle, }, 590 { 591 .name = "C7s", 592 .desc = "MWAIT 0x32", 593 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 594 .exit_latency = 166, 595 .target_residency = 500, 596 .enter = &intel_idle, 597 .enter_s2idle = intel_idle_s2idle, }, 598 { 599 .name = "C8", 600 .desc = "MWAIT 0x40", 601 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 602 .exit_latency = 300, 603 .target_residency = 900, 604 .enter = &intel_idle, 605 .enter_s2idle = intel_idle_s2idle, }, 606 { 607 .name = "C9", 608 .desc = "MWAIT 0x50", 609 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 610 .exit_latency = 600, 611 .target_residency = 1800, 612 .enter = &intel_idle, 613 .enter_s2idle = intel_idle_s2idle, }, 614 { 615 .name = "C10", 616 .desc = "MWAIT 0x60", 617 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 618 .exit_latency = 2600, 619 .target_residency = 7700, 620 .enter = &intel_idle, 621 .enter_s2idle = intel_idle_s2idle, }, 622 { 623 .enter = NULL } 624}; 625static struct cpuidle_state bdw_cstates[] __initdata = { 626 { 627 .name = "C1", 628 .desc = "MWAIT 0x00", 629 .flags = MWAIT2flg(0x00), 630 .exit_latency = 2, 631 .target_residency = 2, 632 .enter = &intel_idle, 633 .enter_s2idle = intel_idle_s2idle, }, 634 { 635 .name = "C1E", 636 .desc = "MWAIT 0x01", 637 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 638 .exit_latency = 10, 639 .target_residency = 20, 640 .enter = &intel_idle, 641 .enter_s2idle = intel_idle_s2idle, }, 642 { 643 .name = "C3", 644 .desc = "MWAIT 0x10", 645 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 646 .exit_latency = 40, 647 .target_residency = 100, 648 .enter = &intel_idle, 649 .enter_s2idle = intel_idle_s2idle, }, 650 { 651 .name = "C6", 652 .desc = "MWAIT 0x20", 653 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 654 .exit_latency = 133, 655 .target_residency = 400, 656 .enter = &intel_idle, 657 .enter_s2idle = intel_idle_s2idle, }, 658 { 659 .name = "C7s", 660 .desc = "MWAIT 0x32", 661 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, 662 .exit_latency = 166, 663 .target_residency = 500, 664 .enter = &intel_idle, 665 .enter_s2idle = intel_idle_s2idle, }, 666 { 667 .name = "C8", 668 .desc = "MWAIT 0x40", 669 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 670 .exit_latency = 300, 671 .target_residency = 900, 672 .enter = &intel_idle, 673 .enter_s2idle = intel_idle_s2idle, }, 674 { 675 .name = "C9", 676 .desc = "MWAIT 0x50", 677 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 678 .exit_latency = 600, 679 .target_residency = 1800, 680 .enter = &intel_idle, 681 .enter_s2idle = intel_idle_s2idle, }, 682 { 683 .name = "C10", 684 .desc = "MWAIT 0x60", 685 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 686 .exit_latency = 2600, 687 .target_residency = 7700, 688 .enter = &intel_idle, 689 .enter_s2idle = intel_idle_s2idle, }, 690 { 691 .enter = NULL } 692}; 693 694static struct cpuidle_state skl_cstates[] __initdata = { 695 { 696 .name = "C1", 697 .desc = "MWAIT 0x00", 698 .flags = MWAIT2flg(0x00), 699 .exit_latency = 2, 700 .target_residency = 2, 701 .enter = &intel_idle, 702 .enter_s2idle = intel_idle_s2idle, }, 703 { 704 .name = "C1E", 705 .desc = "MWAIT 0x01", 706 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 707 .exit_latency = 10, 708 .target_residency = 20, 709 .enter = &intel_idle, 710 .enter_s2idle = intel_idle_s2idle, }, 711 { 712 .name = "C3", 713 .desc = "MWAIT 0x10", 714 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 715 .exit_latency = 70, 716 .target_residency = 100, 717 .enter = &intel_idle, 718 .enter_s2idle = intel_idle_s2idle, }, 719 { 720 .name = "C6", 721 .desc = "MWAIT 0x20", 722 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 723 .exit_latency = 85, 724 .target_residency = 200, 725 .enter = &intel_idle, 726 .enter_s2idle = intel_idle_s2idle, }, 727 { 728 .name = "C7s", 729 .desc = "MWAIT 0x33", 730 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 731 .exit_latency = 124, 732 .target_residency = 800, 733 .enter = &intel_idle, 734 .enter_s2idle = intel_idle_s2idle, }, 735 { 736 .name = "C8", 737 .desc = "MWAIT 0x40", 738 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 739 .exit_latency = 200, 740 .target_residency = 800, 741 .enter = &intel_idle, 742 .enter_s2idle = intel_idle_s2idle, }, 743 { 744 .name = "C9", 745 .desc = "MWAIT 0x50", 746 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 747 .exit_latency = 480, 748 .target_residency = 5000, 749 .enter = &intel_idle, 750 .enter_s2idle = intel_idle_s2idle, }, 751 { 752 .name = "C10", 753 .desc = "MWAIT 0x60", 754 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 755 .exit_latency = 890, 756 .target_residency = 5000, 757 .enter = &intel_idle, 758 .enter_s2idle = intel_idle_s2idle, }, 759 { 760 .enter = NULL } 761}; 762 763static struct cpuidle_state skx_cstates[] __initdata = { 764 { 765 .name = "C1", 766 .desc = "MWAIT 0x00", 767 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 768 .exit_latency = 2, 769 .target_residency = 2, 770 .enter = &intel_idle, 771 .enter_s2idle = intel_idle_s2idle, }, 772 { 773 .name = "C1E", 774 .desc = "MWAIT 0x01", 775 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 776 .exit_latency = 10, 777 .target_residency = 20, 778 .enter = &intel_idle, 779 .enter_s2idle = intel_idle_s2idle, }, 780 { 781 .name = "C6", 782 .desc = "MWAIT 0x20", 783 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, 784 .exit_latency = 133, 785 .target_residency = 600, 786 .enter = &intel_idle, 787 .enter_s2idle = intel_idle_s2idle, }, 788 { 789 .enter = NULL } 790}; 791 792static struct cpuidle_state icx_cstates[] __initdata = { 793 { 794 .name = "C1", 795 .desc = "MWAIT 0x00", 796 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, 797 .exit_latency = 1, 798 .target_residency = 1, 799 .enter = &intel_idle, 800 .enter_s2idle = intel_idle_s2idle, }, 801 { 802 .name = "C1E", 803 .desc = "MWAIT 0x01", 804 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 805 .exit_latency = 4, 806 .target_residency = 4, 807 .enter = &intel_idle, 808 .enter_s2idle = intel_idle_s2idle, }, 809 { 810 .name = "C6", 811 .desc = "MWAIT 0x20", 812 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 813 .exit_latency = 170, 814 .target_residency = 600, 815 .enter = &intel_idle, 816 .enter_s2idle = intel_idle_s2idle, }, 817 { 818 .enter = NULL } 819}; 820 821/* 822 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. 823 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. 824 * But in this case there is effectively no C1, because C1 requests are 825 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 826 * and C1E requests end up with C1, so there is effectively no C1E. 827 * 828 * By default we enable C1E and disable C1 by marking it with 829 * 'CPUIDLE_FLAG_UNUSABLE'. 830 */ 831static struct cpuidle_state adl_cstates[] __initdata = { 832 { 833 .name = "C1", 834 .desc = "MWAIT 0x00", 835 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 836 .exit_latency = 1, 837 .target_residency = 1, 838 .enter = &intel_idle, 839 .enter_s2idle = intel_idle_s2idle, }, 840 { 841 .name = "C1E", 842 .desc = "MWAIT 0x01", 843 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 844 .exit_latency = 2, 845 .target_residency = 4, 846 .enter = &intel_idle, 847 .enter_s2idle = intel_idle_s2idle, }, 848 { 849 .name = "C6", 850 .desc = "MWAIT 0x20", 851 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 852 .exit_latency = 220, 853 .target_residency = 600, 854 .enter = &intel_idle, 855 .enter_s2idle = intel_idle_s2idle, }, 856 { 857 .name = "C8", 858 .desc = "MWAIT 0x40", 859 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 860 .exit_latency = 280, 861 .target_residency = 800, 862 .enter = &intel_idle, 863 .enter_s2idle = intel_idle_s2idle, }, 864 { 865 .name = "C10", 866 .desc = "MWAIT 0x60", 867 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 868 .exit_latency = 680, 869 .target_residency = 2000, 870 .enter = &intel_idle, 871 .enter_s2idle = intel_idle_s2idle, }, 872 { 873 .enter = NULL } 874}; 875 876static struct cpuidle_state adl_l_cstates[] __initdata = { 877 { 878 .name = "C1", 879 .desc = "MWAIT 0x00", 880 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 881 .exit_latency = 1, 882 .target_residency = 1, 883 .enter = &intel_idle, 884 .enter_s2idle = intel_idle_s2idle, }, 885 { 886 .name = "C1E", 887 .desc = "MWAIT 0x01", 888 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 889 .exit_latency = 2, 890 .target_residency = 4, 891 .enter = &intel_idle, 892 .enter_s2idle = intel_idle_s2idle, }, 893 { 894 .name = "C6", 895 .desc = "MWAIT 0x20", 896 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 897 .exit_latency = 170, 898 .target_residency = 500, 899 .enter = &intel_idle, 900 .enter_s2idle = intel_idle_s2idle, }, 901 { 902 .name = "C8", 903 .desc = "MWAIT 0x40", 904 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 905 .exit_latency = 200, 906 .target_residency = 600, 907 .enter = &intel_idle, 908 .enter_s2idle = intel_idle_s2idle, }, 909 { 910 .name = "C10", 911 .desc = "MWAIT 0x60", 912 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 913 .exit_latency = 230, 914 .target_residency = 700, 915 .enter = &intel_idle, 916 .enter_s2idle = intel_idle_s2idle, }, 917 { 918 .enter = NULL } 919}; 920 921static struct cpuidle_state mtl_l_cstates[] __initdata = { 922 { 923 .name = "C1E", 924 .desc = "MWAIT 0x01", 925 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 926 .exit_latency = 1, 927 .target_residency = 1, 928 .enter = &intel_idle, 929 .enter_s2idle = intel_idle_s2idle, }, 930 { 931 .name = "C6", 932 .desc = "MWAIT 0x20", 933 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 934 .exit_latency = 140, 935 .target_residency = 420, 936 .enter = &intel_idle, 937 .enter_s2idle = intel_idle_s2idle, }, 938 { 939 .name = "C10", 940 .desc = "MWAIT 0x60", 941 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 942 .exit_latency = 310, 943 .target_residency = 930, 944 .enter = &intel_idle, 945 .enter_s2idle = intel_idle_s2idle, }, 946 { 947 .enter = NULL } 948}; 949 950static struct cpuidle_state gmt_cstates[] __initdata = { 951 { 952 .name = "C1", 953 .desc = "MWAIT 0x00", 954 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, 955 .exit_latency = 1, 956 .target_residency = 1, 957 .enter = &intel_idle, 958 .enter_s2idle = intel_idle_s2idle, }, 959 { 960 .name = "C1E", 961 .desc = "MWAIT 0x01", 962 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 963 .exit_latency = 2, 964 .target_residency = 4, 965 .enter = &intel_idle, 966 .enter_s2idle = intel_idle_s2idle, }, 967 { 968 .name = "C6", 969 .desc = "MWAIT 0x20", 970 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 971 .exit_latency = 195, 972 .target_residency = 585, 973 .enter = &intel_idle, 974 .enter_s2idle = intel_idle_s2idle, }, 975 { 976 .name = "C8", 977 .desc = "MWAIT 0x40", 978 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 979 .exit_latency = 260, 980 .target_residency = 1040, 981 .enter = &intel_idle, 982 .enter_s2idle = intel_idle_s2idle, }, 983 { 984 .name = "C10", 985 .desc = "MWAIT 0x60", 986 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 987 .exit_latency = 660, 988 .target_residency = 1980, 989 .enter = &intel_idle, 990 .enter_s2idle = intel_idle_s2idle, }, 991 { 992 .enter = NULL } 993}; 994 995static struct cpuidle_state spr_cstates[] __initdata = { 996 { 997 .name = "C1", 998 .desc = "MWAIT 0x00", 999 .flags = MWAIT2flg(0x00), 1000 .exit_latency = 1, 1001 .target_residency = 1, 1002 .enter = &intel_idle, 1003 .enter_s2idle = intel_idle_s2idle, }, 1004 { 1005 .name = "C1E", 1006 .desc = "MWAIT 0x01", 1007 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1008 .exit_latency = 2, 1009 .target_residency = 4, 1010 .enter = &intel_idle, 1011 .enter_s2idle = intel_idle_s2idle, }, 1012 { 1013 .name = "C6", 1014 .desc = "MWAIT 0x20", 1015 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | 1016 CPUIDLE_FLAG_INIT_XSTATE, 1017 .exit_latency = 290, 1018 .target_residency = 800, 1019 .enter = &intel_idle, 1020 .enter_s2idle = intel_idle_s2idle, }, 1021 { 1022 .enter = NULL } 1023}; 1024 1025static struct cpuidle_state atom_cstates[] __initdata = { 1026 { 1027 .name = "C1E", 1028 .desc = "MWAIT 0x00", 1029 .flags = MWAIT2flg(0x00), 1030 .exit_latency = 10, 1031 .target_residency = 20, 1032 .enter = &intel_idle, 1033 .enter_s2idle = intel_idle_s2idle, }, 1034 { 1035 .name = "C2", 1036 .desc = "MWAIT 0x10", 1037 .flags = MWAIT2flg(0x10), 1038 .exit_latency = 20, 1039 .target_residency = 80, 1040 .enter = &intel_idle, 1041 .enter_s2idle = intel_idle_s2idle, }, 1042 { 1043 .name = "C4", 1044 .desc = "MWAIT 0x30", 1045 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1046 .exit_latency = 100, 1047 .target_residency = 400, 1048 .enter = &intel_idle, 1049 .enter_s2idle = intel_idle_s2idle, }, 1050 { 1051 .name = "C6", 1052 .desc = "MWAIT 0x52", 1053 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1054 .exit_latency = 140, 1055 .target_residency = 560, 1056 .enter = &intel_idle, 1057 .enter_s2idle = intel_idle_s2idle, }, 1058 { 1059 .enter = NULL } 1060}; 1061static struct cpuidle_state tangier_cstates[] __initdata = { 1062 { 1063 .name = "C1", 1064 .desc = "MWAIT 0x00", 1065 .flags = MWAIT2flg(0x00), 1066 .exit_latency = 1, 1067 .target_residency = 4, 1068 .enter = &intel_idle, 1069 .enter_s2idle = intel_idle_s2idle, }, 1070 { 1071 .name = "C4", 1072 .desc = "MWAIT 0x30", 1073 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, 1074 .exit_latency = 100, 1075 .target_residency = 400, 1076 .enter = &intel_idle, 1077 .enter_s2idle = intel_idle_s2idle, }, 1078 { 1079 .name = "C6", 1080 .desc = "MWAIT 0x52", 1081 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, 1082 .exit_latency = 140, 1083 .target_residency = 560, 1084 .enter = &intel_idle, 1085 .enter_s2idle = intel_idle_s2idle, }, 1086 { 1087 .name = "C7", 1088 .desc = "MWAIT 0x60", 1089 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1090 .exit_latency = 1200, 1091 .target_residency = 4000, 1092 .enter = &intel_idle, 1093 .enter_s2idle = intel_idle_s2idle, }, 1094 { 1095 .name = "C9", 1096 .desc = "MWAIT 0x64", 1097 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, 1098 .exit_latency = 10000, 1099 .target_residency = 20000, 1100 .enter = &intel_idle, 1101 .enter_s2idle = intel_idle_s2idle, }, 1102 { 1103 .enter = NULL } 1104}; 1105static struct cpuidle_state avn_cstates[] __initdata = { 1106 { 1107 .name = "C1", 1108 .desc = "MWAIT 0x00", 1109 .flags = MWAIT2flg(0x00), 1110 .exit_latency = 2, 1111 .target_residency = 2, 1112 .enter = &intel_idle, 1113 .enter_s2idle = intel_idle_s2idle, }, 1114 { 1115 .name = "C6", 1116 .desc = "MWAIT 0x51", 1117 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, 1118 .exit_latency = 15, 1119 .target_residency = 45, 1120 .enter = &intel_idle, 1121 .enter_s2idle = intel_idle_s2idle, }, 1122 { 1123 .enter = NULL } 1124}; 1125static struct cpuidle_state knl_cstates[] __initdata = { 1126 { 1127 .name = "C1", 1128 .desc = "MWAIT 0x00", 1129 .flags = MWAIT2flg(0x00), 1130 .exit_latency = 1, 1131 .target_residency = 2, 1132 .enter = &intel_idle, 1133 .enter_s2idle = intel_idle_s2idle }, 1134 { 1135 .name = "C6", 1136 .desc = "MWAIT 0x10", 1137 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, 1138 .exit_latency = 120, 1139 .target_residency = 500, 1140 .enter = &intel_idle, 1141 .enter_s2idle = intel_idle_s2idle }, 1142 { 1143 .enter = NULL } 1144}; 1145 1146static struct cpuidle_state bxt_cstates[] __initdata = { 1147 { 1148 .name = "C1", 1149 .desc = "MWAIT 0x00", 1150 .flags = MWAIT2flg(0x00), 1151 .exit_latency = 2, 1152 .target_residency = 2, 1153 .enter = &intel_idle, 1154 .enter_s2idle = intel_idle_s2idle, }, 1155 { 1156 .name = "C1E", 1157 .desc = "MWAIT 0x01", 1158 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1159 .exit_latency = 10, 1160 .target_residency = 20, 1161 .enter = &intel_idle, 1162 .enter_s2idle = intel_idle_s2idle, }, 1163 { 1164 .name = "C6", 1165 .desc = "MWAIT 0x20", 1166 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1167 .exit_latency = 133, 1168 .target_residency = 133, 1169 .enter = &intel_idle, 1170 .enter_s2idle = intel_idle_s2idle, }, 1171 { 1172 .name = "C7s", 1173 .desc = "MWAIT 0x31", 1174 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, 1175 .exit_latency = 155, 1176 .target_residency = 155, 1177 .enter = &intel_idle, 1178 .enter_s2idle = intel_idle_s2idle, }, 1179 { 1180 .name = "C8", 1181 .desc = "MWAIT 0x40", 1182 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, 1183 .exit_latency = 1000, 1184 .target_residency = 1000, 1185 .enter = &intel_idle, 1186 .enter_s2idle = intel_idle_s2idle, }, 1187 { 1188 .name = "C9", 1189 .desc = "MWAIT 0x50", 1190 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, 1191 .exit_latency = 2000, 1192 .target_residency = 2000, 1193 .enter = &intel_idle, 1194 .enter_s2idle = intel_idle_s2idle, }, 1195 { 1196 .name = "C10", 1197 .desc = "MWAIT 0x60", 1198 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, 1199 .exit_latency = 10000, 1200 .target_residency = 10000, 1201 .enter = &intel_idle, 1202 .enter_s2idle = intel_idle_s2idle, }, 1203 { 1204 .enter = NULL } 1205}; 1206 1207static struct cpuidle_state dnv_cstates[] __initdata = { 1208 { 1209 .name = "C1", 1210 .desc = "MWAIT 0x00", 1211 .flags = MWAIT2flg(0x00), 1212 .exit_latency = 2, 1213 .target_residency = 2, 1214 .enter = &intel_idle, 1215 .enter_s2idle = intel_idle_s2idle, }, 1216 { 1217 .name = "C1E", 1218 .desc = "MWAIT 0x01", 1219 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1220 .exit_latency = 10, 1221 .target_residency = 20, 1222 .enter = &intel_idle, 1223 .enter_s2idle = intel_idle_s2idle, }, 1224 { 1225 .name = "C6", 1226 .desc = "MWAIT 0x20", 1227 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1228 .exit_latency = 50, 1229 .target_residency = 500, 1230 .enter = &intel_idle, 1231 .enter_s2idle = intel_idle_s2idle, }, 1232 { 1233 .enter = NULL } 1234}; 1235 1236/* 1237 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support 1238 * C6, and this is indicated in the CPUID mwait leaf. 1239 */ 1240static struct cpuidle_state snr_cstates[] __initdata = { 1241 { 1242 .name = "C1", 1243 .desc = "MWAIT 0x00", 1244 .flags = MWAIT2flg(0x00), 1245 .exit_latency = 2, 1246 .target_residency = 2, 1247 .enter = &intel_idle, 1248 .enter_s2idle = intel_idle_s2idle, }, 1249 { 1250 .name = "C1E", 1251 .desc = "MWAIT 0x01", 1252 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1253 .exit_latency = 15, 1254 .target_residency = 25, 1255 .enter = &intel_idle, 1256 .enter_s2idle = intel_idle_s2idle, }, 1257 { 1258 .name = "C6", 1259 .desc = "MWAIT 0x20", 1260 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, 1261 .exit_latency = 130, 1262 .target_residency = 500, 1263 .enter = &intel_idle, 1264 .enter_s2idle = intel_idle_s2idle, }, 1265 { 1266 .enter = NULL } 1267}; 1268 1269static struct cpuidle_state grr_cstates[] __initdata = { 1270 { 1271 .name = "C1", 1272 .desc = "MWAIT 0x00", 1273 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1274 .exit_latency = 1, 1275 .target_residency = 1, 1276 .enter = &intel_idle, 1277 .enter_s2idle = intel_idle_s2idle, }, 1278 { 1279 .name = "C1E", 1280 .desc = "MWAIT 0x01", 1281 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1282 .exit_latency = 2, 1283 .target_residency = 10, 1284 .enter = &intel_idle, 1285 .enter_s2idle = intel_idle_s2idle, }, 1286 { 1287 .name = "C6S", 1288 .desc = "MWAIT 0x22", 1289 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1290 .exit_latency = 140, 1291 .target_residency = 500, 1292 .enter = &intel_idle, 1293 .enter_s2idle = intel_idle_s2idle, }, 1294 { 1295 .enter = NULL } 1296}; 1297 1298static struct cpuidle_state srf_cstates[] __initdata = { 1299 { 1300 .name = "C1", 1301 .desc = "MWAIT 0x00", 1302 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1303 .exit_latency = 1, 1304 .target_residency = 1, 1305 .enter = &intel_idle, 1306 .enter_s2idle = intel_idle_s2idle, }, 1307 { 1308 .name = "C1E", 1309 .desc = "MWAIT 0x01", 1310 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, 1311 .exit_latency = 2, 1312 .target_residency = 10, 1313 .enter = &intel_idle, 1314 .enter_s2idle = intel_idle_s2idle, }, 1315 { 1316 .name = "C6S", 1317 .desc = "MWAIT 0x22", 1318 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED, 1319 .exit_latency = 270, 1320 .target_residency = 700, 1321 .enter = &intel_idle, 1322 .enter_s2idle = intel_idle_s2idle, }, 1323 { 1324 .name = "C6SP", 1325 .desc = "MWAIT 0x23", 1326 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED, 1327 .exit_latency = 310, 1328 .target_residency = 900, 1329 .enter = &intel_idle, 1330 .enter_s2idle = intel_idle_s2idle, }, 1331 { 1332 .enter = NULL } 1333}; 1334 1335static const struct idle_cpu idle_cpu_nehalem __initconst = { 1336 .state_table = nehalem_cstates, 1337 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1338 .disable_promotion_to_c1e = true, 1339}; 1340 1341static const struct idle_cpu idle_cpu_nhx __initconst = { 1342 .state_table = nehalem_cstates, 1343 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, 1344 .disable_promotion_to_c1e = true, 1345 .use_acpi = true, 1346}; 1347 1348static const struct idle_cpu idle_cpu_atom __initconst = { 1349 .state_table = atom_cstates, 1350}; 1351 1352static const struct idle_cpu idle_cpu_tangier __initconst = { 1353 .state_table = tangier_cstates, 1354}; 1355 1356static const struct idle_cpu idle_cpu_lincroft __initconst = { 1357 .state_table = atom_cstates, 1358 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, 1359}; 1360 1361static const struct idle_cpu idle_cpu_snb __initconst = { 1362 .state_table = snb_cstates, 1363 .disable_promotion_to_c1e = true, 1364}; 1365 1366static const struct idle_cpu idle_cpu_snx __initconst = { 1367 .state_table = snb_cstates, 1368 .disable_promotion_to_c1e = true, 1369 .use_acpi = true, 1370}; 1371 1372static const struct idle_cpu idle_cpu_byt __initconst = { 1373 .state_table = byt_cstates, 1374 .disable_promotion_to_c1e = true, 1375 .byt_auto_demotion_disable_flag = true, 1376}; 1377 1378static const struct idle_cpu idle_cpu_cht __initconst = { 1379 .state_table = cht_cstates, 1380 .disable_promotion_to_c1e = true, 1381 .byt_auto_demotion_disable_flag = true, 1382}; 1383 1384static const struct idle_cpu idle_cpu_ivb __initconst = { 1385 .state_table = ivb_cstates, 1386 .disable_promotion_to_c1e = true, 1387}; 1388 1389static const struct idle_cpu idle_cpu_ivt __initconst = { 1390 .state_table = ivt_cstates, 1391 .disable_promotion_to_c1e = true, 1392 .use_acpi = true, 1393}; 1394 1395static const struct idle_cpu idle_cpu_hsw __initconst = { 1396 .state_table = hsw_cstates, 1397 .disable_promotion_to_c1e = true, 1398}; 1399 1400static const struct idle_cpu idle_cpu_hsx __initconst = { 1401 .state_table = hsw_cstates, 1402 .disable_promotion_to_c1e = true, 1403 .use_acpi = true, 1404}; 1405 1406static const struct idle_cpu idle_cpu_bdw __initconst = { 1407 .state_table = bdw_cstates, 1408 .disable_promotion_to_c1e = true, 1409}; 1410 1411static const struct idle_cpu idle_cpu_bdx __initconst = { 1412 .state_table = bdw_cstates, 1413 .disable_promotion_to_c1e = true, 1414 .use_acpi = true, 1415}; 1416 1417static const struct idle_cpu idle_cpu_skl __initconst = { 1418 .state_table = skl_cstates, 1419 .disable_promotion_to_c1e = true, 1420}; 1421 1422static const struct idle_cpu idle_cpu_skx __initconst = { 1423 .state_table = skx_cstates, 1424 .disable_promotion_to_c1e = true, 1425 .use_acpi = true, 1426}; 1427 1428static const struct idle_cpu idle_cpu_icx __initconst = { 1429 .state_table = icx_cstates, 1430 .disable_promotion_to_c1e = true, 1431 .use_acpi = true, 1432}; 1433 1434static const struct idle_cpu idle_cpu_adl __initconst = { 1435 .state_table = adl_cstates, 1436}; 1437 1438static const struct idle_cpu idle_cpu_adl_l __initconst = { 1439 .state_table = adl_l_cstates, 1440}; 1441 1442static const struct idle_cpu idle_cpu_mtl_l __initconst = { 1443 .state_table = mtl_l_cstates, 1444}; 1445 1446static const struct idle_cpu idle_cpu_gmt __initconst = { 1447 .state_table = gmt_cstates, 1448}; 1449 1450static const struct idle_cpu idle_cpu_spr __initconst = { 1451 .state_table = spr_cstates, 1452 .disable_promotion_to_c1e = true, 1453 .use_acpi = true, 1454}; 1455 1456static const struct idle_cpu idle_cpu_avn __initconst = { 1457 .state_table = avn_cstates, 1458 .disable_promotion_to_c1e = true, 1459 .use_acpi = true, 1460}; 1461 1462static const struct idle_cpu idle_cpu_knl __initconst = { 1463 .state_table = knl_cstates, 1464 .use_acpi = true, 1465}; 1466 1467static const struct idle_cpu idle_cpu_bxt __initconst = { 1468 .state_table = bxt_cstates, 1469 .disable_promotion_to_c1e = true, 1470}; 1471 1472static const struct idle_cpu idle_cpu_dnv __initconst = { 1473 .state_table = dnv_cstates, 1474 .disable_promotion_to_c1e = true, 1475 .use_acpi = true, 1476}; 1477 1478static const struct idle_cpu idle_cpu_snr __initconst = { 1479 .state_table = snr_cstates, 1480 .disable_promotion_to_c1e = true, 1481 .use_acpi = true, 1482}; 1483 1484static const struct idle_cpu idle_cpu_grr __initconst = { 1485 .state_table = grr_cstates, 1486 .disable_promotion_to_c1e = true, 1487 .use_acpi = true, 1488}; 1489 1490static const struct idle_cpu idle_cpu_srf __initconst = { 1491 .state_table = srf_cstates, 1492 .disable_promotion_to_c1e = true, 1493 .use_acpi = true, 1494}; 1495 1496static const struct x86_cpu_id intel_idle_ids[] __initconst = { 1497 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx), 1498 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem), 1499 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem), 1500 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem), 1501 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx), 1502 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx), 1503 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom), 1504 X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft), 1505 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx), 1506 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb), 1507 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx), 1508 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom), 1509 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt), 1510 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier), 1511 X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht), 1512 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb), 1513 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt), 1514 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw), 1515 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx), 1516 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw), 1517 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw), 1518 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn), 1519 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw), 1520 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw), 1521 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx), 1522 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx), 1523 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl), 1524 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl), 1525 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl), 1526 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl), 1527 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx), 1528 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx), 1529 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx), 1530 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &idle_cpu_adl), 1531 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l), 1532 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &idle_cpu_mtl_l), 1533 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &idle_cpu_gmt), 1534 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), 1535 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &idle_cpu_spr), 1536 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), 1537 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), 1538 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), 1539 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), 1540 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv), 1541 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr), 1542 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT, &idle_cpu_grr), 1543 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X, &idle_cpu_srf), 1544 {} 1545}; 1546 1547static const struct x86_cpu_id intel_mwait_ids[] __initconst = { 1548 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), 1549 {} 1550}; 1551 1552static bool __init intel_idle_max_cstate_reached(int cstate) 1553{ 1554 if (cstate + 1 > max_cstate) { 1555 pr_info("max_cstate %d reached\n", max_cstate); 1556 return true; 1557 } 1558 return false; 1559} 1560 1561static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) 1562{ 1563 unsigned long eax = flg2MWAIT(state->flags); 1564 1565 if (boot_cpu_has(X86_FEATURE_ARAT)) 1566 return false; 1567 1568 /* 1569 * Switch over to one-shot tick broadcast if the target C-state 1570 * is deeper than C1. 1571 */ 1572 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); 1573} 1574 1575#ifdef CONFIG_ACPI_PROCESSOR_CSTATE 1576#include <acpi/processor.h> 1577 1578static bool no_acpi __read_mostly; 1579module_param(no_acpi, bool, 0444); 1580MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); 1581 1582static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ 1583module_param_named(use_acpi, force_use_acpi, bool, 0444); 1584MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); 1585 1586static struct acpi_processor_power acpi_state_table __initdata; 1587 1588/** 1589 * intel_idle_cst_usable - Check if the _CST information can be used. 1590 * 1591 * Check if all of the C-states listed by _CST in the max_cstate range are 1592 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. 1593 */ 1594static bool __init intel_idle_cst_usable(void) 1595{ 1596 int cstate, limit; 1597 1598 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), 1599 acpi_state_table.count); 1600 1601 for (cstate = 1; cstate < limit; cstate++) { 1602 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; 1603 1604 if (cx->entry_method != ACPI_CSTATE_FFH) 1605 return false; 1606 } 1607 1608 return true; 1609} 1610 1611static bool __init intel_idle_acpi_cst_extract(void) 1612{ 1613 unsigned int cpu; 1614 1615 if (no_acpi) { 1616 pr_debug("Not allowed to use ACPI _CST\n"); 1617 return false; 1618 } 1619 1620 for_each_possible_cpu(cpu) { 1621 struct acpi_processor *pr = per_cpu(processors, cpu); 1622 1623 if (!pr) 1624 continue; 1625 1626 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) 1627 continue; 1628 1629 acpi_state_table.count++; 1630 1631 if (!intel_idle_cst_usable()) 1632 continue; 1633 1634 if (!acpi_processor_claim_cst_control()) 1635 break; 1636 1637 return true; 1638 } 1639 1640 acpi_state_table.count = 0; 1641 pr_debug("ACPI _CST not found or not usable\n"); 1642 return false; 1643} 1644 1645static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) 1646{ 1647 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1648 1649 /* 1650 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1651 * the interesting states are ACPI_CSTATE_FFH. 1652 */ 1653 for (cstate = 1; cstate < limit; cstate++) { 1654 struct acpi_processor_cx *cx; 1655 struct cpuidle_state *state; 1656 1657 if (intel_idle_max_cstate_reached(cstate - 1)) 1658 break; 1659 1660 cx = &acpi_state_table.states[cstate]; 1661 1662 state = &drv->states[drv->state_count++]; 1663 1664 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); 1665 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); 1666 state->exit_latency = cx->latency; 1667 /* 1668 * For C1-type C-states use the same number for both the exit 1669 * latency and target residency, because that is the case for 1670 * C1 in the majority of the static C-states tables above. 1671 * For the other types of C-states, however, set the target 1672 * residency to 3 times the exit latency which should lead to 1673 * a reasonable balance between energy-efficiency and 1674 * performance in the majority of interesting cases. 1675 */ 1676 state->target_residency = cx->latency; 1677 if (cx->type > ACPI_STATE_C1) 1678 state->target_residency *= 3; 1679 1680 state->flags = MWAIT2flg(cx->address); 1681 if (cx->type > ACPI_STATE_C2) 1682 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; 1683 1684 if (disabled_states_mask & BIT(cstate)) 1685 state->flags |= CPUIDLE_FLAG_OFF; 1686 1687 if (intel_idle_state_needs_timer_stop(state)) 1688 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 1689 1690 state->enter = intel_idle; 1691 state->enter_s2idle = intel_idle_s2idle; 1692 } 1693} 1694 1695static bool __init intel_idle_off_by_default(u32 mwait_hint) 1696{ 1697 int cstate, limit; 1698 1699 /* 1700 * If there are no _CST C-states, do not disable any C-states by 1701 * default. 1702 */ 1703 if (!acpi_state_table.count) 1704 return false; 1705 1706 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); 1707 /* 1708 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of 1709 * the interesting states are ACPI_CSTATE_FFH. 1710 */ 1711 for (cstate = 1; cstate < limit; cstate++) { 1712 if (acpi_state_table.states[cstate].address == mwait_hint) 1713 return false; 1714 } 1715 return true; 1716} 1717#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1718#define force_use_acpi (false) 1719 1720static inline bool intel_idle_acpi_cst_extract(void) { return false; } 1721static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } 1722static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } 1723#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ 1724 1725/** 1726 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. 1727 * 1728 * Tune IVT multi-socket targets. 1729 * Assumption: num_sockets == (max_package_num + 1). 1730 */ 1731static void __init ivt_idle_state_table_update(void) 1732{ 1733 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ 1734 int cpu, package_num, num_sockets = 1; 1735 1736 for_each_online_cpu(cpu) { 1737 package_num = topology_physical_package_id(cpu); 1738 if (package_num + 1 > num_sockets) { 1739 num_sockets = package_num + 1; 1740 1741 if (num_sockets > 4) { 1742 cpuidle_state_table = ivt_cstates_8s; 1743 return; 1744 } 1745 } 1746 } 1747 1748 if (num_sockets > 2) 1749 cpuidle_state_table = ivt_cstates_4s; 1750 1751 /* else, 1 and 2 socket systems use default ivt_cstates */ 1752} 1753 1754/** 1755 * irtl_2_usec - IRTL to microseconds conversion. 1756 * @irtl: IRTL MSR value. 1757 * 1758 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. 1759 */ 1760static unsigned long long __init irtl_2_usec(unsigned long long irtl) 1761{ 1762 static const unsigned int irtl_ns_units[] __initconst = { 1763 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 1764 }; 1765 unsigned long long ns; 1766 1767 if (!irtl) 1768 return 0; 1769 1770 ns = irtl_ns_units[(irtl >> 10) & 0x7]; 1771 1772 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); 1773} 1774 1775/** 1776 * bxt_idle_state_table_update - Fix up the Broxton idle states table. 1777 * 1778 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the 1779 * definitive maximum latency and use the same value for target_residency. 1780 */ 1781static void __init bxt_idle_state_table_update(void) 1782{ 1783 unsigned long long msr; 1784 unsigned int usec; 1785 1786 rdmsrl(MSR_PKGC6_IRTL, msr); 1787 usec = irtl_2_usec(msr); 1788 if (usec) { 1789 bxt_cstates[2].exit_latency = usec; 1790 bxt_cstates[2].target_residency = usec; 1791 } 1792 1793 rdmsrl(MSR_PKGC7_IRTL, msr); 1794 usec = irtl_2_usec(msr); 1795 if (usec) { 1796 bxt_cstates[3].exit_latency = usec; 1797 bxt_cstates[3].target_residency = usec; 1798 } 1799 1800 rdmsrl(MSR_PKGC8_IRTL, msr); 1801 usec = irtl_2_usec(msr); 1802 if (usec) { 1803 bxt_cstates[4].exit_latency = usec; 1804 bxt_cstates[4].target_residency = usec; 1805 } 1806 1807 rdmsrl(MSR_PKGC9_IRTL, msr); 1808 usec = irtl_2_usec(msr); 1809 if (usec) { 1810 bxt_cstates[5].exit_latency = usec; 1811 bxt_cstates[5].target_residency = usec; 1812 } 1813 1814 rdmsrl(MSR_PKGC10_IRTL, msr); 1815 usec = irtl_2_usec(msr); 1816 if (usec) { 1817 bxt_cstates[6].exit_latency = usec; 1818 bxt_cstates[6].target_residency = usec; 1819 } 1820 1821} 1822 1823/** 1824 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. 1825 * 1826 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. 1827 */ 1828static void __init sklh_idle_state_table_update(void) 1829{ 1830 unsigned long long msr; 1831 unsigned int eax, ebx, ecx, edx; 1832 1833 1834 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ 1835 if (max_cstate <= 7) 1836 return; 1837 1838 /* if PC10 not present in CPUID.MWAIT.EDX */ 1839 if ((mwait_substates & (0xF << 28)) == 0) 1840 return; 1841 1842 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1843 1844 /* PC10 is not enabled in PKG C-state limit */ 1845 if ((msr & 0xF) != 8) 1846 return; 1847 1848 ecx = 0; 1849 cpuid(7, &eax, &ebx, &ecx, &edx); 1850 1851 /* if SGX is present */ 1852 if (ebx & (1 << 2)) { 1853 1854 rdmsrl(MSR_IA32_FEAT_CTL, msr); 1855 1856 /* if SGX is enabled */ 1857 if (msr & (1 << 18)) 1858 return; 1859 } 1860 1861 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ 1862 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ 1863} 1864 1865/** 1866 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake 1867 * idle states table. 1868 */ 1869static void __init skx_idle_state_table_update(void) 1870{ 1871 unsigned long long msr; 1872 1873 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1874 1875 /* 1876 * 000b: C0/C1 (no package C-state support) 1877 * 001b: C2 1878 * 010b: C6 (non-retention) 1879 * 011b: C6 (retention) 1880 * 111b: No Package C state limits. 1881 */ 1882 if ((msr & 0x7) < 2) { 1883 /* 1884 * Uses the CC6 + PC0 latency and 3 times of 1885 * latency for target_residency if the PC6 1886 * is disabled in BIOS. This is consistent 1887 * with how intel_idle driver uses _CST 1888 * to set the target_residency. 1889 */ 1890 skx_cstates[2].exit_latency = 92; 1891 skx_cstates[2].target_residency = 276; 1892 } 1893} 1894 1895/** 1896 * adl_idle_state_table_update - Adjust AlderLake idle states table. 1897 */ 1898static void __init adl_idle_state_table_update(void) 1899{ 1900 /* Check if user prefers C1 over C1E. */ 1901 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { 1902 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; 1903 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; 1904 1905 /* Disable C1E by clearing the "C1E promotion" bit. */ 1906 c1e_promotion = C1E_PROMOTION_DISABLE; 1907 return; 1908 } 1909 1910 /* Make sure C1E is enabled by default */ 1911 c1e_promotion = C1E_PROMOTION_ENABLE; 1912} 1913 1914/** 1915 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. 1916 */ 1917static void __init spr_idle_state_table_update(void) 1918{ 1919 unsigned long long msr; 1920 1921 /* 1922 * By default, the C6 state assumes the worst-case scenario of package 1923 * C6. However, if PC6 is disabled, we update the numbers to match 1924 * core C6. 1925 */ 1926 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); 1927 1928 /* Limit value 2 and above allow for PC6. */ 1929 if ((msr & 0x7) < 2) { 1930 spr_cstates[2].exit_latency = 190; 1931 spr_cstates[2].target_residency = 600; 1932 } 1933} 1934 1935static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) 1936{ 1937 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) & 1938 MWAIT_CSTATE_MASK; 1939 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & 1940 MWAIT_SUBSTATE_MASK; 1941 1942 /* Ignore the C-state if there are NO sub-states in CPUID for it. */ 1943 if (num_substates == 0) 1944 return false; 1945 1946 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) 1947 mark_tsc_unstable("TSC halts in idle states deeper than C2"); 1948 1949 return true; 1950} 1951 1952static void state_update_enter_method(struct cpuidle_state *state, int cstate) 1953{ 1954 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) { 1955 /* 1956 * Combining with XSTATE with IBRS or IRQ_ENABLE flags 1957 * is not currently supported but this driver. 1958 */ 1959 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS); 1960 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 1961 state->enter = intel_idle_xstate; 1962 return; 1963 } 1964 1965 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && 1966 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) { 1967 /* 1968 * IBRS mitigation requires that C-states are entered 1969 * with interrupts disabled. 1970 */ 1971 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE)) 1972 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE; 1973 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE); 1974 state->enter = intel_idle_ibrs; 1975 return; 1976 } 1977 1978 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) { 1979 state->enter = intel_idle_irq; 1980 return; 1981 } 1982 1983 if (force_irq_on) { 1984 pr_info("forced intel_idle_irq for state %d\n", cstate); 1985 state->enter = intel_idle_irq; 1986 } 1987} 1988 1989static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) 1990{ 1991 int cstate; 1992 1993 switch (boot_cpu_data.x86_model) { 1994 case INTEL_FAM6_IVYBRIDGE_X: 1995 ivt_idle_state_table_update(); 1996 break; 1997 case INTEL_FAM6_ATOM_GOLDMONT: 1998 case INTEL_FAM6_ATOM_GOLDMONT_PLUS: 1999 bxt_idle_state_table_update(); 2000 break; 2001 case INTEL_FAM6_SKYLAKE: 2002 sklh_idle_state_table_update(); 2003 break; 2004 case INTEL_FAM6_SKYLAKE_X: 2005 skx_idle_state_table_update(); 2006 break; 2007 case INTEL_FAM6_SAPPHIRERAPIDS_X: 2008 case INTEL_FAM6_EMERALDRAPIDS_X: 2009 spr_idle_state_table_update(); 2010 break; 2011 case INTEL_FAM6_ALDERLAKE: 2012 case INTEL_FAM6_ALDERLAKE_L: 2013 case INTEL_FAM6_ATOM_GRACEMONT: 2014 adl_idle_state_table_update(); 2015 break; 2016 } 2017 2018 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { 2019 struct cpuidle_state *state; 2020 unsigned int mwait_hint; 2021 2022 if (intel_idle_max_cstate_reached(cstate)) 2023 break; 2024 2025 if (!cpuidle_state_table[cstate].enter && 2026 !cpuidle_state_table[cstate].enter_s2idle) 2027 break; 2028 2029 /* If marked as unusable, skip this state. */ 2030 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { 2031 pr_debug("state %s is disabled\n", 2032 cpuidle_state_table[cstate].name); 2033 continue; 2034 } 2035 2036 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); 2037 if (!intel_idle_verify_cstate(mwait_hint)) 2038 continue; 2039 2040 /* Structure copy. */ 2041 drv->states[drv->state_count] = cpuidle_state_table[cstate]; 2042 state = &drv->states[drv->state_count]; 2043 2044 state_update_enter_method(state, cstate); 2045 2046 2047 if ((disabled_states_mask & BIT(drv->state_count)) || 2048 ((icpu->use_acpi || force_use_acpi) && 2049 intel_idle_off_by_default(mwait_hint) && 2050 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) 2051 state->flags |= CPUIDLE_FLAG_OFF; 2052 2053 if (intel_idle_state_needs_timer_stop(state)) 2054 state->flags |= CPUIDLE_FLAG_TIMER_STOP; 2055 2056 drv->state_count++; 2057 } 2058 2059 if (icpu->byt_auto_demotion_disable_flag) { 2060 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); 2061 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); 2062 } 2063} 2064 2065/** 2066 * intel_idle_cpuidle_driver_init - Create the list of available idle states. 2067 * @drv: cpuidle driver structure to initialize. 2068 */ 2069static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) 2070{ 2071 cpuidle_poll_state_init(drv); 2072 2073 if (disabled_states_mask & BIT(0)) 2074 drv->states[0].flags |= CPUIDLE_FLAG_OFF; 2075 2076 drv->state_count = 1; 2077 2078 if (icpu) 2079 intel_idle_init_cstates_icpu(drv); 2080 else 2081 intel_idle_init_cstates_acpi(drv); 2082} 2083 2084static void auto_demotion_disable(void) 2085{ 2086 unsigned long long msr_bits; 2087 2088 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2089 msr_bits &= ~auto_demotion_disable_flags; 2090 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); 2091} 2092 2093static void c1e_promotion_enable(void) 2094{ 2095 unsigned long long msr_bits; 2096 2097 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2098 msr_bits |= 0x2; 2099 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2100} 2101 2102static void c1e_promotion_disable(void) 2103{ 2104 unsigned long long msr_bits; 2105 2106 rdmsrl(MSR_IA32_POWER_CTL, msr_bits); 2107 msr_bits &= ~0x2; 2108 wrmsrl(MSR_IA32_POWER_CTL, msr_bits); 2109} 2110 2111/** 2112 * intel_idle_cpu_init - Register the target CPU with the cpuidle core. 2113 * @cpu: CPU to initialize. 2114 * 2115 * Register a cpuidle device object for @cpu and update its MSRs in accordance 2116 * with the processor model flags. 2117 */ 2118static int intel_idle_cpu_init(unsigned int cpu) 2119{ 2120 struct cpuidle_device *dev; 2121 2122 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2123 dev->cpu = cpu; 2124 2125 if (cpuidle_register_device(dev)) { 2126 pr_debug("cpuidle_register_device %d failed!\n", cpu); 2127 return -EIO; 2128 } 2129 2130 if (auto_demotion_disable_flags) 2131 auto_demotion_disable(); 2132 2133 if (c1e_promotion == C1E_PROMOTION_ENABLE) 2134 c1e_promotion_enable(); 2135 else if (c1e_promotion == C1E_PROMOTION_DISABLE) 2136 c1e_promotion_disable(); 2137 2138 return 0; 2139} 2140 2141static int intel_idle_cpu_online(unsigned int cpu) 2142{ 2143 struct cpuidle_device *dev; 2144 2145 if (!boot_cpu_has(X86_FEATURE_ARAT)) 2146 tick_broadcast_enable(); 2147 2148 /* 2149 * Some systems can hotplug a cpu at runtime after 2150 * the kernel has booted, we have to initialize the 2151 * driver in this case 2152 */ 2153 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); 2154 if (!dev->registered) 2155 return intel_idle_cpu_init(cpu); 2156 2157 return 0; 2158} 2159 2160/** 2161 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. 2162 */ 2163static void __init intel_idle_cpuidle_devices_uninit(void) 2164{ 2165 int i; 2166 2167 for_each_online_cpu(i) 2168 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); 2169} 2170 2171static int __init intel_idle_init(void) 2172{ 2173 const struct x86_cpu_id *id; 2174 unsigned int eax, ebx, ecx; 2175 int retval; 2176 2177 /* Do not load intel_idle at all for now if idle= is passed */ 2178 if (boot_option_idle_override != IDLE_NO_OVERRIDE) 2179 return -ENODEV; 2180 2181 if (max_cstate == 0) { 2182 pr_debug("disabled\n"); 2183 return -EPERM; 2184 } 2185 2186 id = x86_match_cpu(intel_idle_ids); 2187 if (id) { 2188 if (!boot_cpu_has(X86_FEATURE_MWAIT)) { 2189 pr_debug("Please enable MWAIT in BIOS SETUP\n"); 2190 return -ENODEV; 2191 } 2192 } else { 2193 id = x86_match_cpu(intel_mwait_ids); 2194 if (!id) 2195 return -ENODEV; 2196 } 2197 2198 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) 2199 return -ENODEV; 2200 2201 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); 2202 2203 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || 2204 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || 2205 !mwait_substates) 2206 return -ENODEV; 2207 2208 pr_debug("MWAIT substates: 0x%x\n", mwait_substates); 2209 2210 icpu = (const struct idle_cpu *)id->driver_data; 2211 if (icpu) { 2212 cpuidle_state_table = icpu->state_table; 2213 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; 2214 if (icpu->disable_promotion_to_c1e) 2215 c1e_promotion = C1E_PROMOTION_DISABLE; 2216 if (icpu->use_acpi || force_use_acpi) 2217 intel_idle_acpi_cst_extract(); 2218 } else if (!intel_idle_acpi_cst_extract()) { 2219 return -ENODEV; 2220 } 2221 2222 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", 2223 boot_cpu_data.x86_model); 2224 2225 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); 2226 if (!intel_idle_cpuidle_devices) 2227 return -ENOMEM; 2228 2229 intel_idle_cpuidle_driver_init(&intel_idle_driver); 2230 2231 retval = cpuidle_register_driver(&intel_idle_driver); 2232 if (retval) { 2233 struct cpuidle_driver *drv = cpuidle_get_driver(); 2234 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), 2235 drv ? drv->name : "none"); 2236 goto init_driver_fail; 2237 } 2238 2239 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", 2240 intel_idle_cpu_online, NULL); 2241 if (retval < 0) 2242 goto hp_setup_fail; 2243 2244 pr_debug("Local APIC timer is reliable in %s\n", 2245 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); 2246 2247 return 0; 2248 2249hp_setup_fail: 2250 intel_idle_cpuidle_devices_uninit(); 2251 cpuidle_unregister_driver(&intel_idle_driver); 2252init_driver_fail: 2253 free_percpu(intel_idle_cpuidle_devices); 2254 return retval; 2255 2256} 2257device_initcall(intel_idle_init); 2258 2259/* 2260 * We are not really modular, but we used to support that. Meaning we also 2261 * support "intel_idle.max_cstate=..." at boot and also a read-only export of 2262 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param 2263 * is the easiest way (currently) to continue doing that. 2264 */ 2265module_param(max_cstate, int, 0444); 2266/* 2267 * The positions of the bits that are set in this number are the indices of the 2268 * idle states to be disabled by default (as reflected by the names of the 2269 * corresponding idle state directories in sysfs, "state0", "state1" ... 2270 * "state<i>" ..., where <i> is the index of the given state). 2271 */ 2272module_param_named(states_off, disabled_states_mask, uint, 0444); 2273MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); 2274/* 2275 * Some platforms come with mutually exclusive C-states, so that if one is 2276 * enabled, the other C-states must not be used. Example: C1 and C1E on 2277 * Sapphire Rapids platform. This parameter allows for selecting the 2278 * preferred C-states among the groups of mutually exclusive C-states - the 2279 * selected C-states will be registered, the other C-states from the mutually 2280 * exclusive group won't be registered. If the platform has no mutually 2281 * exclusive C-states, this parameter has no effect. 2282 */ 2283module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); 2284MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); 2285/* 2286 * Debugging option that forces the driver to enter all C-states with 2287 * interrupts enabled. Does not apply to C-states with 2288 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags. 2289 */ 2290module_param(force_irq_on, bool, 0444); 2291/* 2292 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and 2293 * CPUIDLE_FLAG_IRQ_ENABLE isn't set. 2294 */ 2295module_param(ibrs_off, bool, 0444); 2296MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); 2297