1/* 2 * NMI watchdog support on APIC systems 3 * 4 * Started by Ingo Molnar <mingo@redhat.com> 5 * 6 * Fixes: 7 * Mikael Pettersson : AMD K7 support for local APIC NMI watchdog. 8 * Mikael Pettersson : Power Management for local APIC NMI watchdog. 9 * Mikael Pettersson : Pentium 4 support for local APIC NMI watchdog. 10 * Pavel Machek and 11 * Mikael Pettersson : PM converted to driver model. Disable/enable API. 12 */ 13 14#include <asm/apic.h> 15 16#include <linux/nmi.h> 17#include <linux/mm.h> 18#include <linux/delay.h> 19#include <linux/interrupt.h> 20#include <linux/module.h> 21#include <linux/slab.h> 22#include <linux/sysdev.h> 23#include <linux/sysctl.h> 24#include <linux/percpu.h> 25#include <linux/kprobes.h> 26#include <linux/cpumask.h> 27#include <linux/kernel_stat.h> 28#include <linux/kdebug.h> 29#include <linux/smp.h> 30 31#include <asm/i8259.h> 32#include <asm/io_apic.h> 33#include <asm/proto.h> 34#include <asm/timer.h> 35 36#include <asm/mce.h> 37 38#include <asm/mach_traps.h> 39 40int unknown_nmi_panic; 41int nmi_watchdog_enabled; 42 43/* For reliability, we're prepared to waste bits here. */ 44static DECLARE_BITMAP(backtrace_mask, NR_CPUS) __read_mostly; 45 46/* nmi_active: 47 * >0: the lapic NMI watchdog is active, but can be disabled 48 * <0: the lapic NMI watchdog has not been set up, and cannot 49 * be enabled 50 * 0: the lapic NMI watchdog is disabled, but can be enabled 51 */ 52atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */ 53EXPORT_SYMBOL(nmi_active); 54 55unsigned int nmi_watchdog = NMI_NONE; 56EXPORT_SYMBOL(nmi_watchdog); 57 58static int panic_on_timeout; 59 60static unsigned int nmi_hz = HZ; 61static DEFINE_PER_CPU(short, wd_enabled); 62static int endflag __initdata; 63 64static inline unsigned int get_nmi_count(int cpu) 65{ 66 return per_cpu(irq_stat, cpu).__nmi_count; 67} 68 69static inline int mce_in_progress(void) 70{ 71#if defined(CONFIG_X86_MCE) 72 return atomic_read(&mce_entry) > 0; 73#endif 74 return 0; 75} 76 77/* 78 * Take the local apic timer and PIT/HPET into account. We don't 79 * know which one is active, when we have highres/dyntick on 80 */ 81static inline unsigned int get_timer_irqs(int cpu) 82{ 83 return per_cpu(irq_stat, cpu).apic_timer_irqs + 84 per_cpu(irq_stat, cpu).irq0_irqs; 85} 86 87#ifdef CONFIG_SMP 88/* 89 * The performance counters used by NMI_LOCAL_APIC don't trigger when 90 * the CPU is idle. To make sure the NMI watchdog really ticks on all 91 * CPUs during the test make them busy. 92 */ 93static __init void nmi_cpu_busy(void *data) 94{ 95 local_irq_enable_in_hardirq(); 96 /* 97 * Intentionally don't use cpu_relax here. This is 98 * to make sure that the performance counter really ticks, 99 * even if there is a simulator or similar that catches the 100 * pause instruction. On a real HT machine this is fine because 101 * all other CPUs are busy with "useless" delay loops and don't 102 * care if they get somewhat less cycles. 103 */ 104 while (endflag == 0) 105 mb(); 106} 107#endif 108 109static void report_broken_nmi(int cpu, unsigned int *prev_nmi_count) 110{ 111 printk(KERN_CONT "\n"); 112 113 printk(KERN_WARNING 114 "WARNING: CPU#%d: NMI appears to be stuck (%d->%d)!\n", 115 cpu, prev_nmi_count[cpu], get_nmi_count(cpu)); 116 117 printk(KERN_WARNING 118 "Please report this to bugzilla.kernel.org,\n"); 119 printk(KERN_WARNING 120 "and attach the output of the 'dmesg' command.\n"); 121 122 per_cpu(wd_enabled, cpu) = 0; 123 atomic_dec(&nmi_active); 124} 125 126static void __acpi_nmi_disable(void *__unused) 127{ 128 apic_write(APIC_LVT0, APIC_DM_NMI | APIC_LVT_MASKED); 129} 130 131int __init check_nmi_watchdog(void) 132{ 133 unsigned int *prev_nmi_count; 134 int cpu; 135 136 if (!nmi_watchdog_active() || !atomic_read(&nmi_active)) 137 return 0; 138 139 prev_nmi_count = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); 140 if (!prev_nmi_count) 141 goto error; 142 143 printk(KERN_INFO "Testing NMI watchdog ... "); 144 145#ifdef CONFIG_SMP 146 if (nmi_watchdog == NMI_LOCAL_APIC) 147 smp_call_function(nmi_cpu_busy, (void *)&endflag, 0); 148#endif 149 150 for_each_possible_cpu(cpu) 151 prev_nmi_count[cpu] = get_nmi_count(cpu); 152 local_irq_enable(); 153 mdelay((20 * 1000) / nmi_hz); /* wait 20 ticks */ 154 155 for_each_online_cpu(cpu) { 156 if (!per_cpu(wd_enabled, cpu)) 157 continue; 158 if (get_nmi_count(cpu) - prev_nmi_count[cpu] <= 5) 159 report_broken_nmi(cpu, prev_nmi_count); 160 } 161 endflag = 1; 162 if (!atomic_read(&nmi_active)) { 163 kfree(prev_nmi_count); 164 atomic_set(&nmi_active, -1); 165 goto error; 166 } 167 printk("OK.\n"); 168 169 /* 170 * now that we know it works we can reduce NMI frequency to 171 * something more reasonable; makes a difference in some configs 172 */ 173 if (nmi_watchdog == NMI_LOCAL_APIC) 174 nmi_hz = lapic_adjust_nmi_hz(1); 175 176 kfree(prev_nmi_count); 177 return 0; 178error: 179 if (nmi_watchdog == NMI_IO_APIC) { 180 if (!timer_through_8259) 181 legacy_pic->chip->mask(0); 182 on_each_cpu(__acpi_nmi_disable, NULL, 1); 183 } 184 185#ifdef CONFIG_X86_32 186 timer_ack = 0; 187#endif 188 return -1; 189} 190 191static int __init setup_nmi_watchdog(char *str) 192{ 193 unsigned int nmi; 194 195 if (!strncmp(str, "panic", 5)) { 196 panic_on_timeout = 1; 197 str = strchr(str, ','); 198 if (!str) 199 return 1; 200 ++str; 201 } 202 203 if (!strncmp(str, "lapic", 5)) 204 nmi_watchdog = NMI_LOCAL_APIC; 205 else if (!strncmp(str, "ioapic", 6)) 206 nmi_watchdog = NMI_IO_APIC; 207 else { 208 get_option(&str, &nmi); 209 if (nmi >= NMI_INVALID) 210 return 0; 211 nmi_watchdog = nmi; 212 } 213 214 return 1; 215} 216__setup("nmi_watchdog=", setup_nmi_watchdog); 217 218/* 219 * Suspend/resume support 220 */ 221#ifdef CONFIG_PM 222 223static int nmi_pm_active; /* nmi_active before suspend */ 224 225static int lapic_nmi_suspend(struct sys_device *dev, pm_message_t state) 226{ 227 /* only CPU0 goes here, other CPUs should be offline */ 228 nmi_pm_active = atomic_read(&nmi_active); 229 stop_apic_nmi_watchdog(NULL); 230 BUG_ON(atomic_read(&nmi_active) != 0); 231 return 0; 232} 233 234static int lapic_nmi_resume(struct sys_device *dev) 235{ 236 /* only CPU0 goes here, other CPUs should be offline */ 237 if (nmi_pm_active > 0) { 238 setup_apic_nmi_watchdog(NULL); 239 touch_nmi_watchdog(); 240 } 241 return 0; 242} 243 244static struct sysdev_class nmi_sysclass = { 245 .name = "lapic_nmi", 246 .resume = lapic_nmi_resume, 247 .suspend = lapic_nmi_suspend, 248}; 249 250static struct sys_device device_lapic_nmi = { 251 .id = 0, 252 .cls = &nmi_sysclass, 253}; 254 255static int __init init_lapic_nmi_sysfs(void) 256{ 257 int error; 258 259 /* 260 * should really be a BUG_ON but b/c this is an 261 * init call, it just doesn't work. -dcz 262 */ 263 if (nmi_watchdog != NMI_LOCAL_APIC) 264 return 0; 265 266 if (atomic_read(&nmi_active) < 0) 267 return 0; 268 269 error = sysdev_class_register(&nmi_sysclass); 270 if (!error) 271 error = sysdev_register(&device_lapic_nmi); 272 return error; 273} 274 275/* must come after the local APIC's device_initcall() */ 276late_initcall(init_lapic_nmi_sysfs); 277 278#endif /* CONFIG_PM */ 279 280static void __acpi_nmi_enable(void *__unused) 281{ 282 apic_write(APIC_LVT0, APIC_DM_NMI); 283} 284 285/* 286 * Enable timer based NMIs on all CPUs: 287 */ 288void acpi_nmi_enable(void) 289{ 290 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC) 291 on_each_cpu(__acpi_nmi_enable, NULL, 1); 292} 293 294/* 295 * Disable timer based NMIs on all CPUs: 296 */ 297void acpi_nmi_disable(void) 298{ 299 if (atomic_read(&nmi_active) && nmi_watchdog == NMI_IO_APIC) 300 on_each_cpu(__acpi_nmi_disable, NULL, 1); 301} 302 303/* 304 * This function is called as soon the LAPIC NMI watchdog driver has everything 305 * in place and it's ready to check if the NMIs belong to the NMI watchdog 306 */ 307void cpu_nmi_set_wd_enabled(void) 308{ 309 __get_cpu_var(wd_enabled) = 1; 310} 311 312void setup_apic_nmi_watchdog(void *unused) 313{ 314 if (__get_cpu_var(wd_enabled)) 315 return; 316 317 /* cheap hack to support suspend/resume */ 318 /* if cpu0 is not active neither should the other cpus */ 319 if (smp_processor_id() != 0 && atomic_read(&nmi_active) <= 0) 320 return; 321 322 switch (nmi_watchdog) { 323 case NMI_LOCAL_APIC: 324 if (lapic_watchdog_init(nmi_hz) < 0) { 325 __get_cpu_var(wd_enabled) = 0; 326 return; 327 } 328 /* FALL THROUGH */ 329 case NMI_IO_APIC: 330 __get_cpu_var(wd_enabled) = 1; 331 atomic_inc(&nmi_active); 332 } 333} 334 335void stop_apic_nmi_watchdog(void *unused) 336{ 337 /* only support LOCAL and IO APICs for now */ 338 if (!nmi_watchdog_active()) 339 return; 340 if (__get_cpu_var(wd_enabled) == 0) 341 return; 342 if (nmi_watchdog == NMI_LOCAL_APIC) 343 lapic_watchdog_stop(); 344 else 345 __acpi_nmi_disable(NULL); 346 __get_cpu_var(wd_enabled) = 0; 347 atomic_dec(&nmi_active); 348} 349 350/* 351 * the best way to detect whether a CPU has a 'hard lockup' problem 352 * is to check it's local APIC timer IRQ counts. If they are not 353 * changing then that CPU has some problem. 354 * 355 * as these watchdog NMI IRQs are generated on every CPU, we only 356 * have to check the current processor. 357 * 358 * since NMIs don't listen to _any_ locks, we have to be extremely 359 * careful not to rely on unsafe variables. The printk might lock 360 * up though, so we have to break up any console locks first ... 361 * [when there will be more tty-related locks, break them up here too!] 362 */ 363 364static DEFINE_PER_CPU(unsigned, last_irq_sum); 365static DEFINE_PER_CPU(long, alert_counter); 366static DEFINE_PER_CPU(int, nmi_touch); 367 368void touch_nmi_watchdog(void) 369{ 370 if (nmi_watchdog_active()) { 371 unsigned cpu; 372 373 /* 374 * Tell other CPUs to reset their alert counters. We cannot 375 * do it ourselves because the alert count increase is not 376 * atomic. 377 */ 378 for_each_present_cpu(cpu) { 379 if (per_cpu(nmi_touch, cpu) != 1) 380 per_cpu(nmi_touch, cpu) = 1; 381 } 382 } 383 384 /* 385 * Tickle the softlockup detector too: 386 */ 387 touch_softlockup_watchdog(); 388} 389EXPORT_SYMBOL(touch_nmi_watchdog); 390 391notrace __kprobes int 392nmi_watchdog_tick(struct pt_regs *regs, unsigned reason) 393{ 394 /* 395 * Since current_thread_info()-> is always on the stack, and we 396 * always switch the stack NMI-atomically, it's safe to use 397 * smp_processor_id(). 398 */ 399 unsigned int sum; 400 int touched = 0; 401 int cpu = smp_processor_id(); 402 int rc = 0; 403 404 sum = get_timer_irqs(cpu); 405 406 if (__get_cpu_var(nmi_touch)) { 407 __get_cpu_var(nmi_touch) = 0; 408 touched = 1; 409 } 410 411 /* We can be called before check_nmi_watchdog, hence NULL check. */ 412 if (cpumask_test_cpu(cpu, to_cpumask(backtrace_mask))) { 413 static DEFINE_RAW_SPINLOCK(lock); /* Serialise the printks */ 414 415 raw_spin_lock(&lock); 416 printk(KERN_WARNING "NMI backtrace for cpu %d\n", cpu); 417 show_regs(regs); 418 dump_stack(); 419 raw_spin_unlock(&lock); 420 cpumask_clear_cpu(cpu, to_cpumask(backtrace_mask)); 421 422 rc = 1; 423 } 424 425 /* Could check oops_in_progress here too, but it's safer not to */ 426 if (mce_in_progress()) 427 touched = 1; 428 429 /* if the none of the timers isn't firing, this cpu isn't doing much */ 430 if (!touched && __get_cpu_var(last_irq_sum) == sum) { 431 /* 432 * Ayiee, looks like this CPU is stuck ... 433 * wait a few IRQs (5 seconds) before doing the oops ... 434 */ 435 __this_cpu_inc(alert_counter); 436 if (__this_cpu_read(alert_counter) == 5 * nmi_hz) 437 /* 438 * die_nmi will return ONLY if NOTIFY_STOP happens.. 439 */ 440 die_nmi("BUG: NMI Watchdog detected LOCKUP", 441 regs, panic_on_timeout); 442 } else { 443 __get_cpu_var(last_irq_sum) = sum; 444 __this_cpu_write(alert_counter, 0); 445 } 446 447 /* see if the nmi watchdog went off */ 448 if (!__get_cpu_var(wd_enabled)) 449 return rc; 450 switch (nmi_watchdog) { 451 case NMI_LOCAL_APIC: 452 rc |= lapic_wd_event(nmi_hz); 453 break; 454 case NMI_IO_APIC: 455 /* 456 * don't know how to accurately check for this. 457 * just assume it was a watchdog timer interrupt 458 * This matches the old behaviour. 459 */ 460 rc = 1; 461 break; 462 } 463 return rc; 464} 465 466#ifdef CONFIG_SYSCTL 467 468static void enable_ioapic_nmi_watchdog_single(void *unused) 469{ 470 __get_cpu_var(wd_enabled) = 1; 471 atomic_inc(&nmi_active); 472 __acpi_nmi_enable(NULL); 473} 474 475static void enable_ioapic_nmi_watchdog(void) 476{ 477 on_each_cpu(enable_ioapic_nmi_watchdog_single, NULL, 1); 478 touch_nmi_watchdog(); 479} 480 481static void disable_ioapic_nmi_watchdog(void) 482{ 483 on_each_cpu(stop_apic_nmi_watchdog, NULL, 1); 484} 485 486static int __init setup_unknown_nmi_panic(char *str) 487{ 488 unknown_nmi_panic = 1; 489 return 1; 490} 491__setup("unknown_nmi_panic", setup_unknown_nmi_panic); 492 493static int unknown_nmi_panic_callback(struct pt_regs *regs, int cpu) 494{ 495 unsigned char reason = get_nmi_reason(); 496 char buf[64]; 497 498 sprintf(buf, "NMI received for unknown reason %02x\n", reason); 499 die_nmi(buf, regs, 1); /* Always panic here */ 500 return 0; 501} 502 503/* 504 * proc handler for /proc/sys/kernel/nmi 505 */ 506int proc_nmi_enabled(struct ctl_table *table, int write, 507 void __user *buffer, size_t *length, loff_t *ppos) 508{ 509 int old_state; 510 511 nmi_watchdog_enabled = (atomic_read(&nmi_active) > 0) ? 1 : 0; 512 old_state = nmi_watchdog_enabled; 513 proc_dointvec(table, write, buffer, length, ppos); 514 if (!!old_state == !!nmi_watchdog_enabled) 515 return 0; 516 517 if (atomic_read(&nmi_active) < 0 || !nmi_watchdog_active()) { 518 printk(KERN_WARNING 519 "NMI watchdog is permanently disabled\n"); 520 return -EIO; 521 } 522 523 if (nmi_watchdog == NMI_LOCAL_APIC) { 524 if (nmi_watchdog_enabled) 525 enable_lapic_nmi_watchdog(); 526 else 527 disable_lapic_nmi_watchdog(); 528 } else if (nmi_watchdog == NMI_IO_APIC) { 529 if (nmi_watchdog_enabled) 530 enable_ioapic_nmi_watchdog(); 531 else 532 disable_ioapic_nmi_watchdog(); 533 } else { 534 printk(KERN_WARNING 535 "NMI watchdog doesn't know what hardware to touch\n"); 536 return -EIO; 537 } 538 return 0; 539} 540 541#endif /* CONFIG_SYSCTL */ 542 543int do_nmi_callback(struct pt_regs *regs, int cpu) 544{ 545#ifdef CONFIG_SYSCTL 546 if (unknown_nmi_panic) 547 return unknown_nmi_panic_callback(regs, cpu); 548#endif 549 return 0; 550} 551 552void arch_trigger_all_cpu_backtrace(void) 553{ 554 int i; 555 556 cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask); 557 558 printk(KERN_INFO "sending NMI to all CPUs:\n"); 559 apic->send_IPI_all(NMI_VECTOR); 560 561 /* Wait for up to 10 seconds for all CPUs to do the backtrace */ 562 for (i = 0; i < 10 * 1000; i++) { 563 if (cpumask_empty(to_cpumask(backtrace_mask))) 564 break; 565 mdelay(1); 566 } 567} 568