1/* 2 * Local APIC handling, local APIC timers 3 * 4 * (c) 1999, 2000 Ingo Molnar <mingo@redhat.com> 5 * 6 * Fixes 7 * Maciej W. Rozycki : Bits for genuine 82489DX APICs; 8 * thanks to Eric Gilmore 9 * and Rolf G. Tews 10 * for testing these extensively. 11 * Maciej W. Rozycki : Various updates and fixes. 12 * Mikael Pettersson : Power Management for UP-APIC. 13 * Pavel Machek and 14 * Mikael Pettersson : PM converted to driver model. 15 */ 16 17#include <linux/init.h> 18 19#include <linux/mm.h> 20#include <linux/delay.h> 21#include <linux/bootmem.h> 22#include <linux/interrupt.h> 23#include <linux/mc146818rtc.h> 24#include <linux/kernel_stat.h> 25#include <linux/sysdev.h> 26#include <linux/module.h> 27#include <linux/ioport.h> 28 29#include <asm/atomic.h> 30#include <asm/smp.h> 31#include <asm/mtrr.h> 32#include <asm/mpspec.h> 33#include <asm/pgalloc.h> 34#include <asm/mach_apic.h> 35#include <asm/nmi.h> 36#include <asm/idle.h> 37#include <asm/proto.h> 38#include <asm/timex.h> 39#include <asm/hpet.h> 40#include <asm/apic.h> 41 42int apic_mapped; 43int apic_verbosity; 44int apic_runs_main_timer; 45int apic_calibrate_pmtmr __initdata; 46 47int disable_apic_timer __initdata; 48 49/* Local APIC timer works in C2? */ 50int local_apic_timer_c2_ok; 51EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok); 52 53static struct resource *ioapic_resources; 54static struct resource lapic_resource = { 55 .name = "Local APIC", 56 .flags = IORESOURCE_MEM | IORESOURCE_BUSY, 57}; 58 59/* 60 * cpu_mask that denotes the CPUs that needs timer interrupt coming in as 61 * IPIs in place of local APIC timers 62 */ 63static cpumask_t timer_interrupt_broadcast_ipi_mask; 64 65/* Using APIC to generate smp_local_timer_interrupt? */ 66int using_apic_timer __read_mostly = 0; 67 68static void apic_pm_activate(void); 69 70void apic_wait_icr_idle(void) 71{ 72 while (apic_read(APIC_ICR) & APIC_ICR_BUSY) 73 cpu_relax(); 74} 75 76unsigned int safe_apic_wait_icr_idle(void) 77{ 78 unsigned int send_status; 79 int timeout; 80 81 timeout = 0; 82 do { 83 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY; 84 if (!send_status) 85 break; 86 udelay(100); 87 } while (timeout++ < 1000); 88 89 return send_status; 90} 91 92void enable_NMI_through_LVT0 (void * dummy) 93{ 94 unsigned int v; 95 96 v = APIC_DM_NMI; /* unmask and set to NMI */ 97 apic_write(APIC_LVT0, v); 98} 99 100int get_maxlvt(void) 101{ 102 unsigned int v, maxlvt; 103 104 v = apic_read(APIC_LVR); 105 maxlvt = GET_APIC_MAXLVT(v); 106 return maxlvt; 107} 108 109/* 110 * 'what should we do if we get a hw irq event on an illegal vector'. 111 * each architecture has to answer this themselves. 112 */ 113void ack_bad_irq(unsigned int irq) 114{ 115 printk("unexpected IRQ trap at vector %02x\n", irq); 116 /* 117 * Currently unexpected vectors happen only on SMP and APIC. 118 * We _must_ ack these because every local APIC has only N 119 * irq slots per priority level, and a 'hanging, unacked' IRQ 120 * holds up an irq slot - in excessive cases (when multiple 121 * unexpected vectors occur) that might lock up the APIC 122 * completely. 123 * But don't ack when the APIC is disabled. -AK 124 */ 125 if (!disable_apic) 126 ack_APIC_irq(); 127} 128 129void clear_local_APIC(void) 130{ 131 int maxlvt; 132 unsigned int v; 133 134 maxlvt = get_maxlvt(); 135 136 /* 137 * Masking an LVT entry can trigger a local APIC error 138 * if the vector is zero. Mask LVTERR first to prevent this. 139 */ 140 if (maxlvt >= 3) { 141 v = ERROR_APIC_VECTOR; /* any non-zero vector will do */ 142 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); 143 } 144 /* 145 * Careful: we have to set masks only first to deassert 146 * any level-triggered sources. 147 */ 148 v = apic_read(APIC_LVTT); 149 apic_write(APIC_LVTT, v | APIC_LVT_MASKED); 150 v = apic_read(APIC_LVT0); 151 apic_write(APIC_LVT0, v | APIC_LVT_MASKED); 152 v = apic_read(APIC_LVT1); 153 apic_write(APIC_LVT1, v | APIC_LVT_MASKED); 154 if (maxlvt >= 4) { 155 v = apic_read(APIC_LVTPC); 156 apic_write(APIC_LVTPC, v | APIC_LVT_MASKED); 157 } 158 159 /* 160 * Clean APIC state for other OSs: 161 */ 162 apic_write(APIC_LVTT, APIC_LVT_MASKED); 163 apic_write(APIC_LVT0, APIC_LVT_MASKED); 164 apic_write(APIC_LVT1, APIC_LVT_MASKED); 165 if (maxlvt >= 3) 166 apic_write(APIC_LVTERR, APIC_LVT_MASKED); 167 if (maxlvt >= 4) 168 apic_write(APIC_LVTPC, APIC_LVT_MASKED); 169 apic_write(APIC_ESR, 0); 170 apic_read(APIC_ESR); 171} 172 173void disconnect_bsp_APIC(int virt_wire_setup) 174{ 175 /* Go back to Virtual Wire compatibility mode */ 176 unsigned long value; 177 178 /* For the spurious interrupt use vector F, and enable it */ 179 value = apic_read(APIC_SPIV); 180 value &= ~APIC_VECTOR_MASK; 181 value |= APIC_SPIV_APIC_ENABLED; 182 value |= 0xf; 183 apic_write(APIC_SPIV, value); 184 185 if (!virt_wire_setup) { 186 /* For LVT0 make it edge triggered, active high, external and enabled */ 187 value = apic_read(APIC_LVT0); 188 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 189 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 190 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED ); 191 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 192 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT); 193 apic_write(APIC_LVT0, value); 194 } else { 195 /* Disable LVT0 */ 196 apic_write(APIC_LVT0, APIC_LVT_MASKED); 197 } 198 199 /* For LVT1 make it edge triggered, active high, nmi and enabled */ 200 value = apic_read(APIC_LVT1); 201 value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | 202 APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | 203 APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); 204 value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; 205 value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI); 206 apic_write(APIC_LVT1, value); 207} 208 209void disable_local_APIC(void) 210{ 211 unsigned int value; 212 213 clear_local_APIC(); 214 215 /* 216 * Disable APIC (implies clearing of registers 217 * for 82489DX!). 218 */ 219 value = apic_read(APIC_SPIV); 220 value &= ~APIC_SPIV_APIC_ENABLED; 221 apic_write(APIC_SPIV, value); 222} 223 224/* 225 * This is to verify that we're looking at a real local APIC. 226 * Check these against your board if the CPUs aren't getting 227 * started for no apparent reason. 228 */ 229int __init verify_local_APIC(void) 230{ 231 unsigned int reg0, reg1; 232 233 /* 234 * The version register is read-only in a real APIC. 235 */ 236 reg0 = apic_read(APIC_LVR); 237 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0); 238 apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK); 239 reg1 = apic_read(APIC_LVR); 240 apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1); 241 242 /* 243 * The two version reads above should print the same 244 * numbers. If the second one is different, then we 245 * poke at a non-APIC. 246 */ 247 if (reg1 != reg0) 248 return 0; 249 250 /* 251 * Check if the version looks reasonably. 252 */ 253 reg1 = GET_APIC_VERSION(reg0); 254 if (reg1 == 0x00 || reg1 == 0xff) 255 return 0; 256 reg1 = get_maxlvt(); 257 if (reg1 < 0x02 || reg1 == 0xff) 258 return 0; 259 260 /* 261 * The ID register is read/write in a real APIC. 262 */ 263 reg0 = apic_read(APIC_ID); 264 apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0); 265 apic_write(APIC_ID, reg0 ^ APIC_ID_MASK); 266 reg1 = apic_read(APIC_ID); 267 apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg1); 268 apic_write(APIC_ID, reg0); 269 if (reg1 != (reg0 ^ APIC_ID_MASK)) 270 return 0; 271 272 /* 273 * The next two are just to see if we have sane values. 274 * They're only really relevant if we're in Virtual Wire 275 * compatibility mode, but most boxes are anymore. 276 */ 277 reg0 = apic_read(APIC_LVT0); 278 apic_printk(APIC_DEBUG,"Getting LVT0: %x\n", reg0); 279 reg1 = apic_read(APIC_LVT1); 280 apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1); 281 282 return 1; 283} 284 285void __init sync_Arb_IDs(void) 286{ 287 /* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 */ 288 unsigned int ver = GET_APIC_VERSION(apic_read(APIC_LVR)); 289 if (ver >= 0x14) /* P4 or higher */ 290 return; 291 292 /* 293 * Wait for idle. 294 */ 295 apic_wait_icr_idle(); 296 297 apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n"); 298 apic_write(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG 299 | APIC_DM_INIT); 300} 301 302/* 303 * An initial setup of the virtual wire mode. 304 */ 305void __init init_bsp_APIC(void) 306{ 307 unsigned int value; 308 309 /* 310 * Don't do the setup now if we have a SMP BIOS as the 311 * through-I/O-APIC virtual wire mode might be active. 312 */ 313 if (smp_found_config || !cpu_has_apic) 314 return; 315 316 value = apic_read(APIC_LVR); 317 318 /* 319 * Do not trust the local APIC being empty at bootup. 320 */ 321 clear_local_APIC(); 322 323 /* 324 * Enable APIC. 325 */ 326 value = apic_read(APIC_SPIV); 327 value &= ~APIC_VECTOR_MASK; 328 value |= APIC_SPIV_APIC_ENABLED; 329 value |= APIC_SPIV_FOCUS_DISABLED; 330 value |= SPURIOUS_APIC_VECTOR; 331 apic_write(APIC_SPIV, value); 332 333 /* 334 * Set up the virtual wire mode. 335 */ 336 apic_write(APIC_LVT0, APIC_DM_EXTINT); 337 value = APIC_DM_NMI; 338 apic_write(APIC_LVT1, value); 339} 340 341void __cpuinit setup_local_APIC (void) 342{ 343 unsigned int value, maxlvt; 344 int i, j; 345 346 value = apic_read(APIC_LVR); 347 348 BUILD_BUG_ON((SPURIOUS_APIC_VECTOR & 0x0f) != 0x0f); 349 350 /* 351 * Double-check whether this APIC is really registered. 352 * This is meaningless in clustered apic mode, so we skip it. 353 */ 354 if (!apic_id_registered()) 355 BUG(); 356 357 /* 358 * Intel recommends to set DFR, LDR and TPR before enabling 359 * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel 360 * document number 292116). So here it goes... 361 */ 362 init_apic_ldr(); 363 364 /* 365 * Set Task Priority to 'accept all'. We never change this 366 * later on. 367 */ 368 value = apic_read(APIC_TASKPRI); 369 value &= ~APIC_TPRI_MASK; 370 apic_write(APIC_TASKPRI, value); 371 372 /* 373 * After a crash, we no longer service the interrupts and a pending 374 * interrupt from previous kernel might still have ISR bit set. 375 * 376 * Most probably by now CPU has serviced that pending interrupt and 377 * it might not have done the ack_APIC_irq() because it thought, 378 * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it 379 * does not clear the ISR bit and cpu thinks it has already serivced 380 * the interrupt. Hence a vector might get locked. It was noticed 381 * for timer irq (vector 0x31). Issue an extra EOI to clear ISR. 382 */ 383 for (i = APIC_ISR_NR - 1; i >= 0; i--) { 384 value = apic_read(APIC_ISR + i*0x10); 385 for (j = 31; j >= 0; j--) { 386 if (value & (1<<j)) 387 ack_APIC_irq(); 388 } 389 } 390 391 /* 392 * Now that we are all set up, enable the APIC 393 */ 394 value = apic_read(APIC_SPIV); 395 value &= ~APIC_VECTOR_MASK; 396 /* 397 * Enable APIC 398 */ 399 value |= APIC_SPIV_APIC_ENABLED; 400 401 /* We always use processor focus */ 402 403 /* 404 * Set spurious IRQ vector 405 */ 406 value |= SPURIOUS_APIC_VECTOR; 407 apic_write(APIC_SPIV, value); 408 409 /* 410 * Set up LVT0, LVT1: 411 * 412 * set up through-local-APIC on the BP's LINT0. This is not 413 * strictly necessary in pure symmetric-IO mode, but sometimes 414 * we delegate interrupts to the 8259A. 415 */ 416 /* 417 * TODO: set up through-local-APIC from through-I/O-APIC? --macro 418 */ 419 value = apic_read(APIC_LVT0) & APIC_LVT_MASKED; 420 if (!smp_processor_id() && !value) { 421 value = APIC_DM_EXTINT; 422 apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", smp_processor_id()); 423 } else { 424 value = APIC_DM_EXTINT | APIC_LVT_MASKED; 425 apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", smp_processor_id()); 426 } 427 apic_write(APIC_LVT0, value); 428 429 /* 430 * only the BP should see the LINT1 NMI signal, obviously. 431 */ 432 if (!smp_processor_id()) 433 value = APIC_DM_NMI; 434 else 435 value = APIC_DM_NMI | APIC_LVT_MASKED; 436 apic_write(APIC_LVT1, value); 437 438 { 439 unsigned oldvalue; 440 maxlvt = get_maxlvt(); 441 oldvalue = apic_read(APIC_ESR); 442 value = ERROR_APIC_VECTOR; // enables sending errors 443 apic_write(APIC_LVTERR, value); 444 /* 445 * spec says clear errors after enabling vector. 446 */ 447 if (maxlvt > 3) 448 apic_write(APIC_ESR, 0); 449 value = apic_read(APIC_ESR); 450 if (value != oldvalue) 451 apic_printk(APIC_VERBOSE, 452 "ESR value after enabling vector: %08x, after %08x\n", 453 oldvalue, value); 454 } 455 456 nmi_watchdog_default(); 457 setup_apic_nmi_watchdog(NULL); 458 apic_pm_activate(); 459} 460 461#ifdef CONFIG_PM 462 463static struct { 464 /* 'active' is true if the local APIC was enabled by us and 465 not the BIOS; this signifies that we are also responsible 466 for disabling it before entering apm/acpi suspend */ 467 int active; 468 /* r/w apic fields */ 469 unsigned int apic_id; 470 unsigned int apic_taskpri; 471 unsigned int apic_ldr; 472 unsigned int apic_dfr; 473 unsigned int apic_spiv; 474 unsigned int apic_lvtt; 475 unsigned int apic_lvtpc; 476 unsigned int apic_lvt0; 477 unsigned int apic_lvt1; 478 unsigned int apic_lvterr; 479 unsigned int apic_tmict; 480 unsigned int apic_tdcr; 481 unsigned int apic_thmr; 482} apic_pm_state; 483 484static int lapic_suspend(struct sys_device *dev, pm_message_t state) 485{ 486 unsigned long flags; 487 int maxlvt; 488 489 if (!apic_pm_state.active) 490 return 0; 491 492 maxlvt = get_maxlvt(); 493 494 apic_pm_state.apic_id = apic_read(APIC_ID); 495 apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI); 496 apic_pm_state.apic_ldr = apic_read(APIC_LDR); 497 apic_pm_state.apic_dfr = apic_read(APIC_DFR); 498 apic_pm_state.apic_spiv = apic_read(APIC_SPIV); 499 apic_pm_state.apic_lvtt = apic_read(APIC_LVTT); 500 if (maxlvt >= 4) 501 apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC); 502 apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0); 503 apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1); 504 apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR); 505 apic_pm_state.apic_tmict = apic_read(APIC_TMICT); 506 apic_pm_state.apic_tdcr = apic_read(APIC_TDCR); 507#ifdef CONFIG_X86_MCE_INTEL 508 if (maxlvt >= 5) 509 apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR); 510#endif 511 local_irq_save(flags); 512 disable_local_APIC(); 513 local_irq_restore(flags); 514 return 0; 515} 516 517static int lapic_resume(struct sys_device *dev) 518{ 519 unsigned int l, h; 520 unsigned long flags; 521 int maxlvt; 522 523 if (!apic_pm_state.active) 524 return 0; 525 526 maxlvt = get_maxlvt(); 527 528 local_irq_save(flags); 529 rdmsr(MSR_IA32_APICBASE, l, h); 530 l &= ~MSR_IA32_APICBASE_BASE; 531 l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr; 532 wrmsr(MSR_IA32_APICBASE, l, h); 533 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED); 534 apic_write(APIC_ID, apic_pm_state.apic_id); 535 apic_write(APIC_DFR, apic_pm_state.apic_dfr); 536 apic_write(APIC_LDR, apic_pm_state.apic_ldr); 537 apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri); 538 apic_write(APIC_SPIV, apic_pm_state.apic_spiv); 539 apic_write(APIC_LVT0, apic_pm_state.apic_lvt0); 540 apic_write(APIC_LVT1, apic_pm_state.apic_lvt1); 541#ifdef CONFIG_X86_MCE_INTEL 542 if (maxlvt >= 5) 543 apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr); 544#endif 545 if (maxlvt >= 4) 546 apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc); 547 apic_write(APIC_LVTT, apic_pm_state.apic_lvtt); 548 apic_write(APIC_TDCR, apic_pm_state.apic_tdcr); 549 apic_write(APIC_TMICT, apic_pm_state.apic_tmict); 550 apic_write(APIC_ESR, 0); 551 apic_read(APIC_ESR); 552 apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr); 553 apic_write(APIC_ESR, 0); 554 apic_read(APIC_ESR); 555 local_irq_restore(flags); 556 return 0; 557} 558 559static struct sysdev_class lapic_sysclass = { 560 set_kset_name("lapic"), 561 .resume = lapic_resume, 562 .suspend = lapic_suspend, 563}; 564 565static struct sys_device device_lapic = { 566 .id = 0, 567 .cls = &lapic_sysclass, 568}; 569 570static void __cpuinit apic_pm_activate(void) 571{ 572 apic_pm_state.active = 1; 573} 574 575static int __init init_lapic_sysfs(void) 576{ 577 int error; 578 if (!cpu_has_apic) 579 return 0; 580 error = sysdev_class_register(&lapic_sysclass); 581 if (!error) 582 error = sysdev_register(&device_lapic); 583 return error; 584} 585device_initcall(init_lapic_sysfs); 586 587#else /* CONFIG_PM */ 588 589static void apic_pm_activate(void) { } 590 591#endif /* CONFIG_PM */ 592 593static int __init apic_set_verbosity(char *str) 594{ 595 if (str == NULL) { 596 skip_ioapic_setup = 0; 597 ioapic_force = 1; 598 return 0; 599 } 600 if (strcmp("debug", str) == 0) 601 apic_verbosity = APIC_DEBUG; 602 else if (strcmp("verbose", str) == 0) 603 apic_verbosity = APIC_VERBOSE; 604 else { 605 printk(KERN_WARNING "APIC Verbosity level %s not recognised" 606 " use apic=verbose or apic=debug\n", str); 607 return -EINVAL; 608 } 609 610 return 0; 611} 612early_param("apic", apic_set_verbosity); 613 614/* 615 * Detect and enable local APICs on non-SMP boards. 616 * Original code written by Keir Fraser. 617 * On AMD64 we trust the BIOS - if it says no APIC it is likely 618 * not correctly set up (usually the APIC timer won't work etc.) 619 */ 620 621static int __init detect_init_APIC (void) 622{ 623 if (!cpu_has_apic) { 624 printk(KERN_INFO "No local APIC present\n"); 625 return -1; 626 } 627 628 mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; 629 boot_cpu_id = 0; 630 return 0; 631} 632 633#ifdef CONFIG_X86_IO_APIC 634static struct resource * __init ioapic_setup_resources(void) 635{ 636#define IOAPIC_RESOURCE_NAME_SIZE 11 637 unsigned long n; 638 struct resource *res; 639 char *mem; 640 int i; 641 642 if (nr_ioapics <= 0) 643 return NULL; 644 645 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource); 646 n *= nr_ioapics; 647 648 mem = alloc_bootmem(n); 649 res = (void *)mem; 650 651 if (mem != NULL) { 652 memset(mem, 0, n); 653 mem += sizeof(struct resource) * nr_ioapics; 654 655 for (i = 0; i < nr_ioapics; i++) { 656 res[i].name = mem; 657 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY; 658 sprintf(mem, "IOAPIC %u", i); 659 mem += IOAPIC_RESOURCE_NAME_SIZE; 660 } 661 } 662 663 ioapic_resources = res; 664 665 return res; 666} 667 668static int __init ioapic_insert_resources(void) 669{ 670 int i; 671 struct resource *r = ioapic_resources; 672 673 if (!r) { 674 printk("IO APIC resources could be not be allocated.\n"); 675 return -1; 676 } 677 678 for (i = 0; i < nr_ioapics; i++) { 679 insert_resource(&iomem_resource, r); 680 r++; 681 } 682 683 return 0; 684} 685 686/* Insert the IO APIC resources after PCI initialization has occured to handle 687 * IO APICS that are mapped in on a BAR in PCI space. */ 688late_initcall(ioapic_insert_resources); 689#endif 690 691void __init init_apic_mappings(void) 692{ 693 unsigned long apic_phys; 694 695 /* 696 * If no local APIC can be found then set up a fake all 697 * zeroes page to simulate the local APIC and another 698 * one for the IO-APIC. 699 */ 700 if (!smp_found_config && detect_init_APIC()) { 701 apic_phys = (unsigned long) alloc_bootmem_pages(PAGE_SIZE); 702 apic_phys = __pa(apic_phys); 703 } else 704 apic_phys = mp_lapic_addr; 705 706 set_fixmap_nocache(FIX_APIC_BASE, apic_phys); 707 apic_mapped = 1; 708 apic_printk(APIC_VERBOSE,"mapped APIC to %16lx (%16lx)\n", APIC_BASE, apic_phys); 709 710 /* Put local APIC into the resource map. */ 711 lapic_resource.start = apic_phys; 712 lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1; 713 insert_resource(&iomem_resource, &lapic_resource); 714 715 /* 716 * Fetch the APIC ID of the BSP in case we have a 717 * default configuration (or the MP table is broken). 718 */ 719 boot_cpu_id = GET_APIC_ID(apic_read(APIC_ID)); 720 721 { 722 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0; 723 int i; 724 struct resource *ioapic_res; 725 726 ioapic_res = ioapic_setup_resources(); 727 for (i = 0; i < nr_ioapics; i++) { 728 if (smp_found_config) { 729 ioapic_phys = mp_ioapics[i].mpc_apicaddr; 730 } else { 731 ioapic_phys = (unsigned long) alloc_bootmem_pages(PAGE_SIZE); 732 ioapic_phys = __pa(ioapic_phys); 733 } 734 set_fixmap_nocache(idx, ioapic_phys); 735 apic_printk(APIC_VERBOSE,"mapped IOAPIC to %016lx (%016lx)\n", 736 __fix_to_virt(idx), ioapic_phys); 737 idx++; 738 739 if (ioapic_res != NULL) { 740 ioapic_res->start = ioapic_phys; 741 ioapic_res->end = ioapic_phys + (4 * 1024) - 1; 742 ioapic_res++; 743 } 744 } 745 } 746} 747 748/* 749 * This function sets up the local APIC timer, with a timeout of 750 * 'clocks' APIC bus clock. During calibration we actually call 751 * this function twice on the boot CPU, once with a bogus timeout 752 * value, second time for real. The other (noncalibrating) CPUs 753 * call this function only once, with the real, calibrated value. 754 * 755 * We do reads before writes even if unnecessary, to get around the 756 * P5 APIC double write bug. 757 */ 758 759#define APIC_DIVISOR 16 760 761static void __setup_APIC_LVTT(unsigned int clocks) 762{ 763 unsigned int lvtt_value, tmp_value; 764 int cpu = smp_processor_id(); 765 766 lvtt_value = APIC_LVT_TIMER_PERIODIC | LOCAL_TIMER_VECTOR; 767 768 if (cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask)) 769 lvtt_value |= APIC_LVT_MASKED; 770 771 apic_write(APIC_LVTT, lvtt_value); 772 773 /* 774 * Divide PICLK by 16 775 */ 776 tmp_value = apic_read(APIC_TDCR); 777 apic_write(APIC_TDCR, (tmp_value 778 & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) 779 | APIC_TDR_DIV_16); 780 781 apic_write(APIC_TMICT, clocks/APIC_DIVISOR); 782} 783 784static void setup_APIC_timer(unsigned int clocks) 785{ 786 unsigned long flags; 787 788 local_irq_save(flags); 789 790 /* wait for irq slice */ 791 if (hpet_address && hpet_use_timer) { 792 int trigger = hpet_readl(HPET_T0_CMP); 793 while (hpet_readl(HPET_COUNTER) >= trigger) 794 /* do nothing */ ; 795 while (hpet_readl(HPET_COUNTER) < trigger) 796 /* do nothing */ ; 797 } else { 798 int c1, c2; 799 outb_p(0x00, 0x43); 800 c2 = inb_p(0x40); 801 c2 |= inb_p(0x40) << 8; 802 do { 803 c1 = c2; 804 outb_p(0x00, 0x43); 805 c2 = inb_p(0x40); 806 c2 |= inb_p(0x40) << 8; 807 } while (c2 - c1 < 300); 808 } 809 __setup_APIC_LVTT(clocks); 810 /* Turn off PIT interrupt if we use APIC timer as main timer. 811 Only works with the PM timer right now 812 TBD fix it for HPET too. */ 813 if ((pmtmr_ioport != 0) && 814 smp_processor_id() == boot_cpu_id && 815 apic_runs_main_timer == 1 && 816 !cpu_isset(boot_cpu_id, timer_interrupt_broadcast_ipi_mask)) { 817 stop_timer_interrupt(); 818 apic_runs_main_timer++; 819 } 820 local_irq_restore(flags); 821} 822 823/* 824 * In this function we calibrate APIC bus clocks to the external 825 * timer. Unfortunately we cannot use jiffies and the timer irq 826 * to calibrate, since some later bootup code depends on getting 827 * the first irq? Ugh. 828 * 829 * We want to do the calibration only once since we 830 * want to have local timer irqs syncron. CPUs connected 831 * by the same APIC bus have the very same bus frequency. 832 * And we want to have irqs off anyways, no accidental 833 * APIC irq that way. 834 */ 835 836#define TICK_COUNT 100000000 837 838static int __init calibrate_APIC_clock(void) 839{ 840 unsigned apic, apic_start; 841 unsigned long tsc, tsc_start; 842 int result; 843 /* 844 * Put whatever arbitrary (but long enough) timeout 845 * value into the APIC clock, we just want to get the 846 * counter running for calibration. 847 */ 848 __setup_APIC_LVTT(4000000000); 849 850 apic_start = apic_read(APIC_TMCCT); 851#ifdef CONFIG_X86_PM_TIMER 852 if (apic_calibrate_pmtmr && pmtmr_ioport) { 853 pmtimer_wait(5000); /* 5ms wait */ 854 apic = apic_read(APIC_TMCCT); 855 result = (apic_start - apic) * 1000L / 5; 856 } else 857#endif 858 { 859 rdtscll(tsc_start); 860 861 do { 862 apic = apic_read(APIC_TMCCT); 863 rdtscll(tsc); 864 } while ((tsc - tsc_start) < TICK_COUNT && 865 (apic_start - apic) < TICK_COUNT); 866 867 result = (apic_start - apic) * 1000L * tsc_khz / 868 (tsc - tsc_start); 869 } 870 printk("result %d\n", result); 871 872 873 printk(KERN_INFO "Detected %d.%03d MHz APIC timer.\n", 874 result / 1000 / 1000, result / 1000 % 1000); 875 876 return result * APIC_DIVISOR / HZ; 877} 878 879static unsigned int calibration_result; 880 881void __init setup_boot_APIC_clock (void) 882{ 883 if (disable_apic_timer) { 884 printk(KERN_INFO "Disabling APIC timer\n"); 885 return; 886 } 887 888 printk(KERN_INFO "Using local APIC timer interrupts.\n"); 889 using_apic_timer = 1; 890 891 local_irq_disable(); 892 893 calibration_result = calibrate_APIC_clock(); 894 /* 895 * Now set up the timer for real. 896 */ 897 setup_APIC_timer(calibration_result); 898 899 local_irq_enable(); 900} 901 902void __cpuinit setup_secondary_APIC_clock(void) 903{ 904 local_irq_disable(); 905 setup_APIC_timer(calibration_result); 906 local_irq_enable(); 907} 908 909void disable_APIC_timer(void) 910{ 911 if (using_apic_timer) { 912 unsigned long v; 913 914 v = apic_read(APIC_LVTT); 915 /* 916 * When an illegal vector value (0-15) is written to an LVT 917 * entry and delivery mode is Fixed, the APIC may signal an 918 * illegal vector error, with out regard to whether the mask 919 * bit is set or whether an interrupt is actually seen on input. 920 * 921 * Boot sequence might call this function when the LVTT has 922 * '0' vector value. So make sure vector field is set to 923 * valid value. 924 */ 925 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); 926 apic_write(APIC_LVTT, v); 927 } 928} 929 930void enable_APIC_timer(void) 931{ 932 int cpu = smp_processor_id(); 933 934 if (using_apic_timer && 935 !cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask)) { 936 unsigned long v; 937 938 v = apic_read(APIC_LVTT); 939 apic_write(APIC_LVTT, v & ~APIC_LVT_MASKED); 940 } 941} 942 943void switch_APIC_timer_to_ipi(void *cpumask) 944{ 945 cpumask_t mask = *(cpumask_t *)cpumask; 946 int cpu = smp_processor_id(); 947 948 if (cpu_isset(cpu, mask) && 949 !cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask)) { 950 disable_APIC_timer(); 951 cpu_set(cpu, timer_interrupt_broadcast_ipi_mask); 952 } 953} 954EXPORT_SYMBOL(switch_APIC_timer_to_ipi); 955 956void smp_send_timer_broadcast_ipi(void) 957{ 958 int cpu = smp_processor_id(); 959 cpumask_t mask; 960 961 cpus_and(mask, cpu_online_map, timer_interrupt_broadcast_ipi_mask); 962 963 if (cpu_isset(cpu, mask)) { 964 cpu_clear(cpu, mask); 965 add_pda(apic_timer_irqs, 1); 966 smp_local_timer_interrupt(); 967 } 968 969 if (!cpus_empty(mask)) { 970 send_IPI_mask(mask, LOCAL_TIMER_VECTOR); 971 } 972} 973 974void switch_ipi_to_APIC_timer(void *cpumask) 975{ 976 cpumask_t mask = *(cpumask_t *)cpumask; 977 int cpu = smp_processor_id(); 978 979 if (cpu_isset(cpu, mask) && 980 cpu_isset(cpu, timer_interrupt_broadcast_ipi_mask)) { 981 cpu_clear(cpu, timer_interrupt_broadcast_ipi_mask); 982 enable_APIC_timer(); 983 } 984} 985EXPORT_SYMBOL(switch_ipi_to_APIC_timer); 986 987int setup_profiling_timer(unsigned int multiplier) 988{ 989 return -EINVAL; 990} 991 992void setup_APIC_extened_lvt(unsigned char lvt_off, unsigned char vector, 993 unsigned char msg_type, unsigned char mask) 994{ 995 unsigned long reg = (lvt_off << 4) + K8_APIC_EXT_LVT_BASE; 996 unsigned int v = (mask << 16) | (msg_type << 8) | vector; 997 apic_write(reg, v); 998} 999 1000#undef APIC_DIVISOR 1001 1002/* 1003 * Local timer interrupt handler. It does both profiling and 1004 * process statistics/rescheduling. 1005 * 1006 * We do profiling in every local tick, statistics/rescheduling 1007 * happen only every 'profiling multiplier' ticks. The default 1008 * multiplier is 1 and it can be changed by writing the new multiplier 1009 * value into /proc/profile. 1010 */ 1011 1012void smp_local_timer_interrupt(void) 1013{ 1014 profile_tick(CPU_PROFILING); 1015#ifdef CONFIG_SMP 1016 update_process_times(user_mode(get_irq_regs())); 1017#endif 1018 if (apic_runs_main_timer > 1 && smp_processor_id() == boot_cpu_id) 1019 main_timer_handler(); 1020 /* 1021 * We take the 'long' return path, and there every subsystem 1022 * grabs the appropriate locks (kernel lock/ irq lock). 1023 * 1024 * We might want to decouple profiling from the 'long path', 1025 * and do the profiling totally in assembly. 1026 * 1027 * Currently this isn't too much of an issue (performance wise), 1028 * we can take more than 100K local irqs per second on a 100 MHz P5. 1029 */ 1030} 1031 1032/* 1033 * Local APIC timer interrupt. This is the most natural way for doing 1034 * local interrupts, but local timer interrupts can be emulated by 1035 * broadcast interrupts too. [in case the hw doesn't support APIC timers] 1036 * 1037 * [ if a single-CPU system runs an SMP kernel then we call the local 1038 * interrupt as well. Thus we cannot inline the local irq ... ] 1039 */ 1040void smp_apic_timer_interrupt(struct pt_regs *regs) 1041{ 1042 struct pt_regs *old_regs = set_irq_regs(regs); 1043 1044 /* 1045 * the NMI deadlock-detector uses this. 1046 */ 1047 add_pda(apic_timer_irqs, 1); 1048 1049 /* 1050 * NOTE! We'd better ACK the irq immediately, 1051 * because timer handling can be slow. 1052 */ 1053 ack_APIC_irq(); 1054 /* 1055 * update_process_times() expects us to have done irq_enter(). 1056 * Besides, if we don't timer interrupts ignore the global 1057 * interrupt lock, which is the WrongThing (tm) to do. 1058 */ 1059 exit_idle(); 1060 irq_enter(); 1061 smp_local_timer_interrupt(); 1062 irq_exit(); 1063 set_irq_regs(old_regs); 1064} 1065 1066/* 1067 * apic_is_clustered_box() -- Check if we can expect good TSC 1068 * 1069 * Thus far, the major user of this is IBM's Summit2 series: 1070 * 1071 * Clustered boxes may have unsynced TSC problems if they are 1072 * multi-chassis. Use available data to take a good guess. 1073 * If in doubt, go HPET. 1074 */ 1075__cpuinit int apic_is_clustered_box(void) 1076{ 1077 int i, clusters, zeros; 1078 unsigned id; 1079 DECLARE_BITMAP(clustermap, NUM_APIC_CLUSTERS); 1080 1081 bitmap_zero(clustermap, NUM_APIC_CLUSTERS); 1082 1083 for (i = 0; i < NR_CPUS; i++) { 1084 id = bios_cpu_apicid[i]; 1085 if (id != BAD_APICID) 1086 __set_bit(APIC_CLUSTERID(id), clustermap); 1087 } 1088 1089 /* Problem: Partially populated chassis may not have CPUs in some of 1090 * the APIC clusters they have been allocated. Only present CPUs have 1091 * bios_cpu_apicid entries, thus causing zeroes in the bitmap. Since 1092 * clusters are allocated sequentially, count zeros only if they are 1093 * bounded by ones. 1094 */ 1095 clusters = 0; 1096 zeros = 0; 1097 for (i = 0; i < NUM_APIC_CLUSTERS; i++) { 1098 if (test_bit(i, clustermap)) { 1099 clusters += 1 + zeros; 1100 zeros = 0; 1101 } else 1102 ++zeros; 1103 } 1104 1105 /* 1106 * If clusters > 2, then should be multi-chassis. 1107 * May have to revisit this when multi-core + hyperthreaded CPUs come 1108 * out, but AFAIK this will work even for them. 1109 */ 1110 return (clusters > 2); 1111} 1112 1113/* 1114 * This interrupt should _never_ happen with our APIC/SMP architecture 1115 */ 1116asmlinkage void smp_spurious_interrupt(void) 1117{ 1118 unsigned int v; 1119 exit_idle(); 1120 irq_enter(); 1121 /* 1122 * Check if this really is a spurious interrupt and ACK it 1123 * if it is a vectored one. Just in case... 1124 * Spurious interrupts should not be ACKed. 1125 */ 1126 v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1)); 1127 if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f))) 1128 ack_APIC_irq(); 1129 1130 irq_exit(); 1131} 1132 1133/* 1134 * This interrupt should never happen with our APIC/SMP architecture 1135 */ 1136 1137asmlinkage void smp_error_interrupt(void) 1138{ 1139 unsigned int v, v1; 1140 1141 exit_idle(); 1142 irq_enter(); 1143 /* First tickle the hardware, only then report what went on. -- REW */ 1144 v = apic_read(APIC_ESR); 1145 apic_write(APIC_ESR, 0); 1146 v1 = apic_read(APIC_ESR); 1147 ack_APIC_irq(); 1148 atomic_inc(&irq_err_count); 1149 1150 /* Here is what the APIC error bits mean: 1151 0: Send CS error 1152 1: Receive CS error 1153 2: Send accept error 1154 3: Receive accept error 1155 4: Reserved 1156 5: Send illegal vector 1157 6: Received illegal vector 1158 7: Illegal register address 1159 */ 1160 printk (KERN_DEBUG "APIC error on CPU%d: %02x(%02x)\n", 1161 smp_processor_id(), v , v1); 1162 irq_exit(); 1163} 1164 1165int disable_apic; 1166 1167/* 1168 * This initializes the IO-APIC and APIC hardware if this is 1169 * a UP kernel. 1170 */ 1171int __init APIC_init_uniprocessor (void) 1172{ 1173 if (disable_apic) { 1174 printk(KERN_INFO "Apic disabled\n"); 1175 return -1; 1176 } 1177 if (!cpu_has_apic) { 1178 disable_apic = 1; 1179 printk(KERN_INFO "Apic disabled by BIOS\n"); 1180 return -1; 1181 } 1182 1183 verify_local_APIC(); 1184 1185 phys_cpu_present_map = physid_mask_of_physid(boot_cpu_id); 1186 apic_write(APIC_ID, SET_APIC_ID(boot_cpu_id)); 1187 1188 setup_local_APIC(); 1189 1190 if (smp_found_config && !skip_ioapic_setup && nr_ioapics) 1191 setup_IO_APIC(); 1192 else 1193 nr_ioapics = 0; 1194 setup_boot_APIC_clock(); 1195 check_nmi_watchdog(); 1196 return 0; 1197} 1198 1199static __init int setup_disableapic(char *str) 1200{ 1201 disable_apic = 1; 1202 clear_bit(X86_FEATURE_APIC, boot_cpu_data.x86_capability); 1203 return 0; 1204} 1205early_param("disableapic", setup_disableapic); 1206 1207/* same as disableapic, for compatibility */ 1208static __init int setup_nolapic(char *str) 1209{ 1210 return setup_disableapic(str); 1211} 1212early_param("nolapic", setup_nolapic); 1213 1214static int __init parse_lapic_timer_c2_ok(char *arg) 1215{ 1216 local_apic_timer_c2_ok = 1; 1217 return 0; 1218} 1219early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok); 1220 1221static __init int setup_noapictimer(char *str) 1222{ 1223 if (str[0] != ' ' && str[0] != 0) 1224 return 0; 1225 disable_apic_timer = 1; 1226 return 1; 1227} 1228 1229static __init int setup_apicmaintimer(char *str) 1230{ 1231 apic_runs_main_timer = 1; 1232 nohpet = 1; 1233 return 1; 1234} 1235__setup("apicmaintimer", setup_apicmaintimer); 1236 1237static __init int setup_noapicmaintimer(char *str) 1238{ 1239 apic_runs_main_timer = -1; 1240 return 1; 1241} 1242__setup("noapicmaintimer", setup_noapicmaintimer); 1243 1244static __init int setup_apicpmtimer(char *s) 1245{ 1246 apic_calibrate_pmtmr = 1; 1247 notsc_setup(NULL); 1248 return setup_apicmaintimer(NULL); 1249} 1250__setup("apicpmtimer", setup_apicpmtimer); 1251 1252__setup("noapictimer", setup_noapictimer); 1253