local_apic.c revision 309483
1/*- 2 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3 * Copyright (c) 1996, by Steve Passe 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. The name of the developer may NOT be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 3. Neither the name of the author nor the names of any co-contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * Local APIC support on Pentium and later processors. 32 */ 33 34#include <sys/cdefs.h> 35__FBSDID("$FreeBSD: stable/11/sys/x86/x86/local_apic.c 309483 2016-12-03 17:10:04Z kib $"); 36 37#include "opt_atpic.h" 38#include "opt_hwpmc_hooks.h" 39 40#include "opt_ddb.h" 41 42#include <sys/param.h> 43#include <sys/systm.h> 44#include <sys/bus.h> 45#include <sys/kernel.h> 46#include <sys/lock.h> 47#include <sys/mutex.h> 48#include <sys/pcpu.h> 49#include <sys/proc.h> 50#include <sys/sched.h> 51#include <sys/smp.h> 52#include <sys/sysctl.h> 53#include <sys/timeet.h> 54 55#include <vm/vm.h> 56#include <vm/pmap.h> 57 58#include <x86/apicreg.h> 59#include <machine/clock.h> 60#include <machine/cpufunc.h> 61#include <machine/cputypes.h> 62#include <machine/frame.h> 63#include <machine/intr_machdep.h> 64#include <x86/apicvar.h> 65#include <x86/mca.h> 66#include <machine/md_var.h> 67#include <machine/smp.h> 68#include <machine/specialreg.h> 69#include <x86/init.h> 70 71#ifdef DDB 72#include <sys/interrupt.h> 73#include <ddb/ddb.h> 74#endif 75 76#ifdef __amd64__ 77#define SDT_APIC SDT_SYSIGT 78#define SDT_APICT SDT_SYSIGT 79#define GSEL_APIC 0 80#else 81#define SDT_APIC SDT_SYS386IGT 82#define SDT_APICT SDT_SYS386TGT 83#define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL) 84#endif 85 86/* Sanity checks on IDT vectors. */ 87CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); 88CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); 89CTASSERT(APIC_LOCAL_INTS == 240); 90CTASSERT(IPI_STOP < APIC_SPURIOUS_INT); 91 92/* Magic IRQ values for the timer and syscalls. */ 93#define IRQ_TIMER (NUM_IO_INTS + 1) 94#define IRQ_SYSCALL (NUM_IO_INTS + 2) 95#define IRQ_DTRACE_RET (NUM_IO_INTS + 3) 96#define IRQ_EVTCHN (NUM_IO_INTS + 4) 97 98enum lat_timer_mode { 99 LAT_MODE_UNDEF = 0, 100 LAT_MODE_PERIODIC = 1, 101 LAT_MODE_ONESHOT = 2, 102 LAT_MODE_DEADLINE = 3, 103}; 104 105/* 106 * Support for local APICs. Local APICs manage interrupts on each 107 * individual processor as opposed to I/O APICs which receive interrupts 108 * from I/O devices and then forward them on to the local APICs. 109 * 110 * Local APICs can also send interrupts to each other thus providing the 111 * mechanism for IPIs. 112 */ 113 114struct lvt { 115 u_int lvt_edgetrigger:1; 116 u_int lvt_activehi:1; 117 u_int lvt_masked:1; 118 u_int lvt_active:1; 119 u_int lvt_mode:16; 120 u_int lvt_vector:8; 121}; 122 123struct lapic { 124 struct lvt la_lvts[APIC_LVT_MAX + 1]; 125 u_int la_id:8; 126 u_int la_cluster:4; 127 u_int la_cluster_id:2; 128 u_int la_present:1; 129 u_long *la_timer_count; 130 uint64_t la_timer_period; 131 enum lat_timer_mode la_timer_mode; 132 uint32_t lvt_timer_base; 133 uint32_t lvt_timer_last; 134 /* Include IDT_SYSCALL to make indexing easier. */ 135 int la_ioint_irqs[APIC_NUM_IOINTS + 1]; 136} static lapics[MAX_APIC_ID + 1]; 137 138/* Global defaults for local APIC LVT entries. */ 139static struct lvt lvts[APIC_LVT_MAX + 1] = { 140 { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 }, /* LINT0: masked ExtINT */ 141 { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 }, /* LINT1: NMI */ 142 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT }, /* Timer */ 143 { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT }, /* Error */ 144 { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 }, /* PMC */ 145 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT }, /* Thermal */ 146 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT }, /* CMCI */ 147}; 148 149static inthand_t *ioint_handlers[] = { 150 NULL, /* 0 - 31 */ 151 IDTVEC(apic_isr1), /* 32 - 63 */ 152 IDTVEC(apic_isr2), /* 64 - 95 */ 153 IDTVEC(apic_isr3), /* 96 - 127 */ 154 IDTVEC(apic_isr4), /* 128 - 159 */ 155 IDTVEC(apic_isr5), /* 160 - 191 */ 156 IDTVEC(apic_isr6), /* 192 - 223 */ 157 IDTVEC(apic_isr7), /* 224 - 255 */ 158}; 159 160 161static u_int32_t lapic_timer_divisors[] = { 162 APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, 163 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128 164}; 165 166extern inthand_t IDTVEC(rsvd); 167 168volatile char *lapic_map; 169vm_paddr_t lapic_paddr; 170int x2apic_mode; 171int lapic_eoi_suppression; 172static int lapic_timer_tsc_deadline; 173static u_long lapic_timer_divisor, count_freq; 174static struct eventtimer lapic_et; 175#ifdef SMP 176static uint64_t lapic_ipi_wait_mult; 177#endif 178 179SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options"); 180SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, ""); 181SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD, 182 &lapic_eoi_suppression, 0, ""); 183SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD, 184 &lapic_timer_tsc_deadline, 0, ""); 185 186static uint32_t 187lapic_read32(enum LAPIC_REGISTERS reg) 188{ 189 uint32_t res; 190 191 if (x2apic_mode) { 192 res = rdmsr32(MSR_APIC_000 + reg); 193 } else { 194 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL); 195 } 196 return (res); 197} 198 199static void 200lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val) 201{ 202 203 if (x2apic_mode) { 204 mfence(); 205 wrmsr(MSR_APIC_000 + reg, val); 206 } else { 207 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val; 208 } 209} 210 211static void 212lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val) 213{ 214 215 if (x2apic_mode) { 216 wrmsr(MSR_APIC_000 + reg, val); 217 } else { 218 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val; 219 } 220} 221 222#ifdef SMP 223static uint64_t 224lapic_read_icr(void) 225{ 226 uint64_t v; 227 uint32_t vhi, vlo; 228 229 if (x2apic_mode) { 230 v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO); 231 } else { 232 vhi = lapic_read32(LAPIC_ICR_HI); 233 vlo = lapic_read32(LAPIC_ICR_LO); 234 v = ((uint64_t)vhi << 32) | vlo; 235 } 236 return (v); 237} 238 239static uint64_t 240lapic_read_icr_lo(void) 241{ 242 243 return (lapic_read32(LAPIC_ICR_LO)); 244} 245 246static void 247lapic_write_icr(uint32_t vhi, uint32_t vlo) 248{ 249 uint64_t v; 250 251 if (x2apic_mode) { 252 v = ((uint64_t)vhi << 32) | vlo; 253 mfence(); 254 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v); 255 } else { 256 lapic_write32(LAPIC_ICR_HI, vhi); 257 lapic_write32(LAPIC_ICR_LO, vlo); 258 } 259} 260#endif /* SMP */ 261 262static void 263native_lapic_enable_x2apic(void) 264{ 265 uint64_t apic_base; 266 267 apic_base = rdmsr(MSR_APICBASE); 268 apic_base |= APICBASE_X2APIC | APICBASE_ENABLED; 269 wrmsr(MSR_APICBASE, apic_base); 270} 271 272static bool 273native_lapic_is_x2apic(void) 274{ 275 uint64_t apic_base; 276 277 apic_base = rdmsr(MSR_APICBASE); 278 return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) == 279 (APICBASE_X2APIC | APICBASE_ENABLED)); 280} 281 282static void lapic_enable(void); 283static void lapic_resume(struct pic *pic, bool suspend_cancelled); 284static void lapic_timer_oneshot(struct lapic *); 285static void lapic_timer_oneshot_nointr(struct lapic *, uint32_t); 286static void lapic_timer_periodic(struct lapic *); 287static void lapic_timer_deadline(struct lapic *); 288static void lapic_timer_stop(struct lapic *); 289static void lapic_timer_set_divisor(u_int divisor); 290static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value); 291static int lapic_et_start(struct eventtimer *et, 292 sbintime_t first, sbintime_t period); 293static int lapic_et_stop(struct eventtimer *et); 294static u_int apic_idt_to_irq(u_int apic_id, u_int vector); 295static void lapic_set_tpr(u_int vector); 296 297struct pic lapic_pic = { .pic_resume = lapic_resume }; 298 299/* Forward declarations for apic_ops */ 300static void native_lapic_create(u_int apic_id, int boot_cpu); 301static void native_lapic_init(vm_paddr_t addr); 302static void native_lapic_xapic_mode(void); 303static void native_lapic_setup(int boot); 304static void native_lapic_dump(const char *str); 305static void native_lapic_disable(void); 306static void native_lapic_eoi(void); 307static int native_lapic_id(void); 308static int native_lapic_intr_pending(u_int vector); 309static u_int native_apic_cpuid(u_int apic_id); 310static u_int native_apic_alloc_vector(u_int apic_id, u_int irq); 311static u_int native_apic_alloc_vectors(u_int apic_id, u_int *irqs, 312 u_int count, u_int align); 313static void native_apic_disable_vector(u_int apic_id, u_int vector); 314static void native_apic_enable_vector(u_int apic_id, u_int vector); 315static void native_apic_free_vector(u_int apic_id, u_int vector, u_int irq); 316static void native_lapic_set_logical_id(u_int apic_id, u_int cluster, 317 u_int cluster_id); 318static int native_lapic_enable_pmc(void); 319static void native_lapic_disable_pmc(void); 320static void native_lapic_reenable_pmc(void); 321static void native_lapic_enable_cmc(void); 322static int native_lapic_set_lvt_mask(u_int apic_id, u_int lvt, 323 u_char masked); 324static int native_lapic_set_lvt_mode(u_int apic_id, u_int lvt, 325 uint32_t mode); 326static int native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, 327 enum intr_polarity pol); 328static int native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 329 enum intr_trigger trigger); 330#ifdef SMP 331static void native_lapic_ipi_raw(register_t icrlo, u_int dest); 332static void native_lapic_ipi_vectored(u_int vector, int dest); 333static int native_lapic_ipi_wait(int delay); 334#endif /* SMP */ 335static int native_lapic_ipi_alloc(inthand_t *ipifunc); 336static void native_lapic_ipi_free(int vector); 337 338struct apic_ops apic_ops = { 339 .create = native_lapic_create, 340 .init = native_lapic_init, 341 .xapic_mode = native_lapic_xapic_mode, 342 .is_x2apic = native_lapic_is_x2apic, 343 .setup = native_lapic_setup, 344 .dump = native_lapic_dump, 345 .disable = native_lapic_disable, 346 .eoi = native_lapic_eoi, 347 .id = native_lapic_id, 348 .intr_pending = native_lapic_intr_pending, 349 .set_logical_id = native_lapic_set_logical_id, 350 .cpuid = native_apic_cpuid, 351 .alloc_vector = native_apic_alloc_vector, 352 .alloc_vectors = native_apic_alloc_vectors, 353 .enable_vector = native_apic_enable_vector, 354 .disable_vector = native_apic_disable_vector, 355 .free_vector = native_apic_free_vector, 356 .enable_pmc = native_lapic_enable_pmc, 357 .disable_pmc = native_lapic_disable_pmc, 358 .reenable_pmc = native_lapic_reenable_pmc, 359 .enable_cmc = native_lapic_enable_cmc, 360#ifdef SMP 361 .ipi_raw = native_lapic_ipi_raw, 362 .ipi_vectored = native_lapic_ipi_vectored, 363 .ipi_wait = native_lapic_ipi_wait, 364#endif 365 .ipi_alloc = native_lapic_ipi_alloc, 366 .ipi_free = native_lapic_ipi_free, 367 .set_lvt_mask = native_lapic_set_lvt_mask, 368 .set_lvt_mode = native_lapic_set_lvt_mode, 369 .set_lvt_polarity = native_lapic_set_lvt_polarity, 370 .set_lvt_triggermode = native_lapic_set_lvt_triggermode, 371}; 372 373static uint32_t 374lvt_mode(struct lapic *la, u_int pin, uint32_t value) 375{ 376 struct lvt *lvt; 377 378 KASSERT(pin <= APIC_LVT_MAX, ("%s: pin %u out of range", __func__, pin)); 379 if (la->la_lvts[pin].lvt_active) 380 lvt = &la->la_lvts[pin]; 381 else 382 lvt = &lvts[pin]; 383 384 value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM | 385 APIC_LVT_VECTOR); 386 if (lvt->lvt_edgetrigger == 0) 387 value |= APIC_LVT_TM; 388 if (lvt->lvt_activehi == 0) 389 value |= APIC_LVT_IIPP_INTALO; 390 if (lvt->lvt_masked) 391 value |= APIC_LVT_M; 392 value |= lvt->lvt_mode; 393 switch (lvt->lvt_mode) { 394 case APIC_LVT_DM_NMI: 395 case APIC_LVT_DM_SMI: 396 case APIC_LVT_DM_INIT: 397 case APIC_LVT_DM_EXTINT: 398 if (!lvt->lvt_edgetrigger && bootverbose) { 399 printf("lapic%u: Forcing LINT%u to edge trigger\n", 400 la->la_id, pin); 401 value |= APIC_LVT_TM; 402 } 403 /* Use a vector of 0. */ 404 break; 405 case APIC_LVT_DM_FIXED: 406 value |= lvt->lvt_vector; 407 break; 408 default: 409 panic("bad APIC LVT delivery mode: %#x\n", value); 410 } 411 return (value); 412} 413 414/* 415 * Map the local APIC and setup necessary interrupt vectors. 416 */ 417static void 418native_lapic_init(vm_paddr_t addr) 419{ 420#ifdef SMP 421 uint64_t r, r1, r2, rx; 422#endif 423 uint32_t ver; 424 u_int regs[4]; 425 int i, arat; 426 427 /* 428 * Enable x2APIC mode if possible. Map the local APIC 429 * registers page. 430 * 431 * Keep the LAPIC registers page mapped uncached for x2APIC 432 * mode too, to have direct map page attribute set to 433 * uncached. This is needed to work around CPU errata present 434 * on all Intel processors. 435 */ 436 KASSERT(trunc_page(addr) == addr, 437 ("local APIC not aligned on a page boundary")); 438 lapic_paddr = addr; 439 lapic_map = pmap_mapdev(addr, PAGE_SIZE); 440 if (x2apic_mode) { 441 native_lapic_enable_x2apic(); 442 lapic_map = NULL; 443 } 444 445 /* Setup the spurious interrupt handler. */ 446 setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL, 447 GSEL_APIC); 448 449 /* Perform basic initialization of the BSP's local APIC. */ 450 lapic_enable(); 451 452 /* Set BSP's per-CPU local APIC ID. */ 453 PCPU_SET(apic_id, lapic_id()); 454 455 /* Local APIC timer interrupt. */ 456 setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC); 457 458 /* Local APIC error interrupt. */ 459 setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC); 460 461 /* XXX: Thermal interrupt */ 462 463 /* Local APIC CMCI. */ 464 setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC); 465 466 if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) { 467 arat = 0; 468 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */ 469 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) { 470 do_cpuid(0x06, regs); 471 if ((regs[0] & CPUTPM1_ARAT) != 0) 472 arat = 1; 473 } 474 bzero(&lapic_et, sizeof(lapic_et)); 475 lapic_et.et_name = "LAPIC"; 476 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | 477 ET_FLAGS_PERCPU; 478 lapic_et.et_quality = 600; 479 if (!arat) { 480 lapic_et.et_flags |= ET_FLAGS_C3STOP; 481 lapic_et.et_quality = 100; 482 } 483 if ((cpu_feature & CPUID_TSC) != 0 && 484 (cpu_feature2 & CPUID2_TSCDLT) != 0 && 485 tsc_is_invariant && tsc_freq != 0) { 486 lapic_timer_tsc_deadline = 1; 487 TUNABLE_INT_FETCH("hw.lapic_tsc_deadline", 488 &lapic_timer_tsc_deadline); 489 } 490 491 lapic_et.et_frequency = 0; 492 /* We don't know frequency yet, so trying to guess. */ 493 lapic_et.et_min_period = 0x00001000LL; 494 lapic_et.et_max_period = SBT_1S; 495 lapic_et.et_start = lapic_et_start; 496 lapic_et.et_stop = lapic_et_stop; 497 lapic_et.et_priv = NULL; 498 et_register(&lapic_et); 499 } 500 501 /* 502 * Set lapic_eoi_suppression after lapic_enable(), to not 503 * enable suppression in the hardware prematurely. Note that 504 * we by default enable suppression even when system only has 505 * one IO-APIC, since EOI is broadcasted to all APIC agents, 506 * including CPUs, otherwise. 507 * 508 * It seems that at least some KVM versions report 509 * EOI_SUPPRESSION bit, but auto-EOI does not work. 510 */ 511 ver = lapic_read32(LAPIC_VERSION); 512 if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) { 513 lapic_eoi_suppression = 1; 514 if (vm_guest == VM_GUEST_KVM) { 515 if (bootverbose) 516 printf( 517 "KVM -- disabling lapic eoi suppression\n"); 518 lapic_eoi_suppression = 0; 519 } 520 TUNABLE_INT_FETCH("hw.lapic_eoi_suppression", 521 &lapic_eoi_suppression); 522 } 523 524#ifdef SMP 525#define LOOPS 100000 526 /* 527 * Calibrate the busy loop waiting for IPI ack in xAPIC mode. 528 * lapic_ipi_wait_mult contains the number of iterations which 529 * approximately delay execution for 1 microsecond (the 530 * argument to native_lapic_ipi_wait() is in microseconds). 531 * 532 * We assume that TSC is present and already measured. 533 * Possible TSC frequency jumps are irrelevant to the 534 * calibration loop below, the CPU clock management code is 535 * not yet started, and we do not enter sleep states. 536 */ 537 KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0, 538 ("TSC not initialized")); 539 if (!x2apic_mode) { 540 r = rdtsc(); 541 for (rx = 0; rx < LOOPS; rx++) { 542 (void)lapic_read_icr_lo(); 543 ia32_pause(); 544 } 545 r = rdtsc() - r; 546 r1 = tsc_freq * LOOPS; 547 r2 = r * 1000000; 548 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1; 549 if (bootverbose) { 550 printf("LAPIC: ipi_wait() us multiplier %ju (r %ju " 551 "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult, 552 (uintmax_t)r, (uintmax_t)tsc_freq); 553 } 554 } 555#undef LOOPS 556#endif /* SMP */ 557} 558 559/* 560 * Create a local APIC instance. 561 */ 562static void 563native_lapic_create(u_int apic_id, int boot_cpu) 564{ 565 int i; 566 567 if (apic_id > MAX_APIC_ID) { 568 printf("APIC: Ignoring local APIC with ID %d\n", apic_id); 569 if (boot_cpu) 570 panic("Can't ignore BSP"); 571 return; 572 } 573 KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u", 574 apic_id)); 575 576 /* 577 * Assume no local LVT overrides and a cluster of 0 and 578 * intra-cluster ID of 0. 579 */ 580 lapics[apic_id].la_present = 1; 581 lapics[apic_id].la_id = apic_id; 582 for (i = 0; i <= APIC_LVT_MAX; i++) { 583 lapics[apic_id].la_lvts[i] = lvts[i]; 584 lapics[apic_id].la_lvts[i].lvt_active = 0; 585 } 586 for (i = 0; i <= APIC_NUM_IOINTS; i++) 587 lapics[apic_id].la_ioint_irqs[i] = -1; 588 lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL; 589 lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = 590 IRQ_TIMER; 591#ifdef KDTRACE_HOOKS 592 lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] = 593 IRQ_DTRACE_RET; 594#endif 595#ifdef XENHVM 596 lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN; 597#endif 598 599 600#ifdef SMP 601 cpu_add(apic_id, boot_cpu); 602#endif 603} 604 605/* 606 * Dump contents of local APIC registers 607 */ 608static void 609native_lapic_dump(const char* str) 610{ 611 uint32_t maxlvt; 612 613 maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 614 printf("cpu%d %s:\n", PCPU_GET(cpuid), str); 615 printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x", 616 lapic_read32(LAPIC_ID), lapic_read32(LAPIC_VERSION), 617 lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR)); 618 if ((cpu_feature2 & CPUID2_X2APIC) != 0) 619 printf(" x2APIC: %d", x2apic_mode); 620 printf("\n lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", 621 lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1), 622 lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR)); 623 printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x", 624 lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL), 625 lapic_read32(LAPIC_LVT_ERROR)); 626 if (maxlvt >= APIC_LVT_PMC) 627 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT)); 628 printf("\n"); 629 if (maxlvt >= APIC_LVT_CMCI) 630 printf(" cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI)); 631} 632 633static void 634native_lapic_xapic_mode(void) 635{ 636 register_t saveintr; 637 638 saveintr = intr_disable(); 639 if (x2apic_mode) 640 native_lapic_enable_x2apic(); 641 intr_restore(saveintr); 642} 643 644static void 645native_lapic_setup(int boot) 646{ 647 struct lapic *la; 648 uint32_t maxlvt; 649 register_t saveintr; 650 char buf[MAXCOMLEN + 1]; 651 652 saveintr = intr_disable(); 653 654 la = &lapics[lapic_id()]; 655 KASSERT(la->la_present, ("missing APIC structure")); 656 maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 657 658 /* Initialize the TPR to allow all interrupts. */ 659 lapic_set_tpr(0); 660 661 /* Setup spurious vector and enable the local APIC. */ 662 lapic_enable(); 663 664 /* Program LINT[01] LVT entries. */ 665 lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0, 666 lapic_read32(LAPIC_LVT_LINT0))); 667 lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1, 668 lapic_read32(LAPIC_LVT_LINT1))); 669 670 /* Program the PMC LVT entry if present. */ 671 if (maxlvt >= APIC_LVT_PMC) { 672 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC, 673 LAPIC_LVT_PCINT)); 674 } 675 676 /* Program timer LVT and setup handler. */ 677 la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER, 678 lapic_read32(LAPIC_LVT_TIMER)); 679 la->lvt_timer_last = la->lvt_timer_base; 680 lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base); 681 if (boot) { 682 snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid)); 683 intrcnt_add(buf, &la->la_timer_count); 684 } 685 686 /* Setup the timer if configured. */ 687 if (la->la_timer_mode != LAT_MODE_UNDEF) { 688 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor", 689 lapic_id())); 690 switch (la->la_timer_mode) { 691 case LAT_MODE_PERIODIC: 692 lapic_timer_set_divisor(lapic_timer_divisor); 693 lapic_timer_periodic(la); 694 break; 695 case LAT_MODE_ONESHOT: 696 lapic_timer_set_divisor(lapic_timer_divisor); 697 lapic_timer_oneshot(la); 698 break; 699 case LAT_MODE_DEADLINE: 700 lapic_timer_deadline(la); 701 break; 702 default: 703 panic("corrupted la_timer_mode %p %d", la, 704 la->la_timer_mode); 705 } 706 } 707 708 /* Program error LVT and clear any existing errors. */ 709 lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR, 710 lapic_read32(LAPIC_LVT_ERROR))); 711 lapic_write32(LAPIC_ESR, 0); 712 713 /* XXX: Thermal LVT */ 714 715 /* Program the CMCI LVT entry if present. */ 716 if (maxlvt >= APIC_LVT_CMCI) { 717 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI, 718 lapic_read32(LAPIC_LVT_CMCI))); 719 } 720 721 intr_restore(saveintr); 722} 723 724static void 725native_lapic_reenable_pmc(void) 726{ 727#ifdef HWPMC_HOOKS 728 uint32_t value; 729 730 value = lapic_read32(LAPIC_LVT_PCINT); 731 value &= ~APIC_LVT_M; 732 lapic_write32(LAPIC_LVT_PCINT, value); 733#endif 734} 735 736#ifdef HWPMC_HOOKS 737static void 738lapic_update_pmc(void *dummy) 739{ 740 struct lapic *la; 741 742 la = &lapics[lapic_id()]; 743 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC, 744 lapic_read32(LAPIC_LVT_PCINT))); 745} 746#endif 747 748static int 749native_lapic_enable_pmc(void) 750{ 751#ifdef HWPMC_HOOKS 752 u_int32_t maxlvt; 753 754 /* Fail if the local APIC is not present. */ 755 if (!x2apic_mode && lapic_map == NULL) 756 return (0); 757 758 /* Fail if the PMC LVT is not present. */ 759 maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 760 if (maxlvt < APIC_LVT_PMC) 761 return (0); 762 763 lvts[APIC_LVT_PMC].lvt_masked = 0; 764 765#ifdef EARLY_AP_STARTUP 766 MPASS(mp_ncpus == 1 || smp_started); 767 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 768#else 769#ifdef SMP 770 /* 771 * If hwpmc was loaded at boot time then the APs may not be 772 * started yet. In that case, don't forward the request to 773 * them as they will program the lvt when they start. 774 */ 775 if (smp_started) 776 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 777 else 778#endif 779 lapic_update_pmc(NULL); 780#endif 781 return (1); 782#else 783 return (0); 784#endif 785} 786 787static void 788native_lapic_disable_pmc(void) 789{ 790#ifdef HWPMC_HOOKS 791 u_int32_t maxlvt; 792 793 /* Fail if the local APIC is not present. */ 794 if (!x2apic_mode && lapic_map == NULL) 795 return; 796 797 /* Fail if the PMC LVT is not present. */ 798 maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 799 if (maxlvt < APIC_LVT_PMC) 800 return; 801 802 lvts[APIC_LVT_PMC].lvt_masked = 1; 803 804#ifdef SMP 805 /* The APs should always be started when hwpmc is unloaded. */ 806 KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early")); 807#endif 808 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 809#endif 810} 811 812static void 813lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la) 814{ 815 u_long value; 816 817 /* Start off with a divisor of 2 (power on reset default). */ 818 lapic_timer_divisor = 2; 819 /* Try to calibrate the local APIC timer. */ 820 do { 821 lapic_timer_set_divisor(lapic_timer_divisor); 822 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT); 823 DELAY(1000000); 824 value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER); 825 if (value != APIC_TIMER_MAX_COUNT) 826 break; 827 lapic_timer_divisor <<= 1; 828 } while (lapic_timer_divisor <= 128); 829 if (lapic_timer_divisor > 128) 830 panic("lapic: Divisor too big"); 831 if (bootverbose) { 832 printf("lapic: Divisor %lu, Frequency %lu Hz\n", 833 lapic_timer_divisor, value); 834 } 835 count_freq = value; 836} 837 838static void 839lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused) 840{ 841 842 if (bootverbose) { 843 printf("lapic: deadline tsc mode, Frequency %ju Hz\n", 844 (uintmax_t)tsc_freq); 845 } 846} 847 848static void 849lapic_change_mode(struct eventtimer *et, struct lapic *la, 850 enum lat_timer_mode newmode) 851{ 852 853 if (la->la_timer_mode == newmode) 854 return; 855 switch (newmode) { 856 case LAT_MODE_PERIODIC: 857 lapic_timer_set_divisor(lapic_timer_divisor); 858 et->et_frequency = count_freq; 859 break; 860 case LAT_MODE_DEADLINE: 861 et->et_frequency = tsc_freq; 862 break; 863 case LAT_MODE_ONESHOT: 864 lapic_timer_set_divisor(lapic_timer_divisor); 865 et->et_frequency = count_freq; 866 break; 867 default: 868 panic("lapic_change_mode %d", newmode); 869 } 870 la->la_timer_mode = newmode; 871 et->et_min_period = (0x00000002LLU << 32) / et->et_frequency; 872 et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency; 873} 874 875static int 876lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) 877{ 878 struct lapic *la; 879 880 la = &lapics[PCPU_GET(apic_id)]; 881 if (et->et_frequency == 0) { 882 lapic_calibrate_initcount(et, la); 883 if (lapic_timer_tsc_deadline) 884 lapic_calibrate_deadline(et, la); 885 } 886 if (period != 0) { 887 lapic_change_mode(et, la, LAT_MODE_PERIODIC); 888 la->la_timer_period = ((uint32_t)et->et_frequency * period) >> 889 32; 890 lapic_timer_periodic(la); 891 } else if (lapic_timer_tsc_deadline) { 892 lapic_change_mode(et, la, LAT_MODE_DEADLINE); 893 la->la_timer_period = (et->et_frequency * first) >> 32; 894 lapic_timer_deadline(la); 895 } else { 896 lapic_change_mode(et, la, LAT_MODE_ONESHOT); 897 la->la_timer_period = ((uint32_t)et->et_frequency * first) >> 898 32; 899 lapic_timer_oneshot(la); 900 } 901 return (0); 902} 903 904static int 905lapic_et_stop(struct eventtimer *et) 906{ 907 struct lapic *la; 908 909 la = &lapics[PCPU_GET(apic_id)]; 910 lapic_timer_stop(la); 911 la->la_timer_mode = LAT_MODE_UNDEF; 912 return (0); 913} 914 915static void 916native_lapic_disable(void) 917{ 918 uint32_t value; 919 920 /* Software disable the local APIC. */ 921 value = lapic_read32(LAPIC_SVR); 922 value &= ~APIC_SVR_SWEN; 923 lapic_write32(LAPIC_SVR, value); 924} 925 926static void 927lapic_enable(void) 928{ 929 uint32_t value; 930 931 /* Program the spurious vector to enable the local APIC. */ 932 value = lapic_read32(LAPIC_SVR); 933 value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); 934 value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT; 935 if (lapic_eoi_suppression) 936 value |= APIC_SVR_EOI_SUPPRESSION; 937 lapic_write32(LAPIC_SVR, value); 938} 939 940/* Reset the local APIC on the BSP during resume. */ 941static void 942lapic_resume(struct pic *pic, bool suspend_cancelled) 943{ 944 945 lapic_setup(0); 946} 947 948static int 949native_lapic_id(void) 950{ 951 uint32_t v; 952 953 KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped")); 954 v = lapic_read32(LAPIC_ID); 955 if (!x2apic_mode) 956 v >>= APIC_ID_SHIFT; 957 return (v); 958} 959 960static int 961native_lapic_intr_pending(u_int vector) 962{ 963 uint32_t irr; 964 965 /* 966 * The IRR registers are an array of registers each of which 967 * only describes 32 interrupts in the low 32 bits. Thus, we 968 * divide the vector by 32 to get the register index. 969 * Finally, we modulus the vector by 32 to determine the 970 * individual bit to test. 971 */ 972 irr = lapic_read32(LAPIC_IRR0 + vector / 32); 973 return (irr & 1 << (vector % 32)); 974} 975 976static void 977native_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 978{ 979 struct lapic *la; 980 981 KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist", 982 __func__, apic_id)); 983 KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big", 984 __func__, cluster)); 985 KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID, 986 ("%s: intra cluster id %u too big", __func__, cluster_id)); 987 la = &lapics[apic_id]; 988 la->la_cluster = cluster; 989 la->la_cluster_id = cluster_id; 990} 991 992static int 993native_lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked) 994{ 995 996 if (pin > APIC_LVT_MAX) 997 return (EINVAL); 998 if (apic_id == APIC_ID_ALL) { 999 lvts[pin].lvt_masked = masked; 1000 if (bootverbose) 1001 printf("lapic:"); 1002 } else { 1003 KASSERT(lapics[apic_id].la_present, 1004 ("%s: missing APIC %u", __func__, apic_id)); 1005 lapics[apic_id].la_lvts[pin].lvt_masked = masked; 1006 lapics[apic_id].la_lvts[pin].lvt_active = 1; 1007 if (bootverbose) 1008 printf("lapic%u:", apic_id); 1009 } 1010 if (bootverbose) 1011 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked"); 1012 return (0); 1013} 1014 1015static int 1016native_lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode) 1017{ 1018 struct lvt *lvt; 1019 1020 if (pin > APIC_LVT_MAX) 1021 return (EINVAL); 1022 if (apic_id == APIC_ID_ALL) { 1023 lvt = &lvts[pin]; 1024 if (bootverbose) 1025 printf("lapic:"); 1026 } else { 1027 KASSERT(lapics[apic_id].la_present, 1028 ("%s: missing APIC %u", __func__, apic_id)); 1029 lvt = &lapics[apic_id].la_lvts[pin]; 1030 lvt->lvt_active = 1; 1031 if (bootverbose) 1032 printf("lapic%u:", apic_id); 1033 } 1034 lvt->lvt_mode = mode; 1035 switch (mode) { 1036 case APIC_LVT_DM_NMI: 1037 case APIC_LVT_DM_SMI: 1038 case APIC_LVT_DM_INIT: 1039 case APIC_LVT_DM_EXTINT: 1040 lvt->lvt_edgetrigger = 1; 1041 lvt->lvt_activehi = 1; 1042 if (mode == APIC_LVT_DM_EXTINT) 1043 lvt->lvt_masked = 1; 1044 else 1045 lvt->lvt_masked = 0; 1046 break; 1047 default: 1048 panic("Unsupported delivery mode: 0x%x\n", mode); 1049 } 1050 if (bootverbose) { 1051 printf(" Routing "); 1052 switch (mode) { 1053 case APIC_LVT_DM_NMI: 1054 printf("NMI"); 1055 break; 1056 case APIC_LVT_DM_SMI: 1057 printf("SMI"); 1058 break; 1059 case APIC_LVT_DM_INIT: 1060 printf("INIT"); 1061 break; 1062 case APIC_LVT_DM_EXTINT: 1063 printf("ExtINT"); 1064 break; 1065 } 1066 printf(" -> LINT%u\n", pin); 1067 } 1068 return (0); 1069} 1070 1071static int 1072native_lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol) 1073{ 1074 1075 if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM) 1076 return (EINVAL); 1077 if (apic_id == APIC_ID_ALL) { 1078 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH); 1079 if (bootverbose) 1080 printf("lapic:"); 1081 } else { 1082 KASSERT(lapics[apic_id].la_present, 1083 ("%s: missing APIC %u", __func__, apic_id)); 1084 lapics[apic_id].la_lvts[pin].lvt_active = 1; 1085 lapics[apic_id].la_lvts[pin].lvt_activehi = 1086 (pol == INTR_POLARITY_HIGH); 1087 if (bootverbose) 1088 printf("lapic%u:", apic_id); 1089 } 1090 if (bootverbose) 1091 printf(" LINT%u polarity: %s\n", pin, 1092 pol == INTR_POLARITY_HIGH ? "high" : "low"); 1093 return (0); 1094} 1095 1096static int 1097native_lapic_set_lvt_triggermode(u_int apic_id, u_int pin, 1098 enum intr_trigger trigger) 1099{ 1100 1101 if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM) 1102 return (EINVAL); 1103 if (apic_id == APIC_ID_ALL) { 1104 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE); 1105 if (bootverbose) 1106 printf("lapic:"); 1107 } else { 1108 KASSERT(lapics[apic_id].la_present, 1109 ("%s: missing APIC %u", __func__, apic_id)); 1110 lapics[apic_id].la_lvts[pin].lvt_edgetrigger = 1111 (trigger == INTR_TRIGGER_EDGE); 1112 lapics[apic_id].la_lvts[pin].lvt_active = 1; 1113 if (bootverbose) 1114 printf("lapic%u:", apic_id); 1115 } 1116 if (bootverbose) 1117 printf(" LINT%u trigger: %s\n", pin, 1118 trigger == INTR_TRIGGER_EDGE ? "edge" : "level"); 1119 return (0); 1120} 1121 1122/* 1123 * Adjust the TPR of the current CPU so that it blocks all interrupts below 1124 * the passed in vector. 1125 */ 1126static void 1127lapic_set_tpr(u_int vector) 1128{ 1129#ifdef CHEAP_TPR 1130 lapic_write32(LAPIC_TPR, vector); 1131#else 1132 uint32_t tpr; 1133 1134 tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO; 1135 tpr |= vector; 1136 lapic_write32(LAPIC_TPR, tpr); 1137#endif 1138} 1139 1140static void 1141native_lapic_eoi(void) 1142{ 1143 1144 lapic_write32_nofence(LAPIC_EOI, 0); 1145} 1146 1147void 1148lapic_handle_intr(int vector, struct trapframe *frame) 1149{ 1150 struct intsrc *isrc; 1151 1152 isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id), 1153 vector)); 1154 intr_execute_handlers(isrc, frame); 1155} 1156 1157void 1158lapic_handle_timer(struct trapframe *frame) 1159{ 1160 struct lapic *la; 1161 struct trapframe *oldframe; 1162 struct thread *td; 1163 1164 /* Send EOI first thing. */ 1165 lapic_eoi(); 1166 1167#if defined(SMP) && !defined(SCHED_ULE) 1168 /* 1169 * Don't do any accounting for the disabled HTT cores, since it 1170 * will provide misleading numbers for the userland. 1171 * 1172 * No locking is necessary here, since even if we lose the race 1173 * when hlt_cpus_mask changes it is not a big deal, really. 1174 * 1175 * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask 1176 * and unlike other schedulers it actually schedules threads to 1177 * those CPUs. 1178 */ 1179 if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)) 1180 return; 1181#endif 1182 1183 /* Look up our local APIC structure for the tick counters. */ 1184 la = &lapics[PCPU_GET(apic_id)]; 1185 (*la->la_timer_count)++; 1186 critical_enter(); 1187 if (lapic_et.et_active) { 1188 td = curthread; 1189 td->td_intr_nesting_level++; 1190 oldframe = td->td_intr_frame; 1191 td->td_intr_frame = frame; 1192 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg); 1193 td->td_intr_frame = oldframe; 1194 td->td_intr_nesting_level--; 1195 } 1196 critical_exit(); 1197} 1198 1199static void 1200lapic_timer_set_divisor(u_int divisor) 1201{ 1202 1203 KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor)); 1204 KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors), 1205 ("lapic: invalid divisor %u", divisor)); 1206 lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]); 1207} 1208 1209static void 1210lapic_timer_oneshot(struct lapic *la) 1211{ 1212 uint32_t value; 1213 1214 value = la->lvt_timer_base; 1215 value &= ~(APIC_LVTT_TM | APIC_LVT_M); 1216 value |= APIC_LVTT_TM_ONE_SHOT; 1217 la->lvt_timer_last = value; 1218 lapic_write32(LAPIC_LVT_TIMER, value); 1219 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period); 1220} 1221 1222static void 1223lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count) 1224{ 1225 uint32_t value; 1226 1227 value = la->lvt_timer_base; 1228 value &= ~APIC_LVTT_TM; 1229 value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M; 1230 la->lvt_timer_last = value; 1231 lapic_write32(LAPIC_LVT_TIMER, value); 1232 lapic_write32(LAPIC_ICR_TIMER, count); 1233} 1234 1235static void 1236lapic_timer_periodic(struct lapic *la) 1237{ 1238 uint32_t value; 1239 1240 value = la->lvt_timer_base; 1241 value &= ~(APIC_LVTT_TM | APIC_LVT_M); 1242 value |= APIC_LVTT_TM_PERIODIC; 1243 la->lvt_timer_last = value; 1244 lapic_write32(LAPIC_LVT_TIMER, value); 1245 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period); 1246} 1247 1248static void 1249lapic_timer_deadline(struct lapic *la) 1250{ 1251 uint32_t value; 1252 1253 value = la->lvt_timer_base; 1254 value &= ~(APIC_LVTT_TM | APIC_LVT_M); 1255 value |= APIC_LVTT_TM_TSCDLT; 1256 if (value != la->lvt_timer_last) { 1257 la->lvt_timer_last = value; 1258 lapic_write32_nofence(LAPIC_LVT_TIMER, value); 1259 if (!x2apic_mode) 1260 mfence(); 1261 } 1262 wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc()); 1263} 1264 1265static void 1266lapic_timer_stop(struct lapic *la) 1267{ 1268 uint32_t value; 1269 1270 if (la->la_timer_mode == LAT_MODE_DEADLINE) { 1271 wrmsr(MSR_TSC_DEADLINE, 0); 1272 mfence(); 1273 } else { 1274 value = la->lvt_timer_base; 1275 value &= ~APIC_LVTT_TM; 1276 value |= APIC_LVT_M; 1277 la->lvt_timer_last = value; 1278 lapic_write32(LAPIC_LVT_TIMER, value); 1279 } 1280} 1281 1282void 1283lapic_handle_cmc(void) 1284{ 1285 1286 lapic_eoi(); 1287 cmc_intr(); 1288} 1289 1290/* 1291 * Called from the mca_init() to activate the CMC interrupt if this CPU is 1292 * responsible for monitoring any MC banks for CMC events. Since mca_init() 1293 * is called prior to lapic_setup() during boot, this just needs to unmask 1294 * this CPU's LVT_CMCI entry. 1295 */ 1296static void 1297native_lapic_enable_cmc(void) 1298{ 1299 u_int apic_id; 1300 1301#ifdef DEV_ATPIC 1302 if (!x2apic_mode && lapic_map == NULL) 1303 return; 1304#endif 1305 apic_id = PCPU_GET(apic_id); 1306 KASSERT(lapics[apic_id].la_present, 1307 ("%s: missing APIC %u", __func__, apic_id)); 1308 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0; 1309 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1; 1310 if (bootverbose) 1311 printf("lapic%u: CMCI unmasked\n", apic_id); 1312} 1313 1314void 1315lapic_handle_error(void) 1316{ 1317 uint32_t esr; 1318 1319 /* 1320 * Read the contents of the error status register. Write to 1321 * the register first before reading from it to force the APIC 1322 * to update its value to indicate any errors that have 1323 * occurred since the previous write to the register. 1324 */ 1325 lapic_write32(LAPIC_ESR, 0); 1326 esr = lapic_read32(LAPIC_ESR); 1327 1328 printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr); 1329 lapic_eoi(); 1330} 1331 1332static u_int 1333native_apic_cpuid(u_int apic_id) 1334{ 1335#ifdef SMP 1336 return apic_cpuids[apic_id]; 1337#else 1338 return 0; 1339#endif 1340} 1341 1342/* Request a free IDT vector to be used by the specified IRQ. */ 1343static u_int 1344native_apic_alloc_vector(u_int apic_id, u_int irq) 1345{ 1346 u_int vector; 1347 1348 KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 1349 1350 /* 1351 * Search for a free vector. Currently we just use a very simple 1352 * algorithm to find the first free vector. 1353 */ 1354 mtx_lock_spin(&icu_lock); 1355 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 1356 if (lapics[apic_id].la_ioint_irqs[vector] != -1) 1357 continue; 1358 lapics[apic_id].la_ioint_irqs[vector] = irq; 1359 mtx_unlock_spin(&icu_lock); 1360 return (vector + APIC_IO_INTS); 1361 } 1362 mtx_unlock_spin(&icu_lock); 1363 return (0); 1364} 1365 1366/* 1367 * Request 'count' free contiguous IDT vectors to be used by 'count' 1368 * IRQs. 'count' must be a power of two and the vectors will be 1369 * aligned on a boundary of 'align'. If the request cannot be 1370 * satisfied, 0 is returned. 1371 */ 1372static u_int 1373native_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 1374{ 1375 u_int first, run, vector; 1376 1377 KASSERT(powerof2(count), ("bad count")); 1378 KASSERT(powerof2(align), ("bad align")); 1379 KASSERT(align >= count, ("align < count")); 1380#ifdef INVARIANTS 1381 for (run = 0; run < count; run++) 1382 KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u", 1383 irqs[run], run)); 1384#endif 1385 1386 /* 1387 * Search for 'count' free vectors. As with apic_alloc_vector(), 1388 * this just uses a simple first fit algorithm. 1389 */ 1390 run = 0; 1391 first = 0; 1392 mtx_lock_spin(&icu_lock); 1393 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 1394 1395 /* Vector is in use, end run. */ 1396 if (lapics[apic_id].la_ioint_irqs[vector] != -1) { 1397 run = 0; 1398 first = 0; 1399 continue; 1400 } 1401 1402 /* Start a new run if run == 0 and vector is aligned. */ 1403 if (run == 0) { 1404 if ((vector & (align - 1)) != 0) 1405 continue; 1406 first = vector; 1407 } 1408 run++; 1409 1410 /* Keep looping if the run isn't long enough yet. */ 1411 if (run < count) 1412 continue; 1413 1414 /* Found a run, assign IRQs and return the first vector. */ 1415 for (vector = 0; vector < count; vector++) 1416 lapics[apic_id].la_ioint_irqs[first + vector] = 1417 irqs[vector]; 1418 mtx_unlock_spin(&icu_lock); 1419 return (first + APIC_IO_INTS); 1420 } 1421 mtx_unlock_spin(&icu_lock); 1422 printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count); 1423 return (0); 1424} 1425 1426/* 1427 * Enable a vector for a particular apic_id. Since all lapics share idt 1428 * entries and ioint_handlers this enables the vector on all lapics. lapics 1429 * which do not have the vector configured would report spurious interrupts 1430 * should it fire. 1431 */ 1432static void 1433native_apic_enable_vector(u_int apic_id, u_int vector) 1434{ 1435 1436 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1437 KASSERT(ioint_handlers[vector / 32] != NULL, 1438 ("No ISR handler for vector %u", vector)); 1439#ifdef KDTRACE_HOOKS 1440 KASSERT(vector != IDT_DTRACE_RET, 1441 ("Attempt to overwrite DTrace entry")); 1442#endif 1443 setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL, 1444 GSEL_APIC); 1445} 1446 1447static void 1448native_apic_disable_vector(u_int apic_id, u_int vector) 1449{ 1450 1451 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1452#ifdef KDTRACE_HOOKS 1453 KASSERT(vector != IDT_DTRACE_RET, 1454 ("Attempt to overwrite DTrace entry")); 1455#endif 1456 KASSERT(ioint_handlers[vector / 32] != NULL, 1457 ("No ISR handler for vector %u", vector)); 1458#ifdef notyet 1459 /* 1460 * We can not currently clear the idt entry because other cpus 1461 * may have a valid vector at this offset. 1462 */ 1463 setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC); 1464#endif 1465} 1466 1467/* Release an APIC vector when it's no longer in use. */ 1468static void 1469native_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 1470{ 1471 struct thread *td; 1472 1473 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1474 vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1475 ("Vector %u does not map to an IRQ line", vector)); 1476 KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 1477 KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] == 1478 irq, ("IRQ mismatch")); 1479#ifdef KDTRACE_HOOKS 1480 KASSERT(vector != IDT_DTRACE_RET, 1481 ("Attempt to overwrite DTrace entry")); 1482#endif 1483 1484 /* 1485 * Bind us to the cpu that owned the vector before freeing it so 1486 * we don't lose an interrupt delivery race. 1487 */ 1488 td = curthread; 1489 if (!rebooting) { 1490 thread_lock(td); 1491 if (sched_is_bound(td)) 1492 panic("apic_free_vector: Thread already bound.\n"); 1493 sched_bind(td, apic_cpuid(apic_id)); 1494 thread_unlock(td); 1495 } 1496 mtx_lock_spin(&icu_lock); 1497 lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1; 1498 mtx_unlock_spin(&icu_lock); 1499 if (!rebooting) { 1500 thread_lock(td); 1501 sched_unbind(td); 1502 thread_unlock(td); 1503 } 1504} 1505 1506/* Map an IDT vector (APIC) to an IRQ (interrupt source). */ 1507static u_int 1508apic_idt_to_irq(u_int apic_id, u_int vector) 1509{ 1510 int irq; 1511 1512 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1513 vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1514 ("Vector %u does not map to an IRQ line", vector)); 1515#ifdef KDTRACE_HOOKS 1516 KASSERT(vector != IDT_DTRACE_RET, 1517 ("Attempt to overwrite DTrace entry")); 1518#endif 1519 irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS]; 1520 if (irq < 0) 1521 irq = 0; 1522 return (irq); 1523} 1524 1525#ifdef DDB 1526/* 1527 * Dump data about APIC IDT vector mappings. 1528 */ 1529DB_SHOW_COMMAND(apic, db_show_apic) 1530{ 1531 struct intsrc *isrc; 1532 int i, verbose; 1533 u_int apic_id; 1534 u_int irq; 1535 1536 if (strcmp(modif, "vv") == 0) 1537 verbose = 2; 1538 else if (strcmp(modif, "v") == 0) 1539 verbose = 1; 1540 else 1541 verbose = 0; 1542 for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) { 1543 if (lapics[apic_id].la_present == 0) 1544 continue; 1545 db_printf("Interrupts bound to lapic %u\n", apic_id); 1546 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) { 1547 irq = lapics[apic_id].la_ioint_irqs[i]; 1548 if (irq == -1 || irq == IRQ_SYSCALL) 1549 continue; 1550#ifdef KDTRACE_HOOKS 1551 if (irq == IRQ_DTRACE_RET) 1552 continue; 1553#endif 1554#ifdef XENHVM 1555 if (irq == IRQ_EVTCHN) 1556 continue; 1557#endif 1558 db_printf("vec 0x%2x -> ", i + APIC_IO_INTS); 1559 if (irq == IRQ_TIMER) 1560 db_printf("lapic timer\n"); 1561 else if (irq < NUM_IO_INTS) { 1562 isrc = intr_lookup_source(irq); 1563 if (isrc == NULL || verbose == 0) 1564 db_printf("IRQ %u\n", irq); 1565 else 1566 db_dump_intr_event(isrc->is_event, 1567 verbose == 2); 1568 } else 1569 db_printf("IRQ %u ???\n", irq); 1570 } 1571 } 1572} 1573 1574static void 1575dump_mask(const char *prefix, uint32_t v, int base) 1576{ 1577 int i, first; 1578 1579 first = 1; 1580 for (i = 0; i < 32; i++) 1581 if (v & (1 << i)) { 1582 if (first) { 1583 db_printf("%s:", prefix); 1584 first = 0; 1585 } 1586 db_printf(" %02x", base + i); 1587 } 1588 if (!first) 1589 db_printf("\n"); 1590} 1591 1592/* Show info from the lapic regs for this CPU. */ 1593DB_SHOW_COMMAND(lapic, db_show_lapic) 1594{ 1595 uint32_t v; 1596 1597 db_printf("lapic ID = %d\n", lapic_id()); 1598 v = lapic_read32(LAPIC_VERSION); 1599 db_printf("version = %d.%d\n", (v & APIC_VER_VERSION) >> 4, 1600 v & 0xf); 1601 db_printf("max LVT = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT); 1602 v = lapic_read32(LAPIC_SVR); 1603 db_printf("SVR = %02x (%s)\n", v & APIC_SVR_VECTOR, 1604 v & APIC_SVR_ENABLE ? "enabled" : "disabled"); 1605 db_printf("TPR = %02x\n", lapic_read32(LAPIC_TPR)); 1606 1607#define dump_field(prefix, regn, index) \ 1608 dump_mask(__XSTRING(prefix ## index), \ 1609 lapic_read32(LAPIC_ ## regn ## index), \ 1610 index * 32) 1611 1612 db_printf("In-service Interrupts:\n"); 1613 dump_field(isr, ISR, 0); 1614 dump_field(isr, ISR, 1); 1615 dump_field(isr, ISR, 2); 1616 dump_field(isr, ISR, 3); 1617 dump_field(isr, ISR, 4); 1618 dump_field(isr, ISR, 5); 1619 dump_field(isr, ISR, 6); 1620 dump_field(isr, ISR, 7); 1621 1622 db_printf("TMR Interrupts:\n"); 1623 dump_field(tmr, TMR, 0); 1624 dump_field(tmr, TMR, 1); 1625 dump_field(tmr, TMR, 2); 1626 dump_field(tmr, TMR, 3); 1627 dump_field(tmr, TMR, 4); 1628 dump_field(tmr, TMR, 5); 1629 dump_field(tmr, TMR, 6); 1630 dump_field(tmr, TMR, 7); 1631 1632 db_printf("IRR Interrupts:\n"); 1633 dump_field(irr, IRR, 0); 1634 dump_field(irr, IRR, 1); 1635 dump_field(irr, IRR, 2); 1636 dump_field(irr, IRR, 3); 1637 dump_field(irr, IRR, 4); 1638 dump_field(irr, IRR, 5); 1639 dump_field(irr, IRR, 6); 1640 dump_field(irr, IRR, 7); 1641 1642#undef dump_field 1643} 1644#endif 1645 1646/* 1647 * APIC probing support code. This includes code to manage enumerators. 1648 */ 1649 1650static SLIST_HEAD(, apic_enumerator) enumerators = 1651 SLIST_HEAD_INITIALIZER(enumerators); 1652static struct apic_enumerator *best_enum; 1653 1654void 1655apic_register_enumerator(struct apic_enumerator *enumerator) 1656{ 1657#ifdef INVARIANTS 1658 struct apic_enumerator *apic_enum; 1659 1660 SLIST_FOREACH(apic_enum, &enumerators, apic_next) { 1661 if (apic_enum == enumerator) 1662 panic("%s: Duplicate register of %s", __func__, 1663 enumerator->apic_name); 1664 } 1665#endif 1666 SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next); 1667} 1668 1669/* 1670 * We have to look for CPU's very, very early because certain subsystems 1671 * want to know how many CPU's we have extremely early on in the boot 1672 * process. 1673 */ 1674static void 1675apic_init(void *dummy __unused) 1676{ 1677 struct apic_enumerator *enumerator; 1678 int retval, best; 1679 1680 /* We only support built in local APICs. */ 1681 if (!(cpu_feature & CPUID_APIC)) 1682 return; 1683 1684 /* Don't probe if APIC mode is disabled. */ 1685 if (resource_disabled("apic", 0)) 1686 return; 1687 1688 /* Probe all the enumerators to find the best match. */ 1689 best_enum = NULL; 1690 best = 0; 1691 SLIST_FOREACH(enumerator, &enumerators, apic_next) { 1692 retval = enumerator->apic_probe(); 1693 if (retval > 0) 1694 continue; 1695 if (best_enum == NULL || best < retval) { 1696 best_enum = enumerator; 1697 best = retval; 1698 } 1699 } 1700 if (best_enum == NULL) { 1701 if (bootverbose) 1702 printf("APIC: Could not find any APICs.\n"); 1703#ifndef DEV_ATPIC 1704 panic("running without device atpic requires a local APIC"); 1705#endif 1706 return; 1707 } 1708 1709 if (bootverbose) 1710 printf("APIC: Using the %s enumerator.\n", 1711 best_enum->apic_name); 1712 1713#ifdef I686_CPU 1714 /* 1715 * To work around an errata, we disable the local APIC on some 1716 * CPUs during early startup. We need to turn the local APIC back 1717 * on on such CPUs now. 1718 */ 1719 ppro_reenable_apic(); 1720#endif 1721 1722 /* Probe the CPU's in the system. */ 1723 retval = best_enum->apic_probe_cpus(); 1724 if (retval != 0) 1725 printf("%s: Failed to probe CPUs: returned %d\n", 1726 best_enum->apic_name, retval); 1727 1728} 1729SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL); 1730 1731/* 1732 * Setup the local APIC. We have to do this prior to starting up the APs 1733 * in the SMP case. 1734 */ 1735static void 1736apic_setup_local(void *dummy __unused) 1737{ 1738 int retval; 1739 1740 if (best_enum == NULL) 1741 return; 1742 1743 /* Initialize the local APIC. */ 1744 retval = best_enum->apic_setup_local(); 1745 if (retval != 0) 1746 printf("%s: Failed to setup the local APIC: returned %d\n", 1747 best_enum->apic_name, retval); 1748} 1749SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL); 1750 1751/* 1752 * Setup the I/O APICs. 1753 */ 1754static void 1755apic_setup_io(void *dummy __unused) 1756{ 1757 int retval; 1758 1759 if (best_enum == NULL) 1760 return; 1761 1762 /* 1763 * Local APIC must be registered before other PICs and pseudo PICs 1764 * for proper suspend/resume order. 1765 */ 1766 intr_register_pic(&lapic_pic); 1767 1768 retval = best_enum->apic_setup_io(); 1769 if (retval != 0) 1770 printf("%s: Failed to setup I/O APICs: returned %d\n", 1771 best_enum->apic_name, retval); 1772 1773 /* 1774 * Finish setting up the local APIC on the BSP once we know 1775 * how to properly program the LINT pins. In particular, this 1776 * enables the EOI suppression mode, if LAPIC support it and 1777 * user did not disabled the mode. 1778 */ 1779 lapic_setup(1); 1780 if (bootverbose) 1781 lapic_dump("BSP"); 1782 1783 /* Enable the MSI "pic". */ 1784 init_ops.msi_init(); 1785} 1786SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL); 1787 1788#ifdef SMP 1789/* 1790 * Inter Processor Interrupt functions. The lapic_ipi_*() functions are 1791 * private to the MD code. The public interface for the rest of the 1792 * kernel is defined in mp_machdep.c. 1793 */ 1794 1795/* 1796 * Wait delay microseconds for IPI to be sent. If delay is -1, we 1797 * wait forever. 1798 */ 1799static int 1800native_lapic_ipi_wait(int delay) 1801{ 1802 uint64_t rx; 1803 1804 /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */ 1805 if (x2apic_mode) 1806 return (1); 1807 1808 for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) { 1809 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) == 1810 APIC_DELSTAT_IDLE) 1811 return (1); 1812 ia32_pause(); 1813 } 1814 return (0); 1815} 1816 1817static void 1818native_lapic_ipi_raw(register_t icrlo, u_int dest) 1819{ 1820 uint64_t icr; 1821 uint32_t vhi, vlo; 1822 register_t saveintr; 1823 1824 /* XXX: Need more sanity checking of icrlo? */ 1825 KASSERT(x2apic_mode || lapic_map != NULL, 1826 ("%s called too early", __func__)); 1827 KASSERT(x2apic_mode || 1828 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1829 ("%s: invalid dest field", __func__)); 1830 KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, 1831 ("%s: reserved bits set in ICR LO register", __func__)); 1832 1833 /* Set destination in ICR HI register if it is being used. */ 1834 if (!x2apic_mode) { 1835 saveintr = intr_disable(); 1836 icr = lapic_read_icr(); 1837 } 1838 1839 if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { 1840 if (x2apic_mode) { 1841 vhi = dest; 1842 } else { 1843 vhi = icr >> 32; 1844 vhi &= ~APIC_ID_MASK; 1845 vhi |= dest << APIC_ID_SHIFT; 1846 } 1847 } else { 1848 vhi = 0; 1849 } 1850 1851 /* Program the contents of the IPI and dispatch it. */ 1852 if (x2apic_mode) { 1853 vlo = icrlo; 1854 } else { 1855 vlo = icr; 1856 vlo &= APIC_ICRLO_RESV_MASK; 1857 vlo |= icrlo; 1858 } 1859 lapic_write_icr(vhi, vlo); 1860 if (!x2apic_mode) 1861 intr_restore(saveintr); 1862} 1863 1864#define BEFORE_SPIN 50000 1865#ifdef DETECT_DEADLOCK 1866#define AFTER_SPIN 50 1867#endif 1868 1869static void 1870native_lapic_ipi_vectored(u_int vector, int dest) 1871{ 1872 register_t icrlo, destfield; 1873 1874 KASSERT((vector & ~APIC_VECTOR_MASK) == 0, 1875 ("%s: invalid vector %d", __func__, vector)); 1876 1877 icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT; 1878 1879 /* 1880 * NMI IPIs are just fake vectors used to send a NMI. Use special rules 1881 * regarding NMIs if passed, otherwise specify the vector. 1882 */ 1883 if (vector >= IPI_NMI_FIRST) 1884 icrlo |= APIC_DELMODE_NMI; 1885 else 1886 icrlo |= vector | APIC_DELMODE_FIXED; 1887 destfield = 0; 1888 switch (dest) { 1889 case APIC_IPI_DEST_SELF: 1890 icrlo |= APIC_DEST_SELF; 1891 break; 1892 case APIC_IPI_DEST_ALL: 1893 icrlo |= APIC_DEST_ALLISELF; 1894 break; 1895 case APIC_IPI_DEST_OTHERS: 1896 icrlo |= APIC_DEST_ALLESELF; 1897 break; 1898 default: 1899 KASSERT(x2apic_mode || 1900 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1901 ("%s: invalid destination 0x%x", __func__, dest)); 1902 destfield = dest; 1903 } 1904 1905 /* Wait for an earlier IPI to finish. */ 1906 if (!lapic_ipi_wait(BEFORE_SPIN)) { 1907 if (panicstr != NULL) 1908 return; 1909 else 1910 panic("APIC: Previous IPI is stuck"); 1911 } 1912 1913 lapic_ipi_raw(icrlo, destfield); 1914 1915#ifdef DETECT_DEADLOCK 1916 /* Wait for IPI to be delivered. */ 1917 if (!lapic_ipi_wait(AFTER_SPIN)) { 1918#ifdef needsattention 1919 /* 1920 * XXX FIXME: 1921 * 1922 * The above function waits for the message to actually be 1923 * delivered. It breaks out after an arbitrary timeout 1924 * since the message should eventually be delivered (at 1925 * least in theory) and that if it wasn't we would catch 1926 * the failure with the check above when the next IPI is 1927 * sent. 1928 * 1929 * We could skip this wait entirely, EXCEPT it probably 1930 * protects us from other routines that assume that the 1931 * message was delivered and acted upon when this function 1932 * returns. 1933 */ 1934 printf("APIC: IPI might be stuck\n"); 1935#else /* !needsattention */ 1936 /* Wait until mesage is sent without a timeout. */ 1937 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND) 1938 ia32_pause(); 1939#endif /* needsattention */ 1940 } 1941#endif /* DETECT_DEADLOCK */ 1942} 1943 1944#endif /* SMP */ 1945 1946/* 1947 * Since the IDT is shared by all CPUs the IPI slot update needs to be globally 1948 * visible. 1949 * 1950 * Consider the case where an IPI is generated immediately after allocation: 1951 * vector = lapic_ipi_alloc(ipifunc); 1952 * ipi_selected(other_cpus, vector); 1953 * 1954 * In xAPIC mode a write to ICR_LO has serializing semantics because the 1955 * APIC page is mapped as an uncached region. In x2APIC mode there is an 1956 * explicit 'mfence' before the ICR MSR is written. Therefore in both cases 1957 * the IDT slot update is globally visible before the IPI is delivered. 1958 */ 1959static int 1960native_lapic_ipi_alloc(inthand_t *ipifunc) 1961{ 1962 struct gate_descriptor *ip; 1963 long func; 1964 int idx, vector; 1965 1966 KASSERT(ipifunc != &IDTVEC(rsvd), ("invalid ipifunc %p", ipifunc)); 1967 1968 vector = -1; 1969 mtx_lock_spin(&icu_lock); 1970 for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) { 1971 ip = &idt[idx]; 1972 func = (ip->gd_hioffset << 16) | ip->gd_looffset; 1973 if (func == (uintptr_t)&IDTVEC(rsvd)) { 1974 vector = idx; 1975 setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC); 1976 break; 1977 } 1978 } 1979 mtx_unlock_spin(&icu_lock); 1980 return (vector); 1981} 1982 1983static void 1984native_lapic_ipi_free(int vector) 1985{ 1986 struct gate_descriptor *ip; 1987 long func; 1988 1989 KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST, 1990 ("%s: invalid vector %d", __func__, vector)); 1991 1992 mtx_lock_spin(&icu_lock); 1993 ip = &idt[vector]; 1994 func = (ip->gd_hioffset << 16) | ip->gd_looffset; 1995 KASSERT(func != (uintptr_t)&IDTVEC(rsvd), 1996 ("invalid idtfunc %#lx", func)); 1997 setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC); 1998 mtx_unlock_spin(&icu_lock); 1999} 2000