1121986Sjhb/*- 2121986Sjhb * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3121986Sjhb * Copyright (c) 1996, by Steve Passe 4121986Sjhb * All rights reserved. 5121986Sjhb * 6121986Sjhb * Redistribution and use in source and binary forms, with or without 7121986Sjhb * modification, are permitted provided that the following conditions 8121986Sjhb * are met: 9121986Sjhb * 1. Redistributions of source code must retain the above copyright 10121986Sjhb * notice, this list of conditions and the following disclaimer. 11121986Sjhb * 2. The name of the developer may NOT be used to endorse or promote products 12121986Sjhb * derived from this software without specific prior written permission. 13121986Sjhb * 3. Neither the name of the author nor the names of any co-contributors 14121986Sjhb * may be used to endorse or promote products derived from this software 15121986Sjhb * without specific prior written permission. 16121986Sjhb * 17121986Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18121986Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19121986Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20121986Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21121986Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22121986Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23121986Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24121986Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25121986Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26121986Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27121986Sjhb * SUCH DAMAGE. 28121986Sjhb */ 29121986Sjhb 30121986Sjhb/* 31121986Sjhb * Local APIC support on Pentium and later processors. 32121986Sjhb */ 33121986Sjhb 34121986Sjhb#include <sys/cdefs.h> 35121986Sjhb__FBSDID("$FreeBSD: releng/10.3/sys/x86/x86/local_apic.c 299066 2016-05-04 15:25:47Z delphij $"); 36121986Sjhb 37232232Sjhb#include "opt_atpic.h" 38147565Speter#include "opt_hwpmc_hooks.h" 39179277Sjb#include "opt_kdtrace.h" 40147565Speter 41151979Sjhb#include "opt_ddb.h" 42151979Sjhb 43121986Sjhb#include <sys/param.h> 44121986Sjhb#include <sys/systm.h> 45121986Sjhb#include <sys/bus.h> 46121986Sjhb#include <sys/kernel.h> 47151979Sjhb#include <sys/lock.h> 48151979Sjhb#include <sys/mutex.h> 49121986Sjhb#include <sys/pcpu.h> 50187880Sjeff#include <sys/proc.h> 51187880Sjeff#include <sys/sched.h> 52141538Sjhb#include <sys/smp.h> 53209371Smav#include <sys/timeet.h> 54121986Sjhb 55121986Sjhb#include <vm/vm.h> 56121986Sjhb#include <vm/pmap.h> 57121986Sjhb 58214631Sjhb#include <x86/apicreg.h> 59299066Sdelphij#include <machine/clock.h> 60121986Sjhb#include <machine/cputypes.h> 61121986Sjhb#include <machine/frame.h> 62121986Sjhb#include <machine/intr_machdep.h> 63121986Sjhb#include <machine/apicvar.h> 64214630Sjhb#include <x86/mca.h> 65121986Sjhb#include <machine/md_var.h> 66121986Sjhb#include <machine/smp.h> 67121986Sjhb#include <machine/specialreg.h> 68121986Sjhb 69151979Sjhb#ifdef DDB 70151979Sjhb#include <sys/interrupt.h> 71151979Sjhb#include <ddb/ddb.h> 72151979Sjhb#endif 73151979Sjhb 74208452Smav#ifdef __amd64__ 75208452Smav#define SDT_APIC SDT_SYSIGT 76208452Smav#define SDT_APICT SDT_SYSIGT 77208452Smav#define GSEL_APIC 0 78208452Smav#else 79208452Smav#define SDT_APIC SDT_SYS386IGT 80208452Smav#define SDT_APICT SDT_SYS386TGT 81208452Smav#define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL) 82208452Smav#endif 83208452Smav 84122690Sjhb/* Sanity checks on IDT vectors. */ 85139240SjhbCTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); 86139240SjhbCTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); 87139240SjhbCTASSERT(APIC_LOCAL_INTS == 240); 88122690SjhbCTASSERT(IPI_STOP < APIC_SPURIOUS_INT); 89122690Sjhb 90151979Sjhb/* Magic IRQ values for the timer and syscalls. */ 91151979Sjhb#define IRQ_TIMER (NUM_IO_INTS + 1) 92151979Sjhb#define IRQ_SYSCALL (NUM_IO_INTS + 2) 93212004Srpaulo#define IRQ_DTRACE_RET (NUM_IO_INTS + 3) 94255040Sgibbs#define IRQ_EVTCHN (NUM_IO_INTS + 4) 95151979Sjhb 96121986Sjhb/* 97121986Sjhb * Support for local APICs. Local APICs manage interrupts on each 98121986Sjhb * individual processor as opposed to I/O APICs which receive interrupts 99121986Sjhb * from I/O devices and then forward them on to the local APICs. 100121986Sjhb * 101121986Sjhb * Local APICs can also send interrupts to each other thus providing the 102121986Sjhb * mechanism for IPIs. 103121986Sjhb */ 104121986Sjhb 105121986Sjhbstruct lvt { 106121986Sjhb u_int lvt_edgetrigger:1; 107121986Sjhb u_int lvt_activehi:1; 108121986Sjhb u_int lvt_masked:1; 109121986Sjhb u_int lvt_active:1; 110121986Sjhb u_int lvt_mode:16; 111121986Sjhb u_int lvt_vector:8; 112121986Sjhb}; 113121986Sjhb 114121986Sjhbstruct lapic { 115262141Sjhb struct lvt la_lvts[APIC_LVT_MAX + 1]; 116121986Sjhb u_int la_id:8; 117121986Sjhb u_int la_cluster:4; 118121986Sjhb u_int la_cluster_id:2; 119121986Sjhb u_int la_present:1; 120141538Sjhb u_long *la_timer_count; 121209371Smav u_long la_timer_period; 122209371Smav u_int la_timer_mode; 123239013Smav uint32_t lvt_timer_cache; 124187880Sjeff /* Include IDT_SYSCALL to make indexing easier. */ 125191720Smav int la_ioint_irqs[APIC_NUM_IOINTS + 1]; 126169395Sjhb} static lapics[MAX_APIC_ID + 1]; 127121986Sjhb 128121986Sjhb/* Global defaults for local APIC LVT entries. */ 129262141Sjhbstatic struct lvt lvts[APIC_LVT_MAX + 1] = { 130121986Sjhb { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 }, /* LINT0: masked ExtINT */ 131121986Sjhb { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 }, /* LINT1: NMI */ 132139245Sjhb { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT }, /* Timer */ 133205851Sjhb { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT }, /* Error */ 134196224Sjhb { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 }, /* PMC */ 135139245Sjhb { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT }, /* Thermal */ 136208507Sjhb { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT }, /* CMCI */ 137121986Sjhb}; 138121986Sjhb 139121986Sjhbstatic inthand_t *ioint_handlers[] = { 140121986Sjhb NULL, /* 0 - 31 */ 141121986Sjhb IDTVEC(apic_isr1), /* 32 - 63 */ 142121986Sjhb IDTVEC(apic_isr2), /* 64 - 95 */ 143121986Sjhb IDTVEC(apic_isr3), /* 96 - 127 */ 144121986Sjhb IDTVEC(apic_isr4), /* 128 - 159 */ 145121986Sjhb IDTVEC(apic_isr5), /* 160 - 191 */ 146122690Sjhb IDTVEC(apic_isr6), /* 192 - 223 */ 147122690Sjhb IDTVEC(apic_isr7), /* 224 - 255 */ 148121986Sjhb}; 149121986Sjhb 150151979Sjhb 151195249Sjhbstatic u_int32_t lapic_timer_divisors[] = { 152141538Sjhb APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, 153141538Sjhb APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128 154141538Sjhb}; 155141538Sjhb 156169391Sjhbextern inthand_t IDTVEC(rsvd); 157169391Sjhb 158121986Sjhbvolatile lapic_t *lapic; 159167747Sjhbvm_paddr_t lapic_paddr; 160209371Smavstatic u_long lapic_timer_divisor; 161209371Smavstatic struct eventtimer lapic_et; 162299066Sdelphij#ifdef SMP 163299066Sdelphijstatic uint64_t lapic_ipi_wait_mult; 164299066Sdelphij#endif 165121986Sjhb 166139245Sjhbstatic void lapic_enable(void); 167255726Sgibbsstatic void lapic_resume(struct pic *pic, bool suspend_cancelled); 168239013Smavstatic void lapic_timer_oneshot(struct lapic *, 169239013Smav u_int count, int enable_int); 170239013Smavstatic void lapic_timer_periodic(struct lapic *, 171239013Smav u_int count, int enable_int); 172239013Smavstatic void lapic_timer_stop(struct lapic *); 173141538Sjhbstatic void lapic_timer_set_divisor(u_int divisor); 174139245Sjhbstatic uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value); 175209371Smavstatic int lapic_et_start(struct eventtimer *et, 176247463Smav sbintime_t first, sbintime_t period); 177209371Smavstatic int lapic_et_stop(struct eventtimer *et); 178139245Sjhb 179163219Sjhbstruct pic lapic_pic = { .pic_resume = lapic_resume }; 180163219Sjhb 181121986Sjhbstatic uint32_t 182121986Sjhblvt_mode(struct lapic *la, u_int pin, uint32_t value) 183121986Sjhb{ 184121986Sjhb struct lvt *lvt; 185121986Sjhb 186262141Sjhb KASSERT(pin <= APIC_LVT_MAX, ("%s: pin %u out of range", __func__, pin)); 187121986Sjhb if (la->la_lvts[pin].lvt_active) 188121986Sjhb lvt = &la->la_lvts[pin]; 189121986Sjhb else 190121986Sjhb lvt = &lvts[pin]; 191121986Sjhb 192121986Sjhb value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM | 193121986Sjhb APIC_LVT_VECTOR); 194121986Sjhb if (lvt->lvt_edgetrigger == 0) 195121986Sjhb value |= APIC_LVT_TM; 196121986Sjhb if (lvt->lvt_activehi == 0) 197121986Sjhb value |= APIC_LVT_IIPP_INTALO; 198121986Sjhb if (lvt->lvt_masked) 199121986Sjhb value |= APIC_LVT_M; 200121986Sjhb value |= lvt->lvt_mode; 201121986Sjhb switch (lvt->lvt_mode) { 202121986Sjhb case APIC_LVT_DM_NMI: 203121986Sjhb case APIC_LVT_DM_SMI: 204121986Sjhb case APIC_LVT_DM_INIT: 205121986Sjhb case APIC_LVT_DM_EXTINT: 206121986Sjhb if (!lvt->lvt_edgetrigger) { 207121986Sjhb printf("lapic%u: Forcing LINT%u to edge trigger\n", 208121986Sjhb la->la_id, pin); 209121986Sjhb value |= APIC_LVT_TM; 210121986Sjhb } 211121986Sjhb /* Use a vector of 0. */ 212121986Sjhb break; 213121986Sjhb case APIC_LVT_DM_FIXED: 214121986Sjhb value |= lvt->lvt_vector; 215121986Sjhb break; 216121986Sjhb default: 217121986Sjhb panic("bad APIC LVT delivery mode: %#x\n", value); 218121986Sjhb } 219121986Sjhb return (value); 220121986Sjhb} 221121986Sjhb 222121986Sjhb/* 223121986Sjhb * Map the local APIC and setup necessary interrupt vectors. 224121986Sjhb */ 225121986Sjhbvoid 226167247Sjhblapic_init(vm_paddr_t addr) 227121986Sjhb{ 228299066Sdelphij#ifdef SMP 229299066Sdelphij uint64_t r, r1, r2, rx; 230299066Sdelphij#endif 231209371Smav u_int regs[4]; 232209371Smav int i, arat; 233121986Sjhb 234121986Sjhb /* Map the local APIC and setup the spurious interrupt handler. */ 235121986Sjhb KASSERT(trunc_page(addr) == addr, 236121986Sjhb ("local APIC not aligned on a page boundary")); 237245577Sjhb lapic_paddr = addr; 238156920Sjhb lapic = pmap_mapdev(addr, sizeof(lapic_t)); 239208452Smav setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL, 240208452Smav GSEL_APIC); 241121986Sjhb 242121986Sjhb /* Perform basic initialization of the BSP's local APIC. */ 243139245Sjhb lapic_enable(); 244121986Sjhb 245121986Sjhb /* Set BSP's per-CPU local APIC ID. */ 246121986Sjhb PCPU_SET(apic_id, lapic_id()); 247121986Sjhb 248141538Sjhb /* Local APIC timer interrupt. */ 249208452Smav setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC); 250141538Sjhb 251205851Sjhb /* Local APIC error interrupt. */ 252208452Smav setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC); 253205851Sjhb 254205851Sjhb /* XXX: Thermal interrupt */ 255208507Sjhb 256208507Sjhb /* Local APIC CMCI. */ 257208507Sjhb setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC); 258209371Smav 259209371Smav if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) { 260209371Smav arat = 0; 261209371Smav /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */ 262209371Smav if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) { 263209371Smav do_cpuid(0x06, regs); 264215751Savg if ((regs[0] & CPUTPM1_ARAT) != 0) 265209371Smav arat = 1; 266209371Smav } 267209371Smav bzero(&lapic_et, sizeof(lapic_et)); 268209371Smav lapic_et.et_name = "LAPIC"; 269209371Smav lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | 270209371Smav ET_FLAGS_PERCPU; 271209371Smav lapic_et.et_quality = 600; 272209371Smav if (!arat) { 273209371Smav lapic_et.et_flags |= ET_FLAGS_C3STOP; 274212541Smav lapic_et.et_quality -= 200; 275209371Smav } 276209371Smav lapic_et.et_frequency = 0; 277210290Smav /* We don't know frequency yet, so trying to guess. */ 278247463Smav lapic_et.et_min_period = 0x00001000LL; 279247463Smav lapic_et.et_max_period = SBT_1S; 280209371Smav lapic_et.et_start = lapic_et_start; 281209371Smav lapic_et.et_stop = lapic_et_stop; 282209371Smav lapic_et.et_priv = NULL; 283209371Smav et_register(&lapic_et); 284209371Smav } 285299066Sdelphij 286299066Sdelphij#ifdef SMP 287299066Sdelphij#define LOOPS 1000000 288299066Sdelphij /* 289299066Sdelphij * Calibrate the busy loop waiting for IPI ack in xAPIC mode. 290299066Sdelphij * lapic_ipi_wait_mult contains the number of iterations which 291299066Sdelphij * approximately delay execution for 1 microsecond (the 292299066Sdelphij * argument to native_lapic_ipi_wait() is in microseconds). 293299066Sdelphij * 294299066Sdelphij * We assume that TSC is present and already measured. 295299066Sdelphij * Possible TSC frequency jumps are irrelevant to the 296299066Sdelphij * calibration loop below, the CPU clock management code is 297299066Sdelphij * not yet started, and we do not enter sleep states. 298299066Sdelphij */ 299299066Sdelphij KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0, 300299066Sdelphij ("TSC not initialized")); 301299066Sdelphij r = rdtsc(); 302299066Sdelphij for (rx = 0; rx < LOOPS; rx++) { 303299066Sdelphij (void)lapic->icr_lo; 304299066Sdelphij ia32_pause(); 305299066Sdelphij } 306299066Sdelphij r = rdtsc() - r; 307299066Sdelphij r1 = tsc_freq * LOOPS; 308299066Sdelphij r2 = r * 1000000; 309299066Sdelphij lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1; 310299066Sdelphij if (bootverbose) { 311299066Sdelphij printf("LAPIC: ipi_wait() us multiplier %ju (r %ju tsc %ju)\n", 312299066Sdelphij (uintmax_t)lapic_ipi_wait_mult, (uintmax_t)r, 313299066Sdelphij (uintmax_t)tsc_freq); 314299066Sdelphij } 315299066Sdelphij#undef LOOPS 316299066Sdelphij#endif /* SMP */ 317121986Sjhb} 318121986Sjhb 319121986Sjhb/* 320121986Sjhb * Create a local APIC instance. 321121986Sjhb */ 322121986Sjhbvoid 323121986Sjhblapic_create(u_int apic_id, int boot_cpu) 324121986Sjhb{ 325121986Sjhb int i; 326121986Sjhb 327169395Sjhb if (apic_id > MAX_APIC_ID) { 328121986Sjhb printf("APIC: Ignoring local APIC with ID %d\n", apic_id); 329121986Sjhb if (boot_cpu) 330121986Sjhb panic("Can't ignore BSP"); 331121986Sjhb return; 332121986Sjhb } 333121986Sjhb KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u", 334121986Sjhb apic_id)); 335121986Sjhb 336121986Sjhb /* 337121986Sjhb * Assume no local LVT overrides and a cluster of 0 and 338121986Sjhb * intra-cluster ID of 0. 339121986Sjhb */ 340121986Sjhb lapics[apic_id].la_present = 1; 341121986Sjhb lapics[apic_id].la_id = apic_id; 342262141Sjhb for (i = 0; i <= APIC_LVT_MAX; i++) { 343121986Sjhb lapics[apic_id].la_lvts[i] = lvts[i]; 344121986Sjhb lapics[apic_id].la_lvts[i].lvt_active = 0; 345121986Sjhb } 346191720Smav for (i = 0; i <= APIC_NUM_IOINTS; i++) 347191720Smav lapics[apic_id].la_ioint_irqs[i] = -1; 348187880Sjeff lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL; 349187880Sjeff lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = 350187880Sjeff IRQ_TIMER; 351212004Srpaulo#ifdef KDTRACE_HOOKS 352232232Sjhb lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] = 353232232Sjhb IRQ_DTRACE_RET; 354212004Srpaulo#endif 355255040Sgibbs#ifdef XENHVM 356255040Sgibbs lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN; 357255040Sgibbs#endif 358121986Sjhb 359212004Srpaulo 360121986Sjhb#ifdef SMP 361121986Sjhb cpu_add(apic_id, boot_cpu); 362121986Sjhb#endif 363121986Sjhb} 364121986Sjhb 365121986Sjhb/* 366121986Sjhb * Dump contents of local APIC registers 367121986Sjhb */ 368121986Sjhbvoid 369121986Sjhblapic_dump(const char* str) 370121986Sjhb{ 371215001Sjhb uint32_t maxlvt; 372121986Sjhb 373215001Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 374121986Sjhb printf("cpu%d %s:\n", PCPU_GET(cpuid), str); 375121986Sjhb printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n", 376121986Sjhb lapic->id, lapic->version, lapic->ldr, lapic->dfr); 377121986Sjhb printf(" lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", 378121986Sjhb lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr); 379215001Sjhb printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x", 380215001Sjhb lapic->lvt_timer, lapic->lvt_thermal, lapic->lvt_error); 381262141Sjhb if (maxlvt >= APIC_LVT_PMC) 382215001Sjhb printf(" pmc: 0x%08x", lapic->lvt_pcint); 383215001Sjhb printf("\n"); 384262141Sjhb if (maxlvt >= APIC_LVT_CMCI) 385215001Sjhb printf(" cmci: 0x%08x\n", lapic->lvt_cmci); 386121986Sjhb} 387121986Sjhb 388121986Sjhbvoid 389163219Sjhblapic_setup(int boot) 390121986Sjhb{ 391121986Sjhb struct lapic *la; 392156124Sjhb u_int32_t maxlvt; 393214347Sjhb register_t saveintr; 394141538Sjhb char buf[MAXCOMLEN + 1]; 395121986Sjhb 396121986Sjhb la = &lapics[lapic_id()]; 397121986Sjhb KASSERT(la->la_present, ("missing APIC structure")); 398214347Sjhb saveintr = intr_disable(); 399121986Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 400121986Sjhb 401139240Sjhb /* Initialize the TPR to allow all interrupts. */ 402139240Sjhb lapic_set_tpr(0); 403121986Sjhb 404121986Sjhb /* Setup spurious vector and enable the local APIC. */ 405139245Sjhb lapic_enable(); 406139245Sjhb 407139245Sjhb /* Program LINT[01] LVT entries. */ 408262141Sjhb lapic->lvt_lint0 = lvt_mode(la, APIC_LVT_LINT0, lapic->lvt_lint0); 409262141Sjhb lapic->lvt_lint1 = lvt_mode(la, APIC_LVT_LINT1, lapic->lvt_lint1); 410185933Sjhb 411145256Sjkoshy /* Program the PMC LVT entry if present. */ 412262141Sjhb if (maxlvt >= APIC_LVT_PMC) 413262141Sjhb lapic->lvt_pcint = lvt_mode(la, APIC_LVT_PMC, lapic->lvt_pcint); 414139245Sjhb 415141538Sjhb /* Program timer LVT and setup handler. */ 416239013Smav la->lvt_timer_cache = lapic->lvt_timer = 417262141Sjhb lvt_mode(la, APIC_LVT_TIMER, lapic->lvt_timer); 418163219Sjhb if (boot) { 419209371Smav snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid)); 420163219Sjhb intrcnt_add(buf, &la->la_timer_count); 421163219Sjhb } 422163219Sjhb 423209371Smav /* Setup the timer if configured. */ 424209371Smav if (la->la_timer_mode != 0) { 425209371Smav KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor", 426141538Sjhb lapic_id())); 427141538Sjhb lapic_timer_set_divisor(lapic_timer_divisor); 428209371Smav if (la->la_timer_mode == 1) 429239013Smav lapic_timer_periodic(la, la->la_timer_period, 1); 430209371Smav else 431239013Smav lapic_timer_oneshot(la, la->la_timer_period, 1); 432141538Sjhb } 433139245Sjhb 434205851Sjhb /* Program error LVT and clear any existing errors. */ 435262141Sjhb lapic->lvt_error = lvt_mode(la, APIC_LVT_ERROR, lapic->lvt_error); 436205851Sjhb lapic->esr = 0; 437141538Sjhb 438205851Sjhb /* XXX: Thermal LVT */ 439205851Sjhb 440208507Sjhb /* Program the CMCI LVT entry if present. */ 441262141Sjhb if (maxlvt >= APIC_LVT_CMCI) 442262141Sjhb lapic->lvt_cmci = lvt_mode(la, APIC_LVT_CMCI, lapic->lvt_cmci); 443208507Sjhb 444214347Sjhb intr_restore(saveintr); 445121986Sjhb} 446121986Sjhb 447196224Sjhbvoid 448196224Sjhblapic_reenable_pmc(void) 449196224Sjhb{ 450196224Sjhb#ifdef HWPMC_HOOKS 451196224Sjhb uint32_t value; 452196224Sjhb 453196224Sjhb value = lapic->lvt_pcint; 454196224Sjhb value &= ~APIC_LVT_M; 455196224Sjhb lapic->lvt_pcint = value; 456196224Sjhb#endif 457196224Sjhb} 458196224Sjhb 459196224Sjhb#ifdef HWPMC_HOOKS 460196224Sjhbstatic void 461196224Sjhblapic_update_pmc(void *dummy) 462196224Sjhb{ 463196224Sjhb struct lapic *la; 464196224Sjhb 465196224Sjhb la = &lapics[lapic_id()]; 466262141Sjhb lapic->lvt_pcint = lvt_mode(la, APIC_LVT_PMC, lapic->lvt_pcint); 467196224Sjhb} 468196224Sjhb#endif 469196224Sjhb 470196224Sjhbint 471196224Sjhblapic_enable_pmc(void) 472196224Sjhb{ 473196224Sjhb#ifdef HWPMC_HOOKS 474196224Sjhb u_int32_t maxlvt; 475196224Sjhb 476196224Sjhb /* Fail if the local APIC is not present. */ 477196224Sjhb if (lapic == NULL) 478196224Sjhb return (0); 479196224Sjhb 480196224Sjhb /* Fail if the PMC LVT is not present. */ 481196224Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 482262141Sjhb if (maxlvt < APIC_LVT_PMC) 483196224Sjhb return (0); 484196224Sjhb 485262141Sjhb lvts[APIC_LVT_PMC].lvt_masked = 0; 486196224Sjhb 487196224Sjhb#ifdef SMP 488196224Sjhb /* 489196224Sjhb * If hwpmc was loaded at boot time then the APs may not be 490196224Sjhb * started yet. In that case, don't forward the request to 491196224Sjhb * them as they will program the lvt when they start. 492196224Sjhb */ 493196224Sjhb if (smp_started) 494196224Sjhb smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 495196224Sjhb else 496196224Sjhb#endif 497196224Sjhb lapic_update_pmc(NULL); 498196224Sjhb return (1); 499196224Sjhb#else 500196224Sjhb return (0); 501196224Sjhb#endif 502196224Sjhb} 503196224Sjhb 504196224Sjhbvoid 505196224Sjhblapic_disable_pmc(void) 506196224Sjhb{ 507196224Sjhb#ifdef HWPMC_HOOKS 508196224Sjhb u_int32_t maxlvt; 509196224Sjhb 510196224Sjhb /* Fail if the local APIC is not present. */ 511196224Sjhb if (lapic == NULL) 512196224Sjhb return; 513196224Sjhb 514196224Sjhb /* Fail if the PMC LVT is not present. */ 515196224Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 516262141Sjhb if (maxlvt < APIC_LVT_PMC) 517196224Sjhb return; 518196224Sjhb 519262141Sjhb lvts[APIC_LVT_PMC].lvt_masked = 1; 520196224Sjhb 521196224Sjhb#ifdef SMP 522196224Sjhb /* The APs should always be started when hwpmc is unloaded. */ 523196224Sjhb KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early")); 524196224Sjhb#endif 525196224Sjhb smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 526196224Sjhb#endif 527196224Sjhb} 528196224Sjhb 529209371Smavstatic int 530247463Smavlapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) 531141538Sjhb{ 532209371Smav struct lapic *la; 533141538Sjhb u_long value; 534141538Sjhb 535239013Smav la = &lapics[PCPU_GET(apic_id)]; 536209371Smav if (et->et_frequency == 0) { 537209371Smav /* Start off with a divisor of 2 (power on reset default). */ 538209371Smav lapic_timer_divisor = 2; 539209371Smav /* Try to calibrate the local APIC timer. */ 540209371Smav do { 541209371Smav lapic_timer_set_divisor(lapic_timer_divisor); 542239013Smav lapic_timer_oneshot(la, APIC_TIMER_MAX_COUNT, 0); 543209371Smav DELAY(1000000); 544209371Smav value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer; 545209371Smav if (value != APIC_TIMER_MAX_COUNT) 546209371Smav break; 547209371Smav lapic_timer_divisor <<= 1; 548209371Smav } while (lapic_timer_divisor <= 128); 549209371Smav if (lapic_timer_divisor > 128) 550209371Smav panic("lapic: Divisor too big"); 551209371Smav if (bootverbose) 552209371Smav printf("lapic: Divisor %lu, Frequency %lu Hz\n", 553209371Smav lapic_timer_divisor, value); 554209371Smav et->et_frequency = value; 555247463Smav et->et_min_period = (0x00000002LLU << 32) / et->et_frequency; 556247463Smav et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency; 557204641Sattilio } 558239013Smav if (la->la_timer_mode == 0) 559239013Smav lapic_timer_set_divisor(lapic_timer_divisor); 560247463Smav if (period != 0) { 561209371Smav la->la_timer_mode = 1; 562247463Smav la->la_timer_period = ((uint32_t)et->et_frequency * period) >> 32; 563239013Smav lapic_timer_periodic(la, la->la_timer_period, 1); 564209371Smav } else { 565209371Smav la->la_timer_mode = 2; 566247463Smav la->la_timer_period = ((uint32_t)et->et_frequency * first) >> 32; 567239013Smav lapic_timer_oneshot(la, la->la_timer_period, 1); 568209371Smav } 569209371Smav return (0); 570141538Sjhb} 571141538Sjhb 572209371Smavstatic int 573209371Smavlapic_et_stop(struct eventtimer *et) 574209371Smav{ 575239013Smav struct lapic *la = &lapics[PCPU_GET(apic_id)]; 576209371Smav 577209371Smav la->la_timer_mode = 0; 578239013Smav lapic_timer_stop(la); 579209371Smav return (0); 580209371Smav} 581209371Smav 582121986Sjhbvoid 583121986Sjhblapic_disable(void) 584121986Sjhb{ 585121986Sjhb uint32_t value; 586121986Sjhb 587121986Sjhb /* Software disable the local APIC. */ 588121986Sjhb value = lapic->svr; 589121986Sjhb value &= ~APIC_SVR_SWEN; 590121986Sjhb lapic->svr = value; 591121986Sjhb} 592121986Sjhb 593139245Sjhbstatic void 594139245Sjhblapic_enable(void) 595139245Sjhb{ 596139245Sjhb u_int32_t value; 597139245Sjhb 598139245Sjhb /* Program the spurious vector to enable the local APIC. */ 599139245Sjhb value = lapic->svr; 600139245Sjhb value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); 601139245Sjhb value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT); 602139245Sjhb lapic->svr = value; 603139245Sjhb} 604139245Sjhb 605163219Sjhb/* Reset the local APIC on the BSP during resume. */ 606163219Sjhbstatic void 607255726Sgibbslapic_resume(struct pic *pic, bool suspend_cancelled) 608163219Sjhb{ 609163219Sjhb 610163219Sjhb lapic_setup(0); 611163219Sjhb} 612163219Sjhb 613121986Sjhbint 614121986Sjhblapic_id(void) 615121986Sjhb{ 616121986Sjhb 617121986Sjhb KASSERT(lapic != NULL, ("local APIC is not mapped")); 618121986Sjhb return (lapic->id >> APIC_ID_SHIFT); 619121986Sjhb} 620121986Sjhb 621121986Sjhbint 622121986Sjhblapic_intr_pending(u_int vector) 623121986Sjhb{ 624121986Sjhb volatile u_int32_t *irr; 625121986Sjhb 626121986Sjhb /* 627121986Sjhb * The IRR registers are an array of 128-bit registers each of 628121986Sjhb * which only describes 32 interrupts in the low 32 bits.. Thus, 629121986Sjhb * we divide the vector by 32 to get the 128-bit index. We then 630121986Sjhb * multiply that index by 4 to get the equivalent index from 631121986Sjhb * treating the IRR as an array of 32-bit registers. Finally, we 632121986Sjhb * modulus the vector by 32 to determine the individual bit to 633121986Sjhb * test. 634121986Sjhb */ 635121986Sjhb irr = &lapic->irr0; 636121986Sjhb return (irr[(vector / 32) * 4] & 1 << (vector % 32)); 637121986Sjhb} 638121986Sjhb 639121986Sjhbvoid 640121986Sjhblapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 641121986Sjhb{ 642121986Sjhb struct lapic *la; 643121986Sjhb 644121986Sjhb KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist", 645121986Sjhb __func__, apic_id)); 646121986Sjhb KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big", 647121986Sjhb __func__, cluster)); 648121986Sjhb KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID, 649121986Sjhb ("%s: intra cluster id %u too big", __func__, cluster_id)); 650121986Sjhb la = &lapics[apic_id]; 651121986Sjhb la->la_cluster = cluster; 652121986Sjhb la->la_cluster_id = cluster_id; 653121986Sjhb} 654121986Sjhb 655121986Sjhbint 656121986Sjhblapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked) 657121986Sjhb{ 658121986Sjhb 659262141Sjhb if (pin > APIC_LVT_MAX) 660121986Sjhb return (EINVAL); 661121986Sjhb if (apic_id == APIC_ID_ALL) { 662121986Sjhb lvts[pin].lvt_masked = masked; 663121986Sjhb if (bootverbose) 664121986Sjhb printf("lapic:"); 665121986Sjhb } else { 666121986Sjhb KASSERT(lapics[apic_id].la_present, 667121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 668121986Sjhb lapics[apic_id].la_lvts[pin].lvt_masked = masked; 669121986Sjhb lapics[apic_id].la_lvts[pin].lvt_active = 1; 670121986Sjhb if (bootverbose) 671121986Sjhb printf("lapic%u:", apic_id); 672121986Sjhb } 673121986Sjhb if (bootverbose) 674121986Sjhb printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked"); 675121986Sjhb return (0); 676121986Sjhb} 677121986Sjhb 678121986Sjhbint 679121986Sjhblapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode) 680121986Sjhb{ 681121986Sjhb struct lvt *lvt; 682121986Sjhb 683262141Sjhb if (pin > APIC_LVT_MAX) 684121986Sjhb return (EINVAL); 685121986Sjhb if (apic_id == APIC_ID_ALL) { 686121986Sjhb lvt = &lvts[pin]; 687121986Sjhb if (bootverbose) 688121986Sjhb printf("lapic:"); 689121986Sjhb } else { 690121986Sjhb KASSERT(lapics[apic_id].la_present, 691121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 692121986Sjhb lvt = &lapics[apic_id].la_lvts[pin]; 693121986Sjhb lvt->lvt_active = 1; 694121986Sjhb if (bootverbose) 695121986Sjhb printf("lapic%u:", apic_id); 696121986Sjhb } 697121986Sjhb lvt->lvt_mode = mode; 698121986Sjhb switch (mode) { 699121986Sjhb case APIC_LVT_DM_NMI: 700121986Sjhb case APIC_LVT_DM_SMI: 701121986Sjhb case APIC_LVT_DM_INIT: 702121986Sjhb case APIC_LVT_DM_EXTINT: 703121986Sjhb lvt->lvt_edgetrigger = 1; 704121986Sjhb lvt->lvt_activehi = 1; 705121986Sjhb if (mode == APIC_LVT_DM_EXTINT) 706121986Sjhb lvt->lvt_masked = 1; 707121986Sjhb else 708121986Sjhb lvt->lvt_masked = 0; 709121986Sjhb break; 710121986Sjhb default: 711121986Sjhb panic("Unsupported delivery mode: 0x%x\n", mode); 712121986Sjhb } 713121986Sjhb if (bootverbose) { 714121986Sjhb printf(" Routing "); 715121986Sjhb switch (mode) { 716121986Sjhb case APIC_LVT_DM_NMI: 717121986Sjhb printf("NMI"); 718121986Sjhb break; 719121986Sjhb case APIC_LVT_DM_SMI: 720121986Sjhb printf("SMI"); 721121986Sjhb break; 722121986Sjhb case APIC_LVT_DM_INIT: 723121986Sjhb printf("INIT"); 724121986Sjhb break; 725121986Sjhb case APIC_LVT_DM_EXTINT: 726121986Sjhb printf("ExtINT"); 727121986Sjhb break; 728121986Sjhb } 729121986Sjhb printf(" -> LINT%u\n", pin); 730121986Sjhb } 731121986Sjhb return (0); 732121986Sjhb} 733121986Sjhb 734121986Sjhbint 735128930Sjhblapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol) 736121986Sjhb{ 737121986Sjhb 738262141Sjhb if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM) 739121986Sjhb return (EINVAL); 740121986Sjhb if (apic_id == APIC_ID_ALL) { 741128930Sjhb lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH); 742121986Sjhb if (bootverbose) 743121986Sjhb printf("lapic:"); 744121986Sjhb } else { 745121986Sjhb KASSERT(lapics[apic_id].la_present, 746121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 747121986Sjhb lapics[apic_id].la_lvts[pin].lvt_active = 1; 748128930Sjhb lapics[apic_id].la_lvts[pin].lvt_activehi = 749128930Sjhb (pol == INTR_POLARITY_HIGH); 750121986Sjhb if (bootverbose) 751121986Sjhb printf("lapic%u:", apic_id); 752121986Sjhb } 753121986Sjhb if (bootverbose) 754140254Sjhb printf(" LINT%u polarity: %s\n", pin, 755128930Sjhb pol == INTR_POLARITY_HIGH ? "high" : "low"); 756121986Sjhb return (0); 757121986Sjhb} 758121986Sjhb 759121986Sjhbint 760128930Sjhblapic_set_lvt_triggermode(u_int apic_id, u_int pin, enum intr_trigger trigger) 761121986Sjhb{ 762121986Sjhb 763262141Sjhb if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM) 764121986Sjhb return (EINVAL); 765121986Sjhb if (apic_id == APIC_ID_ALL) { 766128930Sjhb lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE); 767121986Sjhb if (bootverbose) 768121986Sjhb printf("lapic:"); 769121986Sjhb } else { 770121986Sjhb KASSERT(lapics[apic_id].la_present, 771121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 772128930Sjhb lapics[apic_id].la_lvts[pin].lvt_edgetrigger = 773128930Sjhb (trigger == INTR_TRIGGER_EDGE); 774121986Sjhb lapics[apic_id].la_lvts[pin].lvt_active = 1; 775121986Sjhb if (bootverbose) 776121986Sjhb printf("lapic%u:", apic_id); 777121986Sjhb } 778121986Sjhb if (bootverbose) 779121986Sjhb printf(" LINT%u trigger: %s\n", pin, 780128930Sjhb trigger == INTR_TRIGGER_EDGE ? "edge" : "level"); 781121986Sjhb return (0); 782121986Sjhb} 783121986Sjhb 784139240Sjhb/* 785139240Sjhb * Adjust the TPR of the current CPU so that it blocks all interrupts below 786139240Sjhb * the passed in vector. 787139240Sjhb */ 788121986Sjhbvoid 789139240Sjhblapic_set_tpr(u_int vector) 790139240Sjhb{ 791139240Sjhb#ifdef CHEAP_TPR 792139240Sjhb lapic->tpr = vector; 793139240Sjhb#else 794139240Sjhb u_int32_t tpr; 795139240Sjhb 796139240Sjhb tpr = lapic->tpr & ~APIC_TPR_PRIO; 797139240Sjhb tpr |= vector; 798139240Sjhb lapic->tpr = tpr; 799139240Sjhb#endif 800139240Sjhb} 801139240Sjhb 802139240Sjhbvoid 803122572Sjhblapic_eoi(void) 804122572Sjhb{ 805122572Sjhb 806122572Sjhb lapic->eoi = 0; 807122572Sjhb} 808122572Sjhb 809122572Sjhbvoid 810165302Skmacylapic_handle_intr(int vector, struct trapframe *frame) 811121986Sjhb{ 812121986Sjhb struct intsrc *isrc; 813121986Sjhb 814187880Sjeff isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id), 815187880Sjeff vector)); 816165302Skmacy intr_execute_handlers(isrc, frame); 817121986Sjhb} 818121986Sjhb 819141538Sjhbvoid 820165302Skmacylapic_handle_timer(struct trapframe *frame) 821141538Sjhb{ 822141538Sjhb struct lapic *la; 823209371Smav struct trapframe *oldframe; 824209371Smav struct thread *td; 825141538Sjhb 826153141Sjhb /* Send EOI first thing. */ 827153141Sjhb lapic_eoi(); 828153141Sjhb 829162708Ssobomax#if defined(SMP) && !defined(SCHED_ULE) 830162042Ssobomax /* 831162042Ssobomax * Don't do any accounting for the disabled HTT cores, since it 832162042Ssobomax * will provide misleading numbers for the userland. 833162042Ssobomax * 834250576Seadler * No locking is necessary here, since even if we lose the race 835162042Ssobomax * when hlt_cpus_mask changes it is not a big deal, really. 836162713Ssobomax * 837162713Ssobomax * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask 838162713Ssobomax * and unlike other schedulers it actually schedules threads to 839162713Ssobomax * those CPUs. 840162042Ssobomax */ 841222813Sattilio if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)) 842162042Ssobomax return; 843162087Ssobomax#endif 844162042Ssobomax 845153141Sjhb /* Look up our local APIC structure for the tick counters. */ 846141538Sjhb la = &lapics[PCPU_GET(apic_id)]; 847141538Sjhb (*la->la_timer_count)++; 848141538Sjhb critical_enter(); 849209371Smav if (lapic_et.et_active) { 850209371Smav td = curthread; 851210444Smav td->td_intr_nesting_level++; 852209371Smav oldframe = td->td_intr_frame; 853209371Smav td->td_intr_frame = frame; 854209990Smav lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg); 855209371Smav td->td_intr_frame = oldframe; 856210444Smav td->td_intr_nesting_level--; 857209371Smav } 858141538Sjhb critical_exit(); 859141538Sjhb} 860141538Sjhb 861141538Sjhbstatic void 862141538Sjhblapic_timer_set_divisor(u_int divisor) 863141538Sjhb{ 864141538Sjhb 865141538Sjhb KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor)); 866141538Sjhb KASSERT(ffs(divisor) <= sizeof(lapic_timer_divisors) / 867141538Sjhb sizeof(u_int32_t), ("lapic: invalid divisor %u", divisor)); 868141538Sjhb lapic->dcr_timer = lapic_timer_divisors[ffs(divisor) - 1]; 869141538Sjhb} 870141538Sjhb 871141538Sjhbstatic void 872239013Smavlapic_timer_oneshot(struct lapic *la, u_int count, int enable_int) 873141538Sjhb{ 874141538Sjhb u_int32_t value; 875141538Sjhb 876239013Smav value = la->lvt_timer_cache; 877141538Sjhb value &= ~APIC_LVTT_TM; 878141538Sjhb value |= APIC_LVTT_TM_ONE_SHOT; 879221508Smav if (enable_int) 880221508Smav value &= ~APIC_LVT_M; 881141538Sjhb lapic->lvt_timer = value; 882141538Sjhb lapic->icr_timer = count; 883141538Sjhb} 884141538Sjhb 885141538Sjhbstatic void 886239013Smavlapic_timer_periodic(struct lapic *la, u_int count, int enable_int) 887141538Sjhb{ 888141538Sjhb u_int32_t value; 889141538Sjhb 890239013Smav value = la->lvt_timer_cache; 891141538Sjhb value &= ~APIC_LVTT_TM; 892141538Sjhb value |= APIC_LVTT_TM_PERIODIC; 893221508Smav if (enable_int) 894221508Smav value &= ~APIC_LVT_M; 895141538Sjhb lapic->lvt_timer = value; 896141538Sjhb lapic->icr_timer = count; 897141538Sjhb} 898141538Sjhb 899141538Sjhbstatic void 900239013Smavlapic_timer_stop(struct lapic *la) 901209371Smav{ 902209371Smav u_int32_t value; 903209371Smav 904239013Smav value = la->lvt_timer_cache; 905209371Smav value &= ~APIC_LVTT_TM; 906211756Smav value |= APIC_LVT_M; 907209371Smav lapic->lvt_timer = value; 908209371Smav} 909209371Smav 910205851Sjhbvoid 911208507Sjhblapic_handle_cmc(void) 912208507Sjhb{ 913208507Sjhb 914208507Sjhb lapic_eoi(); 915208507Sjhb cmc_intr(); 916208507Sjhb} 917208507Sjhb 918208507Sjhb/* 919208507Sjhb * Called from the mca_init() to activate the CMC interrupt if this CPU is 920208507Sjhb * responsible for monitoring any MC banks for CMC events. Since mca_init() 921208507Sjhb * is called prior to lapic_setup() during boot, this just needs to unmask 922208507Sjhb * this CPU's LVT_CMCI entry. 923208507Sjhb */ 924208507Sjhbvoid 925208507Sjhblapic_enable_cmc(void) 926208507Sjhb{ 927208507Sjhb u_int apic_id; 928208507Sjhb 929232232Sjhb#ifdef DEV_ATPIC 930232232Sjhb if (lapic == NULL) 931232232Sjhb return; 932232232Sjhb#endif 933208507Sjhb apic_id = PCPU_GET(apic_id); 934208507Sjhb KASSERT(lapics[apic_id].la_present, 935208507Sjhb ("%s: missing APIC %u", __func__, apic_id)); 936262141Sjhb lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0; 937262141Sjhb lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1; 938208507Sjhb if (bootverbose) 939208507Sjhb printf("lapic%u: CMCI unmasked\n", apic_id); 940208507Sjhb} 941208507Sjhb 942208507Sjhbvoid 943205851Sjhblapic_handle_error(void) 944205851Sjhb{ 945205851Sjhb u_int32_t esr; 946205851Sjhb 947205851Sjhb /* 948205851Sjhb * Read the contents of the error status register. Write to 949205851Sjhb * the register first before reading from it to force the APIC 950205851Sjhb * to update its value to indicate any errors that have 951205851Sjhb * occurred since the previous write to the register. 952205851Sjhb */ 953205851Sjhb lapic->esr = 0; 954205851Sjhb esr = lapic->esr; 955205851Sjhb 956205851Sjhb printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr); 957205851Sjhb lapic_eoi(); 958205851Sjhb} 959205851Sjhb 960187880Sjeffu_int 961187880Sjeffapic_cpuid(u_int apic_id) 962187880Sjeff{ 963187880Sjeff#ifdef SMP 964187880Sjeff return apic_cpuids[apic_id]; 965187880Sjeff#else 966187880Sjeff return 0; 967187880Sjeff#endif 968187880Sjeff} 969187880Sjeff 970151979Sjhb/* Request a free IDT vector to be used by the specified IRQ. */ 971121986Sjhbu_int 972187880Sjeffapic_alloc_vector(u_int apic_id, u_int irq) 973121986Sjhb{ 974121986Sjhb u_int vector; 975121986Sjhb 976121986Sjhb KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 977151979Sjhb 978151979Sjhb /* 979151979Sjhb * Search for a free vector. Currently we just use a very simple 980151979Sjhb * algorithm to find the first free vector. 981151979Sjhb */ 982151979Sjhb mtx_lock_spin(&icu_lock); 983151979Sjhb for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 984191720Smav if (lapics[apic_id].la_ioint_irqs[vector] != -1) 985151979Sjhb continue; 986187880Sjeff lapics[apic_id].la_ioint_irqs[vector] = irq; 987151979Sjhb mtx_unlock_spin(&icu_lock); 988151979Sjhb return (vector + APIC_IO_INTS); 989151979Sjhb } 990151979Sjhb mtx_unlock_spin(&icu_lock); 991195249Sjhb return (0); 992121986Sjhb} 993121986Sjhb 994164265Sjhb/* 995164265Sjhb * Request 'count' free contiguous IDT vectors to be used by 'count' 996164265Sjhb * IRQs. 'count' must be a power of two and the vectors will be 997164265Sjhb * aligned on a boundary of 'align'. If the request cannot be 998164265Sjhb * satisfied, 0 is returned. 999164265Sjhb */ 1000164265Sjhbu_int 1001187880Sjeffapic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 1002164265Sjhb{ 1003164265Sjhb u_int first, run, vector; 1004164265Sjhb 1005164265Sjhb KASSERT(powerof2(count), ("bad count")); 1006164265Sjhb KASSERT(powerof2(align), ("bad align")); 1007164265Sjhb KASSERT(align >= count, ("align < count")); 1008164265Sjhb#ifdef INVARIANTS 1009164265Sjhb for (run = 0; run < count; run++) 1010164265Sjhb KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u", 1011164265Sjhb irqs[run], run)); 1012164265Sjhb#endif 1013164265Sjhb 1014164265Sjhb /* 1015164265Sjhb * Search for 'count' free vectors. As with apic_alloc_vector(), 1016164265Sjhb * this just uses a simple first fit algorithm. 1017164265Sjhb */ 1018164265Sjhb run = 0; 1019164265Sjhb first = 0; 1020164265Sjhb mtx_lock_spin(&icu_lock); 1021164265Sjhb for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 1022164265Sjhb 1023164265Sjhb /* Vector is in use, end run. */ 1024191720Smav if (lapics[apic_id].la_ioint_irqs[vector] != -1) { 1025164265Sjhb run = 0; 1026164265Sjhb first = 0; 1027164265Sjhb continue; 1028164265Sjhb } 1029164265Sjhb 1030164265Sjhb /* Start a new run if run == 0 and vector is aligned. */ 1031164265Sjhb if (run == 0) { 1032164265Sjhb if ((vector & (align - 1)) != 0) 1033164265Sjhb continue; 1034164265Sjhb first = vector; 1035164265Sjhb } 1036164265Sjhb run++; 1037164265Sjhb 1038164265Sjhb /* Keep looping if the run isn't long enough yet. */ 1039164265Sjhb if (run < count) 1040164265Sjhb continue; 1041164265Sjhb 1042164265Sjhb /* Found a run, assign IRQs and return the first vector. */ 1043164265Sjhb for (vector = 0; vector < count; vector++) 1044187880Sjeff lapics[apic_id].la_ioint_irqs[first + vector] = 1045187880Sjeff irqs[vector]; 1046164265Sjhb mtx_unlock_spin(&icu_lock); 1047164265Sjhb return (first + APIC_IO_INTS); 1048164265Sjhb } 1049164265Sjhb mtx_unlock_spin(&icu_lock); 1050164265Sjhb printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count); 1051164265Sjhb return (0); 1052164265Sjhb} 1053164265Sjhb 1054187880Sjeff/* 1055187880Sjeff * Enable a vector for a particular apic_id. Since all lapics share idt 1056187880Sjeff * entries and ioint_handlers this enables the vector on all lapics. lapics 1057187880Sjeff * which do not have the vector configured would report spurious interrupts 1058187880Sjeff * should it fire. 1059187880Sjeff */ 1060151979Sjhbvoid 1061187880Sjeffapic_enable_vector(u_int apic_id, u_int vector) 1062151979Sjhb{ 1063151979Sjhb 1064151979Sjhb KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1065151979Sjhb KASSERT(ioint_handlers[vector / 32] != NULL, 1066151979Sjhb ("No ISR handler for vector %u", vector)); 1067212004Srpaulo#ifdef KDTRACE_HOOKS 1068212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1069212004Srpaulo ("Attempt to overwrite DTrace entry")); 1070212004Srpaulo#endif 1071208452Smav setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL, 1072208452Smav GSEL_APIC); 1073151979Sjhb} 1074151979Sjhb 1075169391Sjhbvoid 1076187880Sjeffapic_disable_vector(u_int apic_id, u_int vector) 1077169391Sjhb{ 1078169391Sjhb 1079169391Sjhb KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1080212004Srpaulo#ifdef KDTRACE_HOOKS 1081212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1082212004Srpaulo ("Attempt to overwrite DTrace entry")); 1083212004Srpaulo#endif 1084169391Sjhb KASSERT(ioint_handlers[vector / 32] != NULL, 1085169391Sjhb ("No ISR handler for vector %u", vector)); 1086188904Sjeff#ifdef notyet 1087188904Sjeff /* 1088188904Sjeff * We can not currently clear the idt entry because other cpus 1089188904Sjeff * may have a valid vector at this offset. 1090188904Sjeff */ 1091208452Smav setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC); 1092188904Sjeff#endif 1093169391Sjhb} 1094169391Sjhb 1095151979Sjhb/* Release an APIC vector when it's no longer in use. */ 1096151979Sjhbvoid 1097187880Sjeffapic_free_vector(u_int apic_id, u_int vector, u_int irq) 1098151979Sjhb{ 1099187880Sjeff struct thread *td; 1100194889Sjhb 1101151979Sjhb KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1102151979Sjhb vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1103151979Sjhb ("Vector %u does not map to an IRQ line", vector)); 1104151979Sjhb KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 1105187880Sjeff KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] == 1106187880Sjeff irq, ("IRQ mismatch")); 1107212004Srpaulo#ifdef KDTRACE_HOOKS 1108212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1109212004Srpaulo ("Attempt to overwrite DTrace entry")); 1110212004Srpaulo#endif 1111187880Sjeff 1112187880Sjeff /* 1113187880Sjeff * Bind us to the cpu that owned the vector before freeing it so 1114187880Sjeff * we don't lose an interrupt delivery race. 1115187880Sjeff */ 1116187880Sjeff td = curthread; 1117196745Sjhb if (!rebooting) { 1118196745Sjhb thread_lock(td); 1119196745Sjhb if (sched_is_bound(td)) 1120196745Sjhb panic("apic_free_vector: Thread already bound.\n"); 1121196745Sjhb sched_bind(td, apic_cpuid(apic_id)); 1122196745Sjhb thread_unlock(td); 1123196745Sjhb } 1124151979Sjhb mtx_lock_spin(&icu_lock); 1125191720Smav lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1; 1126151979Sjhb mtx_unlock_spin(&icu_lock); 1127196745Sjhb if (!rebooting) { 1128196745Sjhb thread_lock(td); 1129196745Sjhb sched_unbind(td); 1130196745Sjhb thread_unlock(td); 1131196745Sjhb } 1132151979Sjhb} 1133151979Sjhb 1134151979Sjhb/* Map an IDT vector (APIC) to an IRQ (interrupt source). */ 1135121986Sjhbu_int 1136187880Sjeffapic_idt_to_irq(u_int apic_id, u_int vector) 1137121986Sjhb{ 1138191730Smav int irq; 1139121986Sjhb 1140122690Sjhb KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1141151979Sjhb vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1142121986Sjhb ("Vector %u does not map to an IRQ line", vector)); 1143212004Srpaulo#ifdef KDTRACE_HOOKS 1144212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1145212004Srpaulo ("Attempt to overwrite DTrace entry")); 1146212004Srpaulo#endif 1147191730Smav irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS]; 1148191730Smav if (irq < 0) 1149191730Smav irq = 0; 1150191730Smav return (irq); 1151121986Sjhb} 1152121986Sjhb 1153151979Sjhb#ifdef DDB 1154121986Sjhb/* 1155151979Sjhb * Dump data about APIC IDT vector mappings. 1156151979Sjhb */ 1157151979SjhbDB_SHOW_COMMAND(apic, db_show_apic) 1158151979Sjhb{ 1159151979Sjhb struct intsrc *isrc; 1160160312Sjhb int i, verbose; 1161187880Sjeff u_int apic_id; 1162151979Sjhb u_int irq; 1163151979Sjhb 1164151979Sjhb if (strcmp(modif, "vv") == 0) 1165151979Sjhb verbose = 2; 1166151979Sjhb else if (strcmp(modif, "v") == 0) 1167151979Sjhb verbose = 1; 1168151979Sjhb else 1169151979Sjhb verbose = 0; 1170187880Sjeff for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) { 1171187880Sjeff if (lapics[apic_id].la_present == 0) 1172187880Sjeff continue; 1173187880Sjeff db_printf("Interrupts bound to lapic %u\n", apic_id); 1174187880Sjeff for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) { 1175187880Sjeff irq = lapics[apic_id].la_ioint_irqs[i]; 1176191720Smav if (irq == -1 || irq == IRQ_SYSCALL) 1177187880Sjeff continue; 1178212004Srpaulo#ifdef KDTRACE_HOOKS 1179212004Srpaulo if (irq == IRQ_DTRACE_RET) 1180212004Srpaulo continue; 1181212004Srpaulo#endif 1182255040Sgibbs#ifdef XENHVM 1183255040Sgibbs if (irq == IRQ_EVTCHN) 1184255040Sgibbs continue; 1185255040Sgibbs#endif 1186151979Sjhb db_printf("vec 0x%2x -> ", i + APIC_IO_INTS); 1187151979Sjhb if (irq == IRQ_TIMER) 1188151979Sjhb db_printf("lapic timer\n"); 1189151979Sjhb else if (irq < NUM_IO_INTS) { 1190151979Sjhb isrc = intr_lookup_source(irq); 1191151979Sjhb if (isrc == NULL || verbose == 0) 1192151979Sjhb db_printf("IRQ %u\n", irq); 1193151979Sjhb else 1194151979Sjhb db_dump_intr_event(isrc->is_event, 1195151979Sjhb verbose == 2); 1196151979Sjhb } else 1197151979Sjhb db_printf("IRQ %u ???\n", irq); 1198151979Sjhb } 1199151979Sjhb } 1200151979Sjhb} 1201162233Sjhb 1202162233Sjhbstatic void 1203162233Sjhbdump_mask(const char *prefix, uint32_t v, int base) 1204162233Sjhb{ 1205162233Sjhb int i, first; 1206162233Sjhb 1207162233Sjhb first = 1; 1208162233Sjhb for (i = 0; i < 32; i++) 1209162233Sjhb if (v & (1 << i)) { 1210162233Sjhb if (first) { 1211162233Sjhb db_printf("%s:", prefix); 1212162233Sjhb first = 0; 1213162233Sjhb } 1214162233Sjhb db_printf(" %02x", base + i); 1215162233Sjhb } 1216162233Sjhb if (!first) 1217162233Sjhb db_printf("\n"); 1218162233Sjhb} 1219162233Sjhb 1220162233Sjhb/* Show info from the lapic regs for this CPU. */ 1221162233SjhbDB_SHOW_COMMAND(lapic, db_show_lapic) 1222162233Sjhb{ 1223162233Sjhb uint32_t v; 1224162233Sjhb 1225162233Sjhb db_printf("lapic ID = %d\n", lapic_id()); 1226162233Sjhb v = lapic->version; 1227162233Sjhb db_printf("version = %d.%d\n", (v & APIC_VER_VERSION) >> 4, 1228162233Sjhb v & 0xf); 1229162233Sjhb db_printf("max LVT = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT); 1230162233Sjhb v = lapic->svr; 1231162233Sjhb db_printf("SVR = %02x (%s)\n", v & APIC_SVR_VECTOR, 1232162233Sjhb v & APIC_SVR_ENABLE ? "enabled" : "disabled"); 1233162233Sjhb db_printf("TPR = %02x\n", lapic->tpr); 1234162233Sjhb 1235162233Sjhb#define dump_field(prefix, index) \ 1236162233Sjhb dump_mask(__XSTRING(prefix ## index), lapic->prefix ## index, \ 1237162233Sjhb index * 32) 1238162233Sjhb 1239162233Sjhb db_printf("In-service Interrupts:\n"); 1240162233Sjhb dump_field(isr, 0); 1241162233Sjhb dump_field(isr, 1); 1242162233Sjhb dump_field(isr, 2); 1243162233Sjhb dump_field(isr, 3); 1244162233Sjhb dump_field(isr, 4); 1245162233Sjhb dump_field(isr, 5); 1246162233Sjhb dump_field(isr, 6); 1247162233Sjhb dump_field(isr, 7); 1248162233Sjhb 1249162233Sjhb db_printf("TMR Interrupts:\n"); 1250162233Sjhb dump_field(tmr, 0); 1251162233Sjhb dump_field(tmr, 1); 1252162233Sjhb dump_field(tmr, 2); 1253162233Sjhb dump_field(tmr, 3); 1254162233Sjhb dump_field(tmr, 4); 1255162233Sjhb dump_field(tmr, 5); 1256162233Sjhb dump_field(tmr, 6); 1257162233Sjhb dump_field(tmr, 7); 1258162233Sjhb 1259162233Sjhb db_printf("IRR Interrupts:\n"); 1260162233Sjhb dump_field(irr, 0); 1261162233Sjhb dump_field(irr, 1); 1262162233Sjhb dump_field(irr, 2); 1263162233Sjhb dump_field(irr, 3); 1264162233Sjhb dump_field(irr, 4); 1265162233Sjhb dump_field(irr, 5); 1266162233Sjhb dump_field(irr, 6); 1267162233Sjhb dump_field(irr, 7); 1268162233Sjhb 1269162233Sjhb#undef dump_field 1270162233Sjhb} 1271151979Sjhb#endif 1272151979Sjhb 1273151979Sjhb/* 1274121986Sjhb * APIC probing support code. This includes code to manage enumerators. 1275121986Sjhb */ 1276121986Sjhb 1277121986Sjhbstatic SLIST_HEAD(, apic_enumerator) enumerators = 1278121986Sjhb SLIST_HEAD_INITIALIZER(enumerators); 1279121986Sjhbstatic struct apic_enumerator *best_enum; 1280195249Sjhb 1281121986Sjhbvoid 1282121986Sjhbapic_register_enumerator(struct apic_enumerator *enumerator) 1283121986Sjhb{ 1284121986Sjhb#ifdef INVARIANTS 1285121986Sjhb struct apic_enumerator *apic_enum; 1286121986Sjhb 1287121986Sjhb SLIST_FOREACH(apic_enum, &enumerators, apic_next) { 1288121986Sjhb if (apic_enum == enumerator) 1289121986Sjhb panic("%s: Duplicate register of %s", __func__, 1290121986Sjhb enumerator->apic_name); 1291121986Sjhb } 1292121986Sjhb#endif 1293121986Sjhb SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next); 1294121986Sjhb} 1295121986Sjhb 1296121986Sjhb/* 1297208479Smav * We have to look for CPU's very, very early because certain subsystems 1298208479Smav * want to know how many CPU's we have extremely early on in the boot 1299208479Smav * process. 1300121986Sjhb */ 1301121986Sjhbstatic void 1302121986Sjhbapic_init(void *dummy __unused) 1303121986Sjhb{ 1304121986Sjhb struct apic_enumerator *enumerator; 1305121986Sjhb int retval, best; 1306121986Sjhb 1307153383Sjhb /* We only support built in local APICs. */ 1308153383Sjhb if (!(cpu_feature & CPUID_APIC)) 1309121986Sjhb return; 1310121986Sjhb 1311123133Sjhb /* Don't probe if APIC mode is disabled. */ 1312123133Sjhb if (resource_disabled("apic", 0)) 1313123133Sjhb return; 1314123133Sjhb 1315215009Sjhb /* Probe all the enumerators to find the best match. */ 1316121986Sjhb best_enum = NULL; 1317121986Sjhb best = 0; 1318121986Sjhb SLIST_FOREACH(enumerator, &enumerators, apic_next) { 1319121986Sjhb retval = enumerator->apic_probe(); 1320121986Sjhb if (retval > 0) 1321121986Sjhb continue; 1322121986Sjhb if (best_enum == NULL || best < retval) { 1323121986Sjhb best_enum = enumerator; 1324121986Sjhb best = retval; 1325121986Sjhb } 1326121986Sjhb } 1327121986Sjhb if (best_enum == NULL) { 1328121986Sjhb if (bootverbose) 1329121986Sjhb printf("APIC: Could not find any APICs.\n"); 1330232232Sjhb#ifndef DEV_ATPIC 1331232232Sjhb panic("running without device atpic requires a local APIC"); 1332232232Sjhb#endif 1333121986Sjhb return; 1334121986Sjhb } 1335121986Sjhb 1336121986Sjhb if (bootverbose) 1337121986Sjhb printf("APIC: Using the %s enumerator.\n", 1338121986Sjhb best_enum->apic_name); 1339121986Sjhb 1340277493Sjhb#ifdef I686_CPU 1341121986Sjhb /* 1342121986Sjhb * To work around an errata, we disable the local APIC on some 1343121986Sjhb * CPUs during early startup. We need to turn the local APIC back 1344121986Sjhb * on on such CPUs now. 1345121986Sjhb */ 1346276076Sjhb ppro_reenable_apic(); 1347208452Smav#endif 1348123133Sjhb 1349215009Sjhb /* Probe the CPU's in the system. */ 1350123133Sjhb retval = best_enum->apic_probe_cpus(); 1351123133Sjhb if (retval != 0) 1352123133Sjhb printf("%s: Failed to probe CPUs: returned %d\n", 1353123133Sjhb best_enum->apic_name, retval); 1354123133Sjhb 1355208479Smav} 1356208479SmavSYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL); 1357208479Smav 1358208479Smav/* 1359208479Smav * Setup the local APIC. We have to do this prior to starting up the APs 1360208479Smav * in the SMP case. 1361208479Smav */ 1362208479Smavstatic void 1363208479Smavapic_setup_local(void *dummy __unused) 1364208479Smav{ 1365208479Smav int retval; 1366208479Smav 1367208479Smav if (best_enum == NULL) 1368208479Smav return; 1369215009Sjhb 1370215009Sjhb /* Initialize the local APIC. */ 1371121986Sjhb retval = best_enum->apic_setup_local(); 1372121986Sjhb if (retval != 0) 1373121986Sjhb printf("%s: Failed to setup the local APIC: returned %d\n", 1374121986Sjhb best_enum->apic_name, retval); 1375121986Sjhb} 1376215009SjhbSYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL); 1377121986Sjhb 1378121986Sjhb/* 1379121986Sjhb * Setup the I/O APICs. 1380121986Sjhb */ 1381121986Sjhbstatic void 1382121986Sjhbapic_setup_io(void *dummy __unused) 1383121986Sjhb{ 1384121986Sjhb int retval; 1385121986Sjhb 1386121986Sjhb if (best_enum == NULL) 1387121986Sjhb return; 1388246247Savg 1389246247Savg /* 1390246247Savg * Local APIC must be registered before other PICs and pseudo PICs 1391246247Savg * for proper suspend/resume order. 1392246247Savg */ 1393246247Savg#ifndef XEN 1394246247Savg intr_register_pic(&lapic_pic); 1395246247Savg#endif 1396246247Savg 1397121986Sjhb retval = best_enum->apic_setup_io(); 1398121986Sjhb if (retval != 0) 1399121986Sjhb printf("%s: Failed to setup I/O APICs: returned %d\n", 1400121986Sjhb best_enum->apic_name, retval); 1401182902Skmacy#ifdef XEN 1402182902Skmacy return; 1403182902Skmacy#endif 1404121986Sjhb /* 1405121986Sjhb * Finish setting up the local APIC on the BSP once we know how to 1406121986Sjhb * properly program the LINT pins. 1407121986Sjhb */ 1408163219Sjhb lapic_setup(1); 1409121986Sjhb if (bootverbose) 1410121986Sjhb lapic_dump("BSP"); 1411164265Sjhb 1412164265Sjhb /* Enable the MSI "pic". */ 1413164265Sjhb msi_init(); 1414121986Sjhb} 1415177253SrwatsonSYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_SECOND, apic_setup_io, NULL); 1416121986Sjhb 1417121986Sjhb#ifdef SMP 1418121986Sjhb/* 1419121986Sjhb * Inter Processor Interrupt functions. The lapic_ipi_*() functions are 1420208452Smav * private to the MD code. The public interface for the rest of the 1421121986Sjhb * kernel is defined in mp_machdep.c. 1422121986Sjhb */ 1423299066Sdelphij 1424299066Sdelphij/* 1425299066Sdelphij * Wait delay microseconds for IPI to be sent. If delay is -1, we 1426299066Sdelphij * wait forever. 1427299066Sdelphij */ 1428121986Sjhbint 1429121986Sjhblapic_ipi_wait(int delay) 1430121986Sjhb{ 1431299066Sdelphij uint64_t rx; 1432121986Sjhb 1433299066Sdelphij for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) { 1434121986Sjhb if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE) 1435121986Sjhb return (1); 1436299066Sdelphij ia32_pause(); 1437121986Sjhb } 1438121986Sjhb return (0); 1439121986Sjhb} 1440121986Sjhb 1441121986Sjhbvoid 1442121986Sjhblapic_ipi_raw(register_t icrlo, u_int dest) 1443121986Sjhb{ 1444214347Sjhb register_t value, saveintr; 1445121986Sjhb 1446121986Sjhb /* XXX: Need more sanity checking of icrlo? */ 1447121986Sjhb KASSERT(lapic != NULL, ("%s called too early", __func__)); 1448121986Sjhb KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1449121986Sjhb ("%s: invalid dest field", __func__)); 1450121986Sjhb KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, 1451121986Sjhb ("%s: reserved bits set in ICR LO register", __func__)); 1452121986Sjhb 1453121986Sjhb /* Set destination in ICR HI register if it is being used. */ 1454214347Sjhb saveintr = intr_disable(); 1455121986Sjhb if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { 1456121986Sjhb value = lapic->icr_hi; 1457121986Sjhb value &= ~APIC_ID_MASK; 1458121986Sjhb value |= dest << APIC_ID_SHIFT; 1459121986Sjhb lapic->icr_hi = value; 1460121986Sjhb } 1461121986Sjhb 1462121986Sjhb /* Program the contents of the IPI and dispatch it. */ 1463121986Sjhb value = lapic->icr_lo; 1464121986Sjhb value &= APIC_ICRLO_RESV_MASK; 1465121986Sjhb value |= icrlo; 1466121986Sjhb lapic->icr_lo = value; 1467214347Sjhb intr_restore(saveintr); 1468121986Sjhb} 1469121986Sjhb 1470281560Sjhb#define BEFORE_SPIN 50000 1471121986Sjhb#ifdef DETECT_DEADLOCK 1472281560Sjhb#define AFTER_SPIN 50 1473121986Sjhb#endif 1474121986Sjhb 1475121986Sjhbvoid 1476121986Sjhblapic_ipi_vectored(u_int vector, int dest) 1477121986Sjhb{ 1478121986Sjhb register_t icrlo, destfield; 1479121986Sjhb 1480121986Sjhb KASSERT((vector & ~APIC_VECTOR_MASK) == 0, 1481121986Sjhb ("%s: invalid vector %d", __func__, vector)); 1482121986Sjhb 1483281560Sjhb icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT; 1484196196Sattilio 1485196196Sattilio /* 1486196196Sattilio * IPI_STOP_HARD is just a "fake" vector used to send a NMI. 1487196196Sattilio * Use special rules regard NMI if passed, otherwise specify 1488196196Sattilio * the vector. 1489196196Sattilio */ 1490196196Sattilio if (vector == IPI_STOP_HARD) 1491281560Sjhb icrlo |= APIC_DELMODE_NMI; 1492196196Sattilio else 1493281560Sjhb icrlo |= vector | APIC_DELMODE_FIXED; 1494121986Sjhb destfield = 0; 1495121986Sjhb switch (dest) { 1496121986Sjhb case APIC_IPI_DEST_SELF: 1497121986Sjhb icrlo |= APIC_DEST_SELF; 1498121986Sjhb break; 1499121986Sjhb case APIC_IPI_DEST_ALL: 1500121986Sjhb icrlo |= APIC_DEST_ALLISELF; 1501121986Sjhb break; 1502121986Sjhb case APIC_IPI_DEST_OTHERS: 1503121986Sjhb icrlo |= APIC_DEST_ALLESELF; 1504121986Sjhb break; 1505121986Sjhb default: 1506121986Sjhb KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1507121986Sjhb ("%s: invalid destination 0x%x", __func__, dest)); 1508121986Sjhb destfield = dest; 1509121986Sjhb } 1510121986Sjhb 1511125317Sjeff /* Wait for an earlier IPI to finish. */ 1512150176Sjhb if (!lapic_ipi_wait(BEFORE_SPIN)) { 1513150176Sjhb if (panicstr != NULL) 1514150176Sjhb return; 1515150176Sjhb else 1516150176Sjhb panic("APIC: Previous IPI is stuck"); 1517150176Sjhb } 1518121986Sjhb 1519121986Sjhb lapic_ipi_raw(icrlo, destfield); 1520121986Sjhb 1521121986Sjhb#ifdef DETECT_DEADLOCK 1522121986Sjhb /* Wait for IPI to be delivered. */ 1523121986Sjhb if (!lapic_ipi_wait(AFTER_SPIN)) { 1524121986Sjhb#ifdef needsattention 1525121986Sjhb /* 1526121986Sjhb * XXX FIXME: 1527121986Sjhb * 1528121986Sjhb * The above function waits for the message to actually be 1529121986Sjhb * delivered. It breaks out after an arbitrary timeout 1530121986Sjhb * since the message should eventually be delivered (at 1531121986Sjhb * least in theory) and that if it wasn't we would catch 1532121986Sjhb * the failure with the check above when the next IPI is 1533121986Sjhb * sent. 1534121986Sjhb * 1535139240Sjhb * We could skip this wait entirely, EXCEPT it probably 1536121986Sjhb * protects us from other routines that assume that the 1537121986Sjhb * message was delivered and acted upon when this function 1538121986Sjhb * returns. 1539121986Sjhb */ 1540121986Sjhb printf("APIC: IPI might be stuck\n"); 1541121986Sjhb#else /* !needsattention */ 1542121986Sjhb /* Wait until mesage is sent without a timeout. */ 1543121986Sjhb while (lapic->icr_lo & APIC_DELSTAT_PEND) 1544121986Sjhb ia32_pause(); 1545121986Sjhb#endif /* needsattention */ 1546121986Sjhb } 1547121986Sjhb#endif /* DETECT_DEADLOCK */ 1548121986Sjhb} 1549121986Sjhb#endif /* SMP */ 1550