1121986Sjhb/*- 2121986Sjhb * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3121986Sjhb * Copyright (c) 1996, by Steve Passe 4121986Sjhb * All rights reserved. 5121986Sjhb * 6121986Sjhb * Redistribution and use in source and binary forms, with or without 7121986Sjhb * modification, are permitted provided that the following conditions 8121986Sjhb * are met: 9121986Sjhb * 1. Redistributions of source code must retain the above copyright 10121986Sjhb * notice, this list of conditions and the following disclaimer. 11121986Sjhb * 2. The name of the developer may NOT be used to endorse or promote products 12121986Sjhb * derived from this software without specific prior written permission. 13121986Sjhb * 3. Neither the name of the author nor the names of any co-contributors 14121986Sjhb * may be used to endorse or promote products derived from this software 15121986Sjhb * without specific prior written permission. 16121986Sjhb * 17121986Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18121986Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19121986Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20121986Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21121986Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22121986Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23121986Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24121986Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25121986Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26121986Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27121986Sjhb * SUCH DAMAGE. 28121986Sjhb */ 29121986Sjhb 30121986Sjhb/* 31121986Sjhb * Local APIC support on Pentium and later processors. 32121986Sjhb */ 33121986Sjhb 34121986Sjhb#include <sys/cdefs.h> 35121986Sjhb__FBSDID("$FreeBSD$"); 36121986Sjhb 37147565Speter#include "opt_hwpmc_hooks.h" 38179277Sjb#include "opt_kdtrace.h" 39147565Speter 40151979Sjhb#include "opt_ddb.h" 41151979Sjhb 42121986Sjhb#include <sys/param.h> 43121986Sjhb#include <sys/systm.h> 44121986Sjhb#include <sys/bus.h> 45121986Sjhb#include <sys/kernel.h> 46151979Sjhb#include <sys/lock.h> 47151979Sjhb#include <sys/mutex.h> 48121986Sjhb#include <sys/pcpu.h> 49187880Sjeff#include <sys/proc.h> 50187880Sjeff#include <sys/sched.h> 51141538Sjhb#include <sys/smp.h> 52209371Smav#include <sys/timeet.h> 53121986Sjhb 54121986Sjhb#include <vm/vm.h> 55121986Sjhb#include <vm/pmap.h> 56121986Sjhb 57214631Sjhb#include <x86/apicreg.h> 58153666Sjhb#include <machine/cpu.h> 59121986Sjhb#include <machine/cputypes.h> 60121986Sjhb#include <machine/frame.h> 61121986Sjhb#include <machine/intr_machdep.h> 62121986Sjhb#include <machine/apicvar.h> 63214630Sjhb#include <x86/mca.h> 64121986Sjhb#include <machine/md_var.h> 65121986Sjhb#include <machine/smp.h> 66121986Sjhb#include <machine/specialreg.h> 67121986Sjhb 68151979Sjhb#ifdef DDB 69151979Sjhb#include <sys/interrupt.h> 70151979Sjhb#include <ddb/ddb.h> 71151979Sjhb#endif 72151979Sjhb 73208452Smav#ifdef __amd64__ 74208452Smav#define SDT_APIC SDT_SYSIGT 75208452Smav#define SDT_APICT SDT_SYSIGT 76208452Smav#define GSEL_APIC 0 77208452Smav#else 78208452Smav#define SDT_APIC SDT_SYS386IGT 79208452Smav#define SDT_APICT SDT_SYS386TGT 80208452Smav#define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL) 81208452Smav#endif 82208452Smav 83122690Sjhb/* Sanity checks on IDT vectors. */ 84139240SjhbCTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); 85139240SjhbCTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); 86139240SjhbCTASSERT(APIC_LOCAL_INTS == 240); 87122690SjhbCTASSERT(IPI_STOP < APIC_SPURIOUS_INT); 88122690Sjhb 89151979Sjhb/* Magic IRQ values for the timer and syscalls. */ 90151979Sjhb#define IRQ_TIMER (NUM_IO_INTS + 1) 91151979Sjhb#define IRQ_SYSCALL (NUM_IO_INTS + 2) 92212004Srpaulo#define IRQ_DTRACE_RET (NUM_IO_INTS + 3) 93151979Sjhb 94121986Sjhb/* 95121986Sjhb * Support for local APICs. Local APICs manage interrupts on each 96121986Sjhb * individual processor as opposed to I/O APICs which receive interrupts 97121986Sjhb * from I/O devices and then forward them on to the local APICs. 98121986Sjhb * 99121986Sjhb * Local APICs can also send interrupts to each other thus providing the 100121986Sjhb * mechanism for IPIs. 101121986Sjhb */ 102121986Sjhb 103121986Sjhbstruct lvt { 104121986Sjhb u_int lvt_edgetrigger:1; 105121986Sjhb u_int lvt_activehi:1; 106121986Sjhb u_int lvt_masked:1; 107121986Sjhb u_int lvt_active:1; 108121986Sjhb u_int lvt_mode:16; 109121986Sjhb u_int lvt_vector:8; 110121986Sjhb}; 111121986Sjhb 112121986Sjhbstruct lapic { 113121986Sjhb struct lvt la_lvts[LVT_MAX + 1]; 114121986Sjhb u_int la_id:8; 115121986Sjhb u_int la_cluster:4; 116121986Sjhb u_int la_cluster_id:2; 117121986Sjhb u_int la_present:1; 118141538Sjhb u_long *la_timer_count; 119209371Smav u_long la_timer_period; 120209371Smav u_int la_timer_mode; 121187880Sjeff /* Include IDT_SYSCALL to make indexing easier. */ 122191720Smav int la_ioint_irqs[APIC_NUM_IOINTS + 1]; 123169395Sjhb} static lapics[MAX_APIC_ID + 1]; 124121986Sjhb 125121986Sjhb/* Global defaults for local APIC LVT entries. */ 126121986Sjhbstatic struct lvt lvts[LVT_MAX + 1] = { 127121986Sjhb { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 }, /* LINT0: masked ExtINT */ 128121986Sjhb { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 }, /* LINT1: NMI */ 129139245Sjhb { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT }, /* Timer */ 130205851Sjhb { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT }, /* Error */ 131196224Sjhb { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 }, /* PMC */ 132139245Sjhb { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT }, /* Thermal */ 133208507Sjhb { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT }, /* CMCI */ 134121986Sjhb}; 135121986Sjhb 136121986Sjhbstatic inthand_t *ioint_handlers[] = { 137121986Sjhb NULL, /* 0 - 31 */ 138121986Sjhb IDTVEC(apic_isr1), /* 32 - 63 */ 139121986Sjhb IDTVEC(apic_isr2), /* 64 - 95 */ 140121986Sjhb IDTVEC(apic_isr3), /* 96 - 127 */ 141121986Sjhb IDTVEC(apic_isr4), /* 128 - 159 */ 142121986Sjhb IDTVEC(apic_isr5), /* 160 - 191 */ 143122690Sjhb IDTVEC(apic_isr6), /* 192 - 223 */ 144122690Sjhb IDTVEC(apic_isr7), /* 224 - 255 */ 145121986Sjhb}; 146121986Sjhb 147151979Sjhb 148195249Sjhbstatic u_int32_t lapic_timer_divisors[] = { 149141538Sjhb APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, 150141538Sjhb APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128 151141538Sjhb}; 152141538Sjhb 153169391Sjhbextern inthand_t IDTVEC(rsvd); 154169391Sjhb 155121986Sjhbvolatile lapic_t *lapic; 156167747Sjhbvm_paddr_t lapic_paddr; 157209371Smavstatic u_long lapic_timer_divisor; 158209371Smavstatic struct eventtimer lapic_et; 159121986Sjhb 160139245Sjhbstatic void lapic_enable(void); 161163219Sjhbstatic void lapic_resume(struct pic *pic); 162221508Smavstatic void lapic_timer_oneshot(u_int count, int enable_int); 163221508Smavstatic void lapic_timer_periodic(u_int count, int enable_int); 164209371Smavstatic void lapic_timer_stop(void); 165141538Sjhbstatic void lapic_timer_set_divisor(u_int divisor); 166139245Sjhbstatic uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value); 167209371Smavstatic int lapic_et_start(struct eventtimer *et, 168209371Smav struct bintime *first, struct bintime *period); 169209371Smavstatic int lapic_et_stop(struct eventtimer *et); 170139245Sjhb 171163219Sjhbstruct pic lapic_pic = { .pic_resume = lapic_resume }; 172163219Sjhb 173121986Sjhbstatic uint32_t 174121986Sjhblvt_mode(struct lapic *la, u_int pin, uint32_t value) 175121986Sjhb{ 176121986Sjhb struct lvt *lvt; 177121986Sjhb 178121986Sjhb KASSERT(pin <= LVT_MAX, ("%s: pin %u out of range", __func__, pin)); 179121986Sjhb if (la->la_lvts[pin].lvt_active) 180121986Sjhb lvt = &la->la_lvts[pin]; 181121986Sjhb else 182121986Sjhb lvt = &lvts[pin]; 183121986Sjhb 184121986Sjhb value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM | 185121986Sjhb APIC_LVT_VECTOR); 186121986Sjhb if (lvt->lvt_edgetrigger == 0) 187121986Sjhb value |= APIC_LVT_TM; 188121986Sjhb if (lvt->lvt_activehi == 0) 189121986Sjhb value |= APIC_LVT_IIPP_INTALO; 190121986Sjhb if (lvt->lvt_masked) 191121986Sjhb value |= APIC_LVT_M; 192121986Sjhb value |= lvt->lvt_mode; 193121986Sjhb switch (lvt->lvt_mode) { 194121986Sjhb case APIC_LVT_DM_NMI: 195121986Sjhb case APIC_LVT_DM_SMI: 196121986Sjhb case APIC_LVT_DM_INIT: 197121986Sjhb case APIC_LVT_DM_EXTINT: 198121986Sjhb if (!lvt->lvt_edgetrigger) { 199121986Sjhb printf("lapic%u: Forcing LINT%u to edge trigger\n", 200121986Sjhb la->la_id, pin); 201121986Sjhb value |= APIC_LVT_TM; 202121986Sjhb } 203121986Sjhb /* Use a vector of 0. */ 204121986Sjhb break; 205121986Sjhb case APIC_LVT_DM_FIXED: 206121986Sjhb value |= lvt->lvt_vector; 207121986Sjhb break; 208121986Sjhb default: 209121986Sjhb panic("bad APIC LVT delivery mode: %#x\n", value); 210121986Sjhb } 211121986Sjhb return (value); 212121986Sjhb} 213121986Sjhb 214121986Sjhb/* 215121986Sjhb * Map the local APIC and setup necessary interrupt vectors. 216121986Sjhb */ 217121986Sjhbvoid 218167247Sjhblapic_init(vm_paddr_t addr) 219121986Sjhb{ 220209371Smav u_int regs[4]; 221209371Smav int i, arat; 222121986Sjhb 223121986Sjhb /* Map the local APIC and setup the spurious interrupt handler. */ 224121986Sjhb KASSERT(trunc_page(addr) == addr, 225121986Sjhb ("local APIC not aligned on a page boundary")); 226247547Sjhb lapic_paddr = addr; 227156920Sjhb lapic = pmap_mapdev(addr, sizeof(lapic_t)); 228208452Smav setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL, 229208452Smav GSEL_APIC); 230121986Sjhb 231121986Sjhb /* Perform basic initialization of the BSP's local APIC. */ 232139245Sjhb lapic_enable(); 233121986Sjhb 234121986Sjhb /* Set BSP's per-CPU local APIC ID. */ 235121986Sjhb PCPU_SET(apic_id, lapic_id()); 236121986Sjhb 237141538Sjhb /* Local APIC timer interrupt. */ 238208452Smav setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC); 239141538Sjhb 240205851Sjhb /* Local APIC error interrupt. */ 241208452Smav setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC); 242205851Sjhb 243205851Sjhb /* XXX: Thermal interrupt */ 244208507Sjhb 245208507Sjhb /* Local APIC CMCI. */ 246208507Sjhb setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC); 247209371Smav 248209371Smav if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) { 249209371Smav arat = 0; 250209371Smav /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */ 251209371Smav if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) { 252209371Smav do_cpuid(0x06, regs); 253215751Savg if ((regs[0] & CPUTPM1_ARAT) != 0) 254209371Smav arat = 1; 255209371Smav } 256209371Smav bzero(&lapic_et, sizeof(lapic_et)); 257209371Smav lapic_et.et_name = "LAPIC"; 258209371Smav lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | 259209371Smav ET_FLAGS_PERCPU; 260209371Smav lapic_et.et_quality = 600; 261209371Smav if (!arat) { 262209371Smav lapic_et.et_flags |= ET_FLAGS_C3STOP; 263212541Smav lapic_et.et_quality -= 200; 264209371Smav } 265209371Smav lapic_et.et_frequency = 0; 266210290Smav /* We don't know frequency yet, so trying to guess. */ 267210290Smav lapic_et.et_min_period.sec = 0; 268210290Smav lapic_et.et_min_period.frac = 0x00001000LL << 32; 269210290Smav lapic_et.et_max_period.sec = 1; 270210290Smav lapic_et.et_max_period.frac = 0; 271209371Smav lapic_et.et_start = lapic_et_start; 272209371Smav lapic_et.et_stop = lapic_et_stop; 273209371Smav lapic_et.et_priv = NULL; 274209371Smav et_register(&lapic_et); 275209371Smav } 276121986Sjhb} 277121986Sjhb 278121986Sjhb/* 279121986Sjhb * Create a local APIC instance. 280121986Sjhb */ 281121986Sjhbvoid 282121986Sjhblapic_create(u_int apic_id, int boot_cpu) 283121986Sjhb{ 284121986Sjhb int i; 285121986Sjhb 286169395Sjhb if (apic_id > MAX_APIC_ID) { 287121986Sjhb printf("APIC: Ignoring local APIC with ID %d\n", apic_id); 288121986Sjhb if (boot_cpu) 289121986Sjhb panic("Can't ignore BSP"); 290121986Sjhb return; 291121986Sjhb } 292121986Sjhb KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u", 293121986Sjhb apic_id)); 294121986Sjhb 295121986Sjhb /* 296121986Sjhb * Assume no local LVT overrides and a cluster of 0 and 297121986Sjhb * intra-cluster ID of 0. 298121986Sjhb */ 299121986Sjhb lapics[apic_id].la_present = 1; 300121986Sjhb lapics[apic_id].la_id = apic_id; 301208507Sjhb for (i = 0; i <= LVT_MAX; i++) { 302121986Sjhb lapics[apic_id].la_lvts[i] = lvts[i]; 303121986Sjhb lapics[apic_id].la_lvts[i].lvt_active = 0; 304121986Sjhb } 305191720Smav for (i = 0; i <= APIC_NUM_IOINTS; i++) 306191720Smav lapics[apic_id].la_ioint_irqs[i] = -1; 307187880Sjeff lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL; 308187880Sjeff lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = 309187880Sjeff IRQ_TIMER; 310212004Srpaulo#ifdef KDTRACE_HOOKS 311212004Srpaulo lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] = IRQ_DTRACE_RET; 312212004Srpaulo#endif 313121986Sjhb 314212004Srpaulo 315121986Sjhb#ifdef SMP 316121986Sjhb cpu_add(apic_id, boot_cpu); 317121986Sjhb#endif 318121986Sjhb} 319121986Sjhb 320121986Sjhb/* 321121986Sjhb * Dump contents of local APIC registers 322121986Sjhb */ 323121986Sjhbvoid 324121986Sjhblapic_dump(const char* str) 325121986Sjhb{ 326215001Sjhb uint32_t maxlvt; 327121986Sjhb 328215001Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 329121986Sjhb printf("cpu%d %s:\n", PCPU_GET(cpuid), str); 330121986Sjhb printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n", 331121986Sjhb lapic->id, lapic->version, lapic->ldr, lapic->dfr); 332121986Sjhb printf(" lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", 333121986Sjhb lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr); 334215001Sjhb printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x", 335215001Sjhb lapic->lvt_timer, lapic->lvt_thermal, lapic->lvt_error); 336215001Sjhb if (maxlvt >= LVT_PMC) 337215001Sjhb printf(" pmc: 0x%08x", lapic->lvt_pcint); 338215001Sjhb printf("\n"); 339215001Sjhb if (maxlvt >= LVT_CMCI) 340215001Sjhb printf(" cmci: 0x%08x\n", lapic->lvt_cmci); 341121986Sjhb} 342121986Sjhb 343121986Sjhbvoid 344163219Sjhblapic_setup(int boot) 345121986Sjhb{ 346121986Sjhb struct lapic *la; 347156124Sjhb u_int32_t maxlvt; 348214347Sjhb register_t saveintr; 349141538Sjhb char buf[MAXCOMLEN + 1]; 350121986Sjhb 351121986Sjhb la = &lapics[lapic_id()]; 352121986Sjhb KASSERT(la->la_present, ("missing APIC structure")); 353214347Sjhb saveintr = intr_disable(); 354121986Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 355121986Sjhb 356139240Sjhb /* Initialize the TPR to allow all interrupts. */ 357139240Sjhb lapic_set_tpr(0); 358121986Sjhb 359121986Sjhb /* Setup spurious vector and enable the local APIC. */ 360139245Sjhb lapic_enable(); 361139245Sjhb 362139245Sjhb /* Program LINT[01] LVT entries. */ 363139245Sjhb lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0); 364139245Sjhb lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1); 365185933Sjhb 366145256Sjkoshy /* Program the PMC LVT entry if present. */ 367145256Sjkoshy if (maxlvt >= LVT_PMC) 368145256Sjkoshy lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); 369139245Sjhb 370141538Sjhb /* Program timer LVT and setup handler. */ 371141538Sjhb lapic->lvt_timer = lvt_mode(la, LVT_TIMER, lapic->lvt_timer); 372163219Sjhb if (boot) { 373209371Smav snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid)); 374163219Sjhb intrcnt_add(buf, &la->la_timer_count); 375163219Sjhb } 376163219Sjhb 377209371Smav /* Setup the timer if configured. */ 378209371Smav if (la->la_timer_mode != 0) { 379209371Smav KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor", 380141538Sjhb lapic_id())); 381141538Sjhb lapic_timer_set_divisor(lapic_timer_divisor); 382209371Smav if (la->la_timer_mode == 1) 383221508Smav lapic_timer_periodic(la->la_timer_period, 1); 384209371Smav else 385221508Smav lapic_timer_oneshot(la->la_timer_period, 1); 386141538Sjhb } 387139245Sjhb 388205851Sjhb /* Program error LVT and clear any existing errors. */ 389205851Sjhb lapic->lvt_error = lvt_mode(la, LVT_ERROR, lapic->lvt_error); 390205851Sjhb lapic->esr = 0; 391141538Sjhb 392205851Sjhb /* XXX: Thermal LVT */ 393205851Sjhb 394208507Sjhb /* Program the CMCI LVT entry if present. */ 395208507Sjhb if (maxlvt >= LVT_CMCI) 396208507Sjhb lapic->lvt_cmci = lvt_mode(la, LVT_CMCI, lapic->lvt_cmci); 397208507Sjhb 398214347Sjhb intr_restore(saveintr); 399121986Sjhb} 400121986Sjhb 401196224Sjhbvoid 402196224Sjhblapic_reenable_pmc(void) 403196224Sjhb{ 404196224Sjhb#ifdef HWPMC_HOOKS 405196224Sjhb uint32_t value; 406196224Sjhb 407196224Sjhb value = lapic->lvt_pcint; 408196224Sjhb value &= ~APIC_LVT_M; 409196224Sjhb lapic->lvt_pcint = value; 410196224Sjhb#endif 411196224Sjhb} 412196224Sjhb 413196224Sjhb#ifdef HWPMC_HOOKS 414196224Sjhbstatic void 415196224Sjhblapic_update_pmc(void *dummy) 416196224Sjhb{ 417196224Sjhb struct lapic *la; 418196224Sjhb 419196224Sjhb la = &lapics[lapic_id()]; 420196224Sjhb lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); 421196224Sjhb} 422196224Sjhb#endif 423196224Sjhb 424196224Sjhbint 425196224Sjhblapic_enable_pmc(void) 426196224Sjhb{ 427196224Sjhb#ifdef HWPMC_HOOKS 428196224Sjhb u_int32_t maxlvt; 429196224Sjhb 430196224Sjhb /* Fail if the local APIC is not present. */ 431196224Sjhb if (lapic == NULL) 432196224Sjhb return (0); 433196224Sjhb 434196224Sjhb /* Fail if the PMC LVT is not present. */ 435196224Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 436196224Sjhb if (maxlvt < LVT_PMC) 437196224Sjhb return (0); 438196224Sjhb 439196224Sjhb lvts[LVT_PMC].lvt_masked = 0; 440196224Sjhb 441196224Sjhb#ifdef SMP 442196224Sjhb /* 443196224Sjhb * If hwpmc was loaded at boot time then the APs may not be 444196224Sjhb * started yet. In that case, don't forward the request to 445196224Sjhb * them as they will program the lvt when they start. 446196224Sjhb */ 447196224Sjhb if (smp_started) 448196224Sjhb smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 449196224Sjhb else 450196224Sjhb#endif 451196224Sjhb lapic_update_pmc(NULL); 452196224Sjhb return (1); 453196224Sjhb#else 454196224Sjhb return (0); 455196224Sjhb#endif 456196224Sjhb} 457196224Sjhb 458196224Sjhbvoid 459196224Sjhblapic_disable_pmc(void) 460196224Sjhb{ 461196224Sjhb#ifdef HWPMC_HOOKS 462196224Sjhb u_int32_t maxlvt; 463196224Sjhb 464196224Sjhb /* Fail if the local APIC is not present. */ 465196224Sjhb if (lapic == NULL) 466196224Sjhb return; 467196224Sjhb 468196224Sjhb /* Fail if the PMC LVT is not present. */ 469196224Sjhb maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 470196224Sjhb if (maxlvt < LVT_PMC) 471196224Sjhb return; 472196224Sjhb 473196224Sjhb lvts[LVT_PMC].lvt_masked = 1; 474196224Sjhb 475196224Sjhb#ifdef SMP 476196224Sjhb /* The APs should always be started when hwpmc is unloaded. */ 477196224Sjhb KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early")); 478196224Sjhb#endif 479196224Sjhb smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 480196224Sjhb#endif 481196224Sjhb} 482196224Sjhb 483209371Smavstatic int 484209371Smavlapic_et_start(struct eventtimer *et, 485209371Smav struct bintime *first, struct bintime *period) 486141538Sjhb{ 487209371Smav struct lapic *la; 488141538Sjhb u_long value; 489141538Sjhb 490209371Smav if (et->et_frequency == 0) { 491209371Smav /* Start off with a divisor of 2 (power on reset default). */ 492209371Smav lapic_timer_divisor = 2; 493209371Smav /* Try to calibrate the local APIC timer. */ 494209371Smav do { 495209371Smav lapic_timer_set_divisor(lapic_timer_divisor); 496221508Smav lapic_timer_oneshot(APIC_TIMER_MAX_COUNT, 0); 497209371Smav DELAY(1000000); 498209371Smav value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer; 499209371Smav if (value != APIC_TIMER_MAX_COUNT) 500209371Smav break; 501209371Smav lapic_timer_divisor <<= 1; 502209371Smav } while (lapic_timer_divisor <= 128); 503209371Smav if (lapic_timer_divisor > 128) 504209371Smav panic("lapic: Divisor too big"); 505209371Smav if (bootverbose) 506209371Smav printf("lapic: Divisor %lu, Frequency %lu Hz\n", 507209371Smav lapic_timer_divisor, value); 508209371Smav et->et_frequency = value; 509210290Smav et->et_min_period.sec = 0; 510210290Smav et->et_min_period.frac = 511210298Smav ((0x00000002LLU << 32) / et->et_frequency) << 32; 512210298Smav et->et_max_period.sec = 0xfffffffeLLU / et->et_frequency; 513210290Smav et->et_max_period.frac = 514210298Smav ((0xfffffffeLLU << 32) / et->et_frequency) << 32; 515204641Sattilio } 516211756Smav lapic_timer_set_divisor(lapic_timer_divisor); 517209371Smav la = &lapics[lapic_id()]; 518209371Smav if (period != NULL) { 519209371Smav la->la_timer_mode = 1; 520209371Smav la->la_timer_period = 521209371Smav (et->et_frequency * (period->frac >> 32)) >> 32; 522209371Smav if (period->sec != 0) 523209371Smav la->la_timer_period += et->et_frequency * period->sec; 524221508Smav lapic_timer_periodic(la->la_timer_period, 1); 525209371Smav } else { 526209371Smav la->la_timer_mode = 2; 527209371Smav la->la_timer_period = 528209371Smav (et->et_frequency * (first->frac >> 32)) >> 32; 529209371Smav if (first->sec != 0) 530209371Smav la->la_timer_period += et->et_frequency * first->sec; 531221508Smav lapic_timer_oneshot(la->la_timer_period, 1); 532209371Smav } 533209371Smav return (0); 534141538Sjhb} 535141538Sjhb 536209371Smavstatic int 537209371Smavlapic_et_stop(struct eventtimer *et) 538209371Smav{ 539209371Smav struct lapic *la = &lapics[lapic_id()]; 540209371Smav 541209371Smav la->la_timer_mode = 0; 542209371Smav lapic_timer_stop(); 543209371Smav return (0); 544209371Smav} 545209371Smav 546121986Sjhbvoid 547121986Sjhblapic_disable(void) 548121986Sjhb{ 549121986Sjhb uint32_t value; 550121986Sjhb 551121986Sjhb /* Software disable the local APIC. */ 552121986Sjhb value = lapic->svr; 553121986Sjhb value &= ~APIC_SVR_SWEN; 554121986Sjhb lapic->svr = value; 555121986Sjhb} 556121986Sjhb 557139245Sjhbstatic void 558139245Sjhblapic_enable(void) 559139245Sjhb{ 560139245Sjhb u_int32_t value; 561139245Sjhb 562139245Sjhb /* Program the spurious vector to enable the local APIC. */ 563139245Sjhb value = lapic->svr; 564139245Sjhb value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); 565139245Sjhb value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT); 566139245Sjhb lapic->svr = value; 567139245Sjhb} 568139245Sjhb 569163219Sjhb/* Reset the local APIC on the BSP during resume. */ 570163219Sjhbstatic void 571163219Sjhblapic_resume(struct pic *pic) 572163219Sjhb{ 573163219Sjhb 574163219Sjhb lapic_setup(0); 575163219Sjhb} 576163219Sjhb 577121986Sjhbint 578121986Sjhblapic_id(void) 579121986Sjhb{ 580121986Sjhb 581121986Sjhb KASSERT(lapic != NULL, ("local APIC is not mapped")); 582121986Sjhb return (lapic->id >> APIC_ID_SHIFT); 583121986Sjhb} 584121986Sjhb 585121986Sjhbint 586121986Sjhblapic_intr_pending(u_int vector) 587121986Sjhb{ 588121986Sjhb volatile u_int32_t *irr; 589121986Sjhb 590121986Sjhb /* 591121986Sjhb * The IRR registers are an array of 128-bit registers each of 592121986Sjhb * which only describes 32 interrupts in the low 32 bits.. Thus, 593121986Sjhb * we divide the vector by 32 to get the 128-bit index. We then 594121986Sjhb * multiply that index by 4 to get the equivalent index from 595121986Sjhb * treating the IRR as an array of 32-bit registers. Finally, we 596121986Sjhb * modulus the vector by 32 to determine the individual bit to 597121986Sjhb * test. 598121986Sjhb */ 599121986Sjhb irr = &lapic->irr0; 600121986Sjhb return (irr[(vector / 32) * 4] & 1 << (vector % 32)); 601121986Sjhb} 602121986Sjhb 603121986Sjhbvoid 604121986Sjhblapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 605121986Sjhb{ 606121986Sjhb struct lapic *la; 607121986Sjhb 608121986Sjhb KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist", 609121986Sjhb __func__, apic_id)); 610121986Sjhb KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big", 611121986Sjhb __func__, cluster)); 612121986Sjhb KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID, 613121986Sjhb ("%s: intra cluster id %u too big", __func__, cluster_id)); 614121986Sjhb la = &lapics[apic_id]; 615121986Sjhb la->la_cluster = cluster; 616121986Sjhb la->la_cluster_id = cluster_id; 617121986Sjhb} 618121986Sjhb 619121986Sjhbint 620121986Sjhblapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked) 621121986Sjhb{ 622121986Sjhb 623121986Sjhb if (pin > LVT_MAX) 624121986Sjhb return (EINVAL); 625121986Sjhb if (apic_id == APIC_ID_ALL) { 626121986Sjhb lvts[pin].lvt_masked = masked; 627121986Sjhb if (bootverbose) 628121986Sjhb printf("lapic:"); 629121986Sjhb } else { 630121986Sjhb KASSERT(lapics[apic_id].la_present, 631121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 632121986Sjhb lapics[apic_id].la_lvts[pin].lvt_masked = masked; 633121986Sjhb lapics[apic_id].la_lvts[pin].lvt_active = 1; 634121986Sjhb if (bootverbose) 635121986Sjhb printf("lapic%u:", apic_id); 636121986Sjhb } 637121986Sjhb if (bootverbose) 638121986Sjhb printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked"); 639121986Sjhb return (0); 640121986Sjhb} 641121986Sjhb 642121986Sjhbint 643121986Sjhblapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode) 644121986Sjhb{ 645121986Sjhb struct lvt *lvt; 646121986Sjhb 647121986Sjhb if (pin > LVT_MAX) 648121986Sjhb return (EINVAL); 649121986Sjhb if (apic_id == APIC_ID_ALL) { 650121986Sjhb lvt = &lvts[pin]; 651121986Sjhb if (bootverbose) 652121986Sjhb printf("lapic:"); 653121986Sjhb } else { 654121986Sjhb KASSERT(lapics[apic_id].la_present, 655121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 656121986Sjhb lvt = &lapics[apic_id].la_lvts[pin]; 657121986Sjhb lvt->lvt_active = 1; 658121986Sjhb if (bootverbose) 659121986Sjhb printf("lapic%u:", apic_id); 660121986Sjhb } 661121986Sjhb lvt->lvt_mode = mode; 662121986Sjhb switch (mode) { 663121986Sjhb case APIC_LVT_DM_NMI: 664121986Sjhb case APIC_LVT_DM_SMI: 665121986Sjhb case APIC_LVT_DM_INIT: 666121986Sjhb case APIC_LVT_DM_EXTINT: 667121986Sjhb lvt->lvt_edgetrigger = 1; 668121986Sjhb lvt->lvt_activehi = 1; 669121986Sjhb if (mode == APIC_LVT_DM_EXTINT) 670121986Sjhb lvt->lvt_masked = 1; 671121986Sjhb else 672121986Sjhb lvt->lvt_masked = 0; 673121986Sjhb break; 674121986Sjhb default: 675121986Sjhb panic("Unsupported delivery mode: 0x%x\n", mode); 676121986Sjhb } 677121986Sjhb if (bootverbose) { 678121986Sjhb printf(" Routing "); 679121986Sjhb switch (mode) { 680121986Sjhb case APIC_LVT_DM_NMI: 681121986Sjhb printf("NMI"); 682121986Sjhb break; 683121986Sjhb case APIC_LVT_DM_SMI: 684121986Sjhb printf("SMI"); 685121986Sjhb break; 686121986Sjhb case APIC_LVT_DM_INIT: 687121986Sjhb printf("INIT"); 688121986Sjhb break; 689121986Sjhb case APIC_LVT_DM_EXTINT: 690121986Sjhb printf("ExtINT"); 691121986Sjhb break; 692121986Sjhb } 693121986Sjhb printf(" -> LINT%u\n", pin); 694121986Sjhb } 695121986Sjhb return (0); 696121986Sjhb} 697121986Sjhb 698121986Sjhbint 699128930Sjhblapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol) 700121986Sjhb{ 701121986Sjhb 702128930Sjhb if (pin > LVT_MAX || pol == INTR_POLARITY_CONFORM) 703121986Sjhb return (EINVAL); 704121986Sjhb if (apic_id == APIC_ID_ALL) { 705128930Sjhb lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH); 706121986Sjhb if (bootverbose) 707121986Sjhb printf("lapic:"); 708121986Sjhb } else { 709121986Sjhb KASSERT(lapics[apic_id].la_present, 710121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 711121986Sjhb lapics[apic_id].la_lvts[pin].lvt_active = 1; 712128930Sjhb lapics[apic_id].la_lvts[pin].lvt_activehi = 713128930Sjhb (pol == INTR_POLARITY_HIGH); 714121986Sjhb if (bootverbose) 715121986Sjhb printf("lapic%u:", apic_id); 716121986Sjhb } 717121986Sjhb if (bootverbose) 718140254Sjhb printf(" LINT%u polarity: %s\n", pin, 719128930Sjhb pol == INTR_POLARITY_HIGH ? "high" : "low"); 720121986Sjhb return (0); 721121986Sjhb} 722121986Sjhb 723121986Sjhbint 724128930Sjhblapic_set_lvt_triggermode(u_int apic_id, u_int pin, enum intr_trigger trigger) 725121986Sjhb{ 726121986Sjhb 727128930Sjhb if (pin > LVT_MAX || trigger == INTR_TRIGGER_CONFORM) 728121986Sjhb return (EINVAL); 729121986Sjhb if (apic_id == APIC_ID_ALL) { 730128930Sjhb lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE); 731121986Sjhb if (bootverbose) 732121986Sjhb printf("lapic:"); 733121986Sjhb } else { 734121986Sjhb KASSERT(lapics[apic_id].la_present, 735121986Sjhb ("%s: missing APIC %u", __func__, apic_id)); 736128930Sjhb lapics[apic_id].la_lvts[pin].lvt_edgetrigger = 737128930Sjhb (trigger == INTR_TRIGGER_EDGE); 738121986Sjhb lapics[apic_id].la_lvts[pin].lvt_active = 1; 739121986Sjhb if (bootverbose) 740121986Sjhb printf("lapic%u:", apic_id); 741121986Sjhb } 742121986Sjhb if (bootverbose) 743121986Sjhb printf(" LINT%u trigger: %s\n", pin, 744128930Sjhb trigger == INTR_TRIGGER_EDGE ? "edge" : "level"); 745121986Sjhb return (0); 746121986Sjhb} 747121986Sjhb 748139240Sjhb/* 749139240Sjhb * Adjust the TPR of the current CPU so that it blocks all interrupts below 750139240Sjhb * the passed in vector. 751139240Sjhb */ 752121986Sjhbvoid 753139240Sjhblapic_set_tpr(u_int vector) 754139240Sjhb{ 755139240Sjhb#ifdef CHEAP_TPR 756139240Sjhb lapic->tpr = vector; 757139240Sjhb#else 758139240Sjhb u_int32_t tpr; 759139240Sjhb 760139240Sjhb tpr = lapic->tpr & ~APIC_TPR_PRIO; 761139240Sjhb tpr |= vector; 762139240Sjhb lapic->tpr = tpr; 763139240Sjhb#endif 764139240Sjhb} 765139240Sjhb 766139240Sjhbvoid 767122572Sjhblapic_eoi(void) 768122572Sjhb{ 769122572Sjhb 770122572Sjhb lapic->eoi = 0; 771122572Sjhb} 772122572Sjhb 773122572Sjhbvoid 774165302Skmacylapic_handle_intr(int vector, struct trapframe *frame) 775121986Sjhb{ 776121986Sjhb struct intsrc *isrc; 777121986Sjhb 778187880Sjeff isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id), 779187880Sjeff vector)); 780165302Skmacy intr_execute_handlers(isrc, frame); 781121986Sjhb} 782121986Sjhb 783141538Sjhbvoid 784165302Skmacylapic_handle_timer(struct trapframe *frame) 785141538Sjhb{ 786141538Sjhb struct lapic *la; 787209371Smav struct trapframe *oldframe; 788209371Smav struct thread *td; 789141538Sjhb 790153141Sjhb /* Send EOI first thing. */ 791153141Sjhb lapic_eoi(); 792153141Sjhb 793162708Ssobomax#if defined(SMP) && !defined(SCHED_ULE) 794162042Ssobomax /* 795162042Ssobomax * Don't do any accounting for the disabled HTT cores, since it 796162042Ssobomax * will provide misleading numbers for the userland. 797162042Ssobomax * 798258218Smav * No locking is necessary here, since even if we lose the race 799162042Ssobomax * when hlt_cpus_mask changes it is not a big deal, really. 800162713Ssobomax * 801162713Ssobomax * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask 802162713Ssobomax * and unlike other schedulers it actually schedules threads to 803162713Ssobomax * those CPUs. 804162042Ssobomax */ 805222813Sattilio if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)) 806162042Ssobomax return; 807162087Ssobomax#endif 808162042Ssobomax 809153141Sjhb /* Look up our local APIC structure for the tick counters. */ 810141538Sjhb la = &lapics[PCPU_GET(apic_id)]; 811141538Sjhb (*la->la_timer_count)++; 812141538Sjhb critical_enter(); 813209371Smav if (lapic_et.et_active) { 814209371Smav td = curthread; 815210444Smav td->td_intr_nesting_level++; 816209371Smav oldframe = td->td_intr_frame; 817209371Smav td->td_intr_frame = frame; 818209990Smav lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg); 819209371Smav td->td_intr_frame = oldframe; 820210444Smav td->td_intr_nesting_level--; 821209371Smav } 822141538Sjhb critical_exit(); 823141538Sjhb} 824141538Sjhb 825141538Sjhbstatic void 826141538Sjhblapic_timer_set_divisor(u_int divisor) 827141538Sjhb{ 828141538Sjhb 829141538Sjhb KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor)); 830141538Sjhb KASSERT(ffs(divisor) <= sizeof(lapic_timer_divisors) / 831141538Sjhb sizeof(u_int32_t), ("lapic: invalid divisor %u", divisor)); 832141538Sjhb lapic->dcr_timer = lapic_timer_divisors[ffs(divisor) - 1]; 833141538Sjhb} 834141538Sjhb 835141538Sjhbstatic void 836221508Smavlapic_timer_oneshot(u_int count, int enable_int) 837141538Sjhb{ 838141538Sjhb u_int32_t value; 839141538Sjhb 840141538Sjhb value = lapic->lvt_timer; 841141538Sjhb value &= ~APIC_LVTT_TM; 842141538Sjhb value |= APIC_LVTT_TM_ONE_SHOT; 843221508Smav if (enable_int) 844221508Smav value &= ~APIC_LVT_M; 845141538Sjhb lapic->lvt_timer = value; 846141538Sjhb lapic->icr_timer = count; 847141538Sjhb} 848141538Sjhb 849141538Sjhbstatic void 850221508Smavlapic_timer_periodic(u_int count, int enable_int) 851141538Sjhb{ 852141538Sjhb u_int32_t value; 853141538Sjhb 854141538Sjhb value = lapic->lvt_timer; 855141538Sjhb value &= ~APIC_LVTT_TM; 856141538Sjhb value |= APIC_LVTT_TM_PERIODIC; 857221508Smav if (enable_int) 858221508Smav value &= ~APIC_LVT_M; 859141538Sjhb lapic->lvt_timer = value; 860141538Sjhb lapic->icr_timer = count; 861141538Sjhb} 862141538Sjhb 863141538Sjhbstatic void 864209371Smavlapic_timer_stop(void) 865209371Smav{ 866209371Smav u_int32_t value; 867209371Smav 868209371Smav value = lapic->lvt_timer; 869209371Smav value &= ~APIC_LVTT_TM; 870211756Smav value |= APIC_LVT_M; 871209371Smav lapic->lvt_timer = value; 872209371Smav} 873209371Smav 874205851Sjhbvoid 875208507Sjhblapic_handle_cmc(void) 876208507Sjhb{ 877208507Sjhb 878208507Sjhb lapic_eoi(); 879208507Sjhb cmc_intr(); 880208507Sjhb} 881208507Sjhb 882208507Sjhb/* 883208507Sjhb * Called from the mca_init() to activate the CMC interrupt if this CPU is 884208507Sjhb * responsible for monitoring any MC banks for CMC events. Since mca_init() 885208507Sjhb * is called prior to lapic_setup() during boot, this just needs to unmask 886208507Sjhb * this CPU's LVT_CMCI entry. 887208507Sjhb */ 888208507Sjhbvoid 889208507Sjhblapic_enable_cmc(void) 890208507Sjhb{ 891208507Sjhb u_int apic_id; 892208507Sjhb 893208507Sjhb apic_id = PCPU_GET(apic_id); 894208507Sjhb KASSERT(lapics[apic_id].la_present, 895208507Sjhb ("%s: missing APIC %u", __func__, apic_id)); 896208507Sjhb lapics[apic_id].la_lvts[LVT_CMCI].lvt_masked = 0; 897208507Sjhb lapics[apic_id].la_lvts[LVT_CMCI].lvt_active = 1; 898208507Sjhb if (bootverbose) 899208507Sjhb printf("lapic%u: CMCI unmasked\n", apic_id); 900208507Sjhb} 901208507Sjhb 902208507Sjhbvoid 903205851Sjhblapic_handle_error(void) 904205851Sjhb{ 905205851Sjhb u_int32_t esr; 906205851Sjhb 907205851Sjhb /* 908205851Sjhb * Read the contents of the error status register. Write to 909205851Sjhb * the register first before reading from it to force the APIC 910205851Sjhb * to update its value to indicate any errors that have 911205851Sjhb * occurred since the previous write to the register. 912205851Sjhb */ 913205851Sjhb lapic->esr = 0; 914205851Sjhb esr = lapic->esr; 915205851Sjhb 916205851Sjhb printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr); 917205851Sjhb lapic_eoi(); 918205851Sjhb} 919205851Sjhb 920187880Sjeffu_int 921187880Sjeffapic_cpuid(u_int apic_id) 922187880Sjeff{ 923187880Sjeff#ifdef SMP 924187880Sjeff return apic_cpuids[apic_id]; 925187880Sjeff#else 926187880Sjeff return 0; 927187880Sjeff#endif 928187880Sjeff} 929187880Sjeff 930151979Sjhb/* Request a free IDT vector to be used by the specified IRQ. */ 931121986Sjhbu_int 932187880Sjeffapic_alloc_vector(u_int apic_id, u_int irq) 933121986Sjhb{ 934121986Sjhb u_int vector; 935121986Sjhb 936121986Sjhb KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 937151979Sjhb 938151979Sjhb /* 939151979Sjhb * Search for a free vector. Currently we just use a very simple 940151979Sjhb * algorithm to find the first free vector. 941151979Sjhb */ 942151979Sjhb mtx_lock_spin(&icu_lock); 943151979Sjhb for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 944191720Smav if (lapics[apic_id].la_ioint_irqs[vector] != -1) 945151979Sjhb continue; 946187880Sjeff lapics[apic_id].la_ioint_irqs[vector] = irq; 947151979Sjhb mtx_unlock_spin(&icu_lock); 948151979Sjhb return (vector + APIC_IO_INTS); 949151979Sjhb } 950151979Sjhb mtx_unlock_spin(&icu_lock); 951195249Sjhb return (0); 952121986Sjhb} 953121986Sjhb 954164265Sjhb/* 955164265Sjhb * Request 'count' free contiguous IDT vectors to be used by 'count' 956164265Sjhb * IRQs. 'count' must be a power of two and the vectors will be 957164265Sjhb * aligned on a boundary of 'align'. If the request cannot be 958164265Sjhb * satisfied, 0 is returned. 959164265Sjhb */ 960164265Sjhbu_int 961187880Sjeffapic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 962164265Sjhb{ 963164265Sjhb u_int first, run, vector; 964164265Sjhb 965164265Sjhb KASSERT(powerof2(count), ("bad count")); 966164265Sjhb KASSERT(powerof2(align), ("bad align")); 967164265Sjhb KASSERT(align >= count, ("align < count")); 968164265Sjhb#ifdef INVARIANTS 969164265Sjhb for (run = 0; run < count; run++) 970164265Sjhb KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u", 971164265Sjhb irqs[run], run)); 972164265Sjhb#endif 973164265Sjhb 974164265Sjhb /* 975164265Sjhb * Search for 'count' free vectors. As with apic_alloc_vector(), 976164265Sjhb * this just uses a simple first fit algorithm. 977164265Sjhb */ 978164265Sjhb run = 0; 979164265Sjhb first = 0; 980164265Sjhb mtx_lock_spin(&icu_lock); 981164265Sjhb for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 982164265Sjhb 983164265Sjhb /* Vector is in use, end run. */ 984191720Smav if (lapics[apic_id].la_ioint_irqs[vector] != -1) { 985164265Sjhb run = 0; 986164265Sjhb first = 0; 987164265Sjhb continue; 988164265Sjhb } 989164265Sjhb 990164265Sjhb /* Start a new run if run == 0 and vector is aligned. */ 991164265Sjhb if (run == 0) { 992164265Sjhb if ((vector & (align - 1)) != 0) 993164265Sjhb continue; 994164265Sjhb first = vector; 995164265Sjhb } 996164265Sjhb run++; 997164265Sjhb 998164265Sjhb /* Keep looping if the run isn't long enough yet. */ 999164265Sjhb if (run < count) 1000164265Sjhb continue; 1001164265Sjhb 1002164265Sjhb /* Found a run, assign IRQs and return the first vector. */ 1003164265Sjhb for (vector = 0; vector < count; vector++) 1004187880Sjeff lapics[apic_id].la_ioint_irqs[first + vector] = 1005187880Sjeff irqs[vector]; 1006164265Sjhb mtx_unlock_spin(&icu_lock); 1007164265Sjhb return (first + APIC_IO_INTS); 1008164265Sjhb } 1009164265Sjhb mtx_unlock_spin(&icu_lock); 1010164265Sjhb printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count); 1011164265Sjhb return (0); 1012164265Sjhb} 1013164265Sjhb 1014187880Sjeff/* 1015187880Sjeff * Enable a vector for a particular apic_id. Since all lapics share idt 1016187880Sjeff * entries and ioint_handlers this enables the vector on all lapics. lapics 1017187880Sjeff * which do not have the vector configured would report spurious interrupts 1018187880Sjeff * should it fire. 1019187880Sjeff */ 1020151979Sjhbvoid 1021187880Sjeffapic_enable_vector(u_int apic_id, u_int vector) 1022151979Sjhb{ 1023151979Sjhb 1024151979Sjhb KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1025151979Sjhb KASSERT(ioint_handlers[vector / 32] != NULL, 1026151979Sjhb ("No ISR handler for vector %u", vector)); 1027212004Srpaulo#ifdef KDTRACE_HOOKS 1028212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1029212004Srpaulo ("Attempt to overwrite DTrace entry")); 1030212004Srpaulo#endif 1031208452Smav setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL, 1032208452Smav GSEL_APIC); 1033151979Sjhb} 1034151979Sjhb 1035169391Sjhbvoid 1036187880Sjeffapic_disable_vector(u_int apic_id, u_int vector) 1037169391Sjhb{ 1038169391Sjhb 1039169391Sjhb KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1040212004Srpaulo#ifdef KDTRACE_HOOKS 1041212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1042212004Srpaulo ("Attempt to overwrite DTrace entry")); 1043212004Srpaulo#endif 1044169391Sjhb KASSERT(ioint_handlers[vector / 32] != NULL, 1045169391Sjhb ("No ISR handler for vector %u", vector)); 1046188904Sjeff#ifdef notyet 1047188904Sjeff /* 1048188904Sjeff * We can not currently clear the idt entry because other cpus 1049188904Sjeff * may have a valid vector at this offset. 1050188904Sjeff */ 1051208452Smav setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC); 1052188904Sjeff#endif 1053169391Sjhb} 1054169391Sjhb 1055151979Sjhb/* Release an APIC vector when it's no longer in use. */ 1056151979Sjhbvoid 1057187880Sjeffapic_free_vector(u_int apic_id, u_int vector, u_int irq) 1058151979Sjhb{ 1059187880Sjeff struct thread *td; 1060194889Sjhb 1061151979Sjhb KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1062151979Sjhb vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1063151979Sjhb ("Vector %u does not map to an IRQ line", vector)); 1064151979Sjhb KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 1065187880Sjeff KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] == 1066187880Sjeff irq, ("IRQ mismatch")); 1067212004Srpaulo#ifdef KDTRACE_HOOKS 1068212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1069212004Srpaulo ("Attempt to overwrite DTrace entry")); 1070212004Srpaulo#endif 1071187880Sjeff 1072187880Sjeff /* 1073187880Sjeff * Bind us to the cpu that owned the vector before freeing it so 1074187880Sjeff * we don't lose an interrupt delivery race. 1075187880Sjeff */ 1076187880Sjeff td = curthread; 1077196745Sjhb if (!rebooting) { 1078196745Sjhb thread_lock(td); 1079196745Sjhb if (sched_is_bound(td)) 1080196745Sjhb panic("apic_free_vector: Thread already bound.\n"); 1081196745Sjhb sched_bind(td, apic_cpuid(apic_id)); 1082196745Sjhb thread_unlock(td); 1083196745Sjhb } 1084151979Sjhb mtx_lock_spin(&icu_lock); 1085191720Smav lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1; 1086151979Sjhb mtx_unlock_spin(&icu_lock); 1087196745Sjhb if (!rebooting) { 1088196745Sjhb thread_lock(td); 1089196745Sjhb sched_unbind(td); 1090196745Sjhb thread_unlock(td); 1091196745Sjhb } 1092151979Sjhb} 1093151979Sjhb 1094151979Sjhb/* Map an IDT vector (APIC) to an IRQ (interrupt source). */ 1095121986Sjhbu_int 1096187880Sjeffapic_idt_to_irq(u_int apic_id, u_int vector) 1097121986Sjhb{ 1098191730Smav int irq; 1099121986Sjhb 1100122690Sjhb KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1101151979Sjhb vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1102121986Sjhb ("Vector %u does not map to an IRQ line", vector)); 1103212004Srpaulo#ifdef KDTRACE_HOOKS 1104212004Srpaulo KASSERT(vector != IDT_DTRACE_RET, 1105212004Srpaulo ("Attempt to overwrite DTrace entry")); 1106212004Srpaulo#endif 1107191730Smav irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS]; 1108191730Smav if (irq < 0) 1109191730Smav irq = 0; 1110191730Smav return (irq); 1111121986Sjhb} 1112121986Sjhb 1113151979Sjhb#ifdef DDB 1114121986Sjhb/* 1115151979Sjhb * Dump data about APIC IDT vector mappings. 1116151979Sjhb */ 1117151979SjhbDB_SHOW_COMMAND(apic, db_show_apic) 1118151979Sjhb{ 1119151979Sjhb struct intsrc *isrc; 1120160312Sjhb int i, verbose; 1121187880Sjeff u_int apic_id; 1122151979Sjhb u_int irq; 1123151979Sjhb 1124151979Sjhb if (strcmp(modif, "vv") == 0) 1125151979Sjhb verbose = 2; 1126151979Sjhb else if (strcmp(modif, "v") == 0) 1127151979Sjhb verbose = 1; 1128151979Sjhb else 1129151979Sjhb verbose = 0; 1130187880Sjeff for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) { 1131187880Sjeff if (lapics[apic_id].la_present == 0) 1132187880Sjeff continue; 1133187880Sjeff db_printf("Interrupts bound to lapic %u\n", apic_id); 1134187880Sjeff for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) { 1135187880Sjeff irq = lapics[apic_id].la_ioint_irqs[i]; 1136191720Smav if (irq == -1 || irq == IRQ_SYSCALL) 1137187880Sjeff continue; 1138212004Srpaulo#ifdef KDTRACE_HOOKS 1139212004Srpaulo if (irq == IRQ_DTRACE_RET) 1140212004Srpaulo continue; 1141212004Srpaulo#endif 1142151979Sjhb db_printf("vec 0x%2x -> ", i + APIC_IO_INTS); 1143151979Sjhb if (irq == IRQ_TIMER) 1144151979Sjhb db_printf("lapic timer\n"); 1145151979Sjhb else if (irq < NUM_IO_INTS) { 1146151979Sjhb isrc = intr_lookup_source(irq); 1147151979Sjhb if (isrc == NULL || verbose == 0) 1148151979Sjhb db_printf("IRQ %u\n", irq); 1149151979Sjhb else 1150151979Sjhb db_dump_intr_event(isrc->is_event, 1151151979Sjhb verbose == 2); 1152151979Sjhb } else 1153151979Sjhb db_printf("IRQ %u ???\n", irq); 1154151979Sjhb } 1155151979Sjhb } 1156151979Sjhb} 1157162233Sjhb 1158162233Sjhbstatic void 1159162233Sjhbdump_mask(const char *prefix, uint32_t v, int base) 1160162233Sjhb{ 1161162233Sjhb int i, first; 1162162233Sjhb 1163162233Sjhb first = 1; 1164162233Sjhb for (i = 0; i < 32; i++) 1165162233Sjhb if (v & (1 << i)) { 1166162233Sjhb if (first) { 1167162233Sjhb db_printf("%s:", prefix); 1168162233Sjhb first = 0; 1169162233Sjhb } 1170162233Sjhb db_printf(" %02x", base + i); 1171162233Sjhb } 1172162233Sjhb if (!first) 1173162233Sjhb db_printf("\n"); 1174162233Sjhb} 1175162233Sjhb 1176162233Sjhb/* Show info from the lapic regs for this CPU. */ 1177162233SjhbDB_SHOW_COMMAND(lapic, db_show_lapic) 1178162233Sjhb{ 1179162233Sjhb uint32_t v; 1180162233Sjhb 1181162233Sjhb db_printf("lapic ID = %d\n", lapic_id()); 1182162233Sjhb v = lapic->version; 1183162233Sjhb db_printf("version = %d.%d\n", (v & APIC_VER_VERSION) >> 4, 1184162233Sjhb v & 0xf); 1185162233Sjhb db_printf("max LVT = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT); 1186162233Sjhb v = lapic->svr; 1187162233Sjhb db_printf("SVR = %02x (%s)\n", v & APIC_SVR_VECTOR, 1188162233Sjhb v & APIC_SVR_ENABLE ? "enabled" : "disabled"); 1189162233Sjhb db_printf("TPR = %02x\n", lapic->tpr); 1190162233Sjhb 1191162233Sjhb#define dump_field(prefix, index) \ 1192162233Sjhb dump_mask(__XSTRING(prefix ## index), lapic->prefix ## index, \ 1193162233Sjhb index * 32) 1194162233Sjhb 1195162233Sjhb db_printf("In-service Interrupts:\n"); 1196162233Sjhb dump_field(isr, 0); 1197162233Sjhb dump_field(isr, 1); 1198162233Sjhb dump_field(isr, 2); 1199162233Sjhb dump_field(isr, 3); 1200162233Sjhb dump_field(isr, 4); 1201162233Sjhb dump_field(isr, 5); 1202162233Sjhb dump_field(isr, 6); 1203162233Sjhb dump_field(isr, 7); 1204162233Sjhb 1205162233Sjhb db_printf("TMR Interrupts:\n"); 1206162233Sjhb dump_field(tmr, 0); 1207162233Sjhb dump_field(tmr, 1); 1208162233Sjhb dump_field(tmr, 2); 1209162233Sjhb dump_field(tmr, 3); 1210162233Sjhb dump_field(tmr, 4); 1211162233Sjhb dump_field(tmr, 5); 1212162233Sjhb dump_field(tmr, 6); 1213162233Sjhb dump_field(tmr, 7); 1214162233Sjhb 1215162233Sjhb db_printf("IRR Interrupts:\n"); 1216162233Sjhb dump_field(irr, 0); 1217162233Sjhb dump_field(irr, 1); 1218162233Sjhb dump_field(irr, 2); 1219162233Sjhb dump_field(irr, 3); 1220162233Sjhb dump_field(irr, 4); 1221162233Sjhb dump_field(irr, 5); 1222162233Sjhb dump_field(irr, 6); 1223162233Sjhb dump_field(irr, 7); 1224162233Sjhb 1225162233Sjhb#undef dump_field 1226162233Sjhb} 1227151979Sjhb#endif 1228151979Sjhb 1229151979Sjhb/* 1230121986Sjhb * APIC probing support code. This includes code to manage enumerators. 1231121986Sjhb */ 1232121986Sjhb 1233121986Sjhbstatic SLIST_HEAD(, apic_enumerator) enumerators = 1234121986Sjhb SLIST_HEAD_INITIALIZER(enumerators); 1235121986Sjhbstatic struct apic_enumerator *best_enum; 1236195249Sjhb 1237121986Sjhbvoid 1238121986Sjhbapic_register_enumerator(struct apic_enumerator *enumerator) 1239121986Sjhb{ 1240121986Sjhb#ifdef INVARIANTS 1241121986Sjhb struct apic_enumerator *apic_enum; 1242121986Sjhb 1243121986Sjhb SLIST_FOREACH(apic_enum, &enumerators, apic_next) { 1244121986Sjhb if (apic_enum == enumerator) 1245121986Sjhb panic("%s: Duplicate register of %s", __func__, 1246121986Sjhb enumerator->apic_name); 1247121986Sjhb } 1248121986Sjhb#endif 1249121986Sjhb SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next); 1250121986Sjhb} 1251121986Sjhb 1252121986Sjhb/* 1253208479Smav * We have to look for CPU's very, very early because certain subsystems 1254208479Smav * want to know how many CPU's we have extremely early on in the boot 1255208479Smav * process. 1256121986Sjhb */ 1257121986Sjhbstatic void 1258121986Sjhbapic_init(void *dummy __unused) 1259121986Sjhb{ 1260121986Sjhb struct apic_enumerator *enumerator; 1261208452Smav#ifndef __amd64__ 1262123133Sjhb uint64_t apic_base; 1263208452Smav#endif 1264121986Sjhb int retval, best; 1265121986Sjhb 1266153383Sjhb /* We only support built in local APICs. */ 1267153383Sjhb if (!(cpu_feature & CPUID_APIC)) 1268121986Sjhb return; 1269121986Sjhb 1270123133Sjhb /* Don't probe if APIC mode is disabled. */ 1271123133Sjhb if (resource_disabled("apic", 0)) 1272123133Sjhb return; 1273123133Sjhb 1274215009Sjhb /* Probe all the enumerators to find the best match. */ 1275121986Sjhb best_enum = NULL; 1276121986Sjhb best = 0; 1277121986Sjhb SLIST_FOREACH(enumerator, &enumerators, apic_next) { 1278121986Sjhb retval = enumerator->apic_probe(); 1279121986Sjhb if (retval > 0) 1280121986Sjhb continue; 1281121986Sjhb if (best_enum == NULL || best < retval) { 1282121986Sjhb best_enum = enumerator; 1283121986Sjhb best = retval; 1284121986Sjhb } 1285121986Sjhb } 1286121986Sjhb if (best_enum == NULL) { 1287121986Sjhb if (bootverbose) 1288121986Sjhb printf("APIC: Could not find any APICs.\n"); 1289121986Sjhb return; 1290121986Sjhb } 1291121986Sjhb 1292121986Sjhb if (bootverbose) 1293121986Sjhb printf("APIC: Using the %s enumerator.\n", 1294121986Sjhb best_enum->apic_name); 1295121986Sjhb 1296208452Smav#ifndef __amd64__ 1297121986Sjhb /* 1298121986Sjhb * To work around an errata, we disable the local APIC on some 1299121986Sjhb * CPUs during early startup. We need to turn the local APIC back 1300121986Sjhb * on on such CPUs now. 1301121986Sjhb */ 1302185341Sjkim if (cpu == CPU_686 && cpu_vendor_id == CPU_VENDOR_INTEL && 1303121986Sjhb (cpu_id & 0xff0) == 0x610) { 1304121986Sjhb apic_base = rdmsr(MSR_APICBASE); 1305121986Sjhb apic_base |= APICBASE_ENABLED; 1306121986Sjhb wrmsr(MSR_APICBASE, apic_base); 1307121986Sjhb } 1308208452Smav#endif 1309123133Sjhb 1310215009Sjhb /* Probe the CPU's in the system. */ 1311123133Sjhb retval = best_enum->apic_probe_cpus(); 1312123133Sjhb if (retval != 0) 1313123133Sjhb printf("%s: Failed to probe CPUs: returned %d\n", 1314123133Sjhb best_enum->apic_name, retval); 1315123133Sjhb 1316208479Smav} 1317208479SmavSYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL); 1318208479Smav 1319208479Smav/* 1320208479Smav * Setup the local APIC. We have to do this prior to starting up the APs 1321208479Smav * in the SMP case. 1322208479Smav */ 1323208479Smavstatic void 1324208479Smavapic_setup_local(void *dummy __unused) 1325208479Smav{ 1326208479Smav int retval; 1327208479Smav 1328208479Smav if (best_enum == NULL) 1329208479Smav return; 1330215009Sjhb 1331215009Sjhb /* Initialize the local APIC. */ 1332121986Sjhb retval = best_enum->apic_setup_local(); 1333121986Sjhb if (retval != 0) 1334121986Sjhb printf("%s: Failed to setup the local APIC: returned %d\n", 1335121986Sjhb best_enum->apic_name, retval); 1336121986Sjhb} 1337215009SjhbSYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL); 1338121986Sjhb 1339121986Sjhb/* 1340121986Sjhb * Setup the I/O APICs. 1341121986Sjhb */ 1342121986Sjhbstatic void 1343121986Sjhbapic_setup_io(void *dummy __unused) 1344121986Sjhb{ 1345121986Sjhb int retval; 1346121986Sjhb 1347121986Sjhb if (best_enum == NULL) 1348121986Sjhb return; 1349247877Savg 1350247877Savg /* 1351247877Savg * Local APIC must be registered before other PICs and pseudo PICs 1352247877Savg * for proper suspend/resume order. 1353247877Savg */ 1354247877Savg#ifndef XEN 1355247877Savg intr_register_pic(&lapic_pic); 1356247877Savg#endif 1357247877Savg 1358121986Sjhb retval = best_enum->apic_setup_io(); 1359121986Sjhb if (retval != 0) 1360121986Sjhb printf("%s: Failed to setup I/O APICs: returned %d\n", 1361121986Sjhb best_enum->apic_name, retval); 1362182902Skmacy#ifdef XEN 1363182902Skmacy return; 1364182902Skmacy#endif 1365121986Sjhb /* 1366121986Sjhb * Finish setting up the local APIC on the BSP once we know how to 1367121986Sjhb * properly program the LINT pins. 1368121986Sjhb */ 1369163219Sjhb lapic_setup(1); 1370121986Sjhb if (bootverbose) 1371121986Sjhb lapic_dump("BSP"); 1372164265Sjhb 1373164265Sjhb /* Enable the MSI "pic". */ 1374164265Sjhb msi_init(); 1375121986Sjhb} 1376177253SrwatsonSYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_SECOND, apic_setup_io, NULL); 1377121986Sjhb 1378121986Sjhb#ifdef SMP 1379121986Sjhb/* 1380121986Sjhb * Inter Processor Interrupt functions. The lapic_ipi_*() functions are 1381208452Smav * private to the MD code. The public interface for the rest of the 1382121986Sjhb * kernel is defined in mp_machdep.c. 1383121986Sjhb */ 1384121986Sjhbint 1385121986Sjhblapic_ipi_wait(int delay) 1386121986Sjhb{ 1387121986Sjhb int x, incr; 1388121986Sjhb 1389121986Sjhb /* 1390121986Sjhb * Wait delay loops for IPI to be sent. This is highly bogus 1391121986Sjhb * since this is sensitive to CPU clock speed. If delay is 1392121986Sjhb * -1, we wait forever. 1393121986Sjhb */ 1394121986Sjhb if (delay == -1) { 1395121986Sjhb incr = 0; 1396121986Sjhb delay = 1; 1397121986Sjhb } else 1398121986Sjhb incr = 1; 1399121986Sjhb for (x = 0; x < delay; x += incr) { 1400121986Sjhb if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE) 1401121986Sjhb return (1); 1402121986Sjhb ia32_pause(); 1403121986Sjhb } 1404121986Sjhb return (0); 1405121986Sjhb} 1406121986Sjhb 1407121986Sjhbvoid 1408121986Sjhblapic_ipi_raw(register_t icrlo, u_int dest) 1409121986Sjhb{ 1410214347Sjhb register_t value, saveintr; 1411121986Sjhb 1412121986Sjhb /* XXX: Need more sanity checking of icrlo? */ 1413121986Sjhb KASSERT(lapic != NULL, ("%s called too early", __func__)); 1414121986Sjhb KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1415121986Sjhb ("%s: invalid dest field", __func__)); 1416121986Sjhb KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, 1417121986Sjhb ("%s: reserved bits set in ICR LO register", __func__)); 1418121986Sjhb 1419121986Sjhb /* Set destination in ICR HI register if it is being used. */ 1420214347Sjhb saveintr = intr_disable(); 1421121986Sjhb if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { 1422121986Sjhb value = lapic->icr_hi; 1423121986Sjhb value &= ~APIC_ID_MASK; 1424121986Sjhb value |= dest << APIC_ID_SHIFT; 1425121986Sjhb lapic->icr_hi = value; 1426121986Sjhb } 1427121986Sjhb 1428121986Sjhb /* Program the contents of the IPI and dispatch it. */ 1429121986Sjhb value = lapic->icr_lo; 1430121986Sjhb value &= APIC_ICRLO_RESV_MASK; 1431121986Sjhb value |= icrlo; 1432121986Sjhb lapic->icr_lo = value; 1433214347Sjhb intr_restore(saveintr); 1434121986Sjhb} 1435121986Sjhb 1436125317Sjeff#define BEFORE_SPIN 1000000 1437121986Sjhb#ifdef DETECT_DEADLOCK 1438121986Sjhb#define AFTER_SPIN 1000 1439121986Sjhb#endif 1440121986Sjhb 1441121986Sjhbvoid 1442121986Sjhblapic_ipi_vectored(u_int vector, int dest) 1443121986Sjhb{ 1444121986Sjhb register_t icrlo, destfield; 1445121986Sjhb 1446121986Sjhb KASSERT((vector & ~APIC_VECTOR_MASK) == 0, 1447121986Sjhb ("%s: invalid vector %d", __func__, vector)); 1448121986Sjhb 1449196196Sattilio icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE; 1450196196Sattilio 1451196196Sattilio /* 1452196196Sattilio * IPI_STOP_HARD is just a "fake" vector used to send a NMI. 1453196196Sattilio * Use special rules regard NMI if passed, otherwise specify 1454196196Sattilio * the vector. 1455196196Sattilio */ 1456196196Sattilio if (vector == IPI_STOP_HARD) 1457196196Sattilio icrlo |= APIC_DELMODE_NMI | APIC_LEVEL_ASSERT; 1458196196Sattilio else 1459196196Sattilio icrlo |= vector | APIC_DELMODE_FIXED | APIC_LEVEL_DEASSERT; 1460121986Sjhb destfield = 0; 1461121986Sjhb switch (dest) { 1462121986Sjhb case APIC_IPI_DEST_SELF: 1463121986Sjhb icrlo |= APIC_DEST_SELF; 1464121986Sjhb break; 1465121986Sjhb case APIC_IPI_DEST_ALL: 1466121986Sjhb icrlo |= APIC_DEST_ALLISELF; 1467121986Sjhb break; 1468121986Sjhb case APIC_IPI_DEST_OTHERS: 1469121986Sjhb icrlo |= APIC_DEST_ALLESELF; 1470121986Sjhb break; 1471121986Sjhb default: 1472121986Sjhb KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1473121986Sjhb ("%s: invalid destination 0x%x", __func__, dest)); 1474121986Sjhb destfield = dest; 1475121986Sjhb } 1476121986Sjhb 1477125317Sjeff /* Wait for an earlier IPI to finish. */ 1478150176Sjhb if (!lapic_ipi_wait(BEFORE_SPIN)) { 1479150176Sjhb if (panicstr != NULL) 1480150176Sjhb return; 1481150176Sjhb else 1482150176Sjhb panic("APIC: Previous IPI is stuck"); 1483150176Sjhb } 1484121986Sjhb 1485121986Sjhb lapic_ipi_raw(icrlo, destfield); 1486121986Sjhb 1487121986Sjhb#ifdef DETECT_DEADLOCK 1488121986Sjhb /* Wait for IPI to be delivered. */ 1489121986Sjhb if (!lapic_ipi_wait(AFTER_SPIN)) { 1490121986Sjhb#ifdef needsattention 1491121986Sjhb /* 1492121986Sjhb * XXX FIXME: 1493121986Sjhb * 1494121986Sjhb * The above function waits for the message to actually be 1495121986Sjhb * delivered. It breaks out after an arbitrary timeout 1496121986Sjhb * since the message should eventually be delivered (at 1497121986Sjhb * least in theory) and that if it wasn't we would catch 1498121986Sjhb * the failure with the check above when the next IPI is 1499121986Sjhb * sent. 1500121986Sjhb * 1501139240Sjhb * We could skip this wait entirely, EXCEPT it probably 1502121986Sjhb * protects us from other routines that assume that the 1503121986Sjhb * message was delivered and acted upon when this function 1504121986Sjhb * returns. 1505121986Sjhb */ 1506121986Sjhb printf("APIC: IPI might be stuck\n"); 1507121986Sjhb#else /* !needsattention */ 1508121986Sjhb /* Wait until mesage is sent without a timeout. */ 1509121986Sjhb while (lapic->icr_lo & APIC_DELSTAT_PEND) 1510121986Sjhb ia32_pause(); 1511121986Sjhb#endif /* needsattention */ 1512121986Sjhb } 1513121986Sjhb#endif /* DETECT_DEADLOCK */ 1514121986Sjhb} 1515121986Sjhb#endif /* SMP */ 1516