156083Skris/*- 2296341Sdelphij * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3296341Sdelphij * All rights reserved. 4296341Sdelphij * 5296341Sdelphij * Redistribution and use in source and binary forms, with or without 656083Skris * modification, are permitted provided that the following conditions 789837Skris * are met: 856083Skris * 1. Redistributions of source code must retain the above copyright 9296341Sdelphij * notice, this list of conditions and the following disclaimer. 10296341Sdelphij * 2. Redistributions in binary form must reproduce the above copyright 11296341Sdelphij * notice, this list of conditions and the following disclaimer in the 12296341Sdelphij * documentation and/or other materials provided with the distribution. 13296341Sdelphij * 3. Neither the name of the author nor the names of any co-contributors 14296341Sdelphij * may be used to endorse or promote products derived from this software 15296341Sdelphij * without specific prior written permission. 16296341Sdelphij * 17296341Sdelphij * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1889837Skris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1989837Skris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20273399Sdelphij * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2189837Skris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22109998Smarkm * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23296341Sdelphij * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24296341Sdelphij * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25296341Sdelphij * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26296341Sdelphij * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27296341Sdelphij * SUCH DAMAGE. 28296341Sdelphij * 29296341Sdelphij * $FreeBSD$ 3056083Skris */ 31238405Sjkim 32296341Sdelphij/* 3356083Skris * Machine dependent interrupt code for x86. For x86, we have to 3456083Skris * deal with different PICs. Thus, we use the passed in vector to lookup 35296341Sdelphij * an interrupt source associated with that vector. The interrupt source 36296341Sdelphij * describes which PIC the source belongs to and includes methods to handle 37296341Sdelphij * that source. 38296341Sdelphij */ 39296341Sdelphij 40296341Sdelphij#include "opt_atpic.h" 4156083Skris#include "opt_ddb.h" 42296341Sdelphij 43296341Sdelphij#include <sys/param.h> 44296341Sdelphij#include <sys/bus.h> 45296341Sdelphij#include <sys/interrupt.h> 46296341Sdelphij#include <sys/ktr.h> 4756083Skris#include <sys/kernel.h> 48296341Sdelphij#include <sys/lock.h> 49296341Sdelphij#include <sys/mutex.h> 50296341Sdelphij#include <sys/proc.h> 51296341Sdelphij#include <sys/smp.h> 5256083Skris#include <sys/syslog.h> 53296341Sdelphij#include <sys/systm.h> 54296341Sdelphij#include <machine/clock.h> 55296341Sdelphij#include <machine/intr_machdep.h> 5689837Skris#include <machine/smp.h> 57296341Sdelphij#ifdef DDB 58296341Sdelphij#include <ddb/ddb.h> 59296341Sdelphij#endif 60296341Sdelphij 61296341Sdelphij#ifndef DEV_ATPIC 62296341Sdelphij#include <machine/segments.h> 63296341Sdelphij#include <machine/frame.h> 64296341Sdelphij#include <dev/ic/i8259.h> 65296341Sdelphij#include <x86/isa/icu.h> 66296341Sdelphij#ifdef PC98 67296341Sdelphij#include <pc98/cbus/cbus.h> 68296341Sdelphij#else 69296341Sdelphij#include <x86/isa/isa.h> 7056083Skris#endif 71296341Sdelphij#endif 72296341Sdelphij 73296341Sdelphij#define MAX_STRAY_LOG 5 74296341Sdelphij 75296341Sdelphijtypedef void (*mask_fn)(void *); 76205128Ssimon 77296341Sdelphijstatic int intrcnt_index; 78296341Sdelphijstatic struct intsrc *interrupt_sources[NUM_IO_INTS]; 79296341Sdelphijstatic struct mtx intr_table_lock; 80296341Sdelphijstatic struct mtx intrcnt_lock; 8156083Skrisstatic TAILQ_HEAD(pics_head, pic) pics; 82296341Sdelphij 83296341Sdelphij#ifdef SMP 84296341Sdelphijstatic int assign_cpu; 85296341Sdelphij#endif 8656083Skris 87296341Sdelphiju_long intrcnt[INTRCNT_COUNT]; 88296341Sdelphijchar intrnames[INTRCNT_COUNT * (MAXCOMLEN + 1)]; 89296341Sdelphijsize_t sintrcnt = sizeof(intrcnt); 9056083Skrissize_t sintrnames = sizeof(intrnames); 9156083Skris 92296341Sdelphijstatic int intr_assign_cpu(void *arg, u_char cpu); 93296341Sdelphijstatic void intr_disable_src(void *arg); 94296341Sdelphijstatic void intr_init(void *__dummy); 95296341Sdelphijstatic int intr_pic_registered(struct pic *pic); 96296341Sdelphijstatic void intrcnt_setname(const char *name, int index); 97296341Sdelphijstatic void intrcnt_updatename(struct intsrc *is); 98296341Sdelphijstatic void intrcnt_register(struct intsrc *is); 99296341Sdelphij 100296341Sdelphijstatic int 101296341Sdelphijintr_pic_registered(struct pic *pic) 102296341Sdelphij{ 103296341Sdelphij struct pic *p; 10456083Skris 105296341Sdelphij TAILQ_FOREACH(p, &pics, pics) { 106296341Sdelphij if (p == pic) 107273399Sdelphij return (1); 108296341Sdelphij } 109296341Sdelphij return (0); 110296341Sdelphij} 111296341Sdelphij 112296341Sdelphij/* 113296341Sdelphij * Register a new interrupt controller (PIC). This is to support suspend 114296341Sdelphij * and resume where we suspend/resume controllers rather than individual 115296341Sdelphij * sources. This also allows controllers with no active sources (such as 116296341Sdelphij * 8259As in a system using the APICs) to participate in suspend and resume. 117296341Sdelphij */ 11856083Skrisint 119296341Sdelphijintr_register_pic(struct pic *pic) 120296341Sdelphij{ 121296341Sdelphij int error; 122296341Sdelphij 123296341Sdelphij mtx_lock(&intr_table_lock); 124296341Sdelphij if (intr_pic_registered(pic)) 125296341Sdelphij error = EBUSY; 12656083Skris else { 127296341Sdelphij TAILQ_INSERT_TAIL(&pics, pic, pics); 128296341Sdelphij error = 0; 129296341Sdelphij } 130296341Sdelphij mtx_unlock(&intr_table_lock); 131296341Sdelphij return (error); 132296341Sdelphij} 133296341Sdelphij 134296341Sdelphij/* 135296341Sdelphij * Register a new interrupt source with the global interrupt system. 136296341Sdelphij * The global interrupts need to be disabled when this function is 137194206Ssimon * called. 138296341Sdelphij */ 139296341Sdelphijint 140296341Sdelphijintr_register_source(struct intsrc *isrc) 141296341Sdelphij{ 142296341Sdelphij int error, vector; 143296341Sdelphij 144194206Ssimon KASSERT(intr_pic_registered(isrc->is_pic), ("unregistered PIC")); 145296341Sdelphij vector = isrc->is_pic->pic_vector(isrc); 146296341Sdelphij if (interrupt_sources[vector] != NULL) 147273399Sdelphij return (EEXIST); 148296341Sdelphij error = intr_event_create(&isrc->is_event, isrc, 0, vector, 149296341Sdelphij intr_disable_src, (mask_fn)isrc->is_pic->pic_enable_source, 150296341Sdelphij (mask_fn)isrc->is_pic->pic_eoi_source, intr_assign_cpu, "irq%d:", 151296341Sdelphij vector); 152273399Sdelphij if (error) 153296341Sdelphij return (error); 154296341Sdelphij mtx_lock(&intr_table_lock); 155296341Sdelphij if (interrupt_sources[vector] != NULL) { 156296341Sdelphij mtx_unlock(&intr_table_lock); 15789837Skris intr_event_destroy(isrc->is_event); 158296341Sdelphij return (EEXIST); 159296341Sdelphij } 16089837Skris intrcnt_register(isrc); 161296341Sdelphij interrupt_sources[vector] = isrc; 162296341Sdelphij isrc->is_handlers = 0; 163273399Sdelphij mtx_unlock(&intr_table_lock); 164296341Sdelphij return (0); 165296341Sdelphij} 166296341Sdelphij 167296341Sdelphijstruct intsrc * 168296341Sdelphijintr_lookup_source(int vector) 169296341Sdelphij{ 170296341Sdelphij 171296341Sdelphij return (interrupt_sources[vector]); 172296341Sdelphij} 173296341Sdelphij 174296341Sdelphijint 175296341Sdelphijintr_add_handler(const char *name, int vector, driver_filter_t filter, 176273399Sdelphij driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) 177296341Sdelphij{ 178273399Sdelphij struct intsrc *isrc; 179296341Sdelphij int error; 180296341Sdelphij 181296341Sdelphij isrc = intr_lookup_source(vector); 182296341Sdelphij if (isrc == NULL) 183296341Sdelphij return (EINVAL); 184296341Sdelphij error = intr_event_add_handler(isrc->is_event, name, filter, handler, 185296341Sdelphij arg, intr_priority(flags), flags, cookiep); 186273399Sdelphij if (error == 0) { 187296341Sdelphij mtx_lock(&intr_table_lock); 188296341Sdelphij intrcnt_updatename(isrc); 189273399Sdelphij isrc->is_handlers++; 190296341Sdelphij if (isrc->is_handlers == 1) { 191296341Sdelphij isrc->is_pic->pic_enable_intr(isrc); 192296341Sdelphij isrc->is_pic->pic_enable_source(isrc); 193296341Sdelphij } 194296341Sdelphij mtx_unlock(&intr_table_lock); 195296341Sdelphij } 196296341Sdelphij return (error); 19779998Skris} 198296341Sdelphij 199296341Sdelphijint 200296341Sdelphijintr_remove_handler(void *cookie) 201296341Sdelphij{ 202296341Sdelphij struct intsrc *isrc; 203296341Sdelphij int error; 204296341Sdelphij 205296341Sdelphij isrc = intr_handler_source(cookie); 206296341Sdelphij error = intr_event_remove_handler(cookie); 207296341Sdelphij if (error == 0) { 208296341Sdelphij mtx_lock(&intr_table_lock); 209296341Sdelphij isrc->is_handlers--; 210296341Sdelphij if (isrc->is_handlers == 0) { 21156083Skris isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI); 212160814Ssimon isrc->is_pic->pic_disable_intr(isrc); 213296341Sdelphij } 214296341Sdelphij intrcnt_updatename(isrc); 215296341Sdelphij mtx_unlock(&intr_table_lock); 216296341Sdelphij } 217296341Sdelphij return (error); 218296341Sdelphij} 219296341Sdelphij 220296341Sdelphijint 22156083Skrisintr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol) 222296341Sdelphij{ 223296341Sdelphij struct intsrc *isrc; 224296341Sdelphij 225296341Sdelphij isrc = intr_lookup_source(vector); 226296341Sdelphij if (isrc == NULL) 227296341Sdelphij return (EINVAL); 228296341Sdelphij return (isrc->is_pic->pic_config_intr(isrc, trig, pol)); 229296341Sdelphij} 230296341Sdelphij 231296341Sdelphijstatic void 232296341Sdelphijintr_disable_src(void *arg) 233296341Sdelphij{ 234296341Sdelphij struct intsrc *isrc; 235296341Sdelphij 236296341Sdelphij isrc = arg; 237296341Sdelphij isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 238296341Sdelphij} 239296341Sdelphij 240296341Sdelphijvoid 241296341Sdelphijintr_execute_handlers(struct intsrc *isrc, struct trapframe *frame) 242296341Sdelphij{ 243296341Sdelphij struct intr_event *ie; 244296341Sdelphij int vector; 245296341Sdelphij 246296341Sdelphij /* 247296341Sdelphij * We count software interrupts when we process them. The 248296341Sdelphij * code here follows previous practice, but there's an 249296341Sdelphij * argument for counting hardware interrupts when they're 250296341Sdelphij * processed too. 251160814Ssimon */ 252238405Sjkim (*isrc->is_count)++; 253296341Sdelphij PCPU_INC(cnt.v_intr); 254296341Sdelphij 255296341Sdelphij ie = isrc->is_event; 256296341Sdelphij 25756083Skris /* 258 * XXX: We assume that IRQ 0 is only used for the ISA timer 259 * device (clk). 260 */ 261 vector = isrc->is_pic->pic_vector(isrc); 262 if (vector == 0) 263 clkintr_pending = 1; 264 265 /* 266 * For stray interrupts, mask and EOI the source, bump the 267 * stray count, and log the condition. 268 */ 269 if (intr_event_handle(ie, frame) != 0) { 270 isrc->is_pic->pic_disable_source(isrc, PIC_EOI); 271 (*isrc->is_straycount)++; 272 if (*isrc->is_straycount < MAX_STRAY_LOG) 273 log(LOG_ERR, "stray irq%d\n", vector); 274 else if (*isrc->is_straycount == MAX_STRAY_LOG) 275 log(LOG_CRIT, 276 "too many stray irq %d's: not logging anymore\n", 277 vector); 278 } 279} 280 281void 282intr_resume(bool suspend_cancelled) 283{ 284 struct pic *pic; 285 286#ifndef DEV_ATPIC 287 atpic_reset(); 288#endif 289 mtx_lock(&intr_table_lock); 290 TAILQ_FOREACH(pic, &pics, pics) { 291 if (pic->pic_resume != NULL) 292 pic->pic_resume(pic, suspend_cancelled); 293 } 294 mtx_unlock(&intr_table_lock); 295} 296 297void 298intr_suspend(void) 299{ 300 struct pic *pic; 301 302 mtx_lock(&intr_table_lock); 303 TAILQ_FOREACH_REVERSE(pic, &pics, pics_head, pics) { 304 if (pic->pic_suspend != NULL) 305 pic->pic_suspend(pic); 306 } 307 mtx_unlock(&intr_table_lock); 308} 309 310static int 311intr_assign_cpu(void *arg, u_char cpu) 312{ 313#ifdef SMP 314 struct intsrc *isrc; 315 int error; 316 317 /* 318 * Don't do anything during early boot. We will pick up the 319 * assignment once the APs are started. 320 */ 321 if (assign_cpu && cpu != NOCPU) { 322 isrc = arg; 323 mtx_lock(&intr_table_lock); 324 error = isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); 325 mtx_unlock(&intr_table_lock); 326 } else 327 error = 0; 328 return (error); 329#else 330 return (EOPNOTSUPP); 331#endif 332} 333 334static void 335intrcnt_setname(const char *name, int index) 336{ 337 338 snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", 339 MAXCOMLEN, name); 340} 341 342static void 343intrcnt_updatename(struct intsrc *is) 344{ 345 346 intrcnt_setname(is->is_event->ie_fullname, is->is_index); 347} 348 349static void 350intrcnt_register(struct intsrc *is) 351{ 352 char straystr[MAXCOMLEN + 1]; 353 354 KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__)); 355 mtx_lock_spin(&intrcnt_lock); 356 is->is_index = intrcnt_index; 357 intrcnt_index += 2; 358 snprintf(straystr, MAXCOMLEN + 1, "stray irq%d", 359 is->is_pic->pic_vector(is)); 360 intrcnt_updatename(is); 361 is->is_count = &intrcnt[is->is_index]; 362 intrcnt_setname(straystr, is->is_index + 1); 363 is->is_straycount = &intrcnt[is->is_index + 1]; 364 mtx_unlock_spin(&intrcnt_lock); 365} 366 367void 368intrcnt_add(const char *name, u_long **countp) 369{ 370 371 mtx_lock_spin(&intrcnt_lock); 372 *countp = &intrcnt[intrcnt_index]; 373 intrcnt_setname(name, intrcnt_index); 374 intrcnt_index++; 375 mtx_unlock_spin(&intrcnt_lock); 376} 377 378static void 379intr_init(void *dummy __unused) 380{ 381 382 intrcnt_setname("???", 0); 383 intrcnt_index = 1; 384 TAILQ_INIT(&pics); 385 mtx_init(&intr_table_lock, "intr sources", NULL, MTX_DEF); 386 mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); 387} 388SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); 389 390#ifndef DEV_ATPIC 391/* Initialize the two 8259A's to a known-good shutdown state. */ 392void 393atpic_reset(void) 394{ 395 396 outb(IO_ICU1, ICW1_RESET | ICW1_IC4); 397 outb(IO_ICU1 + ICU_IMR_OFFSET, IDT_IO_INTS); 398 outb(IO_ICU1 + ICU_IMR_OFFSET, IRQ_MASK(ICU_SLAVEID)); 399 outb(IO_ICU1 + ICU_IMR_OFFSET, MASTER_MODE); 400 outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff); 401 outb(IO_ICU1, OCW3_SEL | OCW3_RR); 402 403 outb(IO_ICU2, ICW1_RESET | ICW1_IC4); 404 outb(IO_ICU2 + ICU_IMR_OFFSET, IDT_IO_INTS + 8); 405 outb(IO_ICU2 + ICU_IMR_OFFSET, ICU_SLAVEID); 406 outb(IO_ICU2 + ICU_IMR_OFFSET, SLAVE_MODE); 407 outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff); 408 outb(IO_ICU2, OCW3_SEL | OCW3_RR); 409} 410#endif 411 412/* Add a description to an active interrupt handler. */ 413int 414intr_describe(u_int vector, void *ih, const char *descr) 415{ 416 struct intsrc *isrc; 417 int error; 418 419 isrc = intr_lookup_source(vector); 420 if (isrc == NULL) 421 return (EINVAL); 422 error = intr_event_describe_handler(isrc->is_event, ih, descr); 423 if (error) 424 return (error); 425 intrcnt_updatename(isrc); 426 return (0); 427} 428 429#ifdef DDB 430/* 431 * Dump data about interrupt handlers 432 */ 433DB_SHOW_COMMAND(irqs, db_show_irqs) 434{ 435 struct intsrc **isrc; 436 int i, verbose; 437 438 if (strcmp(modif, "v") == 0) 439 verbose = 1; 440 else 441 verbose = 0; 442 isrc = interrupt_sources; 443 for (i = 0; i < NUM_IO_INTS && !db_pager_quit; i++, isrc++) 444 if (*isrc != NULL) 445 db_dump_intr_event((*isrc)->is_event, verbose); 446} 447#endif 448 449#ifdef SMP 450/* 451 * Support for balancing interrupt sources across CPUs. For now we just 452 * allocate CPUs round-robin. 453 */ 454 455static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); 456static int current_cpu; 457 458/* 459 * Return the CPU that the next interrupt source should use. For now 460 * this just returns the next local APIC according to round-robin. 461 */ 462u_int 463intr_next_cpu(void) 464{ 465 u_int apic_id; 466 467 /* Leave all interrupts on the BSP during boot. */ 468 if (!assign_cpu) 469 return (PCPU_GET(apic_id)); 470 471 mtx_lock_spin(&icu_lock); 472 apic_id = cpu_apic_ids[current_cpu]; 473 do { 474 current_cpu++; 475 if (current_cpu > mp_maxid) 476 current_cpu = 0; 477 } while (!CPU_ISSET(current_cpu, &intr_cpus)); 478 mtx_unlock_spin(&icu_lock); 479 return (apic_id); 480} 481 482/* Attempt to bind the specified IRQ to the specified CPU. */ 483int 484intr_bind(u_int vector, u_char cpu) 485{ 486 struct intsrc *isrc; 487 488 isrc = intr_lookup_source(vector); 489 if (isrc == NULL) 490 return (EINVAL); 491 return (intr_event_bind(isrc->is_event, cpu)); 492} 493 494/* 495 * Add a CPU to our mask of valid CPUs that can be destinations of 496 * interrupts. 497 */ 498void 499intr_add_cpu(u_int cpu) 500{ 501 502 if (cpu >= MAXCPU) 503 panic("%s: Invalid CPU ID", __func__); 504 if (bootverbose) 505 printf("INTR: Adding local APIC %d as a target\n", 506 cpu_apic_ids[cpu]); 507 508 CPU_SET(cpu, &intr_cpus); 509} 510 511/* 512 * Distribute all the interrupt sources among the available CPUs once the 513 * AP's have been launched. 514 */ 515static void 516intr_shuffle_irqs(void *arg __unused) 517{ 518 struct intsrc *isrc; 519 int i; 520 521#ifdef XEN 522 /* 523 * Doesn't work yet 524 */ 525 return; 526#endif 527 528 /* Don't bother on UP. */ 529 if (mp_ncpus == 1) 530 return; 531 532 /* Round-robin assign a CPU to each enabled source. */ 533 mtx_lock(&intr_table_lock); 534 assign_cpu = 1; 535 for (i = 0; i < NUM_IO_INTS; i++) { 536 isrc = interrupt_sources[i]; 537 if (isrc != NULL && isrc->is_handlers > 0) { 538 /* 539 * If this event is already bound to a CPU, 540 * then assign the source to that CPU instead 541 * of picking one via round-robin. Note that 542 * this is careful to only advance the 543 * round-robin if the CPU assignment succeeds. 544 */ 545 if (isrc->is_event->ie_cpu != NOCPU) 546 (void)isrc->is_pic->pic_assign_cpu(isrc, 547 cpu_apic_ids[isrc->is_event->ie_cpu]); 548 else if (isrc->is_pic->pic_assign_cpu(isrc, 549 cpu_apic_ids[current_cpu]) == 0) 550 (void)intr_next_cpu(); 551 552 } 553 } 554 mtx_unlock(&intr_table_lock); 555} 556SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, 557 NULL); 558#else 559/* 560 * Always route interrupts to the current processor in the UP case. 561 */ 562u_int 563intr_next_cpu(void) 564{ 565 566 return (PCPU_GET(apic_id)); 567} 568#endif 569