kern_intr.c revision 177253
1/*- 2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: head/sys/kern/kern_intr.c 177253 2008-03-16 10:58:09Z rwatson $"); 29 30#include "opt_ddb.h" 31 32#include <sys/param.h> 33#include <sys/bus.h> 34#include <sys/conf.h> 35#include <sys/rtprio.h> 36#include <sys/systm.h> 37#include <sys/interrupt.h> 38#include <sys/kernel.h> 39#include <sys/kthread.h> 40#include <sys/ktr.h> 41#include <sys/limits.h> 42#include <sys/lock.h> 43#include <sys/malloc.h> 44#include <sys/mutex.h> 45#include <sys/proc.h> 46#include <sys/random.h> 47#include <sys/resourcevar.h> 48#include <sys/sched.h> 49#include <sys/smp.h> 50#include <sys/sysctl.h> 51#include <sys/unistd.h> 52#include <sys/vmmeter.h> 53#include <machine/atomic.h> 54#include <machine/cpu.h> 55#include <machine/md_var.h> 56#include <machine/stdarg.h> 57#ifdef DDB 58#include <ddb/ddb.h> 59#include <ddb/db_sym.h> 60#endif 61 62/* 63 * Describe an interrupt thread. There is one of these per interrupt event. 64 */ 65struct intr_thread { 66 struct intr_event *it_event; 67 struct thread *it_thread; /* Kernel thread. */ 68 int it_flags; /* (j) IT_* flags. */ 69 int it_need; /* Needs service. */ 70}; 71 72/* Interrupt thread flags kept in it_flags */ 73#define IT_DEAD 0x000001 /* Thread is waiting to exit. */ 74 75struct intr_entropy { 76 struct thread *td; 77 uintptr_t event; 78}; 79 80struct intr_event *clk_intr_event; 81struct intr_event *tty_intr_event; 82void *softclock_ih; 83void *vm_ih; 84struct proc *intrproc; 85 86static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 87 88static int intr_storm_threshold = 1000; 89TUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold); 90SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW, 91 &intr_storm_threshold, 0, 92 "Number of consecutive interrupts before storm protection is enabled"); 93static TAILQ_HEAD(, intr_event) event_list = 94 TAILQ_HEAD_INITIALIZER(event_list); 95 96static void intr_event_update(struct intr_event *ie); 97#ifdef INTR_FILTER 98static struct intr_thread *ithread_create(const char *name, 99 struct intr_handler *ih); 100#else 101static struct intr_thread *ithread_create(const char *name); 102#endif 103static void ithread_destroy(struct intr_thread *ithread); 104static void ithread_execute_handlers(struct proc *p, 105 struct intr_event *ie); 106#ifdef INTR_FILTER 107static void priv_ithread_execute_handler(struct proc *p, 108 struct intr_handler *ih); 109#endif 110static void ithread_loop(void *); 111static void ithread_update(struct intr_thread *ithd); 112static void start_softintr(void *); 113 114/* Map an interrupt type to an ithread priority. */ 115u_char 116intr_priority(enum intr_type flags) 117{ 118 u_char pri; 119 120 flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 121 INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); 122 switch (flags) { 123 case INTR_TYPE_TTY: 124 pri = PI_TTYLOW; 125 break; 126 case INTR_TYPE_BIO: 127 /* 128 * XXX We need to refine this. BSD/OS distinguishes 129 * between tape and disk priorities. 130 */ 131 pri = PI_DISK; 132 break; 133 case INTR_TYPE_NET: 134 pri = PI_NET; 135 break; 136 case INTR_TYPE_CAM: 137 pri = PI_DISK; /* XXX or PI_CAM? */ 138 break; 139 case INTR_TYPE_AV: /* Audio/video */ 140 pri = PI_AV; 141 break; 142 case INTR_TYPE_CLK: 143 pri = PI_REALTIME; 144 break; 145 case INTR_TYPE_MISC: 146 pri = PI_DULL; /* don't care */ 147 break; 148 default: 149 /* We didn't specify an interrupt level. */ 150 panic("intr_priority: no interrupt type in flags"); 151 } 152 153 return pri; 154} 155 156/* 157 * Update an ithread based on the associated intr_event. 158 */ 159static void 160ithread_update(struct intr_thread *ithd) 161{ 162 struct intr_event *ie; 163 struct thread *td; 164 u_char pri; 165 166 ie = ithd->it_event; 167 td = ithd->it_thread; 168 169 /* Determine the overall priority of this event. */ 170 if (TAILQ_EMPTY(&ie->ie_handlers)) 171 pri = PRI_MAX_ITHD; 172 else 173 pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri; 174 175 /* Update name and priority. */ 176 strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name)); 177 thread_lock(td); 178 sched_prio(td, pri); 179 thread_unlock(td); 180} 181 182/* 183 * Regenerate the full name of an interrupt event and update its priority. 184 */ 185static void 186intr_event_update(struct intr_event *ie) 187{ 188 struct intr_handler *ih; 189 char *last; 190 int missed, space; 191 192 /* Start off with no entropy and just the name of the event. */ 193 mtx_assert(&ie->ie_lock, MA_OWNED); 194 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 195 ie->ie_flags &= ~IE_ENTROPY; 196 missed = 0; 197 space = 1; 198 199 /* Run through all the handlers updating values. */ 200 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 201 if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 < 202 sizeof(ie->ie_fullname)) { 203 strcat(ie->ie_fullname, " "); 204 strcat(ie->ie_fullname, ih->ih_name); 205 space = 0; 206 } else 207 missed++; 208 if (ih->ih_flags & IH_ENTROPY) 209 ie->ie_flags |= IE_ENTROPY; 210 } 211 212 /* 213 * If the handler names were too long, add +'s to indicate missing 214 * names. If we run out of room and still have +'s to add, change 215 * the last character from a + to a *. 216 */ 217 last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2]; 218 while (missed-- > 0) { 219 if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) { 220 if (*last == '+') { 221 *last = '*'; 222 break; 223 } else 224 *last = '+'; 225 } else if (space) { 226 strcat(ie->ie_fullname, " +"); 227 space = 0; 228 } else 229 strcat(ie->ie_fullname, "+"); 230 } 231 232 /* 233 * If this event has an ithread, update it's priority and 234 * name. 235 */ 236 if (ie->ie_thread != NULL) 237 ithread_update(ie->ie_thread); 238 CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname); 239} 240 241#ifndef INTR_FILTER 242int 243intr_event_create(struct intr_event **event, void *source, int flags, 244 void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, 245 ...) 246{ 247 struct intr_event *ie; 248 va_list ap; 249 250 /* The only valid flag during creation is IE_SOFT. */ 251 if ((flags & ~IE_SOFT) != 0) 252 return (EINVAL); 253 ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); 254 ie->ie_source = source; 255 ie->ie_enable = enable; 256 ie->ie_assign_cpu = assign_cpu; 257 ie->ie_flags = flags; 258 ie->ie_cpu = NOCPU; 259 TAILQ_INIT(&ie->ie_handlers); 260 mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); 261 262 va_start(ap, fmt); 263 vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); 264 va_end(ap); 265 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 266 mtx_pool_lock(mtxpool_sleep, &event_list); 267 TAILQ_INSERT_TAIL(&event_list, ie, ie_list); 268 mtx_pool_unlock(mtxpool_sleep, &event_list); 269 if (event != NULL) 270 *event = ie; 271 CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); 272 return (0); 273} 274#else 275int 276intr_event_create(struct intr_event **event, void *source, int flags, 277 void (*enable)(void *), void (*eoi)(void *), void (*disab)(void *), 278 int (*assign_cpu)(void *, u_char), const char *fmt, ...) 279{ 280 struct intr_event *ie; 281 va_list ap; 282 283 /* The only valid flag during creation is IE_SOFT. */ 284 if ((flags & ~IE_SOFT) != 0) 285 return (EINVAL); 286 ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); 287 ie->ie_source = source; 288 ie->ie_enable = enable; 289 ie->ie_assign_cpu = assign_cpu; 290 ie->ie_eoi = eoi; 291 ie->ie_disab = disab; 292 ie->ie_flags = flags; 293 ie->ie_cpu = NOCPU; 294 TAILQ_INIT(&ie->ie_handlers); 295 mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); 296 297 va_start(ap, fmt); 298 vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); 299 va_end(ap); 300 strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 301 mtx_pool_lock(mtxpool_sleep, &event_list); 302 TAILQ_INSERT_TAIL(&event_list, ie, ie_list); 303 mtx_pool_unlock(mtxpool_sleep, &event_list); 304 if (event != NULL) 305 *event = ie; 306 CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); 307 return (0); 308} 309#endif 310 311/* 312 * Bind an interrupt event to the specified CPU. Note that not all 313 * platforms support binding an interrupt to a CPU. For those 314 * platforms this request will fail. For supported platforms, any 315 * associated ithreads as well as the primary interrupt context will 316 * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds 317 * the interrupt event. 318 */ 319int 320intr_event_bind(struct intr_event *ie, u_char cpu) 321{ 322 struct thread *td; 323 int error; 324 325 /* Need a CPU to bind to. */ 326 if (cpu != NOCPU && CPU_ABSENT(cpu)) 327 return (EINVAL); 328 329 if (ie->ie_assign_cpu == NULL) 330 return (EOPNOTSUPP); 331 332 /* Don't allow a bind request if the interrupt is already bound. */ 333 mtx_lock(&ie->ie_lock); 334 if (ie->ie_cpu != NOCPU && cpu != NOCPU) { 335 mtx_unlock(&ie->ie_lock); 336 return (EBUSY); 337 } 338 mtx_unlock(&ie->ie_lock); 339 340 error = ie->ie_assign_cpu(ie->ie_source, cpu); 341 if (error) 342 return (error); 343 mtx_lock(&ie->ie_lock); 344 if (ie->ie_thread != NULL) 345 td = ie->ie_thread->it_thread; 346 else 347 td = NULL; 348 if (td != NULL) 349 thread_lock(td); 350 ie->ie_cpu = cpu; 351 if (td != NULL) 352 thread_unlock(td); 353 mtx_unlock(&ie->ie_lock); 354 return (0); 355} 356 357int 358intr_event_destroy(struct intr_event *ie) 359{ 360 361 mtx_lock(&ie->ie_lock); 362 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 363 mtx_unlock(&ie->ie_lock); 364 return (EBUSY); 365 } 366 mtx_pool_lock(mtxpool_sleep, &event_list); 367 TAILQ_REMOVE(&event_list, ie, ie_list); 368 mtx_pool_unlock(mtxpool_sleep, &event_list); 369#ifndef notyet 370 if (ie->ie_thread != NULL) { 371 ithread_destroy(ie->ie_thread); 372 ie->ie_thread = NULL; 373 } 374#endif 375 mtx_unlock(&ie->ie_lock); 376 mtx_destroy(&ie->ie_lock); 377 free(ie, M_ITHREAD); 378 return (0); 379} 380 381#ifndef INTR_FILTER 382static struct intr_thread * 383ithread_create(const char *name) 384{ 385 struct intr_thread *ithd; 386 struct thread *td; 387 int error; 388 389 ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 390 391 error = kproc_kthread_add(ithread_loop, ithd, &intrproc, 392 &td, RFSTOPPED | RFHIGHPID, 393 0, "intr", "%s", name); 394 if (error) 395 panic("kproc_create() failed with %d", error); 396 thread_lock(td); 397 sched_class(td, PRI_ITHD); 398 TD_SET_IWAIT(td); 399 thread_unlock(td); 400 td->td_pflags |= TDP_ITHREAD; 401 ithd->it_thread = td; 402 CTR2(KTR_INTR, "%s: created %s", __func__, name); 403 return (ithd); 404} 405#else 406static struct intr_thread * 407ithread_create(const char *name, struct intr_handler *ih) 408{ 409 struct intr_thread *ithd; 410 struct thread *td; 411 int error; 412 413 ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 414 415 error = kproc_kthread_add(ithread_loop, ih, &intrproc, 416 &td, RFSTOPPED | RFHIGHPID, 417 0, "intr", "%s", name); 418 if (error) 419 panic("kproc_create() failed with %d", error); 420 thread_lock(td); 421 sched_class(td, PRI_ITHD); 422 TD_SET_IWAIT(td); 423 thread_unlock(td); 424 td->td_pflags |= TDP_ITHREAD; 425 ithd->it_thread = td; 426 CTR2(KTR_INTR, "%s: created %s", __func__, name); 427 return (ithd); 428} 429#endif 430 431static void 432ithread_destroy(struct intr_thread *ithread) 433{ 434 struct thread *td; 435 436 CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name); 437 td = ithread->it_thread; 438 thread_lock(td); 439 ithread->it_flags |= IT_DEAD; 440 if (TD_AWAITING_INTR(td)) { 441 TD_CLR_IWAIT(td); 442 sched_add(td, SRQ_INTR); 443 } 444 thread_unlock(td); 445} 446 447#ifndef INTR_FILTER 448int 449intr_event_add_handler(struct intr_event *ie, const char *name, 450 driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 451 enum intr_type flags, void **cookiep) 452{ 453 struct intr_handler *ih, *temp_ih; 454 struct intr_thread *it; 455 456 if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 457 return (EINVAL); 458 459 /* Allocate and populate an interrupt handler structure. */ 460 ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 461 ih->ih_filter = filter; 462 ih->ih_handler = handler; 463 ih->ih_argument = arg; 464 ih->ih_name = name; 465 ih->ih_event = ie; 466 ih->ih_pri = pri; 467 if (flags & INTR_EXCL) 468 ih->ih_flags = IH_EXCLUSIVE; 469 if (flags & INTR_MPSAFE) 470 ih->ih_flags |= IH_MPSAFE; 471 if (flags & INTR_ENTROPY) 472 ih->ih_flags |= IH_ENTROPY; 473 474 /* We can only have one exclusive handler in a event. */ 475 mtx_lock(&ie->ie_lock); 476 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 477 if ((flags & INTR_EXCL) || 478 (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 479 mtx_unlock(&ie->ie_lock); 480 free(ih, M_ITHREAD); 481 return (EINVAL); 482 } 483 } 484 485 /* Add the new handler to the event in priority order. */ 486 TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 487 if (temp_ih->ih_pri > ih->ih_pri) 488 break; 489 } 490 if (temp_ih == NULL) 491 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 492 else 493 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 494 intr_event_update(ie); 495 496 /* Create a thread if we need one. */ 497 while (ie->ie_thread == NULL && handler != NULL) { 498 if (ie->ie_flags & IE_ADDING_THREAD) 499 msleep(ie, &ie->ie_lock, 0, "ithread", 0); 500 else { 501 ie->ie_flags |= IE_ADDING_THREAD; 502 mtx_unlock(&ie->ie_lock); 503 it = ithread_create("intr: newborn"); 504 mtx_lock(&ie->ie_lock); 505 ie->ie_flags &= ~IE_ADDING_THREAD; 506 ie->ie_thread = it; 507 it->it_event = ie; 508 ithread_update(it); 509 wakeup(ie); 510 } 511 } 512 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 513 ie->ie_name); 514 mtx_unlock(&ie->ie_lock); 515 516 if (cookiep != NULL) 517 *cookiep = ih; 518 return (0); 519} 520#else 521int 522intr_event_add_handler(struct intr_event *ie, const char *name, 523 driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 524 enum intr_type flags, void **cookiep) 525{ 526 struct intr_handler *ih, *temp_ih; 527 struct intr_thread *it; 528 529 if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 530 return (EINVAL); 531 532 /* Allocate and populate an interrupt handler structure. */ 533 ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 534 ih->ih_filter = filter; 535 ih->ih_handler = handler; 536 ih->ih_argument = arg; 537 ih->ih_name = name; 538 ih->ih_event = ie; 539 ih->ih_pri = pri; 540 if (flags & INTR_EXCL) 541 ih->ih_flags = IH_EXCLUSIVE; 542 if (flags & INTR_MPSAFE) 543 ih->ih_flags |= IH_MPSAFE; 544 if (flags & INTR_ENTROPY) 545 ih->ih_flags |= IH_ENTROPY; 546 547 /* We can only have one exclusive handler in a event. */ 548 mtx_lock(&ie->ie_lock); 549 if (!TAILQ_EMPTY(&ie->ie_handlers)) { 550 if ((flags & INTR_EXCL) || 551 (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 552 mtx_unlock(&ie->ie_lock); 553 free(ih, M_ITHREAD); 554 return (EINVAL); 555 } 556 } 557 558 /* Add the new handler to the event in priority order. */ 559 TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 560 if (temp_ih->ih_pri > ih->ih_pri) 561 break; 562 } 563 if (temp_ih == NULL) 564 TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 565 else 566 TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 567 intr_event_update(ie); 568 569 /* For filtered handlers, create a private ithread to run on. */ 570 if (filter != NULL && handler != NULL) { 571 mtx_unlock(&ie->ie_lock); 572 it = ithread_create("intr: newborn", ih); 573 mtx_lock(&ie->ie_lock); 574 it->it_event = ie; 575 ih->ih_thread = it; 576 ithread_update(it); // XXX - do we really need this?!?!? 577 } else { /* Create the global per-event thread if we need one. */ 578 while (ie->ie_thread == NULL && handler != NULL) { 579 if (ie->ie_flags & IE_ADDING_THREAD) 580 msleep(ie, &ie->ie_lock, 0, "ithread", 0); 581 else { 582 ie->ie_flags |= IE_ADDING_THREAD; 583 mtx_unlock(&ie->ie_lock); 584 it = ithread_create("intr: newborn", ih); 585 mtx_lock(&ie->ie_lock); 586 ie->ie_flags &= ~IE_ADDING_THREAD; 587 ie->ie_thread = it; 588 it->it_event = ie; 589 ithread_update(it); 590 wakeup(ie); 591 } 592 } 593 } 594 CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 595 ie->ie_name); 596 mtx_unlock(&ie->ie_lock); 597 598 if (cookiep != NULL) 599 *cookiep = ih; 600 return (0); 601} 602#endif 603 604/* 605 * Return the ie_source field from the intr_event an intr_handler is 606 * associated with. 607 */ 608void * 609intr_handler_source(void *cookie) 610{ 611 struct intr_handler *ih; 612 struct intr_event *ie; 613 614 ih = (struct intr_handler *)cookie; 615 if (ih == NULL) 616 return (NULL); 617 ie = ih->ih_event; 618 KASSERT(ie != NULL, 619 ("interrupt handler \"%s\" has a NULL interrupt event", 620 ih->ih_name)); 621 return (ie->ie_source); 622} 623 624#ifndef INTR_FILTER 625int 626intr_event_remove_handler(void *cookie) 627{ 628 struct intr_handler *handler = (struct intr_handler *)cookie; 629 struct intr_event *ie; 630#ifdef INVARIANTS 631 struct intr_handler *ih; 632#endif 633#ifdef notyet 634 int dead; 635#endif 636 637 if (handler == NULL) 638 return (EINVAL); 639 ie = handler->ih_event; 640 KASSERT(ie != NULL, 641 ("interrupt handler \"%s\" has a NULL interrupt event", 642 handler->ih_name)); 643 mtx_lock(&ie->ie_lock); 644 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 645 ie->ie_name); 646#ifdef INVARIANTS 647 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 648 if (ih == handler) 649 goto ok; 650 mtx_unlock(&ie->ie_lock); 651 panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 652 ih->ih_name, ie->ie_name); 653ok: 654#endif 655 /* 656 * If there is no ithread, then just remove the handler and return. 657 * XXX: Note that an INTR_FAST handler might be running on another 658 * CPU! 659 */ 660 if (ie->ie_thread == NULL) { 661 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 662 mtx_unlock(&ie->ie_lock); 663 free(handler, M_ITHREAD); 664 return (0); 665 } 666 667 /* 668 * If the interrupt thread is already running, then just mark this 669 * handler as being dead and let the ithread do the actual removal. 670 * 671 * During a cold boot while cold is set, msleep() does not sleep, 672 * so we have to remove the handler here rather than letting the 673 * thread do it. 674 */ 675 thread_lock(ie->ie_thread->it_thread); 676 if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) { 677 handler->ih_flags |= IH_DEAD; 678 679 /* 680 * Ensure that the thread will process the handler list 681 * again and remove this handler if it has already passed 682 * it on the list. 683 */ 684 ie->ie_thread->it_need = 1; 685 } else 686 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 687 thread_unlock(ie->ie_thread->it_thread); 688 while (handler->ih_flags & IH_DEAD) 689 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 690 intr_event_update(ie); 691#ifdef notyet 692 /* 693 * XXX: This could be bad in the case of ppbus(8). Also, I think 694 * this could lead to races of stale data when servicing an 695 * interrupt. 696 */ 697 dead = 1; 698 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 699 if (!(ih->ih_flags & IH_FAST)) { 700 dead = 0; 701 break; 702 } 703 } 704 if (dead) { 705 ithread_destroy(ie->ie_thread); 706 ie->ie_thread = NULL; 707 } 708#endif 709 mtx_unlock(&ie->ie_lock); 710 free(handler, M_ITHREAD); 711 return (0); 712} 713 714int 715intr_event_schedule_thread(struct intr_event *ie) 716{ 717 struct intr_entropy entropy; 718 struct intr_thread *it; 719 struct thread *td; 720 struct thread *ctd; 721 struct proc *p; 722 723 /* 724 * If no ithread or no handlers, then we have a stray interrupt. 725 */ 726 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || 727 ie->ie_thread == NULL) 728 return (EINVAL); 729 730 ctd = curthread; 731 it = ie->ie_thread; 732 td = it->it_thread; 733 p = td->td_proc; 734 735 /* 736 * If any of the handlers for this ithread claim to be good 737 * sources of entropy, then gather some. 738 */ 739 if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 740 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 741 p->p_pid, td->td_name); 742 entropy.event = (uintptr_t)ie; 743 entropy.td = ctd; 744 random_harvest(&entropy, sizeof(entropy), 2, 0, 745 RANDOM_INTERRUPT); 746 } 747 748 KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 749 750 /* 751 * Set it_need to tell the thread to keep running if it is already 752 * running. Then, lock the thread and see if we actually need to 753 * put it on the runqueue. 754 */ 755 it->it_need = 1; 756 thread_lock(td); 757 if (TD_AWAITING_INTR(td)) { 758 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 759 td->td_name); 760 TD_CLR_IWAIT(td); 761 sched_add(td, SRQ_INTR); 762 } else { 763 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 764 __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 765 } 766 thread_unlock(td); 767 768 return (0); 769} 770#else 771int 772intr_event_remove_handler(void *cookie) 773{ 774 struct intr_handler *handler = (struct intr_handler *)cookie; 775 struct intr_event *ie; 776 struct intr_thread *it; 777#ifdef INVARIANTS 778 struct intr_handler *ih; 779#endif 780#ifdef notyet 781 int dead; 782#endif 783 784 if (handler == NULL) 785 return (EINVAL); 786 ie = handler->ih_event; 787 KASSERT(ie != NULL, 788 ("interrupt handler \"%s\" has a NULL interrupt event", 789 handler->ih_name)); 790 mtx_lock(&ie->ie_lock); 791 CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 792 ie->ie_name); 793#ifdef INVARIANTS 794 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 795 if (ih == handler) 796 goto ok; 797 mtx_unlock(&ie->ie_lock); 798 panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 799 ih->ih_name, ie->ie_name); 800ok: 801#endif 802 /* 803 * If there are no ithreads (per event and per handler), then 804 * just remove the handler and return. 805 * XXX: Note that an INTR_FAST handler might be running on another CPU! 806 */ 807 if (ie->ie_thread == NULL && handler->ih_thread == NULL) { 808 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 809 mtx_unlock(&ie->ie_lock); 810 free(handler, M_ITHREAD); 811 return (0); 812 } 813 814 /* Private or global ithread? */ 815 it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread; 816 /* 817 * If the interrupt thread is already running, then just mark this 818 * handler as being dead and let the ithread do the actual removal. 819 * 820 * During a cold boot while cold is set, msleep() does not sleep, 821 * so we have to remove the handler here rather than letting the 822 * thread do it. 823 */ 824 thread_lock(it->it_thread); 825 if (!TD_AWAITING_INTR(it->it_thread) && !cold) { 826 handler->ih_flags |= IH_DEAD; 827 828 /* 829 * Ensure that the thread will process the handler list 830 * again and remove this handler if it has already passed 831 * it on the list. 832 */ 833 it->it_need = 1; 834 } else 835 TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 836 thread_unlock(it->it_thread); 837 while (handler->ih_flags & IH_DEAD) 838 msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 839 /* 840 * At this point, the handler has been disconnected from the event, 841 * so we can kill the private ithread if any. 842 */ 843 if (handler->ih_thread) { 844 ithread_destroy(handler->ih_thread); 845 handler->ih_thread = NULL; 846 } 847 intr_event_update(ie); 848#ifdef notyet 849 /* 850 * XXX: This could be bad in the case of ppbus(8). Also, I think 851 * this could lead to races of stale data when servicing an 852 * interrupt. 853 */ 854 dead = 1; 855 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 856 if (handler != NULL) { 857 dead = 0; 858 break; 859 } 860 } 861 if (dead) { 862 ithread_destroy(ie->ie_thread); 863 ie->ie_thread = NULL; 864 } 865#endif 866 mtx_unlock(&ie->ie_lock); 867 free(handler, M_ITHREAD); 868 return (0); 869} 870 871int 872intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it) 873{ 874 struct intr_entropy entropy; 875 struct thread *td; 876 struct thread *ctd; 877 struct proc *p; 878 879 /* 880 * If no ithread or no handlers, then we have a stray interrupt. 881 */ 882 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL) 883 return (EINVAL); 884 885 ctd = curthread; 886 td = it->it_thread; 887 p = td->td_proc; 888 889 /* 890 * If any of the handlers for this ithread claim to be good 891 * sources of entropy, then gather some. 892 */ 893 if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 894 CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 895 p->p_pid, td->td_name); 896 entropy.event = (uintptr_t)ie; 897 entropy.td = ctd; 898 random_harvest(&entropy, sizeof(entropy), 2, 0, 899 RANDOM_INTERRUPT); 900 } 901 902 KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 903 904 /* 905 * Set it_need to tell the thread to keep running if it is already 906 * running. Then, lock the thread and see if we actually need to 907 * put it on the runqueue. 908 */ 909 it->it_need = 1; 910 thread_lock(td); 911 if (TD_AWAITING_INTR(td)) { 912 CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 913 td->td_name); 914 TD_CLR_IWAIT(td); 915 sched_add(td, SRQ_INTR); 916 } else { 917 CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 918 __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 919 } 920 thread_unlock(td); 921 922 return (0); 923} 924#endif 925 926/* 927 * Add a software interrupt handler to a specified event. If a given event 928 * is not specified, then a new event is created. 929 */ 930int 931swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, 932 void *arg, int pri, enum intr_type flags, void **cookiep) 933{ 934 struct intr_event *ie; 935 int error; 936 937 if (flags & INTR_ENTROPY) 938 return (EINVAL); 939 940 ie = (eventp != NULL) ? *eventp : NULL; 941 942 if (ie != NULL) { 943 if (!(ie->ie_flags & IE_SOFT)) 944 return (EINVAL); 945 } else { 946#ifdef INTR_FILTER 947 error = intr_event_create(&ie, NULL, IE_SOFT, 948 NULL, NULL, NULL, NULL, "swi%d:", pri); 949#else 950 error = intr_event_create(&ie, NULL, IE_SOFT, 951 NULL, NULL, "swi%d:", pri); 952#endif 953 if (error) 954 return (error); 955 if (eventp != NULL) 956 *eventp = ie; 957 } 958 return (intr_event_add_handler(ie, name, NULL, handler, arg, 959 (pri * RQ_PPQ) + PI_SOFT, flags, cookiep)); 960} 961 962/* 963 * Schedule a software interrupt thread. 964 */ 965void 966swi_sched(void *cookie, int flags) 967{ 968 struct intr_handler *ih = (struct intr_handler *)cookie; 969 struct intr_event *ie = ih->ih_event; 970 int error; 971 972 CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, 973 ih->ih_need); 974 975 /* 976 * Set ih_need for this handler so that if the ithread is already 977 * running it will execute this handler on the next pass. Otherwise, 978 * it will execute it the next time it runs. 979 */ 980 atomic_store_rel_int(&ih->ih_need, 1); 981 982 if (!(flags & SWI_DELAY)) { 983 PCPU_INC(cnt.v_soft); 984#ifdef INTR_FILTER 985 error = intr_event_schedule_thread(ie, ie->ie_thread); 986#else 987 error = intr_event_schedule_thread(ie); 988#endif 989 KASSERT(error == 0, ("stray software interrupt")); 990 } 991} 992 993/* 994 * Remove a software interrupt handler. Currently this code does not 995 * remove the associated interrupt event if it becomes empty. Calling code 996 * may do so manually via intr_event_destroy(), but that's not really 997 * an optimal interface. 998 */ 999int 1000swi_remove(void *cookie) 1001{ 1002 1003 return (intr_event_remove_handler(cookie)); 1004} 1005 1006#ifdef INTR_FILTER 1007static void 1008priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih) 1009{ 1010 struct intr_event *ie; 1011 1012 ie = ih->ih_event; 1013 /* 1014 * If this handler is marked for death, remove it from 1015 * the list of handlers and wake up the sleeper. 1016 */ 1017 if (ih->ih_flags & IH_DEAD) { 1018 mtx_lock(&ie->ie_lock); 1019 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1020 ih->ih_flags &= ~IH_DEAD; 1021 wakeup(ih); 1022 mtx_unlock(&ie->ie_lock); 1023 return; 1024 } 1025 1026 /* Execute this handler. */ 1027 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1028 __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, 1029 ih->ih_name, ih->ih_flags); 1030 1031 if (!(ih->ih_flags & IH_MPSAFE)) 1032 mtx_lock(&Giant); 1033 ih->ih_handler(ih->ih_argument); 1034 if (!(ih->ih_flags & IH_MPSAFE)) 1035 mtx_unlock(&Giant); 1036} 1037#endif 1038 1039static void 1040ithread_execute_handlers(struct proc *p, struct intr_event *ie) 1041{ 1042 struct intr_handler *ih, *ihn; 1043 1044 /* Interrupt handlers should not sleep. */ 1045 if (!(ie->ie_flags & IE_SOFT)) 1046 THREAD_NO_SLEEPING(); 1047 TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { 1048 1049 /* 1050 * If this handler is marked for death, remove it from 1051 * the list of handlers and wake up the sleeper. 1052 */ 1053 if (ih->ih_flags & IH_DEAD) { 1054 mtx_lock(&ie->ie_lock); 1055 TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1056 ih->ih_flags &= ~IH_DEAD; 1057 wakeup(ih); 1058 mtx_unlock(&ie->ie_lock); 1059 continue; 1060 } 1061 1062 /* Skip filter only handlers */ 1063 if (ih->ih_handler == NULL) 1064 continue; 1065 1066 /* 1067 * For software interrupt threads, we only execute 1068 * handlers that have their need flag set. Hardware 1069 * interrupt threads always invoke all of their handlers. 1070 */ 1071 if (ie->ie_flags & IE_SOFT) { 1072 if (!ih->ih_need) 1073 continue; 1074 else 1075 atomic_store_rel_int(&ih->ih_need, 0); 1076 } 1077 1078 /* Execute this handler. */ 1079 CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1080 __func__, p->p_pid, (void *)ih->ih_handler, 1081 ih->ih_argument, ih->ih_name, ih->ih_flags); 1082 1083 if (!(ih->ih_flags & IH_MPSAFE)) 1084 mtx_lock(&Giant); 1085 ih->ih_handler(ih->ih_argument); 1086 if (!(ih->ih_flags & IH_MPSAFE)) 1087 mtx_unlock(&Giant); 1088 } 1089 if (!(ie->ie_flags & IE_SOFT)) 1090 THREAD_SLEEPING_OK(); 1091 1092 /* 1093 * Interrupt storm handling: 1094 * 1095 * If this interrupt source is currently storming, then throttle 1096 * it to only fire the handler once per clock tick. 1097 * 1098 * If this interrupt source is not currently storming, but the 1099 * number of back to back interrupts exceeds the storm threshold, 1100 * then enter storming mode. 1101 */ 1102 if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold && 1103 !(ie->ie_flags & IE_SOFT)) { 1104 /* Report the message only once every second. */ 1105 if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) { 1106 printf( 1107 "interrupt storm detected on \"%s\"; throttling interrupt source\n", 1108 ie->ie_name); 1109 } 1110 pause("istorm", 1); 1111 } else 1112 ie->ie_count++; 1113 1114 /* 1115 * Now that all the handlers have had a chance to run, reenable 1116 * the interrupt source. 1117 */ 1118 if (ie->ie_enable != NULL) 1119 ie->ie_enable(ie->ie_source); 1120} 1121 1122#ifndef INTR_FILTER 1123/* 1124 * This is the main code for interrupt threads. 1125 */ 1126static void 1127ithread_loop(void *arg) 1128{ 1129 struct intr_thread *ithd; 1130 struct intr_event *ie; 1131 struct thread *td; 1132 struct proc *p; 1133 u_char cpu; 1134 1135 td = curthread; 1136 p = td->td_proc; 1137 ithd = (struct intr_thread *)arg; 1138 KASSERT(ithd->it_thread == td, 1139 ("%s: ithread and proc linkage out of sync", __func__)); 1140 ie = ithd->it_event; 1141 ie->ie_count = 0; 1142 cpu = NOCPU; 1143 1144 /* 1145 * As long as we have interrupts outstanding, go through the 1146 * list of handlers, giving each one a go at it. 1147 */ 1148 for (;;) { 1149 /* 1150 * If we are an orphaned thread, then just die. 1151 */ 1152 if (ithd->it_flags & IT_DEAD) { 1153 CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1154 p->p_pid, td->td_name); 1155 free(ithd, M_ITHREAD); 1156 kthread_exit(); 1157 } 1158 1159 /* 1160 * Service interrupts. If another interrupt arrives while 1161 * we are running, it will set it_need to note that we 1162 * should make another pass. 1163 */ 1164 while (ithd->it_need) { 1165 /* 1166 * This might need a full read and write barrier 1167 * to make sure that this write posts before any 1168 * of the memory or device accesses in the 1169 * handlers. 1170 */ 1171 atomic_store_rel_int(&ithd->it_need, 0); 1172 ithread_execute_handlers(p, ie); 1173 } 1174 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1175 mtx_assert(&Giant, MA_NOTOWNED); 1176 1177 /* 1178 * Processed all our interrupts. Now get the sched 1179 * lock. This may take a while and it_need may get 1180 * set again, so we have to check it again. 1181 */ 1182 thread_lock(td); 1183 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) { 1184 TD_SET_IWAIT(td); 1185 ie->ie_count = 0; 1186 mi_switch(SW_VOL, NULL); 1187 } 1188 1189#ifdef SMP 1190 /* 1191 * Ensure we are bound to the correct CPU. We can't 1192 * move ithreads until SMP is running however, so just 1193 * leave interrupts on the boor CPU during boot. 1194 */ 1195 if (ie->ie_cpu != cpu && smp_started) { 1196 cpu = ie->ie_cpu; 1197 if (cpu == NOCPU) 1198 sched_unbind(td); 1199 else 1200 sched_bind(td, cpu); 1201 } 1202#endif 1203 thread_unlock(td); 1204 } 1205} 1206#else 1207/* 1208 * This is the main code for interrupt threads. 1209 */ 1210static void 1211ithread_loop(void *arg) 1212{ 1213 struct intr_thread *ithd; 1214 struct intr_handler *ih; 1215 struct intr_event *ie; 1216 struct thread *td; 1217 struct proc *p; 1218 int priv; 1219 u_char cpu; 1220 1221 td = curthread; 1222 p = td->td_proc; 1223 ih = (struct intr_handler *)arg; 1224 priv = (ih->ih_thread != NULL) ? 1 : 0; 1225 ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread; 1226 KASSERT(ithd->it_thread == td, 1227 ("%s: ithread and proc linkage out of sync", __func__)); 1228 ie = ithd->it_event; 1229 ie->ie_count = 0; 1230 cpu = NOCPU; 1231 1232 /* 1233 * As long as we have interrupts outstanding, go through the 1234 * list of handlers, giving each one a go at it. 1235 */ 1236 for (;;) { 1237 /* 1238 * If we are an orphaned thread, then just die. 1239 */ 1240 if (ithd->it_flags & IT_DEAD) { 1241 CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1242 p->p_pid, td->td_name); 1243 free(ithd, M_ITHREAD); 1244 kthread_exit(); 1245 } 1246 1247 /* 1248 * Service interrupts. If another interrupt arrives while 1249 * we are running, it will set it_need to note that we 1250 * should make another pass. 1251 */ 1252 while (ithd->it_need) { 1253 /* 1254 * This might need a full read and write barrier 1255 * to make sure that this write posts before any 1256 * of the memory or device accesses in the 1257 * handlers. 1258 */ 1259 atomic_store_rel_int(&ithd->it_need, 0); 1260 if (priv) 1261 priv_ithread_execute_handler(p, ih); 1262 else 1263 ithread_execute_handlers(p, ie); 1264 } 1265 WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1266 mtx_assert(&Giant, MA_NOTOWNED); 1267 1268 /* 1269 * Processed all our interrupts. Now get the sched 1270 * lock. This may take a while and it_need may get 1271 * set again, so we have to check it again. 1272 */ 1273 thread_lock(td); 1274 if (!ithd->it_need && !(ithd->it_flags & IT_DEAD)) { 1275 TD_SET_IWAIT(td); 1276 ie->ie_count = 0; 1277 mi_switch(SW_VOL, NULL); 1278 } 1279 1280#ifdef SMP 1281 /* 1282 * Ensure we are bound to the correct CPU. We can't 1283 * move ithreads until SMP is running however, so just 1284 * leave interrupts on the boor CPU during boot. 1285 */ 1286 if (!priv && ie->ie_cpu != cpu && smp_started) { 1287 cpu = ie->ie_cpu; 1288 if (cpu == NOCPU) 1289 sched_unbind(td); 1290 else 1291 sched_bind(td, cpu); 1292 } 1293#endif 1294 thread_unlock(td); 1295 } 1296} 1297 1298/* 1299 * Main loop for interrupt filter. 1300 * 1301 * Some architectures (i386, amd64 and arm) require the optional frame 1302 * parameter, and use it as the main argument for fast handler execution 1303 * when ih_argument == NULL. 1304 * 1305 * Return value: 1306 * o FILTER_STRAY: No filter recognized the event, and no 1307 * filter-less handler is registered on this 1308 * line. 1309 * o FILTER_HANDLED: A filter claimed the event and served it. 1310 * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at 1311 * least one filter-less handler on this line. 1312 * o FILTER_HANDLED | 1313 * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for 1314 * scheduling the per-handler ithread. 1315 * 1316 * In case an ithread has to be scheduled, in *ithd there will be a 1317 * pointer to a struct intr_thread containing the thread to be 1318 * scheduled. 1319 */ 1320 1321int 1322intr_filter_loop(struct intr_event *ie, struct trapframe *frame, 1323 struct intr_thread **ithd) 1324{ 1325 struct intr_handler *ih; 1326 void *arg; 1327 int ret, thread_only; 1328 1329 ret = 0; 1330 thread_only = 0; 1331 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1332 /* 1333 * Execute fast interrupt handlers directly. 1334 * To support clock handlers, if a handler registers 1335 * with a NULL argument, then we pass it a pointer to 1336 * a trapframe as its argument. 1337 */ 1338 arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument); 1339 1340 CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__, 1341 ih->ih_filter, ih->ih_handler, arg, ih->ih_name); 1342 1343 if (ih->ih_filter != NULL) 1344 ret = ih->ih_filter(arg); 1345 else { 1346 thread_only = 1; 1347 continue; 1348 } 1349 1350 if (ret & FILTER_STRAY) 1351 continue; 1352 else { 1353 *ithd = ih->ih_thread; 1354 return (ret); 1355 } 1356 } 1357 1358 /* 1359 * No filters handled the interrupt and we have at least 1360 * one handler without a filter. In this case, we schedule 1361 * all of the filter-less handlers to run in the ithread. 1362 */ 1363 if (thread_only) { 1364 *ithd = ie->ie_thread; 1365 return (FILTER_SCHEDULE_THREAD); 1366 } 1367 return (FILTER_STRAY); 1368} 1369 1370/* 1371 * Main interrupt handling body. 1372 * 1373 * Input: 1374 * o ie: the event connected to this interrupt. 1375 * o frame: some archs (i.e. i386) pass a frame to some. 1376 * handlers as their main argument. 1377 * Return value: 1378 * o 0: everything ok. 1379 * o EINVAL: stray interrupt. 1380 */ 1381int 1382intr_event_handle(struct intr_event *ie, struct trapframe *frame) 1383{ 1384 struct intr_thread *ithd; 1385 struct thread *td; 1386 int thread; 1387 1388 ithd = NULL; 1389 td = curthread; 1390 1391 if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1392 return (EINVAL); 1393 1394 td->td_intr_nesting_level++; 1395 thread = 0; 1396 critical_enter(); 1397 thread = intr_filter_loop(ie, frame, &ithd); 1398 1399 /* 1400 * If the interrupt was fully served, send it an EOI but leave 1401 * it unmasked. Otherwise, mask the source as well as sending 1402 * it an EOI. 1403 */ 1404 if (thread & FILTER_HANDLED) { 1405 if (ie->ie_eoi != NULL) 1406 ie->ie_eoi(ie->ie_source); 1407 } else { 1408 if (ie->ie_disab != NULL) 1409 ie->ie_disab(ie->ie_source); 1410 } 1411 critical_exit(); 1412 1413 /* Interrupt storm logic */ 1414 if (thread & FILTER_STRAY) { 1415 ie->ie_count++; 1416 if (ie->ie_count < intr_storm_threshold) 1417 printf("Interrupt stray detection not present\n"); 1418 } 1419 1420 /* Schedule an ithread if needed. */ 1421 if (thread & FILTER_SCHEDULE_THREAD) { 1422 if (intr_event_schedule_thread(ie, ithd) != 0) 1423 panic("%s: impossible stray interrupt", __func__); 1424 } 1425 td->td_intr_nesting_level--; 1426 return (0); 1427} 1428#endif 1429 1430#ifdef DDB 1431/* 1432 * Dump details about an interrupt handler 1433 */ 1434static void 1435db_dump_intrhand(struct intr_handler *ih) 1436{ 1437 int comma; 1438 1439 db_printf("\t%-10s ", ih->ih_name); 1440 switch (ih->ih_pri) { 1441 case PI_REALTIME: 1442 db_printf("CLK "); 1443 break; 1444 case PI_AV: 1445 db_printf("AV "); 1446 break; 1447 case PI_TTYHIGH: 1448 case PI_TTYLOW: 1449 db_printf("TTY "); 1450 break; 1451 case PI_TAPE: 1452 db_printf("TAPE"); 1453 break; 1454 case PI_NET: 1455 db_printf("NET "); 1456 break; 1457 case PI_DISK: 1458 case PI_DISKLOW: 1459 db_printf("DISK"); 1460 break; 1461 case PI_DULL: 1462 db_printf("DULL"); 1463 break; 1464 default: 1465 if (ih->ih_pri >= PI_SOFT) 1466 db_printf("SWI "); 1467 else 1468 db_printf("%4u", ih->ih_pri); 1469 break; 1470 } 1471 db_printf(" "); 1472 db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); 1473 db_printf("(%p)", ih->ih_argument); 1474 if (ih->ih_need || 1475 (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | 1476 IH_MPSAFE)) != 0) { 1477 db_printf(" {"); 1478 comma = 0; 1479 if (ih->ih_flags & IH_EXCLUSIVE) { 1480 if (comma) 1481 db_printf(", "); 1482 db_printf("EXCL"); 1483 comma = 1; 1484 } 1485 if (ih->ih_flags & IH_ENTROPY) { 1486 if (comma) 1487 db_printf(", "); 1488 db_printf("ENTROPY"); 1489 comma = 1; 1490 } 1491 if (ih->ih_flags & IH_DEAD) { 1492 if (comma) 1493 db_printf(", "); 1494 db_printf("DEAD"); 1495 comma = 1; 1496 } 1497 if (ih->ih_flags & IH_MPSAFE) { 1498 if (comma) 1499 db_printf(", "); 1500 db_printf("MPSAFE"); 1501 comma = 1; 1502 } 1503 if (ih->ih_need) { 1504 if (comma) 1505 db_printf(", "); 1506 db_printf("NEED"); 1507 } 1508 db_printf("}"); 1509 } 1510 db_printf("\n"); 1511} 1512 1513/* 1514 * Dump details about a event. 1515 */ 1516void 1517db_dump_intr_event(struct intr_event *ie, int handlers) 1518{ 1519 struct intr_handler *ih; 1520 struct intr_thread *it; 1521 int comma; 1522 1523 db_printf("%s ", ie->ie_fullname); 1524 it = ie->ie_thread; 1525 if (it != NULL) 1526 db_printf("(pid %d)", it->it_thread->td_proc->p_pid); 1527 else 1528 db_printf("(no thread)"); 1529 if (ie->ie_cpu != NOCPU) 1530 db_printf(" (CPU %d)", ie->ie_cpu); 1531 if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || 1532 (it != NULL && it->it_need)) { 1533 db_printf(" {"); 1534 comma = 0; 1535 if (ie->ie_flags & IE_SOFT) { 1536 db_printf("SOFT"); 1537 comma = 1; 1538 } 1539 if (ie->ie_flags & IE_ENTROPY) { 1540 if (comma) 1541 db_printf(", "); 1542 db_printf("ENTROPY"); 1543 comma = 1; 1544 } 1545 if (ie->ie_flags & IE_ADDING_THREAD) { 1546 if (comma) 1547 db_printf(", "); 1548 db_printf("ADDING_THREAD"); 1549 comma = 1; 1550 } 1551 if (it != NULL && it->it_need) { 1552 if (comma) 1553 db_printf(", "); 1554 db_printf("NEED"); 1555 } 1556 db_printf("}"); 1557 } 1558 db_printf("\n"); 1559 1560 if (handlers) 1561 TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 1562 db_dump_intrhand(ih); 1563} 1564 1565/* 1566 * Dump data about interrupt handlers 1567 */ 1568DB_SHOW_COMMAND(intr, db_show_intr) 1569{ 1570 struct intr_event *ie; 1571 int all, verbose; 1572 1573 verbose = index(modif, 'v') != NULL; 1574 all = index(modif, 'a') != NULL; 1575 TAILQ_FOREACH(ie, &event_list, ie_list) { 1576 if (!all && TAILQ_EMPTY(&ie->ie_handlers)) 1577 continue; 1578 db_dump_intr_event(ie, verbose); 1579 if (db_pager_quit) 1580 break; 1581 } 1582} 1583#endif /* DDB */ 1584 1585/* 1586 * Start standard software interrupt threads 1587 */ 1588static void 1589start_softintr(void *dummy) 1590{ 1591 struct proc *p; 1592 1593 if (swi_add(&clk_intr_event, "clock", softclock, NULL, SWI_CLOCK, 1594 INTR_MPSAFE, &softclock_ih) || 1595 swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) 1596 panic("died while creating standard software ithreads"); 1597 1598 p = clk_intr_event->ie_thread->it_thread->td_proc; 1599 PROC_LOCK(p); 1600 p->p_flag |= P_NOLOAD; 1601 PROC_UNLOCK(p); 1602} 1603SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, 1604 NULL); 1605 1606/* 1607 * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. 1608 * The data for this machine dependent, and the declarations are in machine 1609 * dependent code. The layout of intrnames and intrcnt however is machine 1610 * independent. 1611 * 1612 * We do not know the length of intrcnt and intrnames at compile time, so 1613 * calculate things at run time. 1614 */ 1615static int 1616sysctl_intrnames(SYSCTL_HANDLER_ARGS) 1617{ 1618 return (sysctl_handle_opaque(oidp, intrnames, eintrnames - intrnames, 1619 req)); 1620} 1621 1622SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, 1623 NULL, 0, sysctl_intrnames, "", "Interrupt Names"); 1624 1625static int 1626sysctl_intrcnt(SYSCTL_HANDLER_ARGS) 1627{ 1628 return (sysctl_handle_opaque(oidp, intrcnt, 1629 (char *)eintrcnt - (char *)intrcnt, req)); 1630} 1631 1632SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, 1633 NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); 1634 1635#ifdef DDB 1636/* 1637 * DDB command to dump the interrupt statistics. 1638 */ 1639DB_SHOW_COMMAND(intrcnt, db_show_intrcnt) 1640{ 1641 u_long *i; 1642 char *cp; 1643 1644 cp = intrnames; 1645 for (i = intrcnt; i != eintrcnt && !db_pager_quit; i++) { 1646 if (*cp == '\0') 1647 break; 1648 if (*i != 0) 1649 db_printf("%s\t%lu\n", cp, *i); 1650 cp += strlen(cp) + 1; 1651 } 1652} 1653#endif 1654