kern_intr.c revision 219819
1139804Simp/*- 226156Sse * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 326156Sse * All rights reserved. 426156Sse * 526156Sse * Redistribution and use in source and binary forms, with or without 626156Sse * modification, are permitted provided that the following conditions 726156Sse * are met: 826156Sse * 1. Redistributions of source code must retain the above copyright 926156Sse * notice unmodified, this list of conditions, and the following 1026156Sse * disclaimer. 1126156Sse * 2. Redistributions in binary form must reproduce the above copyright 1226156Sse * notice, this list of conditions and the following disclaimer in the 1326156Sse * documentation and/or other materials provided with the distribution. 1426156Sse * 1526156Sse * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1626156Sse * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1726156Sse * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 1826156Sse * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 1926156Sse * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2026156Sse * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2126156Sse * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2226156Sse * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2326156Sse * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2426156Sse * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2526156Sse */ 2626156Sse 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_intr.c 219819 2011-03-21 09:40:01Z jeff $"); 2936887Sdfr 30121482Sjhb#include "opt_ddb.h" 31121482Sjhb 3241059Speter#include <sys/param.h> 3365822Sjhb#include <sys/bus.h> 34110860Salfred#include <sys/conf.h> 35178092Sjeff#include <sys/cpuset.h> 3665822Sjhb#include <sys/rtprio.h> 3741059Speter#include <sys/systm.h> 3866698Sjhb#include <sys/interrupt.h> 3966698Sjhb#include <sys/kernel.h> 4066698Sjhb#include <sys/kthread.h> 4166698Sjhb#include <sys/ktr.h> 42130128Sbde#include <sys/limits.h> 4374914Sjhb#include <sys/lock.h> 4426156Sse#include <sys/malloc.h> 4567365Sjhb#include <sys/mutex.h> 46195249Sjhb#include <sys/priv.h> 4766698Sjhb#include <sys/proc.h> 4872759Sjhb#include <sys/random.h> 4972237Sjhb#include <sys/resourcevar.h> 50139451Sjhb#include <sys/sched.h> 51177181Sjhb#include <sys/smp.h> 5277582Stmm#include <sys/sysctl.h> 53182024Skmacy#include <sys/syslog.h> 5466698Sjhb#include <sys/unistd.h> 5566698Sjhb#include <sys/vmmeter.h> 5666698Sjhb#include <machine/atomic.h> 5766698Sjhb#include <machine/cpu.h> 5867551Sjhb#include <machine/md_var.h> 5972237Sjhb#include <machine/stdarg.h> 60121482Sjhb#ifdef DDB 61121482Sjhb#include <ddb/ddb.h> 62121482Sjhb#include <ddb/db_sym.h> 63121482Sjhb#endif 6426156Sse 65151658Sjhb/* 66151658Sjhb * Describe an interrupt thread. There is one of these per interrupt event. 67151658Sjhb */ 68151658Sjhbstruct intr_thread { 69151658Sjhb struct intr_event *it_event; 70151658Sjhb struct thread *it_thread; /* Kernel thread. */ 71151658Sjhb int it_flags; /* (j) IT_* flags. */ 72151658Sjhb int it_need; /* Needs service. */ 7372759Sjhb}; 7472759Sjhb 75151658Sjhb/* Interrupt thread flags kept in it_flags */ 76151658Sjhb#define IT_DEAD 0x000001 /* Thread is waiting to exit. */ 77219819Sjeff#define IT_WAIT 0x000002 /* Thread is waiting for completion. */ 78151658Sjhb 79151658Sjhbstruct intr_entropy { 80151658Sjhb struct thread *td; 81151658Sjhb uintptr_t event; 82151658Sjhb}; 83151658Sjhb 84151658Sjhbstruct intr_event *clk_intr_event; 85151658Sjhbstruct intr_event *tty_intr_event; 86128339Sbdevoid *vm_ih; 87173004Sjulianstruct proc *intrproc; 8838244Sbde 8972237Sjhbstatic MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 9072237Sjhb 91168850Snjlstatic int intr_storm_threshold = 1000; 92128331SjhbTUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold); 93128331SjhbSYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW, 94128331Sjhb &intr_storm_threshold, 0, 95128339Sbde "Number of consecutive interrupts before storm protection is enabled"); 96151658Sjhbstatic TAILQ_HEAD(, intr_event) event_list = 97151658Sjhb TAILQ_HEAD_INITIALIZER(event_list); 98178092Sjeffstatic struct mtx event_lock; 99178092SjeffMTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); 100128331Sjhb 101151658Sjhbstatic void intr_event_update(struct intr_event *ie); 102169320Spiso#ifdef INTR_FILTER 103177940Sjhbstatic int intr_event_schedule_thread(struct intr_event *ie, 104177940Sjhb struct intr_thread *ithd); 105177940Sjhbstatic int intr_filter_loop(struct intr_event *ie, 106177940Sjhb struct trapframe *frame, struct intr_thread **ithd); 107169320Spisostatic struct intr_thread *ithread_create(const char *name, 108169320Spiso struct intr_handler *ih); 109169320Spiso#else 110177940Sjhbstatic int intr_event_schedule_thread(struct intr_event *ie); 111151658Sjhbstatic struct intr_thread *ithread_create(const char *name); 112169320Spiso#endif 113151658Sjhbstatic void ithread_destroy(struct intr_thread *ithread); 114169320Spisostatic void ithread_execute_handlers(struct proc *p, 115169320Spiso struct intr_event *ie); 116169320Spiso#ifdef INTR_FILTER 117169320Spisostatic void priv_ithread_execute_handler(struct proc *p, 118169320Spiso struct intr_handler *ih); 119169320Spiso#endif 120128339Sbdestatic void ithread_loop(void *); 121151658Sjhbstatic void ithread_update(struct intr_thread *ithd); 122128339Sbdestatic void start_softintr(void *); 123128339Sbde 124165124Sjhb/* Map an interrupt type to an ithread priority. */ 12572237Sjhbu_char 126151658Sjhbintr_priority(enum intr_type flags) 12765822Sjhb{ 12872237Sjhb u_char pri; 12965822Sjhb 13072237Sjhb flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 13178365Speter INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); 13265822Sjhb switch (flags) { 13372237Sjhb case INTR_TYPE_TTY: 134217292Sjhb pri = PI_TTY; 13565822Sjhb break; 13665822Sjhb case INTR_TYPE_BIO: 13765822Sjhb pri = PI_DISK; 13865822Sjhb break; 13965822Sjhb case INTR_TYPE_NET: 14065822Sjhb pri = PI_NET; 14165822Sjhb break; 14265822Sjhb case INTR_TYPE_CAM: 143217292Sjhb pri = PI_DISK; 14465822Sjhb break; 145217292Sjhb case INTR_TYPE_AV: 14678365Speter pri = PI_AV; 14778365Speter break; 14872237Sjhb case INTR_TYPE_CLK: 14972237Sjhb pri = PI_REALTIME; 15072237Sjhb break; 15165822Sjhb case INTR_TYPE_MISC: 15265822Sjhb pri = PI_DULL; /* don't care */ 15365822Sjhb break; 15465822Sjhb default: 15572237Sjhb /* We didn't specify an interrupt level. */ 156151658Sjhb panic("intr_priority: no interrupt type in flags"); 15765822Sjhb } 15865822Sjhb 15965822Sjhb return pri; 16065822Sjhb} 16165822Sjhb 16272237Sjhb/* 163151658Sjhb * Update an ithread based on the associated intr_event. 16472237Sjhb */ 16572237Sjhbstatic void 166151658Sjhbithread_update(struct intr_thread *ithd) 16772237Sjhb{ 168151658Sjhb struct intr_event *ie; 16983366Sjulian struct thread *td; 170151658Sjhb u_char pri; 17167551Sjhb 172151658Sjhb ie = ithd->it_event; 173151658Sjhb td = ithd->it_thread; 17472237Sjhb 175151658Sjhb /* Determine the overall priority of this event. */ 176151658Sjhb if (TAILQ_EMPTY(&ie->ie_handlers)) 177151658Sjhb pri = PRI_MAX_ITHD; 178151658Sjhb else 179151658Sjhb pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri; 180105354Srobert 181151658Sjhb /* Update name and priority. */ 182173004Sjulian strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name)); 183170307Sjeff thread_lock(td); 184151658Sjhb sched_prio(td, pri); 185170307Sjeff thread_unlock(td); 186151658Sjhb} 187151658Sjhb 188151658Sjhb/* 189151658Sjhb * Regenerate the full name of an interrupt event and update its priority. 190151658Sjhb */ 191151658Sjhbstatic void 192151658Sjhbintr_event_update(struct intr_event *ie) 193151658Sjhb{ 194151658Sjhb struct intr_handler *ih; 195151658Sjhb char *last; 196151658Sjhb int missed, space; 197151658Sjhb 198151658Sjhb /* Start off with no entropy and just the name of the event. */ 199151658Sjhb mtx_assert(&ie->ie_lock, MA_OWNED); 200151658Sjhb strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 201151658Sjhb ie->ie_flags &= ~IE_ENTROPY; 202137267Sjhb missed = 0; 203151658Sjhb space = 1; 204151658Sjhb 205151658Sjhb /* Run through all the handlers updating values. */ 206151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 207151658Sjhb if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 < 208151658Sjhb sizeof(ie->ie_fullname)) { 209151658Sjhb strcat(ie->ie_fullname, " "); 210151658Sjhb strcat(ie->ie_fullname, ih->ih_name); 211151658Sjhb space = 0; 212137267Sjhb } else 213137267Sjhb missed++; 214137267Sjhb if (ih->ih_flags & IH_ENTROPY) 215151658Sjhb ie->ie_flags |= IE_ENTROPY; 216137267Sjhb } 217151658Sjhb 218151658Sjhb /* 219151658Sjhb * If the handler names were too long, add +'s to indicate missing 220151658Sjhb * names. If we run out of room and still have +'s to add, change 221151658Sjhb * the last character from a + to a *. 222151658Sjhb */ 223151658Sjhb last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2]; 224137267Sjhb while (missed-- > 0) { 225151658Sjhb if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) { 226151658Sjhb if (*last == '+') { 227151658Sjhb *last = '*'; 228151658Sjhb break; 229151658Sjhb } else 230151658Sjhb *last = '+'; 231151658Sjhb } else if (space) { 232151658Sjhb strcat(ie->ie_fullname, " +"); 233151658Sjhb space = 0; 23472237Sjhb } else 235151658Sjhb strcat(ie->ie_fullname, "+"); 23672237Sjhb } 237151658Sjhb 238151658Sjhb /* 239151658Sjhb * If this event has an ithread, update it's priority and 240151658Sjhb * name. 241151658Sjhb */ 242151658Sjhb if (ie->ie_thread != NULL) 243151658Sjhb ithread_update(ie->ie_thread); 244151658Sjhb CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname); 24572237Sjhb} 24672237Sjhb 24772237Sjhbint 248183298Sobrienintr_event_create(struct intr_event **event, void *source, int flags, int irq, 249177940Sjhb void (*pre_ithread)(void *), void (*post_ithread)(void *), 250177940Sjhb void (*post_filter)(void *), int (*assign_cpu)(void *, u_char), 251177940Sjhb const char *fmt, ...) 252169320Spiso{ 253169320Spiso struct intr_event *ie; 254169320Spiso va_list ap; 25572237Sjhb 256169320Spiso /* The only valid flag during creation is IE_SOFT. */ 257169320Spiso if ((flags & ~IE_SOFT) != 0) 258169320Spiso return (EINVAL); 259169320Spiso ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); 260169320Spiso ie->ie_source = source; 261177940Sjhb ie->ie_pre_ithread = pre_ithread; 262177940Sjhb ie->ie_post_ithread = post_ithread; 263177940Sjhb ie->ie_post_filter = post_filter; 264177181Sjhb ie->ie_assign_cpu = assign_cpu; 265169320Spiso ie->ie_flags = flags; 266178092Sjeff ie->ie_irq = irq; 267177181Sjhb ie->ie_cpu = NOCPU; 268169320Spiso TAILQ_INIT(&ie->ie_handlers); 269169320Spiso mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); 270169320Spiso 271169320Spiso va_start(ap, fmt); 272169320Spiso vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); 273169320Spiso va_end(ap); 274169320Spiso strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 275178092Sjeff mtx_lock(&event_lock); 276169320Spiso TAILQ_INSERT_TAIL(&event_list, ie, ie_list); 277178092Sjeff mtx_unlock(&event_lock); 278169320Spiso if (event != NULL) 279169320Spiso *event = ie; 280169320Spiso CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); 281169320Spiso return (0); 282169320Spiso} 283169320Spiso 284177181Sjhb/* 285177181Sjhb * Bind an interrupt event to the specified CPU. Note that not all 286177181Sjhb * platforms support binding an interrupt to a CPU. For those 287177181Sjhb * platforms this request will fail. For supported platforms, any 288177181Sjhb * associated ithreads as well as the primary interrupt context will 289177181Sjhb * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds 290177181Sjhb * the interrupt event. 291177181Sjhb */ 292151658Sjhbint 293177181Sjhbintr_event_bind(struct intr_event *ie, u_char cpu) 294177181Sjhb{ 295178092Sjeff cpuset_t mask; 296178092Sjeff lwpid_t id; 297177181Sjhb int error; 298177181Sjhb 299177181Sjhb /* Need a CPU to bind to. */ 300177181Sjhb if (cpu != NOCPU && CPU_ABSENT(cpu)) 301177181Sjhb return (EINVAL); 302177181Sjhb 303177181Sjhb if (ie->ie_assign_cpu == NULL) 304177181Sjhb return (EOPNOTSUPP); 305195249Sjhb 306195249Sjhb error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR); 307195249Sjhb if (error) 308195249Sjhb return (error); 309195249Sjhb 310178092Sjeff /* 311195249Sjhb * If we have any ithreads try to set their mask first to verify 312195249Sjhb * permissions, etc. 313178092Sjeff */ 314177181Sjhb mtx_lock(&ie->ie_lock); 315178092Sjeff if (ie->ie_thread != NULL) { 316178092Sjeff CPU_ZERO(&mask); 317178092Sjeff if (cpu == NOCPU) 318178092Sjeff CPU_COPY(cpuset_root, &mask); 319178092Sjeff else 320178092Sjeff CPU_SET(cpu, &mask); 321178092Sjeff id = ie->ie_thread->it_thread->td_tid; 322177181Sjhb mtx_unlock(&ie->ie_lock); 323178092Sjeff error = cpuset_setthread(id, &mask); 324178092Sjeff if (error) 325178092Sjeff return (error); 326178092Sjeff } else 327178092Sjeff mtx_unlock(&ie->ie_lock); 328177181Sjhb error = ie->ie_assign_cpu(ie->ie_source, cpu); 329195249Sjhb if (error) { 330195249Sjhb mtx_lock(&ie->ie_lock); 331195249Sjhb if (ie->ie_thread != NULL) { 332195249Sjhb CPU_ZERO(&mask); 333195249Sjhb if (ie->ie_cpu == NOCPU) 334195249Sjhb CPU_COPY(cpuset_root, &mask); 335195249Sjhb else 336195249Sjhb CPU_SET(cpu, &mask); 337195249Sjhb id = ie->ie_thread->it_thread->td_tid; 338195249Sjhb mtx_unlock(&ie->ie_lock); 339195249Sjhb (void)cpuset_setthread(id, &mask); 340195249Sjhb } else 341195249Sjhb mtx_unlock(&ie->ie_lock); 342177181Sjhb return (error); 343195249Sjhb } 344195249Sjhb 345177181Sjhb mtx_lock(&ie->ie_lock); 346177181Sjhb ie->ie_cpu = cpu; 347177181Sjhb mtx_unlock(&ie->ie_lock); 348178092Sjeff 349178092Sjeff return (error); 350178092Sjeff} 351178092Sjeff 352178092Sjeffstatic struct intr_event * 353178092Sjeffintr_lookup(int irq) 354178092Sjeff{ 355178092Sjeff struct intr_event *ie; 356178092Sjeff 357178092Sjeff mtx_lock(&event_lock); 358178092Sjeff TAILQ_FOREACH(ie, &event_list, ie_list) 359178092Sjeff if (ie->ie_irq == irq && 360178092Sjeff (ie->ie_flags & IE_SOFT) == 0 && 361178092Sjeff TAILQ_FIRST(&ie->ie_handlers) != NULL) 362178092Sjeff break; 363178092Sjeff mtx_unlock(&event_lock); 364178092Sjeff return (ie); 365178092Sjeff} 366178092Sjeff 367178092Sjeffint 368178092Sjeffintr_setaffinity(int irq, void *m) 369178092Sjeff{ 370178092Sjeff struct intr_event *ie; 371178092Sjeff cpuset_t *mask; 372178092Sjeff u_char cpu; 373178092Sjeff int n; 374178092Sjeff 375178092Sjeff mask = m; 376178092Sjeff cpu = NOCPU; 377178092Sjeff /* 378178092Sjeff * If we're setting all cpus we can unbind. Otherwise make sure 379178092Sjeff * only one cpu is in the set. 380178092Sjeff */ 381178092Sjeff if (CPU_CMP(cpuset_root, mask)) { 382178092Sjeff for (n = 0; n < CPU_SETSIZE; n++) { 383178092Sjeff if (!CPU_ISSET(n, mask)) 384178092Sjeff continue; 385178092Sjeff if (cpu != NOCPU) 386178092Sjeff return (EINVAL); 387178092Sjeff cpu = (u_char)n; 388178092Sjeff } 389178092Sjeff } 390178092Sjeff ie = intr_lookup(irq); 391178092Sjeff if (ie == NULL) 392178092Sjeff return (ESRCH); 393194987Sjhb return (intr_event_bind(ie, cpu)); 394178092Sjeff} 395178092Sjeff 396178092Sjeffint 397178092Sjeffintr_getaffinity(int irq, void *m) 398178092Sjeff{ 399178092Sjeff struct intr_event *ie; 400178092Sjeff cpuset_t *mask; 401178092Sjeff 402178092Sjeff mask = m; 403178092Sjeff ie = intr_lookup(irq); 404178092Sjeff if (ie == NULL) 405178092Sjeff return (ESRCH); 406178092Sjeff CPU_ZERO(mask); 407178092Sjeff mtx_lock(&ie->ie_lock); 408178092Sjeff if (ie->ie_cpu == NOCPU) 409178092Sjeff CPU_COPY(cpuset_root, mask); 410178092Sjeff else 411178092Sjeff CPU_SET(ie->ie_cpu, mask); 412178092Sjeff mtx_unlock(&ie->ie_lock); 413177181Sjhb return (0); 414177181Sjhb} 415177181Sjhb 416177181Sjhbint 417151658Sjhbintr_event_destroy(struct intr_event *ie) 418151658Sjhb{ 419151658Sjhb 420178092Sjeff mtx_lock(&event_lock); 421151658Sjhb mtx_lock(&ie->ie_lock); 422151658Sjhb if (!TAILQ_EMPTY(&ie->ie_handlers)) { 423151658Sjhb mtx_unlock(&ie->ie_lock); 424178092Sjeff mtx_unlock(&event_lock); 425151658Sjhb return (EBUSY); 426151658Sjhb } 427151658Sjhb TAILQ_REMOVE(&event_list, ie, ie_list); 428157728Sjhb#ifndef notyet 429157728Sjhb if (ie->ie_thread != NULL) { 430157728Sjhb ithread_destroy(ie->ie_thread); 431157728Sjhb ie->ie_thread = NULL; 432157728Sjhb } 433157728Sjhb#endif 434151658Sjhb mtx_unlock(&ie->ie_lock); 435178092Sjeff mtx_unlock(&event_lock); 436151658Sjhb mtx_destroy(&ie->ie_lock); 437151658Sjhb free(ie, M_ITHREAD); 438151658Sjhb return (0); 439151658Sjhb} 440151658Sjhb 441169320Spiso#ifndef INTR_FILTER 442151658Sjhbstatic struct intr_thread * 443151658Sjhbithread_create(const char *name) 444151658Sjhb{ 445151658Sjhb struct intr_thread *ithd; 446151658Sjhb struct thread *td; 447151658Sjhb int error; 448151658Sjhb 449151658Sjhb ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 450151658Sjhb 451173004Sjulian error = kproc_kthread_add(ithread_loop, ithd, &intrproc, 452173004Sjulian &td, RFSTOPPED | RFHIGHPID, 453173051Sjulian 0, "intr", "%s", name); 454151658Sjhb if (error) 455172836Sjulian panic("kproc_create() failed with %d", error); 456170307Sjeff thread_lock(td); 457164936Sjulian sched_class(td, PRI_ITHD); 458103216Sjulian TD_SET_IWAIT(td); 459170307Sjeff thread_unlock(td); 460151658Sjhb td->td_pflags |= TDP_ITHREAD; 461151658Sjhb ithd->it_thread = td; 462151658Sjhb CTR2(KTR_INTR, "%s: created %s", __func__, name); 463151658Sjhb return (ithd); 46472237Sjhb} 465169320Spiso#else 466169320Spisostatic struct intr_thread * 467169320Spisoithread_create(const char *name, struct intr_handler *ih) 468169320Spiso{ 469169320Spiso struct intr_thread *ithd; 470169320Spiso struct thread *td; 471169320Spiso int error; 47272237Sjhb 473169320Spiso ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 474169320Spiso 475173153Sjulian error = kproc_kthread_add(ithread_loop, ih, &intrproc, 476173004Sjulian &td, RFSTOPPED | RFHIGHPID, 477173051Sjulian 0, "intr", "%s", name); 478169320Spiso if (error) 479172836Sjulian panic("kproc_create() failed with %d", error); 480170307Sjeff thread_lock(td); 481169320Spiso sched_class(td, PRI_ITHD); 482169320Spiso TD_SET_IWAIT(td); 483170307Sjeff thread_unlock(td); 484169320Spiso td->td_pflags |= TDP_ITHREAD; 485169320Spiso ithd->it_thread = td; 486169320Spiso CTR2(KTR_INTR, "%s: created %s", __func__, name); 487169320Spiso return (ithd); 488169320Spiso} 489169320Spiso#endif 490169320Spiso 491151658Sjhbstatic void 492151658Sjhbithread_destroy(struct intr_thread *ithread) 49372237Sjhb{ 49483366Sjulian struct thread *td; 49572237Sjhb 496157784Sscottl CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name); 497151658Sjhb td = ithread->it_thread; 498170307Sjeff thread_lock(td); 49976771Sjhb ithread->it_flags |= IT_DEAD; 500103216Sjulian if (TD_AWAITING_INTR(td)) { 501103216Sjulian TD_CLR_IWAIT(td); 502166188Sjeff sched_add(td, SRQ_INTR); 50372237Sjhb } 504170307Sjeff thread_unlock(td); 50572237Sjhb} 50672237Sjhb 507169320Spiso#ifndef INTR_FILTER 50872237Sjhbint 509151658Sjhbintr_event_add_handler(struct intr_event *ie, const char *name, 510166901Spiso driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 511166901Spiso enum intr_type flags, void **cookiep) 51272237Sjhb{ 513151658Sjhb struct intr_handler *ih, *temp_ih; 514151658Sjhb struct intr_thread *it; 51572237Sjhb 516166901Spiso if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 51772237Sjhb return (EINVAL); 51872237Sjhb 519151658Sjhb /* Allocate and populate an interrupt handler structure. */ 520151658Sjhb ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 521166901Spiso ih->ih_filter = filter; 52272237Sjhb ih->ih_handler = handler; 52372237Sjhb ih->ih_argument = arg; 524198134Sjhb strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 525151658Sjhb ih->ih_event = ie; 52672237Sjhb ih->ih_pri = pri; 527166901Spiso if (flags & INTR_EXCL) 52872237Sjhb ih->ih_flags = IH_EXCLUSIVE; 52972237Sjhb if (flags & INTR_MPSAFE) 53072237Sjhb ih->ih_flags |= IH_MPSAFE; 53172237Sjhb if (flags & INTR_ENTROPY) 53272237Sjhb ih->ih_flags |= IH_ENTROPY; 53372237Sjhb 534151658Sjhb /* We can only have one exclusive handler in a event. */ 535151658Sjhb mtx_lock(&ie->ie_lock); 536151658Sjhb if (!TAILQ_EMPTY(&ie->ie_handlers)) { 537151658Sjhb if ((flags & INTR_EXCL) || 538151658Sjhb (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 539151658Sjhb mtx_unlock(&ie->ie_lock); 540151658Sjhb free(ih, M_ITHREAD); 541151658Sjhb return (EINVAL); 542151658Sjhb } 543122002Sjhb } 54472237Sjhb 545151658Sjhb /* Add the new handler to the event in priority order. */ 546151658Sjhb TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 547151658Sjhb if (temp_ih->ih_pri > ih->ih_pri) 548151658Sjhb break; 549151658Sjhb } 55072237Sjhb if (temp_ih == NULL) 551151658Sjhb TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 55272237Sjhb else 55372237Sjhb TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 554151658Sjhb intr_event_update(ie); 55572237Sjhb 556151658Sjhb /* Create a thread if we need one. */ 557166901Spiso while (ie->ie_thread == NULL && handler != NULL) { 558151658Sjhb if (ie->ie_flags & IE_ADDING_THREAD) 559157815Sjhb msleep(ie, &ie->ie_lock, 0, "ithread", 0); 560151658Sjhb else { 561151658Sjhb ie->ie_flags |= IE_ADDING_THREAD; 562151658Sjhb mtx_unlock(&ie->ie_lock); 563151658Sjhb it = ithread_create("intr: newborn"); 564151658Sjhb mtx_lock(&ie->ie_lock); 565151658Sjhb ie->ie_flags &= ~IE_ADDING_THREAD; 566151658Sjhb ie->ie_thread = it; 567151658Sjhb it->it_event = ie; 568151658Sjhb ithread_update(it); 569151658Sjhb wakeup(ie); 570151658Sjhb } 571151658Sjhb } 572151658Sjhb CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 573151658Sjhb ie->ie_name); 574151658Sjhb mtx_unlock(&ie->ie_lock); 575151658Sjhb 57672237Sjhb if (cookiep != NULL) 57772237Sjhb *cookiep = ih; 57872237Sjhb return (0); 57972237Sjhb} 580169320Spiso#else 581169320Spisoint 582169320Spisointr_event_add_handler(struct intr_event *ie, const char *name, 583169320Spiso driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 584169320Spiso enum intr_type flags, void **cookiep) 585169320Spiso{ 586169320Spiso struct intr_handler *ih, *temp_ih; 587169320Spiso struct intr_thread *it; 58872237Sjhb 589169320Spiso if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 590169320Spiso return (EINVAL); 591169320Spiso 592169320Spiso /* Allocate and populate an interrupt handler structure. */ 593169320Spiso ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 594169320Spiso ih->ih_filter = filter; 595169320Spiso ih->ih_handler = handler; 596169320Spiso ih->ih_argument = arg; 597198134Sjhb strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 598169320Spiso ih->ih_event = ie; 599169320Spiso ih->ih_pri = pri; 600169320Spiso if (flags & INTR_EXCL) 601169320Spiso ih->ih_flags = IH_EXCLUSIVE; 602169320Spiso if (flags & INTR_MPSAFE) 603169320Spiso ih->ih_flags |= IH_MPSAFE; 604169320Spiso if (flags & INTR_ENTROPY) 605169320Spiso ih->ih_flags |= IH_ENTROPY; 606169320Spiso 607169320Spiso /* We can only have one exclusive handler in a event. */ 608169320Spiso mtx_lock(&ie->ie_lock); 609169320Spiso if (!TAILQ_EMPTY(&ie->ie_handlers)) { 610169320Spiso if ((flags & INTR_EXCL) || 611169320Spiso (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 612169320Spiso mtx_unlock(&ie->ie_lock); 613169320Spiso free(ih, M_ITHREAD); 614169320Spiso return (EINVAL); 615169320Spiso } 616169320Spiso } 617169320Spiso 618169320Spiso /* Add the new handler to the event in priority order. */ 619169320Spiso TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 620169320Spiso if (temp_ih->ih_pri > ih->ih_pri) 621169320Spiso break; 622169320Spiso } 623169320Spiso if (temp_ih == NULL) 624169320Spiso TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 625169320Spiso else 626169320Spiso TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 627169320Spiso intr_event_update(ie); 628169320Spiso 629169320Spiso /* For filtered handlers, create a private ithread to run on. */ 630169320Spiso if (filter != NULL && handler != NULL) { 631169320Spiso mtx_unlock(&ie->ie_lock); 632169320Spiso it = ithread_create("intr: newborn", ih); 633169320Spiso mtx_lock(&ie->ie_lock); 634169320Spiso it->it_event = ie; 635169320Spiso ih->ih_thread = it; 636169320Spiso ithread_update(it); // XXX - do we really need this?!?!? 637169320Spiso } else { /* Create the global per-event thread if we need one. */ 638169320Spiso while (ie->ie_thread == NULL && handler != NULL) { 639169320Spiso if (ie->ie_flags & IE_ADDING_THREAD) 640169320Spiso msleep(ie, &ie->ie_lock, 0, "ithread", 0); 641169320Spiso else { 642169320Spiso ie->ie_flags |= IE_ADDING_THREAD; 643169320Spiso mtx_unlock(&ie->ie_lock); 644169320Spiso it = ithread_create("intr: newborn", ih); 645169320Spiso mtx_lock(&ie->ie_lock); 646169320Spiso ie->ie_flags &= ~IE_ADDING_THREAD; 647169320Spiso ie->ie_thread = it; 648169320Spiso it->it_event = ie; 649169320Spiso ithread_update(it); 650169320Spiso wakeup(ie); 651169320Spiso } 652169320Spiso } 653169320Spiso } 654169320Spiso CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 655169320Spiso ie->ie_name); 656169320Spiso mtx_unlock(&ie->ie_lock); 657169320Spiso 658169320Spiso if (cookiep != NULL) 659169320Spiso *cookiep = ih; 660169320Spiso return (0); 661169320Spiso} 662169320Spiso#endif 663169320Spiso 664165125Sjhb/* 665198134Sjhb * Append a description preceded by a ':' to the name of the specified 666198134Sjhb * interrupt handler. 667198134Sjhb */ 668198134Sjhbint 669198134Sjhbintr_event_describe_handler(struct intr_event *ie, void *cookie, 670198134Sjhb const char *descr) 671198134Sjhb{ 672198134Sjhb struct intr_handler *ih; 673198134Sjhb size_t space; 674198134Sjhb char *start; 675198134Sjhb 676198134Sjhb mtx_lock(&ie->ie_lock); 677198134Sjhb#ifdef INVARIANTS 678198134Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 679198134Sjhb if (ih == cookie) 680198134Sjhb break; 681198134Sjhb } 682198134Sjhb if (ih == NULL) { 683198134Sjhb mtx_unlock(&ie->ie_lock); 684198149Sjhb panic("handler %p not found in interrupt event %p", cookie, ie); 685198134Sjhb } 686198134Sjhb#endif 687198134Sjhb ih = cookie; 688198134Sjhb 689198134Sjhb /* 690198134Sjhb * Look for an existing description by checking for an 691198134Sjhb * existing ":". This assumes device names do not include 692198134Sjhb * colons. If one is found, prepare to insert the new 693198134Sjhb * description at that point. If one is not found, find the 694198134Sjhb * end of the name to use as the insertion point. 695198134Sjhb */ 696198134Sjhb start = index(ih->ih_name, ':'); 697198134Sjhb if (start == NULL) 698198134Sjhb start = index(ih->ih_name, 0); 699198134Sjhb 700198134Sjhb /* 701198134Sjhb * See if there is enough remaining room in the string for the 702198134Sjhb * description + ":". The "- 1" leaves room for the trailing 703198134Sjhb * '\0'. The "+ 1" accounts for the colon. 704198134Sjhb */ 705198134Sjhb space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1; 706198134Sjhb if (strlen(descr) + 1 > space) { 707198134Sjhb mtx_unlock(&ie->ie_lock); 708198134Sjhb return (ENOSPC); 709198134Sjhb } 710198134Sjhb 711198134Sjhb /* Append a colon followed by the description. */ 712198134Sjhb *start = ':'; 713198134Sjhb strcpy(start + 1, descr); 714198134Sjhb intr_event_update(ie); 715198134Sjhb mtx_unlock(&ie->ie_lock); 716198134Sjhb return (0); 717198134Sjhb} 718198134Sjhb 719198134Sjhb/* 720165125Sjhb * Return the ie_source field from the intr_event an intr_handler is 721165125Sjhb * associated with. 722165125Sjhb */ 723165125Sjhbvoid * 724165125Sjhbintr_handler_source(void *cookie) 725165125Sjhb{ 726165125Sjhb struct intr_handler *ih; 727165125Sjhb struct intr_event *ie; 728165125Sjhb 729165125Sjhb ih = (struct intr_handler *)cookie; 730165125Sjhb if (ih == NULL) 731165125Sjhb return (NULL); 732165125Sjhb ie = ih->ih_event; 733165125Sjhb KASSERT(ie != NULL, 734165125Sjhb ("interrupt handler \"%s\" has a NULL interrupt event", 735165125Sjhb ih->ih_name)); 736165125Sjhb return (ie->ie_source); 737165125Sjhb} 738165125Sjhb 739219819Sjeff/* 740219819Sjeff * Sleep until an ithread finishes executing an interrupt handler. 741219819Sjeff * 742219819Sjeff * XXX Doesn't currently handle interrupt filters or fast interrupt 743219819Sjeff * handlers. This is intended for compatibility with linux drivers 744219819Sjeff * only. Do not use in BSD code. 745219819Sjeff */ 746219819Sjeffvoid 747219819Sjeff_intr_drain(int irq) 748219819Sjeff{ 749219819Sjeff struct mtx *mtx; 750219819Sjeff struct intr_event *ie; 751219819Sjeff struct intr_thread *ithd; 752219819Sjeff struct thread *td; 753219819Sjeff 754219819Sjeff ie = intr_lookup(irq); 755219819Sjeff if (ie == NULL) 756219819Sjeff return; 757219819Sjeff if (ie->ie_thread == NULL) 758219819Sjeff return; 759219819Sjeff ithd = ie->ie_thread; 760219819Sjeff td = ithd->it_thread; 761219819Sjeff thread_lock(td); 762219819Sjeff mtx = td->td_lock; 763219819Sjeff if (!TD_AWAITING_INTR(td)) { 764219819Sjeff ithd->it_flags |= IT_WAIT; 765219819Sjeff msleep_spin(ithd, mtx, "isync", 0); 766219819Sjeff } 767219819Sjeff mtx_unlock_spin(mtx); 768219819Sjeff return; 769219819Sjeff} 770219819Sjeff 771219819Sjeff 772169320Spiso#ifndef INTR_FILTER 77372237Sjhbint 774151658Sjhbintr_event_remove_handler(void *cookie) 77572237Sjhb{ 776151658Sjhb struct intr_handler *handler = (struct intr_handler *)cookie; 777151658Sjhb struct intr_event *ie; 77872237Sjhb#ifdef INVARIANTS 779151658Sjhb struct intr_handler *ih; 78072237Sjhb#endif 781151658Sjhb#ifdef notyet 782151658Sjhb int dead; 783151658Sjhb#endif 78472237Sjhb 78572759Sjhb if (handler == NULL) 78672237Sjhb return (EINVAL); 787151658Sjhb ie = handler->ih_event; 788151658Sjhb KASSERT(ie != NULL, 789151658Sjhb ("interrupt handler \"%s\" has a NULL interrupt event", 790165124Sjhb handler->ih_name)); 791151658Sjhb mtx_lock(&ie->ie_lock); 79287593Sobrien CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 793151658Sjhb ie->ie_name); 79472237Sjhb#ifdef INVARIANTS 795151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 79672759Sjhb if (ih == handler) 79772759Sjhb goto ok; 798151658Sjhb mtx_unlock(&ie->ie_lock); 799151658Sjhb panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 800151658Sjhb ih->ih_name, ie->ie_name); 80172759Sjhbok: 80272237Sjhb#endif 80372839Sjhb /* 804151658Sjhb * If there is no ithread, then just remove the handler and return. 805151658Sjhb * XXX: Note that an INTR_FAST handler might be running on another 806151658Sjhb * CPU! 807151658Sjhb */ 808151658Sjhb if (ie->ie_thread == NULL) { 809151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 810151658Sjhb mtx_unlock(&ie->ie_lock); 811151658Sjhb free(handler, M_ITHREAD); 812151658Sjhb return (0); 813151658Sjhb } 814151658Sjhb 815151658Sjhb /* 81672839Sjhb * If the interrupt thread is already running, then just mark this 81772839Sjhb * handler as being dead and let the ithread do the actual removal. 818124505Struckman * 819124505Struckman * During a cold boot while cold is set, msleep() does not sleep, 820124505Struckman * so we have to remove the handler here rather than letting the 821124505Struckman * thread do it. 82272839Sjhb */ 823170307Sjeff thread_lock(ie->ie_thread->it_thread); 824151658Sjhb if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) { 82572839Sjhb handler->ih_flags |= IH_DEAD; 82672839Sjhb 82772839Sjhb /* 82872839Sjhb * Ensure that the thread will process the handler list 82972839Sjhb * again and remove this handler if it has already passed 83072839Sjhb * it on the list. 83172839Sjhb */ 832151658Sjhb ie->ie_thread->it_need = 1; 833151658Sjhb } else 834151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 835170307Sjeff thread_unlock(ie->ie_thread->it_thread); 836151658Sjhb while (handler->ih_flags & IH_DEAD) 837157815Sjhb msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 838151658Sjhb intr_event_update(ie); 839151658Sjhb#ifdef notyet 840151658Sjhb /* 841151658Sjhb * XXX: This could be bad in the case of ppbus(8). Also, I think 842151658Sjhb * this could lead to races of stale data when servicing an 843151658Sjhb * interrupt. 844151658Sjhb */ 845151658Sjhb dead = 1; 846151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 847151658Sjhb if (!(ih->ih_flags & IH_FAST)) { 848151658Sjhb dead = 0; 849151658Sjhb break; 850151658Sjhb } 851151658Sjhb } 852151658Sjhb if (dead) { 853151658Sjhb ithread_destroy(ie->ie_thread); 854151658Sjhb ie->ie_thread = NULL; 855151658Sjhb } 856151658Sjhb#endif 857151658Sjhb mtx_unlock(&ie->ie_lock); 85876771Sjhb free(handler, M_ITHREAD); 85972237Sjhb return (0); 86072237Sjhb} 86172237Sjhb 862177940Sjhbstatic int 863151658Sjhbintr_event_schedule_thread(struct intr_event *ie) 86472759Sjhb{ 865151658Sjhb struct intr_entropy entropy; 866151658Sjhb struct intr_thread *it; 86783366Sjulian struct thread *td; 868101176Sjulian struct thread *ctd; 86972759Sjhb struct proc *p; 87072759Sjhb 87172759Sjhb /* 87272759Sjhb * If no ithread or no handlers, then we have a stray interrupt. 87372759Sjhb */ 874151658Sjhb if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || 875151658Sjhb ie->ie_thread == NULL) 87672759Sjhb return (EINVAL); 87772759Sjhb 878101176Sjulian ctd = curthread; 879151658Sjhb it = ie->ie_thread; 880151658Sjhb td = it->it_thread; 881133191Srwatson p = td->td_proc; 882151658Sjhb 88372759Sjhb /* 88472759Sjhb * If any of the handlers for this ithread claim to be good 88572759Sjhb * sources of entropy, then gather some. 88672759Sjhb */ 887151658Sjhb if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 888133191Srwatson CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 889173004Sjulian p->p_pid, td->td_name); 890151658Sjhb entropy.event = (uintptr_t)ie; 891151658Sjhb entropy.td = ctd; 89272759Sjhb random_harvest(&entropy, sizeof(entropy), 2, 0, 89372759Sjhb RANDOM_INTERRUPT); 89472759Sjhb } 89572759Sjhb 896151658Sjhb KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 89772759Sjhb 89872759Sjhb /* 89972759Sjhb * Set it_need to tell the thread to keep running if it is already 900170307Sjeff * running. Then, lock the thread and see if we actually need to 901170307Sjeff * put it on the runqueue. 90272759Sjhb */ 903151658Sjhb it->it_need = 1; 904170307Sjeff thread_lock(td); 905103216Sjulian if (TD_AWAITING_INTR(td)) { 906151658Sjhb CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 907173004Sjulian td->td_name); 908103216Sjulian TD_CLR_IWAIT(td); 909166188Sjeff sched_add(td, SRQ_INTR); 91072759Sjhb } else { 911151658Sjhb CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 912173004Sjulian __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 91372759Sjhb } 914170307Sjeff thread_unlock(td); 91572759Sjhb 91672759Sjhb return (0); 91772759Sjhb} 918169320Spiso#else 919169320Spisoint 920169320Spisointr_event_remove_handler(void *cookie) 921169320Spiso{ 922169320Spiso struct intr_handler *handler = (struct intr_handler *)cookie; 923169320Spiso struct intr_event *ie; 924169320Spiso struct intr_thread *it; 925169320Spiso#ifdef INVARIANTS 926169320Spiso struct intr_handler *ih; 927169320Spiso#endif 928169320Spiso#ifdef notyet 929169320Spiso int dead; 930169320Spiso#endif 93172759Sjhb 932169320Spiso if (handler == NULL) 933169320Spiso return (EINVAL); 934169320Spiso ie = handler->ih_event; 935169320Spiso KASSERT(ie != NULL, 936169320Spiso ("interrupt handler \"%s\" has a NULL interrupt event", 937169320Spiso handler->ih_name)); 938169320Spiso mtx_lock(&ie->ie_lock); 939169320Spiso CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 940169320Spiso ie->ie_name); 941169320Spiso#ifdef INVARIANTS 942169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 943169320Spiso if (ih == handler) 944169320Spiso goto ok; 945169320Spiso mtx_unlock(&ie->ie_lock); 946169320Spiso panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 947169320Spiso ih->ih_name, ie->ie_name); 948169320Spisook: 949169320Spiso#endif 950169320Spiso /* 951169320Spiso * If there are no ithreads (per event and per handler), then 952169320Spiso * just remove the handler and return. 953169320Spiso * XXX: Note that an INTR_FAST handler might be running on another CPU! 954169320Spiso */ 955169320Spiso if (ie->ie_thread == NULL && handler->ih_thread == NULL) { 956169320Spiso TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 957169320Spiso mtx_unlock(&ie->ie_lock); 958169320Spiso free(handler, M_ITHREAD); 959169320Spiso return (0); 960169320Spiso } 961169320Spiso 962169320Spiso /* Private or global ithread? */ 963169320Spiso it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread; 964169320Spiso /* 965169320Spiso * If the interrupt thread is already running, then just mark this 966169320Spiso * handler as being dead and let the ithread do the actual removal. 967169320Spiso * 968169320Spiso * During a cold boot while cold is set, msleep() does not sleep, 969169320Spiso * so we have to remove the handler here rather than letting the 970169320Spiso * thread do it. 971169320Spiso */ 972170307Sjeff thread_lock(it->it_thread); 973169320Spiso if (!TD_AWAITING_INTR(it->it_thread) && !cold) { 974169320Spiso handler->ih_flags |= IH_DEAD; 975169320Spiso 976169320Spiso /* 977169320Spiso * Ensure that the thread will process the handler list 978169320Spiso * again and remove this handler if it has already passed 979169320Spiso * it on the list. 980169320Spiso */ 981169320Spiso it->it_need = 1; 982169320Spiso } else 983169320Spiso TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 984170307Sjeff thread_unlock(it->it_thread); 985169320Spiso while (handler->ih_flags & IH_DEAD) 986169320Spiso msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 987169320Spiso /* 988169320Spiso * At this point, the handler has been disconnected from the event, 989169320Spiso * so we can kill the private ithread if any. 990169320Spiso */ 991169320Spiso if (handler->ih_thread) { 992169320Spiso ithread_destroy(handler->ih_thread); 993169320Spiso handler->ih_thread = NULL; 994169320Spiso } 995169320Spiso intr_event_update(ie); 996169320Spiso#ifdef notyet 997169320Spiso /* 998169320Spiso * XXX: This could be bad in the case of ppbus(8). Also, I think 999169320Spiso * this could lead to races of stale data when servicing an 1000169320Spiso * interrupt. 1001169320Spiso */ 1002169320Spiso dead = 1; 1003169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1004169320Spiso if (handler != NULL) { 1005169320Spiso dead = 0; 1006169320Spiso break; 1007169320Spiso } 1008169320Spiso } 1009169320Spiso if (dead) { 1010169320Spiso ithread_destroy(ie->ie_thread); 1011169320Spiso ie->ie_thread = NULL; 1012169320Spiso } 1013169320Spiso#endif 1014169320Spiso mtx_unlock(&ie->ie_lock); 1015169320Spiso free(handler, M_ITHREAD); 1016169320Spiso return (0); 1017169320Spiso} 1018169320Spiso 1019177940Sjhbstatic int 1020169320Spisointr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it) 1021169320Spiso{ 1022169320Spiso struct intr_entropy entropy; 1023169320Spiso struct thread *td; 1024169320Spiso struct thread *ctd; 1025169320Spiso struct proc *p; 1026169320Spiso 1027169320Spiso /* 1028169320Spiso * If no ithread or no handlers, then we have a stray interrupt. 1029169320Spiso */ 1030169320Spiso if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL) 1031169320Spiso return (EINVAL); 1032169320Spiso 1033169320Spiso ctd = curthread; 1034169320Spiso td = it->it_thread; 1035169320Spiso p = td->td_proc; 1036169320Spiso 1037169320Spiso /* 1038169320Spiso * If any of the handlers for this ithread claim to be good 1039169320Spiso * sources of entropy, then gather some. 1040169320Spiso */ 1041169320Spiso if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 1042169320Spiso CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 1043173004Sjulian p->p_pid, td->td_name); 1044169320Spiso entropy.event = (uintptr_t)ie; 1045169320Spiso entropy.td = ctd; 1046169320Spiso random_harvest(&entropy, sizeof(entropy), 2, 0, 1047169320Spiso RANDOM_INTERRUPT); 1048169320Spiso } 1049169320Spiso 1050169320Spiso KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 1051169320Spiso 1052169320Spiso /* 1053169320Spiso * Set it_need to tell the thread to keep running if it is already 1054170307Sjeff * running. Then, lock the thread and see if we actually need to 1055170307Sjeff * put it on the runqueue. 1056169320Spiso */ 1057169320Spiso it->it_need = 1; 1058170307Sjeff thread_lock(td); 1059169320Spiso if (TD_AWAITING_INTR(td)) { 1060169320Spiso CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 1061173122Sjulian td->td_name); 1062169320Spiso TD_CLR_IWAIT(td); 1063169320Spiso sched_add(td, SRQ_INTR); 1064169320Spiso } else { 1065169320Spiso CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 1066173004Sjulian __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 1067169320Spiso } 1068170307Sjeff thread_unlock(td); 1069169320Spiso 1070169320Spiso return (0); 1071169320Spiso} 1072169320Spiso#endif 1073169320Spiso 1074151699Sjhb/* 1075192305Srwatson * Allow interrupt event binding for software interrupt handlers -- a no-op, 1076192305Srwatson * since interrupts are generated in software rather than being directed by 1077192305Srwatson * a PIC. 1078192305Srwatson */ 1079192305Srwatsonstatic int 1080192305Srwatsonswi_assign_cpu(void *arg, u_char cpu) 1081192305Srwatson{ 1082192305Srwatson 1083192305Srwatson return (0); 1084192305Srwatson} 1085192305Srwatson 1086192305Srwatson/* 1087151699Sjhb * Add a software interrupt handler to a specified event. If a given event 1088151699Sjhb * is not specified, then a new event is created. 1089151699Sjhb */ 109072759Sjhbint 1091151658Sjhbswi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, 109272237Sjhb void *arg, int pri, enum intr_type flags, void **cookiep) 109372237Sjhb{ 1094198854Sattilio struct thread *td; 1095151658Sjhb struct intr_event *ie; 109672237Sjhb int error; 109766698Sjhb 1098169320Spiso if (flags & INTR_ENTROPY) 109972759Sjhb return (EINVAL); 110072759Sjhb 1101151658Sjhb ie = (eventp != NULL) ? *eventp : NULL; 110266698Sjhb 1103151658Sjhb if (ie != NULL) { 1104151658Sjhb if (!(ie->ie_flags & IE_SOFT)) 1105151658Sjhb return (EINVAL); 110672759Sjhb } else { 1107178092Sjeff error = intr_event_create(&ie, NULL, IE_SOFT, 0, 1108192305Srwatson NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri); 110967551Sjhb if (error) 111072237Sjhb return (error); 1111151658Sjhb if (eventp != NULL) 1112151658Sjhb *eventp = ie; 111366698Sjhb } 1114177859Sjeff error = intr_event_add_handler(ie, name, NULL, handler, arg, 1115217292Sjhb PI_SWI(pri), flags, cookiep); 1116177859Sjeff if (error) 1117177859Sjeff return (error); 1118177859Sjeff if (pri == SWI_CLOCK) { 1119198854Sattilio td = ie->ie_thread->it_thread; 1120198854Sattilio thread_lock(td); 1121198854Sattilio td->td_flags |= TDF_NOLOAD; 1122198854Sattilio thread_unlock(td); 1123177859Sjeff } 1124177859Sjeff return (0); 112566698Sjhb} 112666698Sjhb 112766698Sjhb/* 1128151658Sjhb * Schedule a software interrupt thread. 112966698Sjhb */ 113067551Sjhbvoid 113172237Sjhbswi_sched(void *cookie, int flags) 113266698Sjhb{ 1133151658Sjhb struct intr_handler *ih = (struct intr_handler *)cookie; 1134151658Sjhb struct intr_event *ie = ih->ih_event; 113572759Sjhb int error; 113666698Sjhb 1137151658Sjhb CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, 1138151658Sjhb ih->ih_need); 1139151658Sjhb 114067551Sjhb /* 114172759Sjhb * Set ih_need for this handler so that if the ithread is already 114272759Sjhb * running it will execute this handler on the next pass. Otherwise, 114372759Sjhb * it will execute it the next time it runs. 114467551Sjhb */ 114572237Sjhb atomic_store_rel_int(&ih->ih_need, 1); 1146163474Sbde 114772237Sjhb if (!(flags & SWI_DELAY)) { 1148170291Sattilio PCPU_INC(cnt.v_soft); 1149169320Spiso#ifdef INTR_FILTER 1150169320Spiso error = intr_event_schedule_thread(ie, ie->ie_thread); 1151169320Spiso#else 1152151658Sjhb error = intr_event_schedule_thread(ie); 1153169320Spiso#endif 115472759Sjhb KASSERT(error == 0, ("stray software interrupt")); 115566698Sjhb } 115666698Sjhb} 115766698Sjhb 1158151699Sjhb/* 1159151699Sjhb * Remove a software interrupt handler. Currently this code does not 1160151699Sjhb * remove the associated interrupt event if it becomes empty. Calling code 1161151699Sjhb * may do so manually via intr_event_destroy(), but that's not really 1162151699Sjhb * an optimal interface. 1163151699Sjhb */ 1164151699Sjhbint 1165151699Sjhbswi_remove(void *cookie) 1166151699Sjhb{ 1167151699Sjhb 1168151699Sjhb return (intr_event_remove_handler(cookie)); 1169151699Sjhb} 1170151699Sjhb 1171169320Spiso#ifdef INTR_FILTER 1172151658Sjhbstatic void 1173169320Spisopriv_ithread_execute_handler(struct proc *p, struct intr_handler *ih) 1174169320Spiso{ 1175169320Spiso struct intr_event *ie; 1176169320Spiso 1177169320Spiso ie = ih->ih_event; 1178169320Spiso /* 1179169320Spiso * If this handler is marked for death, remove it from 1180169320Spiso * the list of handlers and wake up the sleeper. 1181169320Spiso */ 1182169320Spiso if (ih->ih_flags & IH_DEAD) { 1183169320Spiso mtx_lock(&ie->ie_lock); 1184169320Spiso TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1185169320Spiso ih->ih_flags &= ~IH_DEAD; 1186169320Spiso wakeup(ih); 1187169320Spiso mtx_unlock(&ie->ie_lock); 1188169320Spiso return; 1189169320Spiso } 1190169320Spiso 1191169320Spiso /* Execute this handler. */ 1192169320Spiso CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1193169320Spiso __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, 1194169320Spiso ih->ih_name, ih->ih_flags); 1195169320Spiso 1196169320Spiso if (!(ih->ih_flags & IH_MPSAFE)) 1197169320Spiso mtx_lock(&Giant); 1198169320Spiso ih->ih_handler(ih->ih_argument); 1199169320Spiso if (!(ih->ih_flags & IH_MPSAFE)) 1200169320Spiso mtx_unlock(&Giant); 1201169320Spiso} 1202169320Spiso#endif 1203169320Spiso 1204183052Sjhb/* 1205183052Sjhb * This is a public function for use by drivers that mux interrupt 1206183052Sjhb * handlers for child devices from their interrupt handler. 1207183052Sjhb */ 1208183052Sjhbvoid 1209183052Sjhbintr_event_execute_handlers(struct proc *p, struct intr_event *ie) 1210151658Sjhb{ 1211151658Sjhb struct intr_handler *ih, *ihn; 1212151658Sjhb 1213151658Sjhb TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { 1214151658Sjhb /* 1215151658Sjhb * If this handler is marked for death, remove it from 1216151658Sjhb * the list of handlers and wake up the sleeper. 1217151658Sjhb */ 1218151658Sjhb if (ih->ih_flags & IH_DEAD) { 1219151658Sjhb mtx_lock(&ie->ie_lock); 1220151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1221151658Sjhb ih->ih_flags &= ~IH_DEAD; 1222151658Sjhb wakeup(ih); 1223151658Sjhb mtx_unlock(&ie->ie_lock); 1224151658Sjhb continue; 1225151658Sjhb } 1226151658Sjhb 1227167080Spiso /* Skip filter only handlers */ 1228167080Spiso if (ih->ih_handler == NULL) 1229167080Spiso continue; 1230167080Spiso 1231151658Sjhb /* 1232151658Sjhb * For software interrupt threads, we only execute 1233151658Sjhb * handlers that have their need flag set. Hardware 1234151658Sjhb * interrupt threads always invoke all of their handlers. 1235151658Sjhb */ 1236151658Sjhb if (ie->ie_flags & IE_SOFT) { 1237151658Sjhb if (!ih->ih_need) 1238151658Sjhb continue; 1239151658Sjhb else 1240151658Sjhb atomic_store_rel_int(&ih->ih_need, 0); 1241151658Sjhb } 1242151658Sjhb 1243151658Sjhb /* Execute this handler. */ 1244151658Sjhb CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1245169320Spiso __func__, p->p_pid, (void *)ih->ih_handler, 1246169320Spiso ih->ih_argument, ih->ih_name, ih->ih_flags); 1247151658Sjhb 1248151658Sjhb if (!(ih->ih_flags & IH_MPSAFE)) 1249151658Sjhb mtx_lock(&Giant); 1250151658Sjhb ih->ih_handler(ih->ih_argument); 1251151658Sjhb if (!(ih->ih_flags & IH_MPSAFE)) 1252151658Sjhb mtx_unlock(&Giant); 1253151658Sjhb } 1254183052Sjhb} 1255183052Sjhb 1256183052Sjhbstatic void 1257183052Sjhbithread_execute_handlers(struct proc *p, struct intr_event *ie) 1258183052Sjhb{ 1259183052Sjhb 1260183052Sjhb /* Interrupt handlers should not sleep. */ 1261151658Sjhb if (!(ie->ie_flags & IE_SOFT)) 1262183052Sjhb THREAD_NO_SLEEPING(); 1263183052Sjhb intr_event_execute_handlers(p, ie); 1264183052Sjhb if (!(ie->ie_flags & IE_SOFT)) 1265151658Sjhb THREAD_SLEEPING_OK(); 1266151658Sjhb 1267151658Sjhb /* 1268151658Sjhb * Interrupt storm handling: 1269151658Sjhb * 1270151658Sjhb * If this interrupt source is currently storming, then throttle 1271151658Sjhb * it to only fire the handler once per clock tick. 1272151658Sjhb * 1273151658Sjhb * If this interrupt source is not currently storming, but the 1274151658Sjhb * number of back to back interrupts exceeds the storm threshold, 1275151658Sjhb * then enter storming mode. 1276151658Sjhb */ 1277167173Sjhb if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold && 1278167173Sjhb !(ie->ie_flags & IE_SOFT)) { 1279168850Snjl /* Report the message only once every second. */ 1280168850Snjl if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) { 1281151658Sjhb printf( 1282168850Snjl "interrupt storm detected on \"%s\"; throttling interrupt source\n", 1283151658Sjhb ie->ie_name); 1284151658Sjhb } 1285167173Sjhb pause("istorm", 1); 1286151658Sjhb } else 1287151658Sjhb ie->ie_count++; 1288151658Sjhb 1289151658Sjhb /* 1290151658Sjhb * Now that all the handlers have had a chance to run, reenable 1291151658Sjhb * the interrupt source. 1292151658Sjhb */ 1293177940Sjhb if (ie->ie_post_ithread != NULL) 1294177940Sjhb ie->ie_post_ithread(ie->ie_source); 1295151658Sjhb} 1296151658Sjhb 1297169320Spiso#ifndef INTR_FILTER 129866698Sjhb/* 129972237Sjhb * This is the main code for interrupt threads. 130066698Sjhb */ 1301104094Sphkstatic void 130272237Sjhbithread_loop(void *arg) 130366698Sjhb{ 1304151658Sjhb struct intr_thread *ithd; 1305151658Sjhb struct intr_event *ie; 130683366Sjulian struct thread *td; 130772237Sjhb struct proc *p; 1308219819Sjeff int wake; 1309151658Sjhb 131083366Sjulian td = curthread; 131183366Sjulian p = td->td_proc; 1312151658Sjhb ithd = (struct intr_thread *)arg; 1313151658Sjhb KASSERT(ithd->it_thread == td, 131487593Sobrien ("%s: ithread and proc linkage out of sync", __func__)); 1315151658Sjhb ie = ithd->it_event; 1316151658Sjhb ie->ie_count = 0; 1317219819Sjeff wake = 0; 131866698Sjhb 131967551Sjhb /* 132067551Sjhb * As long as we have interrupts outstanding, go through the 132167551Sjhb * list of handlers, giving each one a go at it. 132267551Sjhb */ 132366698Sjhb for (;;) { 132472237Sjhb /* 132572237Sjhb * If we are an orphaned thread, then just die. 132672237Sjhb */ 132772237Sjhb if (ithd->it_flags & IT_DEAD) { 1328151658Sjhb CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1329173004Sjulian p->p_pid, td->td_name); 133072237Sjhb free(ithd, M_ITHREAD); 1331173044Sjulian kthread_exit(); 133272237Sjhb } 133372237Sjhb 1334151658Sjhb /* 1335151658Sjhb * Service interrupts. If another interrupt arrives while 1336151658Sjhb * we are running, it will set it_need to note that we 1337151658Sjhb * should make another pass. 1338151658Sjhb */ 133972237Sjhb while (ithd->it_need) { 134067551Sjhb /* 1341151658Sjhb * This might need a full read and write barrier 1342151658Sjhb * to make sure that this write posts before any 1343151658Sjhb * of the memory or device accesses in the 1344151658Sjhb * handlers. 134567551Sjhb */ 134672237Sjhb atomic_store_rel_int(&ithd->it_need, 0); 1347151658Sjhb ithread_execute_handlers(p, ie); 134866698Sjhb } 1349128331Sjhb WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1350128331Sjhb mtx_assert(&Giant, MA_NOTOWNED); 135167551Sjhb 135266698Sjhb /* 135366698Sjhb * Processed all our interrupts. Now get the sched 135467551Sjhb * lock. This may take a while and it_need may get 135566698Sjhb * set again, so we have to check it again. 135666698Sjhb */ 1357170307Sjeff thread_lock(td); 1358219819Sjeff if (!ithd->it_need && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { 1359128331Sjhb TD_SET_IWAIT(td); 1360151658Sjhb ie->ie_count = 0; 1361178272Sjeff mi_switch(SW_VOL | SWT_IWAIT, NULL); 136266698Sjhb } 1363219819Sjeff if (ithd->it_flags & IT_WAIT) { 1364219819Sjeff wake = 1; 1365219819Sjeff ithd->it_flags &= ~IT_WAIT; 1366219819Sjeff } 1367170307Sjeff thread_unlock(td); 1368219819Sjeff if (wake) { 1369219819Sjeff wakeup(ithd); 1370219819Sjeff wake = 0; 1371219819Sjeff } 137266698Sjhb } 137366698Sjhb} 1374177940Sjhb 1375177940Sjhb/* 1376177940Sjhb * Main interrupt handling body. 1377177940Sjhb * 1378177940Sjhb * Input: 1379177940Sjhb * o ie: the event connected to this interrupt. 1380177940Sjhb * o frame: some archs (i.e. i386) pass a frame to some. 1381177940Sjhb * handlers as their main argument. 1382177940Sjhb * Return value: 1383177940Sjhb * o 0: everything ok. 1384177940Sjhb * o EINVAL: stray interrupt. 1385177940Sjhb */ 1386177940Sjhbint 1387177940Sjhbintr_event_handle(struct intr_event *ie, struct trapframe *frame) 1388177940Sjhb{ 1389177940Sjhb struct intr_handler *ih; 1390208988Smav struct trapframe *oldframe; 1391177940Sjhb struct thread *td; 1392177940Sjhb int error, ret, thread; 1393177940Sjhb 1394177940Sjhb td = curthread; 1395177940Sjhb 1396177940Sjhb /* An interrupt with no event or handlers is a stray interrupt. */ 1397177940Sjhb if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1398177940Sjhb return (EINVAL); 1399177940Sjhb 1400177940Sjhb /* 1401177940Sjhb * Execute fast interrupt handlers directly. 1402177940Sjhb * To support clock handlers, if a handler registers 1403177940Sjhb * with a NULL argument, then we pass it a pointer to 1404177940Sjhb * a trapframe as its argument. 1405177940Sjhb */ 1406177940Sjhb td->td_intr_nesting_level++; 1407177940Sjhb thread = 0; 1408177940Sjhb ret = 0; 1409177940Sjhb critical_enter(); 1410208988Smav oldframe = td->td_intr_frame; 1411208988Smav td->td_intr_frame = frame; 1412177940Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1413177940Sjhb if (ih->ih_filter == NULL) { 1414177940Sjhb thread = 1; 1415177940Sjhb continue; 1416177940Sjhb } 1417177940Sjhb CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, 1418177940Sjhb ih->ih_filter, ih->ih_argument == NULL ? frame : 1419177940Sjhb ih->ih_argument, ih->ih_name); 1420177940Sjhb if (ih->ih_argument == NULL) 1421177940Sjhb ret = ih->ih_filter(frame); 1422177940Sjhb else 1423177940Sjhb ret = ih->ih_filter(ih->ih_argument); 1424203061Savg KASSERT(ret == FILTER_STRAY || 1425203061Savg ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1426203061Savg (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1427203061Savg ("%s: incorrect return value %#x from %s", __func__, ret, 1428203061Savg ih->ih_name)); 1429203061Savg 1430177940Sjhb /* 1431177940Sjhb * Wrapper handler special handling: 1432177940Sjhb * 1433177940Sjhb * in some particular cases (like pccard and pccbb), 1434177940Sjhb * the _real_ device handler is wrapped in a couple of 1435177940Sjhb * functions - a filter wrapper and an ithread wrapper. 1436177940Sjhb * In this case (and just in this case), the filter wrapper 1437177940Sjhb * could ask the system to schedule the ithread and mask 1438177940Sjhb * the interrupt source if the wrapped handler is composed 1439177940Sjhb * of just an ithread handler. 1440177940Sjhb * 1441177940Sjhb * TODO: write a generic wrapper to avoid people rolling 1442177940Sjhb * their own 1443177940Sjhb */ 1444177940Sjhb if (!thread) { 1445177940Sjhb if (ret == FILTER_SCHEDULE_THREAD) 1446177940Sjhb thread = 1; 1447177940Sjhb } 1448177940Sjhb } 1449208988Smav td->td_intr_frame = oldframe; 1450177940Sjhb 1451177940Sjhb if (thread) { 1452177940Sjhb if (ie->ie_pre_ithread != NULL) 1453177940Sjhb ie->ie_pre_ithread(ie->ie_source); 1454177940Sjhb } else { 1455177940Sjhb if (ie->ie_post_filter != NULL) 1456177940Sjhb ie->ie_post_filter(ie->ie_source); 1457177940Sjhb } 1458177940Sjhb 1459177940Sjhb /* Schedule the ithread if needed. */ 1460177940Sjhb if (thread) { 1461177940Sjhb error = intr_event_schedule_thread(ie); 1462182024Skmacy#ifndef XEN 1463177940Sjhb KASSERT(error == 0, ("bad stray interrupt")); 1464182024Skmacy#else 1465182024Skmacy if (error != 0) 1466182024Skmacy log(LOG_WARNING, "bad stray interrupt"); 1467182024Skmacy#endif 1468177940Sjhb } 1469177940Sjhb critical_exit(); 1470177940Sjhb td->td_intr_nesting_level--; 1471177940Sjhb return (0); 1472177940Sjhb} 1473169320Spiso#else 1474169320Spiso/* 1475169320Spiso * This is the main code for interrupt threads. 1476169320Spiso */ 1477169320Spisostatic void 1478169320Spisoithread_loop(void *arg) 1479169320Spiso{ 1480169320Spiso struct intr_thread *ithd; 1481169320Spiso struct intr_handler *ih; 1482169320Spiso struct intr_event *ie; 1483169320Spiso struct thread *td; 1484169320Spiso struct proc *p; 1485169320Spiso int priv; 1486219819Sjeff int wake; 148766698Sjhb 1488169320Spiso td = curthread; 1489169320Spiso p = td->td_proc; 1490169320Spiso ih = (struct intr_handler *)arg; 1491169320Spiso priv = (ih->ih_thread != NULL) ? 1 : 0; 1492169320Spiso ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread; 1493169320Spiso KASSERT(ithd->it_thread == td, 1494169320Spiso ("%s: ithread and proc linkage out of sync", __func__)); 1495169320Spiso ie = ithd->it_event; 1496169320Spiso ie->ie_count = 0; 1497219819Sjeff wake = 0; 1498169320Spiso 1499169320Spiso /* 1500169320Spiso * As long as we have interrupts outstanding, go through the 1501169320Spiso * list of handlers, giving each one a go at it. 1502169320Spiso */ 1503169320Spiso for (;;) { 1504169320Spiso /* 1505169320Spiso * If we are an orphaned thread, then just die. 1506169320Spiso */ 1507169320Spiso if (ithd->it_flags & IT_DEAD) { 1508169320Spiso CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1509173004Sjulian p->p_pid, td->td_name); 1510169320Spiso free(ithd, M_ITHREAD); 1511173044Sjulian kthread_exit(); 1512169320Spiso } 1513169320Spiso 1514169320Spiso /* 1515169320Spiso * Service interrupts. If another interrupt arrives while 1516169320Spiso * we are running, it will set it_need to note that we 1517169320Spiso * should make another pass. 1518169320Spiso */ 1519169320Spiso while (ithd->it_need) { 1520169320Spiso /* 1521169320Spiso * This might need a full read and write barrier 1522169320Spiso * to make sure that this write posts before any 1523169320Spiso * of the memory or device accesses in the 1524169320Spiso * handlers. 1525169320Spiso */ 1526169320Spiso atomic_store_rel_int(&ithd->it_need, 0); 1527169320Spiso if (priv) 1528169320Spiso priv_ithread_execute_handler(p, ih); 1529169320Spiso else 1530169320Spiso ithread_execute_handlers(p, ie); 1531169320Spiso } 1532169320Spiso WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1533169320Spiso mtx_assert(&Giant, MA_NOTOWNED); 1534169320Spiso 1535169320Spiso /* 1536169320Spiso * Processed all our interrupts. Now get the sched 1537169320Spiso * lock. This may take a while and it_need may get 1538169320Spiso * set again, so we have to check it again. 1539169320Spiso */ 1540170307Sjeff thread_lock(td); 1541219819Sjeff if (!ithd->it_need && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { 1542169320Spiso TD_SET_IWAIT(td); 1543169320Spiso ie->ie_count = 0; 1544178272Sjeff mi_switch(SW_VOL | SWT_IWAIT, NULL); 1545169320Spiso } 1546219819Sjeff if (ithd->it_flags & IT_WAIT) { 1547219819Sjeff wake = 1; 1548219819Sjeff ithd->it_flags &= ~IT_WAIT; 1549219819Sjeff } 1550170307Sjeff thread_unlock(td); 1551219819Sjeff if (wake) { 1552219819Sjeff wakeup(ithd); 1553219819Sjeff wake = 0; 1554219819Sjeff } 1555169320Spiso } 1556169320Spiso} 1557169320Spiso 1558169320Spiso/* 1559169320Spiso * Main loop for interrupt filter. 1560169320Spiso * 1561169320Spiso * Some architectures (i386, amd64 and arm) require the optional frame 1562169320Spiso * parameter, and use it as the main argument for fast handler execution 1563169320Spiso * when ih_argument == NULL. 1564169320Spiso * 1565169320Spiso * Return value: 1566169320Spiso * o FILTER_STRAY: No filter recognized the event, and no 1567169320Spiso * filter-less handler is registered on this 1568169320Spiso * line. 1569169320Spiso * o FILTER_HANDLED: A filter claimed the event and served it. 1570169320Spiso * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at 1571169320Spiso * least one filter-less handler on this line. 1572169320Spiso * o FILTER_HANDLED | 1573169320Spiso * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for 1574169320Spiso * scheduling the per-handler ithread. 1575169320Spiso * 1576169320Spiso * In case an ithread has to be scheduled, in *ithd there will be a 1577169320Spiso * pointer to a struct intr_thread containing the thread to be 1578169320Spiso * scheduled. 1579169320Spiso */ 1580169320Spiso 1581177940Sjhbstatic int 1582169320Spisointr_filter_loop(struct intr_event *ie, struct trapframe *frame, 1583169320Spiso struct intr_thread **ithd) 1584169320Spiso{ 1585169320Spiso struct intr_handler *ih; 1586169320Spiso void *arg; 1587169320Spiso int ret, thread_only; 1588169320Spiso 1589169320Spiso ret = 0; 1590169320Spiso thread_only = 0; 1591169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1592169320Spiso /* 1593169320Spiso * Execute fast interrupt handlers directly. 1594169320Spiso * To support clock handlers, if a handler registers 1595169320Spiso * with a NULL argument, then we pass it a pointer to 1596169320Spiso * a trapframe as its argument. 1597169320Spiso */ 1598169320Spiso arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument); 1599169320Spiso 1600169320Spiso CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__, 1601169320Spiso ih->ih_filter, ih->ih_handler, arg, ih->ih_name); 1602169320Spiso 1603169320Spiso if (ih->ih_filter != NULL) 1604169320Spiso ret = ih->ih_filter(arg); 1605169320Spiso else { 1606169320Spiso thread_only = 1; 1607169320Spiso continue; 1608169320Spiso } 1609203061Savg KASSERT(ret == FILTER_STRAY || 1610203061Savg ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1611203061Savg (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1612203061Savg ("%s: incorrect return value %#x from %s", __func__, ret, 1613203061Savg ih->ih_name)); 1614169320Spiso if (ret & FILTER_STRAY) 1615169320Spiso continue; 1616169320Spiso else { 1617169320Spiso *ithd = ih->ih_thread; 1618169320Spiso return (ret); 1619169320Spiso } 1620169320Spiso } 1621169320Spiso 1622169320Spiso /* 1623169320Spiso * No filters handled the interrupt and we have at least 1624169320Spiso * one handler without a filter. In this case, we schedule 1625169320Spiso * all of the filter-less handlers to run in the ithread. 1626169320Spiso */ 1627169320Spiso if (thread_only) { 1628169320Spiso *ithd = ie->ie_thread; 1629169320Spiso return (FILTER_SCHEDULE_THREAD); 1630169320Spiso } 1631169320Spiso return (FILTER_STRAY); 1632169320Spiso} 1633169320Spiso 1634169320Spiso/* 1635169320Spiso * Main interrupt handling body. 1636169320Spiso * 1637169320Spiso * Input: 1638169320Spiso * o ie: the event connected to this interrupt. 1639169320Spiso * o frame: some archs (i.e. i386) pass a frame to some. 1640169320Spiso * handlers as their main argument. 1641169320Spiso * Return value: 1642169320Spiso * o 0: everything ok. 1643169320Spiso * o EINVAL: stray interrupt. 1644169320Spiso */ 1645169320Spisoint 1646169320Spisointr_event_handle(struct intr_event *ie, struct trapframe *frame) 1647169320Spiso{ 1648169320Spiso struct intr_thread *ithd; 1649208988Smav struct trapframe *oldframe; 1650169320Spiso struct thread *td; 1651169320Spiso int thread; 1652169320Spiso 1653169320Spiso ithd = NULL; 1654169320Spiso td = curthread; 1655169320Spiso 1656169320Spiso if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1657169320Spiso return (EINVAL); 1658169320Spiso 1659169320Spiso td->td_intr_nesting_level++; 1660169320Spiso thread = 0; 1661169320Spiso critical_enter(); 1662208988Smav oldframe = td->td_intr_frame; 1663208988Smav td->td_intr_frame = frame; 1664177940Sjhb thread = intr_filter_loop(ie, frame, &ithd); 1665169320Spiso if (thread & FILTER_HANDLED) { 1666177940Sjhb if (ie->ie_post_filter != NULL) 1667177940Sjhb ie->ie_post_filter(ie->ie_source); 1668169320Spiso } else { 1669177940Sjhb if (ie->ie_pre_ithread != NULL) 1670177940Sjhb ie->ie_pre_ithread(ie->ie_source); 1671169320Spiso } 1672208988Smav td->td_intr_frame = oldframe; 1673169320Spiso critical_exit(); 1674169320Spiso 1675169320Spiso /* Interrupt storm logic */ 1676169320Spiso if (thread & FILTER_STRAY) { 1677169320Spiso ie->ie_count++; 1678169320Spiso if (ie->ie_count < intr_storm_threshold) 1679169320Spiso printf("Interrupt stray detection not present\n"); 1680169320Spiso } 1681169320Spiso 1682169320Spiso /* Schedule an ithread if needed. */ 1683169320Spiso if (thread & FILTER_SCHEDULE_THREAD) { 1684169320Spiso if (intr_event_schedule_thread(ie, ithd) != 0) 1685169320Spiso panic("%s: impossible stray interrupt", __func__); 1686169320Spiso } 1687169320Spiso td->td_intr_nesting_level--; 1688169320Spiso return (0); 1689169320Spiso} 1690169320Spiso#endif 1691169320Spiso 1692121482Sjhb#ifdef DDB 169372237Sjhb/* 1694121482Sjhb * Dump details about an interrupt handler 1695121482Sjhb */ 1696121482Sjhbstatic void 1697151658Sjhbdb_dump_intrhand(struct intr_handler *ih) 1698121482Sjhb{ 1699121482Sjhb int comma; 1700121482Sjhb 1701121482Sjhb db_printf("\t%-10s ", ih->ih_name); 1702121482Sjhb switch (ih->ih_pri) { 1703121482Sjhb case PI_REALTIME: 1704121482Sjhb db_printf("CLK "); 1705121482Sjhb break; 1706121482Sjhb case PI_AV: 1707121482Sjhb db_printf("AV "); 1708121482Sjhb break; 1709217292Sjhb case PI_TTY: 1710121482Sjhb db_printf("TTY "); 1711121482Sjhb break; 1712121482Sjhb case PI_NET: 1713121482Sjhb db_printf("NET "); 1714121482Sjhb break; 1715121482Sjhb case PI_DISK: 1716121482Sjhb db_printf("DISK"); 1717121482Sjhb break; 1718121482Sjhb case PI_DULL: 1719121482Sjhb db_printf("DULL"); 1720121482Sjhb break; 1721121482Sjhb default: 1722121482Sjhb if (ih->ih_pri >= PI_SOFT) 1723121482Sjhb db_printf("SWI "); 1724121482Sjhb else 1725121482Sjhb db_printf("%4u", ih->ih_pri); 1726121482Sjhb break; 1727121482Sjhb } 1728121482Sjhb db_printf(" "); 1729121482Sjhb db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); 1730121482Sjhb db_printf("(%p)", ih->ih_argument); 1731121482Sjhb if (ih->ih_need || 1732166901Spiso (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | 1733121482Sjhb IH_MPSAFE)) != 0) { 1734121482Sjhb db_printf(" {"); 1735121482Sjhb comma = 0; 1736121482Sjhb if (ih->ih_flags & IH_EXCLUSIVE) { 1737121482Sjhb if (comma) 1738121482Sjhb db_printf(", "); 1739121482Sjhb db_printf("EXCL"); 1740121482Sjhb comma = 1; 1741121482Sjhb } 1742121482Sjhb if (ih->ih_flags & IH_ENTROPY) { 1743121482Sjhb if (comma) 1744121482Sjhb db_printf(", "); 1745121482Sjhb db_printf("ENTROPY"); 1746121482Sjhb comma = 1; 1747121482Sjhb } 1748121482Sjhb if (ih->ih_flags & IH_DEAD) { 1749121482Sjhb if (comma) 1750121482Sjhb db_printf(", "); 1751121482Sjhb db_printf("DEAD"); 1752121482Sjhb comma = 1; 1753121482Sjhb } 1754121482Sjhb if (ih->ih_flags & IH_MPSAFE) { 1755121482Sjhb if (comma) 1756121482Sjhb db_printf(", "); 1757121482Sjhb db_printf("MPSAFE"); 1758121482Sjhb comma = 1; 1759121482Sjhb } 1760121482Sjhb if (ih->ih_need) { 1761121482Sjhb if (comma) 1762121482Sjhb db_printf(", "); 1763121482Sjhb db_printf("NEED"); 1764121482Sjhb } 1765121482Sjhb db_printf("}"); 1766121482Sjhb } 1767121482Sjhb db_printf("\n"); 1768121482Sjhb} 1769121482Sjhb 1770121482Sjhb/* 1771151658Sjhb * Dump details about a event. 1772121482Sjhb */ 1773121482Sjhbvoid 1774151658Sjhbdb_dump_intr_event(struct intr_event *ie, int handlers) 1775121482Sjhb{ 1776151658Sjhb struct intr_handler *ih; 1777151658Sjhb struct intr_thread *it; 1778121482Sjhb int comma; 1779121482Sjhb 1780151658Sjhb db_printf("%s ", ie->ie_fullname); 1781151658Sjhb it = ie->ie_thread; 1782151658Sjhb if (it != NULL) 1783151658Sjhb db_printf("(pid %d)", it->it_thread->td_proc->p_pid); 1784151658Sjhb else 1785151658Sjhb db_printf("(no thread)"); 1786151658Sjhb if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || 1787151658Sjhb (it != NULL && it->it_need)) { 1788121482Sjhb db_printf(" {"); 1789121482Sjhb comma = 0; 1790151658Sjhb if (ie->ie_flags & IE_SOFT) { 1791121482Sjhb db_printf("SOFT"); 1792121482Sjhb comma = 1; 1793121482Sjhb } 1794151658Sjhb if (ie->ie_flags & IE_ENTROPY) { 1795121482Sjhb if (comma) 1796121482Sjhb db_printf(", "); 1797121482Sjhb db_printf("ENTROPY"); 1798121482Sjhb comma = 1; 1799121482Sjhb } 1800151658Sjhb if (ie->ie_flags & IE_ADDING_THREAD) { 1801121482Sjhb if (comma) 1802121482Sjhb db_printf(", "); 1803151658Sjhb db_printf("ADDING_THREAD"); 1804121482Sjhb comma = 1; 1805121482Sjhb } 1806151658Sjhb if (it != NULL && it->it_need) { 1807121482Sjhb if (comma) 1808121482Sjhb db_printf(", "); 1809121482Sjhb db_printf("NEED"); 1810121482Sjhb } 1811121482Sjhb db_printf("}"); 1812121482Sjhb } 1813121482Sjhb db_printf("\n"); 1814121482Sjhb 1815121482Sjhb if (handlers) 1816151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 1817121482Sjhb db_dump_intrhand(ih); 1818121482Sjhb} 1819151658Sjhb 1820151658Sjhb/* 1821151658Sjhb * Dump data about interrupt handlers 1822151658Sjhb */ 1823151658SjhbDB_SHOW_COMMAND(intr, db_show_intr) 1824151658Sjhb{ 1825151658Sjhb struct intr_event *ie; 1826160312Sjhb int all, verbose; 1827151658Sjhb 1828151658Sjhb verbose = index(modif, 'v') != NULL; 1829151658Sjhb all = index(modif, 'a') != NULL; 1830151658Sjhb TAILQ_FOREACH(ie, &event_list, ie_list) { 1831151658Sjhb if (!all && TAILQ_EMPTY(&ie->ie_handlers)) 1832151658Sjhb continue; 1833151658Sjhb db_dump_intr_event(ie, verbose); 1834160312Sjhb if (db_pager_quit) 1835160312Sjhb break; 1836151658Sjhb } 1837151658Sjhb} 1838121482Sjhb#endif /* DDB */ 1839121482Sjhb 1840121482Sjhb/* 184167551Sjhb * Start standard software interrupt threads 184266698Sjhb */ 184367551Sjhbstatic void 184472237Sjhbstart_softintr(void *dummy) 184567551Sjhb{ 184672237Sjhb 1847177859Sjeff if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) 1848177859Sjeff panic("died while creating vm swi ithread"); 184966698Sjhb} 1850177253SrwatsonSYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, 1851177253Srwatson NULL); 185266698Sjhb 1853151658Sjhb/* 185477582Stmm * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. 185577582Stmm * The data for this machine dependent, and the declarations are in machine 185677582Stmm * dependent code. The layout of intrnames and intrcnt however is machine 185777582Stmm * independent. 185877582Stmm * 185977582Stmm * We do not know the length of intrcnt and intrnames at compile time, so 186077582Stmm * calculate things at run time. 186177582Stmm */ 186277582Stmmstatic int 186377582Stmmsysctl_intrnames(SYSCTL_HANDLER_ARGS) 186477582Stmm{ 1865151658Sjhb return (sysctl_handle_opaque(oidp, intrnames, eintrnames - intrnames, 186677582Stmm req)); 186777582Stmm} 186877582Stmm 186977582StmmSYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, 187077582Stmm NULL, 0, sysctl_intrnames, "", "Interrupt Names"); 187177582Stmm 187277582Stmmstatic int 187377582Stmmsysctl_intrcnt(SYSCTL_HANDLER_ARGS) 187477582Stmm{ 1875151658Sjhb return (sysctl_handle_opaque(oidp, intrcnt, 187677582Stmm (char *)eintrcnt - (char *)intrcnt, req)); 187777582Stmm} 187877582Stmm 187977582StmmSYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, 188077582Stmm NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); 1881121482Sjhb 1882121482Sjhb#ifdef DDB 1883121482Sjhb/* 1884121482Sjhb * DDB command to dump the interrupt statistics. 1885121482Sjhb */ 1886121482SjhbDB_SHOW_COMMAND(intrcnt, db_show_intrcnt) 1887121482Sjhb{ 1888121482Sjhb u_long *i; 1889121482Sjhb char *cp; 1890121482Sjhb 1891121482Sjhb cp = intrnames; 1892160312Sjhb for (i = intrcnt; i != eintrcnt && !db_pager_quit; i++) { 1893121482Sjhb if (*cp == '\0') 1894121482Sjhb break; 1895121482Sjhb if (*i != 0) 1896121482Sjhb db_printf("%s\t%lu\n", cp, *i); 1897121482Sjhb cp += strlen(cp) + 1; 1898121482Sjhb } 1899121482Sjhb} 1900121482Sjhb#endif 1901