kern_intr.c revision 224187
1139804Simp/*- 226156Sse * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 326156Sse * All rights reserved. 426156Sse * 526156Sse * Redistribution and use in source and binary forms, with or without 626156Sse * modification, are permitted provided that the following conditions 726156Sse * are met: 826156Sse * 1. Redistributions of source code must retain the above copyright 926156Sse * notice unmodified, this list of conditions, and the following 1026156Sse * disclaimer. 1126156Sse * 2. Redistributions in binary form must reproduce the above copyright 1226156Sse * notice, this list of conditions and the following disclaimer in the 1326156Sse * documentation and/or other materials provided with the distribution. 1426156Sse * 1526156Sse * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1626156Sse * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1726156Sse * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 1826156Sse * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 1926156Sse * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2026156Sse * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2126156Sse * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2226156Sse * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2326156Sse * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2426156Sse * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2526156Sse */ 2626156Sse 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_intr.c 224187 2011-07-18 15:19:40Z attilio $"); 2936887Sdfr 30121482Sjhb#include "opt_ddb.h" 31121482Sjhb 3241059Speter#include <sys/param.h> 3365822Sjhb#include <sys/bus.h> 34110860Salfred#include <sys/conf.h> 35178092Sjeff#include <sys/cpuset.h> 3665822Sjhb#include <sys/rtprio.h> 3741059Speter#include <sys/systm.h> 3866698Sjhb#include <sys/interrupt.h> 3966698Sjhb#include <sys/kernel.h> 4066698Sjhb#include <sys/kthread.h> 4166698Sjhb#include <sys/ktr.h> 42130128Sbde#include <sys/limits.h> 4374914Sjhb#include <sys/lock.h> 4426156Sse#include <sys/malloc.h> 4567365Sjhb#include <sys/mutex.h> 46195249Sjhb#include <sys/priv.h> 4766698Sjhb#include <sys/proc.h> 4872759Sjhb#include <sys/random.h> 4972237Sjhb#include <sys/resourcevar.h> 50139451Sjhb#include <sys/sched.h> 51177181Sjhb#include <sys/smp.h> 5277582Stmm#include <sys/sysctl.h> 53182024Skmacy#include <sys/syslog.h> 5466698Sjhb#include <sys/unistd.h> 5566698Sjhb#include <sys/vmmeter.h> 5666698Sjhb#include <machine/atomic.h> 5766698Sjhb#include <machine/cpu.h> 5867551Sjhb#include <machine/md_var.h> 5972237Sjhb#include <machine/stdarg.h> 60121482Sjhb#ifdef DDB 61121482Sjhb#include <ddb/ddb.h> 62121482Sjhb#include <ddb/db_sym.h> 63121482Sjhb#endif 6426156Sse 65151658Sjhb/* 66151658Sjhb * Describe an interrupt thread. There is one of these per interrupt event. 67151658Sjhb */ 68151658Sjhbstruct intr_thread { 69151658Sjhb struct intr_event *it_event; 70151658Sjhb struct thread *it_thread; /* Kernel thread. */ 71151658Sjhb int it_flags; /* (j) IT_* flags. */ 72151658Sjhb int it_need; /* Needs service. */ 7372759Sjhb}; 7472759Sjhb 75151658Sjhb/* Interrupt thread flags kept in it_flags */ 76151658Sjhb#define IT_DEAD 0x000001 /* Thread is waiting to exit. */ 77219819Sjeff#define IT_WAIT 0x000002 /* Thread is waiting for completion. */ 78151658Sjhb 79151658Sjhbstruct intr_entropy { 80151658Sjhb struct thread *td; 81151658Sjhb uintptr_t event; 82151658Sjhb}; 83151658Sjhb 84151658Sjhbstruct intr_event *clk_intr_event; 85151658Sjhbstruct intr_event *tty_intr_event; 86128339Sbdevoid *vm_ih; 87173004Sjulianstruct proc *intrproc; 8838244Sbde 8972237Sjhbstatic MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 9072237Sjhb 91168850Snjlstatic int intr_storm_threshold = 1000; 92128331SjhbTUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold); 93128331SjhbSYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW, 94128331Sjhb &intr_storm_threshold, 0, 95128339Sbde "Number of consecutive interrupts before storm protection is enabled"); 96151658Sjhbstatic TAILQ_HEAD(, intr_event) event_list = 97151658Sjhb TAILQ_HEAD_INITIALIZER(event_list); 98178092Sjeffstatic struct mtx event_lock; 99178092SjeffMTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); 100128331Sjhb 101151658Sjhbstatic void intr_event_update(struct intr_event *ie); 102169320Spiso#ifdef INTR_FILTER 103177940Sjhbstatic int intr_event_schedule_thread(struct intr_event *ie, 104177940Sjhb struct intr_thread *ithd); 105177940Sjhbstatic int intr_filter_loop(struct intr_event *ie, 106177940Sjhb struct trapframe *frame, struct intr_thread **ithd); 107169320Spisostatic struct intr_thread *ithread_create(const char *name, 108169320Spiso struct intr_handler *ih); 109169320Spiso#else 110177940Sjhbstatic int intr_event_schedule_thread(struct intr_event *ie); 111151658Sjhbstatic struct intr_thread *ithread_create(const char *name); 112169320Spiso#endif 113151658Sjhbstatic void ithread_destroy(struct intr_thread *ithread); 114169320Spisostatic void ithread_execute_handlers(struct proc *p, 115169320Spiso struct intr_event *ie); 116169320Spiso#ifdef INTR_FILTER 117169320Spisostatic void priv_ithread_execute_handler(struct proc *p, 118169320Spiso struct intr_handler *ih); 119169320Spiso#endif 120128339Sbdestatic void ithread_loop(void *); 121151658Sjhbstatic void ithread_update(struct intr_thread *ithd); 122128339Sbdestatic void start_softintr(void *); 123128339Sbde 124165124Sjhb/* Map an interrupt type to an ithread priority. */ 12572237Sjhbu_char 126151658Sjhbintr_priority(enum intr_type flags) 12765822Sjhb{ 12872237Sjhb u_char pri; 12965822Sjhb 13072237Sjhb flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 13178365Speter INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); 13265822Sjhb switch (flags) { 13372237Sjhb case INTR_TYPE_TTY: 134217292Sjhb pri = PI_TTY; 13565822Sjhb break; 13665822Sjhb case INTR_TYPE_BIO: 13765822Sjhb pri = PI_DISK; 13865822Sjhb break; 13965822Sjhb case INTR_TYPE_NET: 14065822Sjhb pri = PI_NET; 14165822Sjhb break; 14265822Sjhb case INTR_TYPE_CAM: 143217292Sjhb pri = PI_DISK; 14465822Sjhb break; 145217292Sjhb case INTR_TYPE_AV: 14678365Speter pri = PI_AV; 14778365Speter break; 14872237Sjhb case INTR_TYPE_CLK: 14972237Sjhb pri = PI_REALTIME; 15072237Sjhb break; 15165822Sjhb case INTR_TYPE_MISC: 15265822Sjhb pri = PI_DULL; /* don't care */ 15365822Sjhb break; 15465822Sjhb default: 15572237Sjhb /* We didn't specify an interrupt level. */ 156151658Sjhb panic("intr_priority: no interrupt type in flags"); 15765822Sjhb } 15865822Sjhb 15965822Sjhb return pri; 16065822Sjhb} 16165822Sjhb 16272237Sjhb/* 163151658Sjhb * Update an ithread based on the associated intr_event. 16472237Sjhb */ 16572237Sjhbstatic void 166151658Sjhbithread_update(struct intr_thread *ithd) 16772237Sjhb{ 168151658Sjhb struct intr_event *ie; 16983366Sjulian struct thread *td; 170151658Sjhb u_char pri; 17167551Sjhb 172151658Sjhb ie = ithd->it_event; 173151658Sjhb td = ithd->it_thread; 17472237Sjhb 175151658Sjhb /* Determine the overall priority of this event. */ 176151658Sjhb if (TAILQ_EMPTY(&ie->ie_handlers)) 177151658Sjhb pri = PRI_MAX_ITHD; 178151658Sjhb else 179151658Sjhb pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri; 180105354Srobert 181151658Sjhb /* Update name and priority. */ 182173004Sjulian strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name)); 183170307Sjeff thread_lock(td); 184151658Sjhb sched_prio(td, pri); 185170307Sjeff thread_unlock(td); 186151658Sjhb} 187151658Sjhb 188151658Sjhb/* 189151658Sjhb * Regenerate the full name of an interrupt event and update its priority. 190151658Sjhb */ 191151658Sjhbstatic void 192151658Sjhbintr_event_update(struct intr_event *ie) 193151658Sjhb{ 194151658Sjhb struct intr_handler *ih; 195151658Sjhb char *last; 196151658Sjhb int missed, space; 197151658Sjhb 198151658Sjhb /* Start off with no entropy and just the name of the event. */ 199151658Sjhb mtx_assert(&ie->ie_lock, MA_OWNED); 200151658Sjhb strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 201151658Sjhb ie->ie_flags &= ~IE_ENTROPY; 202137267Sjhb missed = 0; 203151658Sjhb space = 1; 204151658Sjhb 205151658Sjhb /* Run through all the handlers updating values. */ 206151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 207151658Sjhb if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 < 208151658Sjhb sizeof(ie->ie_fullname)) { 209151658Sjhb strcat(ie->ie_fullname, " "); 210151658Sjhb strcat(ie->ie_fullname, ih->ih_name); 211151658Sjhb space = 0; 212137267Sjhb } else 213137267Sjhb missed++; 214137267Sjhb if (ih->ih_flags & IH_ENTROPY) 215151658Sjhb ie->ie_flags |= IE_ENTROPY; 216137267Sjhb } 217151658Sjhb 218151658Sjhb /* 219151658Sjhb * If the handler names were too long, add +'s to indicate missing 220151658Sjhb * names. If we run out of room and still have +'s to add, change 221151658Sjhb * the last character from a + to a *. 222151658Sjhb */ 223151658Sjhb last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2]; 224137267Sjhb while (missed-- > 0) { 225151658Sjhb if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) { 226151658Sjhb if (*last == '+') { 227151658Sjhb *last = '*'; 228151658Sjhb break; 229151658Sjhb } else 230151658Sjhb *last = '+'; 231151658Sjhb } else if (space) { 232151658Sjhb strcat(ie->ie_fullname, " +"); 233151658Sjhb space = 0; 23472237Sjhb } else 235151658Sjhb strcat(ie->ie_fullname, "+"); 23672237Sjhb } 237151658Sjhb 238151658Sjhb /* 239151658Sjhb * If this event has an ithread, update it's priority and 240151658Sjhb * name. 241151658Sjhb */ 242151658Sjhb if (ie->ie_thread != NULL) 243151658Sjhb ithread_update(ie->ie_thread); 244151658Sjhb CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname); 24572237Sjhb} 24672237Sjhb 24772237Sjhbint 248183298Sobrienintr_event_create(struct intr_event **event, void *source, int flags, int irq, 249177940Sjhb void (*pre_ithread)(void *), void (*post_ithread)(void *), 250177940Sjhb void (*post_filter)(void *), int (*assign_cpu)(void *, u_char), 251177940Sjhb const char *fmt, ...) 252169320Spiso{ 253169320Spiso struct intr_event *ie; 254169320Spiso va_list ap; 25572237Sjhb 256169320Spiso /* The only valid flag during creation is IE_SOFT. */ 257169320Spiso if ((flags & ~IE_SOFT) != 0) 258169320Spiso return (EINVAL); 259169320Spiso ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); 260169320Spiso ie->ie_source = source; 261177940Sjhb ie->ie_pre_ithread = pre_ithread; 262177940Sjhb ie->ie_post_ithread = post_ithread; 263177940Sjhb ie->ie_post_filter = post_filter; 264177181Sjhb ie->ie_assign_cpu = assign_cpu; 265169320Spiso ie->ie_flags = flags; 266178092Sjeff ie->ie_irq = irq; 267177181Sjhb ie->ie_cpu = NOCPU; 268169320Spiso TAILQ_INIT(&ie->ie_handlers); 269169320Spiso mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); 270169320Spiso 271169320Spiso va_start(ap, fmt); 272169320Spiso vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); 273169320Spiso va_end(ap); 274169320Spiso strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 275178092Sjeff mtx_lock(&event_lock); 276169320Spiso TAILQ_INSERT_TAIL(&event_list, ie, ie_list); 277178092Sjeff mtx_unlock(&event_lock); 278169320Spiso if (event != NULL) 279169320Spiso *event = ie; 280169320Spiso CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); 281169320Spiso return (0); 282169320Spiso} 283169320Spiso 284177181Sjhb/* 285177181Sjhb * Bind an interrupt event to the specified CPU. Note that not all 286177181Sjhb * platforms support binding an interrupt to a CPU. For those 287177181Sjhb * platforms this request will fail. For supported platforms, any 288177181Sjhb * associated ithreads as well as the primary interrupt context will 289177181Sjhb * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds 290177181Sjhb * the interrupt event. 291177181Sjhb */ 292151658Sjhbint 293177181Sjhbintr_event_bind(struct intr_event *ie, u_char cpu) 294177181Sjhb{ 295178092Sjeff cpuset_t mask; 296178092Sjeff lwpid_t id; 297177181Sjhb int error; 298177181Sjhb 299177181Sjhb /* Need a CPU to bind to. */ 300177181Sjhb if (cpu != NOCPU && CPU_ABSENT(cpu)) 301177181Sjhb return (EINVAL); 302177181Sjhb 303177181Sjhb if (ie->ie_assign_cpu == NULL) 304177181Sjhb return (EOPNOTSUPP); 305195249Sjhb 306195249Sjhb error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR); 307195249Sjhb if (error) 308195249Sjhb return (error); 309195249Sjhb 310178092Sjeff /* 311195249Sjhb * If we have any ithreads try to set their mask first to verify 312195249Sjhb * permissions, etc. 313178092Sjeff */ 314177181Sjhb mtx_lock(&ie->ie_lock); 315178092Sjeff if (ie->ie_thread != NULL) { 316178092Sjeff CPU_ZERO(&mask); 317178092Sjeff if (cpu == NOCPU) 318178092Sjeff CPU_COPY(cpuset_root, &mask); 319178092Sjeff else 320178092Sjeff CPU_SET(cpu, &mask); 321178092Sjeff id = ie->ie_thread->it_thread->td_tid; 322177181Sjhb mtx_unlock(&ie->ie_lock); 323178092Sjeff error = cpuset_setthread(id, &mask); 324178092Sjeff if (error) 325178092Sjeff return (error); 326178092Sjeff } else 327178092Sjeff mtx_unlock(&ie->ie_lock); 328177181Sjhb error = ie->ie_assign_cpu(ie->ie_source, cpu); 329195249Sjhb if (error) { 330195249Sjhb mtx_lock(&ie->ie_lock); 331195249Sjhb if (ie->ie_thread != NULL) { 332195249Sjhb CPU_ZERO(&mask); 333195249Sjhb if (ie->ie_cpu == NOCPU) 334195249Sjhb CPU_COPY(cpuset_root, &mask); 335195249Sjhb else 336195249Sjhb CPU_SET(cpu, &mask); 337195249Sjhb id = ie->ie_thread->it_thread->td_tid; 338195249Sjhb mtx_unlock(&ie->ie_lock); 339195249Sjhb (void)cpuset_setthread(id, &mask); 340195249Sjhb } else 341195249Sjhb mtx_unlock(&ie->ie_lock); 342177181Sjhb return (error); 343195249Sjhb } 344195249Sjhb 345177181Sjhb mtx_lock(&ie->ie_lock); 346177181Sjhb ie->ie_cpu = cpu; 347177181Sjhb mtx_unlock(&ie->ie_lock); 348178092Sjeff 349178092Sjeff return (error); 350178092Sjeff} 351178092Sjeff 352178092Sjeffstatic struct intr_event * 353178092Sjeffintr_lookup(int irq) 354178092Sjeff{ 355178092Sjeff struct intr_event *ie; 356178092Sjeff 357178092Sjeff mtx_lock(&event_lock); 358178092Sjeff TAILQ_FOREACH(ie, &event_list, ie_list) 359178092Sjeff if (ie->ie_irq == irq && 360178092Sjeff (ie->ie_flags & IE_SOFT) == 0 && 361178092Sjeff TAILQ_FIRST(&ie->ie_handlers) != NULL) 362178092Sjeff break; 363178092Sjeff mtx_unlock(&event_lock); 364178092Sjeff return (ie); 365178092Sjeff} 366178092Sjeff 367178092Sjeffint 368178092Sjeffintr_setaffinity(int irq, void *m) 369178092Sjeff{ 370178092Sjeff struct intr_event *ie; 371178092Sjeff cpuset_t *mask; 372178092Sjeff u_char cpu; 373178092Sjeff int n; 374178092Sjeff 375178092Sjeff mask = m; 376178092Sjeff cpu = NOCPU; 377178092Sjeff /* 378178092Sjeff * If we're setting all cpus we can unbind. Otherwise make sure 379178092Sjeff * only one cpu is in the set. 380178092Sjeff */ 381178092Sjeff if (CPU_CMP(cpuset_root, mask)) { 382178092Sjeff for (n = 0; n < CPU_SETSIZE; n++) { 383178092Sjeff if (!CPU_ISSET(n, mask)) 384178092Sjeff continue; 385178092Sjeff if (cpu != NOCPU) 386178092Sjeff return (EINVAL); 387178092Sjeff cpu = (u_char)n; 388178092Sjeff } 389178092Sjeff } 390178092Sjeff ie = intr_lookup(irq); 391178092Sjeff if (ie == NULL) 392178092Sjeff return (ESRCH); 393194987Sjhb return (intr_event_bind(ie, cpu)); 394178092Sjeff} 395178092Sjeff 396178092Sjeffint 397178092Sjeffintr_getaffinity(int irq, void *m) 398178092Sjeff{ 399178092Sjeff struct intr_event *ie; 400178092Sjeff cpuset_t *mask; 401178092Sjeff 402178092Sjeff mask = m; 403178092Sjeff ie = intr_lookup(irq); 404178092Sjeff if (ie == NULL) 405178092Sjeff return (ESRCH); 406178092Sjeff CPU_ZERO(mask); 407178092Sjeff mtx_lock(&ie->ie_lock); 408178092Sjeff if (ie->ie_cpu == NOCPU) 409178092Sjeff CPU_COPY(cpuset_root, mask); 410178092Sjeff else 411178092Sjeff CPU_SET(ie->ie_cpu, mask); 412178092Sjeff mtx_unlock(&ie->ie_lock); 413177181Sjhb return (0); 414177181Sjhb} 415177181Sjhb 416177181Sjhbint 417151658Sjhbintr_event_destroy(struct intr_event *ie) 418151658Sjhb{ 419151658Sjhb 420178092Sjeff mtx_lock(&event_lock); 421151658Sjhb mtx_lock(&ie->ie_lock); 422151658Sjhb if (!TAILQ_EMPTY(&ie->ie_handlers)) { 423151658Sjhb mtx_unlock(&ie->ie_lock); 424178092Sjeff mtx_unlock(&event_lock); 425151658Sjhb return (EBUSY); 426151658Sjhb } 427151658Sjhb TAILQ_REMOVE(&event_list, ie, ie_list); 428157728Sjhb#ifndef notyet 429157728Sjhb if (ie->ie_thread != NULL) { 430157728Sjhb ithread_destroy(ie->ie_thread); 431157728Sjhb ie->ie_thread = NULL; 432157728Sjhb } 433157728Sjhb#endif 434151658Sjhb mtx_unlock(&ie->ie_lock); 435178092Sjeff mtx_unlock(&event_lock); 436151658Sjhb mtx_destroy(&ie->ie_lock); 437151658Sjhb free(ie, M_ITHREAD); 438151658Sjhb return (0); 439151658Sjhb} 440151658Sjhb 441169320Spiso#ifndef INTR_FILTER 442151658Sjhbstatic struct intr_thread * 443151658Sjhbithread_create(const char *name) 444151658Sjhb{ 445151658Sjhb struct intr_thread *ithd; 446151658Sjhb struct thread *td; 447151658Sjhb int error; 448151658Sjhb 449151658Sjhb ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 450151658Sjhb 451173004Sjulian error = kproc_kthread_add(ithread_loop, ithd, &intrproc, 452173004Sjulian &td, RFSTOPPED | RFHIGHPID, 453173051Sjulian 0, "intr", "%s", name); 454151658Sjhb if (error) 455172836Sjulian panic("kproc_create() failed with %d", error); 456170307Sjeff thread_lock(td); 457164936Sjulian sched_class(td, PRI_ITHD); 458103216Sjulian TD_SET_IWAIT(td); 459170307Sjeff thread_unlock(td); 460151658Sjhb td->td_pflags |= TDP_ITHREAD; 461151658Sjhb ithd->it_thread = td; 462151658Sjhb CTR2(KTR_INTR, "%s: created %s", __func__, name); 463151658Sjhb return (ithd); 46472237Sjhb} 465169320Spiso#else 466169320Spisostatic struct intr_thread * 467169320Spisoithread_create(const char *name, struct intr_handler *ih) 468169320Spiso{ 469169320Spiso struct intr_thread *ithd; 470169320Spiso struct thread *td; 471169320Spiso int error; 47272237Sjhb 473169320Spiso ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 474169320Spiso 475173153Sjulian error = kproc_kthread_add(ithread_loop, ih, &intrproc, 476173004Sjulian &td, RFSTOPPED | RFHIGHPID, 477173051Sjulian 0, "intr", "%s", name); 478169320Spiso if (error) 479172836Sjulian panic("kproc_create() failed with %d", error); 480170307Sjeff thread_lock(td); 481169320Spiso sched_class(td, PRI_ITHD); 482169320Spiso TD_SET_IWAIT(td); 483170307Sjeff thread_unlock(td); 484169320Spiso td->td_pflags |= TDP_ITHREAD; 485169320Spiso ithd->it_thread = td; 486169320Spiso CTR2(KTR_INTR, "%s: created %s", __func__, name); 487169320Spiso return (ithd); 488169320Spiso} 489169320Spiso#endif 490169320Spiso 491151658Sjhbstatic void 492151658Sjhbithread_destroy(struct intr_thread *ithread) 49372237Sjhb{ 49483366Sjulian struct thread *td; 49572237Sjhb 496157784Sscottl CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name); 497151658Sjhb td = ithread->it_thread; 498170307Sjeff thread_lock(td); 49976771Sjhb ithread->it_flags |= IT_DEAD; 500103216Sjulian if (TD_AWAITING_INTR(td)) { 501103216Sjulian TD_CLR_IWAIT(td); 502166188Sjeff sched_add(td, SRQ_INTR); 50372237Sjhb } 504170307Sjeff thread_unlock(td); 50572237Sjhb} 50672237Sjhb 507169320Spiso#ifndef INTR_FILTER 50872237Sjhbint 509151658Sjhbintr_event_add_handler(struct intr_event *ie, const char *name, 510166901Spiso driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 511166901Spiso enum intr_type flags, void **cookiep) 51272237Sjhb{ 513151658Sjhb struct intr_handler *ih, *temp_ih; 514151658Sjhb struct intr_thread *it; 51572237Sjhb 516166901Spiso if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 51772237Sjhb return (EINVAL); 51872237Sjhb 519151658Sjhb /* Allocate and populate an interrupt handler structure. */ 520151658Sjhb ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 521166901Spiso ih->ih_filter = filter; 52272237Sjhb ih->ih_handler = handler; 52372237Sjhb ih->ih_argument = arg; 524198134Sjhb strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 525151658Sjhb ih->ih_event = ie; 52672237Sjhb ih->ih_pri = pri; 527166901Spiso if (flags & INTR_EXCL) 52872237Sjhb ih->ih_flags = IH_EXCLUSIVE; 52972237Sjhb if (flags & INTR_MPSAFE) 53072237Sjhb ih->ih_flags |= IH_MPSAFE; 53172237Sjhb if (flags & INTR_ENTROPY) 53272237Sjhb ih->ih_flags |= IH_ENTROPY; 53372237Sjhb 534151658Sjhb /* We can only have one exclusive handler in a event. */ 535151658Sjhb mtx_lock(&ie->ie_lock); 536151658Sjhb if (!TAILQ_EMPTY(&ie->ie_handlers)) { 537151658Sjhb if ((flags & INTR_EXCL) || 538151658Sjhb (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 539151658Sjhb mtx_unlock(&ie->ie_lock); 540151658Sjhb free(ih, M_ITHREAD); 541151658Sjhb return (EINVAL); 542151658Sjhb } 543122002Sjhb } 54472237Sjhb 545151658Sjhb /* Add the new handler to the event in priority order. */ 546151658Sjhb TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 547151658Sjhb if (temp_ih->ih_pri > ih->ih_pri) 548151658Sjhb break; 549151658Sjhb } 55072237Sjhb if (temp_ih == NULL) 551151658Sjhb TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 55272237Sjhb else 55372237Sjhb TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 554151658Sjhb intr_event_update(ie); 55572237Sjhb 556151658Sjhb /* Create a thread if we need one. */ 557166901Spiso while (ie->ie_thread == NULL && handler != NULL) { 558151658Sjhb if (ie->ie_flags & IE_ADDING_THREAD) 559157815Sjhb msleep(ie, &ie->ie_lock, 0, "ithread", 0); 560151658Sjhb else { 561151658Sjhb ie->ie_flags |= IE_ADDING_THREAD; 562151658Sjhb mtx_unlock(&ie->ie_lock); 563151658Sjhb it = ithread_create("intr: newborn"); 564151658Sjhb mtx_lock(&ie->ie_lock); 565151658Sjhb ie->ie_flags &= ~IE_ADDING_THREAD; 566151658Sjhb ie->ie_thread = it; 567151658Sjhb it->it_event = ie; 568151658Sjhb ithread_update(it); 569151658Sjhb wakeup(ie); 570151658Sjhb } 571151658Sjhb } 572151658Sjhb CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 573151658Sjhb ie->ie_name); 574151658Sjhb mtx_unlock(&ie->ie_lock); 575151658Sjhb 57672237Sjhb if (cookiep != NULL) 57772237Sjhb *cookiep = ih; 57872237Sjhb return (0); 57972237Sjhb} 580169320Spiso#else 581169320Spisoint 582169320Spisointr_event_add_handler(struct intr_event *ie, const char *name, 583169320Spiso driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 584169320Spiso enum intr_type flags, void **cookiep) 585169320Spiso{ 586169320Spiso struct intr_handler *ih, *temp_ih; 587169320Spiso struct intr_thread *it; 58872237Sjhb 589169320Spiso if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 590169320Spiso return (EINVAL); 591169320Spiso 592169320Spiso /* Allocate and populate an interrupt handler structure. */ 593169320Spiso ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 594169320Spiso ih->ih_filter = filter; 595169320Spiso ih->ih_handler = handler; 596169320Spiso ih->ih_argument = arg; 597198134Sjhb strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 598169320Spiso ih->ih_event = ie; 599169320Spiso ih->ih_pri = pri; 600169320Spiso if (flags & INTR_EXCL) 601169320Spiso ih->ih_flags = IH_EXCLUSIVE; 602169320Spiso if (flags & INTR_MPSAFE) 603169320Spiso ih->ih_flags |= IH_MPSAFE; 604169320Spiso if (flags & INTR_ENTROPY) 605169320Spiso ih->ih_flags |= IH_ENTROPY; 606169320Spiso 607169320Spiso /* We can only have one exclusive handler in a event. */ 608169320Spiso mtx_lock(&ie->ie_lock); 609169320Spiso if (!TAILQ_EMPTY(&ie->ie_handlers)) { 610169320Spiso if ((flags & INTR_EXCL) || 611169320Spiso (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 612169320Spiso mtx_unlock(&ie->ie_lock); 613169320Spiso free(ih, M_ITHREAD); 614169320Spiso return (EINVAL); 615169320Spiso } 616169320Spiso } 617169320Spiso 618169320Spiso /* Add the new handler to the event in priority order. */ 619169320Spiso TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 620169320Spiso if (temp_ih->ih_pri > ih->ih_pri) 621169320Spiso break; 622169320Spiso } 623169320Spiso if (temp_ih == NULL) 624169320Spiso TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 625169320Spiso else 626169320Spiso TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 627169320Spiso intr_event_update(ie); 628169320Spiso 629169320Spiso /* For filtered handlers, create a private ithread to run on. */ 630169320Spiso if (filter != NULL && handler != NULL) { 631169320Spiso mtx_unlock(&ie->ie_lock); 632169320Spiso it = ithread_create("intr: newborn", ih); 633169320Spiso mtx_lock(&ie->ie_lock); 634169320Spiso it->it_event = ie; 635169320Spiso ih->ih_thread = it; 636169320Spiso ithread_update(it); // XXX - do we really need this?!?!? 637169320Spiso } else { /* Create the global per-event thread if we need one. */ 638169320Spiso while (ie->ie_thread == NULL && handler != NULL) { 639169320Spiso if (ie->ie_flags & IE_ADDING_THREAD) 640169320Spiso msleep(ie, &ie->ie_lock, 0, "ithread", 0); 641169320Spiso else { 642169320Spiso ie->ie_flags |= IE_ADDING_THREAD; 643169320Spiso mtx_unlock(&ie->ie_lock); 644169320Spiso it = ithread_create("intr: newborn", ih); 645169320Spiso mtx_lock(&ie->ie_lock); 646169320Spiso ie->ie_flags &= ~IE_ADDING_THREAD; 647169320Spiso ie->ie_thread = it; 648169320Spiso it->it_event = ie; 649169320Spiso ithread_update(it); 650169320Spiso wakeup(ie); 651169320Spiso } 652169320Spiso } 653169320Spiso } 654169320Spiso CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 655169320Spiso ie->ie_name); 656169320Spiso mtx_unlock(&ie->ie_lock); 657169320Spiso 658169320Spiso if (cookiep != NULL) 659169320Spiso *cookiep = ih; 660169320Spiso return (0); 661169320Spiso} 662169320Spiso#endif 663169320Spiso 664165125Sjhb/* 665198134Sjhb * Append a description preceded by a ':' to the name of the specified 666198134Sjhb * interrupt handler. 667198134Sjhb */ 668198134Sjhbint 669198134Sjhbintr_event_describe_handler(struct intr_event *ie, void *cookie, 670198134Sjhb const char *descr) 671198134Sjhb{ 672198134Sjhb struct intr_handler *ih; 673198134Sjhb size_t space; 674198134Sjhb char *start; 675198134Sjhb 676198134Sjhb mtx_lock(&ie->ie_lock); 677198134Sjhb#ifdef INVARIANTS 678198134Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 679198134Sjhb if (ih == cookie) 680198134Sjhb break; 681198134Sjhb } 682198134Sjhb if (ih == NULL) { 683198134Sjhb mtx_unlock(&ie->ie_lock); 684198149Sjhb panic("handler %p not found in interrupt event %p", cookie, ie); 685198134Sjhb } 686198134Sjhb#endif 687198134Sjhb ih = cookie; 688198134Sjhb 689198134Sjhb /* 690198134Sjhb * Look for an existing description by checking for an 691198134Sjhb * existing ":". This assumes device names do not include 692198134Sjhb * colons. If one is found, prepare to insert the new 693198134Sjhb * description at that point. If one is not found, find the 694198134Sjhb * end of the name to use as the insertion point. 695198134Sjhb */ 696198134Sjhb start = index(ih->ih_name, ':'); 697198134Sjhb if (start == NULL) 698198134Sjhb start = index(ih->ih_name, 0); 699198134Sjhb 700198134Sjhb /* 701198134Sjhb * See if there is enough remaining room in the string for the 702198134Sjhb * description + ":". The "- 1" leaves room for the trailing 703198134Sjhb * '\0'. The "+ 1" accounts for the colon. 704198134Sjhb */ 705198134Sjhb space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1; 706198134Sjhb if (strlen(descr) + 1 > space) { 707198134Sjhb mtx_unlock(&ie->ie_lock); 708198134Sjhb return (ENOSPC); 709198134Sjhb } 710198134Sjhb 711198134Sjhb /* Append a colon followed by the description. */ 712198134Sjhb *start = ':'; 713198134Sjhb strcpy(start + 1, descr); 714198134Sjhb intr_event_update(ie); 715198134Sjhb mtx_unlock(&ie->ie_lock); 716198134Sjhb return (0); 717198134Sjhb} 718198134Sjhb 719198134Sjhb/* 720165125Sjhb * Return the ie_source field from the intr_event an intr_handler is 721165125Sjhb * associated with. 722165125Sjhb */ 723165125Sjhbvoid * 724165125Sjhbintr_handler_source(void *cookie) 725165125Sjhb{ 726165125Sjhb struct intr_handler *ih; 727165125Sjhb struct intr_event *ie; 728165125Sjhb 729165125Sjhb ih = (struct intr_handler *)cookie; 730165125Sjhb if (ih == NULL) 731165125Sjhb return (NULL); 732165125Sjhb ie = ih->ih_event; 733165125Sjhb KASSERT(ie != NULL, 734165125Sjhb ("interrupt handler \"%s\" has a NULL interrupt event", 735165125Sjhb ih->ih_name)); 736165125Sjhb return (ie->ie_source); 737165125Sjhb} 738165125Sjhb 739219819Sjeff/* 740219819Sjeff * Sleep until an ithread finishes executing an interrupt handler. 741219819Sjeff * 742219819Sjeff * XXX Doesn't currently handle interrupt filters or fast interrupt 743219819Sjeff * handlers. This is intended for compatibility with linux drivers 744219819Sjeff * only. Do not use in BSD code. 745219819Sjeff */ 746219819Sjeffvoid 747219819Sjeff_intr_drain(int irq) 748219819Sjeff{ 749219819Sjeff struct intr_event *ie; 750219819Sjeff struct intr_thread *ithd; 751219819Sjeff struct thread *td; 752219819Sjeff 753219819Sjeff ie = intr_lookup(irq); 754219819Sjeff if (ie == NULL) 755219819Sjeff return; 756219819Sjeff if (ie->ie_thread == NULL) 757219819Sjeff return; 758219819Sjeff ithd = ie->ie_thread; 759219819Sjeff td = ithd->it_thread; 760221055Sjeff /* 761221055Sjeff * We set the flag and wait for it to be cleared to avoid 762221055Sjeff * long delays with potentially busy interrupt handlers 763221055Sjeff * were we to only sample TD_AWAITING_INTR() every tick. 764221055Sjeff */ 765219819Sjeff thread_lock(td); 766219819Sjeff if (!TD_AWAITING_INTR(td)) { 767219819Sjeff ithd->it_flags |= IT_WAIT; 768221055Sjeff while (ithd->it_flags & IT_WAIT) { 769221055Sjeff thread_unlock(td); 770221055Sjeff pause("idrain", 1); 771221055Sjeff thread_lock(td); 772221055Sjeff } 773219819Sjeff } 774221055Sjeff thread_unlock(td); 775219819Sjeff return; 776219819Sjeff} 777219819Sjeff 778219819Sjeff 779169320Spiso#ifndef INTR_FILTER 78072237Sjhbint 781151658Sjhbintr_event_remove_handler(void *cookie) 78272237Sjhb{ 783151658Sjhb struct intr_handler *handler = (struct intr_handler *)cookie; 784151658Sjhb struct intr_event *ie; 78572237Sjhb#ifdef INVARIANTS 786151658Sjhb struct intr_handler *ih; 78772237Sjhb#endif 788151658Sjhb#ifdef notyet 789151658Sjhb int dead; 790151658Sjhb#endif 79172237Sjhb 79272759Sjhb if (handler == NULL) 79372237Sjhb return (EINVAL); 794151658Sjhb ie = handler->ih_event; 795151658Sjhb KASSERT(ie != NULL, 796151658Sjhb ("interrupt handler \"%s\" has a NULL interrupt event", 797165124Sjhb handler->ih_name)); 798151658Sjhb mtx_lock(&ie->ie_lock); 79987593Sobrien CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 800151658Sjhb ie->ie_name); 80172237Sjhb#ifdef INVARIANTS 802151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 80372759Sjhb if (ih == handler) 80472759Sjhb goto ok; 805151658Sjhb mtx_unlock(&ie->ie_lock); 806151658Sjhb panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 807151658Sjhb ih->ih_name, ie->ie_name); 80872759Sjhbok: 80972237Sjhb#endif 81072839Sjhb /* 811151658Sjhb * If there is no ithread, then just remove the handler and return. 812151658Sjhb * XXX: Note that an INTR_FAST handler might be running on another 813151658Sjhb * CPU! 814151658Sjhb */ 815151658Sjhb if (ie->ie_thread == NULL) { 816151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 817151658Sjhb mtx_unlock(&ie->ie_lock); 818151658Sjhb free(handler, M_ITHREAD); 819151658Sjhb return (0); 820151658Sjhb } 821151658Sjhb 822151658Sjhb /* 82372839Sjhb * If the interrupt thread is already running, then just mark this 82472839Sjhb * handler as being dead and let the ithread do the actual removal. 825124505Struckman * 826124505Struckman * During a cold boot while cold is set, msleep() does not sleep, 827124505Struckman * so we have to remove the handler here rather than letting the 828124505Struckman * thread do it. 82972839Sjhb */ 830170307Sjeff thread_lock(ie->ie_thread->it_thread); 831151658Sjhb if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) { 83272839Sjhb handler->ih_flags |= IH_DEAD; 83372839Sjhb 83472839Sjhb /* 83572839Sjhb * Ensure that the thread will process the handler list 83672839Sjhb * again and remove this handler if it has already passed 83772839Sjhb * it on the list. 83872839Sjhb */ 839151658Sjhb ie->ie_thread->it_need = 1; 840151658Sjhb } else 841151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 842170307Sjeff thread_unlock(ie->ie_thread->it_thread); 843151658Sjhb while (handler->ih_flags & IH_DEAD) 844157815Sjhb msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 845151658Sjhb intr_event_update(ie); 846151658Sjhb#ifdef notyet 847151658Sjhb /* 848151658Sjhb * XXX: This could be bad in the case of ppbus(8). Also, I think 849151658Sjhb * this could lead to races of stale data when servicing an 850151658Sjhb * interrupt. 851151658Sjhb */ 852151658Sjhb dead = 1; 853151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 854151658Sjhb if (!(ih->ih_flags & IH_FAST)) { 855151658Sjhb dead = 0; 856151658Sjhb break; 857151658Sjhb } 858151658Sjhb } 859151658Sjhb if (dead) { 860151658Sjhb ithread_destroy(ie->ie_thread); 861151658Sjhb ie->ie_thread = NULL; 862151658Sjhb } 863151658Sjhb#endif 864151658Sjhb mtx_unlock(&ie->ie_lock); 86576771Sjhb free(handler, M_ITHREAD); 86672237Sjhb return (0); 86772237Sjhb} 86872237Sjhb 869177940Sjhbstatic int 870151658Sjhbintr_event_schedule_thread(struct intr_event *ie) 87172759Sjhb{ 872151658Sjhb struct intr_entropy entropy; 873151658Sjhb struct intr_thread *it; 87483366Sjulian struct thread *td; 875101176Sjulian struct thread *ctd; 87672759Sjhb struct proc *p; 87772759Sjhb 87872759Sjhb /* 87972759Sjhb * If no ithread or no handlers, then we have a stray interrupt. 88072759Sjhb */ 881151658Sjhb if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || 882151658Sjhb ie->ie_thread == NULL) 88372759Sjhb return (EINVAL); 88472759Sjhb 885101176Sjulian ctd = curthread; 886151658Sjhb it = ie->ie_thread; 887151658Sjhb td = it->it_thread; 888133191Srwatson p = td->td_proc; 889151658Sjhb 89072759Sjhb /* 89172759Sjhb * If any of the handlers for this ithread claim to be good 89272759Sjhb * sources of entropy, then gather some. 89372759Sjhb */ 894151658Sjhb if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 895133191Srwatson CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 896173004Sjulian p->p_pid, td->td_name); 897151658Sjhb entropy.event = (uintptr_t)ie; 898151658Sjhb entropy.td = ctd; 89972759Sjhb random_harvest(&entropy, sizeof(entropy), 2, 0, 90072759Sjhb RANDOM_INTERRUPT); 90172759Sjhb } 90272759Sjhb 903151658Sjhb KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 90472759Sjhb 90572759Sjhb /* 90672759Sjhb * Set it_need to tell the thread to keep running if it is already 907170307Sjeff * running. Then, lock the thread and see if we actually need to 908170307Sjeff * put it on the runqueue. 90972759Sjhb */ 910151658Sjhb it->it_need = 1; 911170307Sjeff thread_lock(td); 912103216Sjulian if (TD_AWAITING_INTR(td)) { 913151658Sjhb CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 914173004Sjulian td->td_name); 915103216Sjulian TD_CLR_IWAIT(td); 916166188Sjeff sched_add(td, SRQ_INTR); 91772759Sjhb } else { 918151658Sjhb CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 919173004Sjulian __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 92072759Sjhb } 921170307Sjeff thread_unlock(td); 92272759Sjhb 92372759Sjhb return (0); 92472759Sjhb} 925169320Spiso#else 926169320Spisoint 927169320Spisointr_event_remove_handler(void *cookie) 928169320Spiso{ 929169320Spiso struct intr_handler *handler = (struct intr_handler *)cookie; 930169320Spiso struct intr_event *ie; 931169320Spiso struct intr_thread *it; 932169320Spiso#ifdef INVARIANTS 933169320Spiso struct intr_handler *ih; 934169320Spiso#endif 935169320Spiso#ifdef notyet 936169320Spiso int dead; 937169320Spiso#endif 93872759Sjhb 939169320Spiso if (handler == NULL) 940169320Spiso return (EINVAL); 941169320Spiso ie = handler->ih_event; 942169320Spiso KASSERT(ie != NULL, 943169320Spiso ("interrupt handler \"%s\" has a NULL interrupt event", 944169320Spiso handler->ih_name)); 945169320Spiso mtx_lock(&ie->ie_lock); 946169320Spiso CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 947169320Spiso ie->ie_name); 948169320Spiso#ifdef INVARIANTS 949169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 950169320Spiso if (ih == handler) 951169320Spiso goto ok; 952169320Spiso mtx_unlock(&ie->ie_lock); 953169320Spiso panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 954169320Spiso ih->ih_name, ie->ie_name); 955169320Spisook: 956169320Spiso#endif 957169320Spiso /* 958169320Spiso * If there are no ithreads (per event and per handler), then 959169320Spiso * just remove the handler and return. 960169320Spiso * XXX: Note that an INTR_FAST handler might be running on another CPU! 961169320Spiso */ 962169320Spiso if (ie->ie_thread == NULL && handler->ih_thread == NULL) { 963169320Spiso TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 964169320Spiso mtx_unlock(&ie->ie_lock); 965169320Spiso free(handler, M_ITHREAD); 966169320Spiso return (0); 967169320Spiso } 968169320Spiso 969169320Spiso /* Private or global ithread? */ 970169320Spiso it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread; 971169320Spiso /* 972169320Spiso * If the interrupt thread is already running, then just mark this 973169320Spiso * handler as being dead and let the ithread do the actual removal. 974169320Spiso * 975169320Spiso * During a cold boot while cold is set, msleep() does not sleep, 976169320Spiso * so we have to remove the handler here rather than letting the 977169320Spiso * thread do it. 978169320Spiso */ 979170307Sjeff thread_lock(it->it_thread); 980169320Spiso if (!TD_AWAITING_INTR(it->it_thread) && !cold) { 981169320Spiso handler->ih_flags |= IH_DEAD; 982169320Spiso 983169320Spiso /* 984169320Spiso * Ensure that the thread will process the handler list 985169320Spiso * again and remove this handler if it has already passed 986169320Spiso * it on the list. 987169320Spiso */ 988169320Spiso it->it_need = 1; 989169320Spiso } else 990169320Spiso TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 991170307Sjeff thread_unlock(it->it_thread); 992169320Spiso while (handler->ih_flags & IH_DEAD) 993169320Spiso msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 994169320Spiso /* 995169320Spiso * At this point, the handler has been disconnected from the event, 996169320Spiso * so we can kill the private ithread if any. 997169320Spiso */ 998169320Spiso if (handler->ih_thread) { 999169320Spiso ithread_destroy(handler->ih_thread); 1000169320Spiso handler->ih_thread = NULL; 1001169320Spiso } 1002169320Spiso intr_event_update(ie); 1003169320Spiso#ifdef notyet 1004169320Spiso /* 1005169320Spiso * XXX: This could be bad in the case of ppbus(8). Also, I think 1006169320Spiso * this could lead to races of stale data when servicing an 1007169320Spiso * interrupt. 1008169320Spiso */ 1009169320Spiso dead = 1; 1010169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1011169320Spiso if (handler != NULL) { 1012169320Spiso dead = 0; 1013169320Spiso break; 1014169320Spiso } 1015169320Spiso } 1016169320Spiso if (dead) { 1017169320Spiso ithread_destroy(ie->ie_thread); 1018169320Spiso ie->ie_thread = NULL; 1019169320Spiso } 1020169320Spiso#endif 1021169320Spiso mtx_unlock(&ie->ie_lock); 1022169320Spiso free(handler, M_ITHREAD); 1023169320Spiso return (0); 1024169320Spiso} 1025169320Spiso 1026177940Sjhbstatic int 1027169320Spisointr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it) 1028169320Spiso{ 1029169320Spiso struct intr_entropy entropy; 1030169320Spiso struct thread *td; 1031169320Spiso struct thread *ctd; 1032169320Spiso struct proc *p; 1033169320Spiso 1034169320Spiso /* 1035169320Spiso * If no ithread or no handlers, then we have a stray interrupt. 1036169320Spiso */ 1037169320Spiso if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL) 1038169320Spiso return (EINVAL); 1039169320Spiso 1040169320Spiso ctd = curthread; 1041169320Spiso td = it->it_thread; 1042169320Spiso p = td->td_proc; 1043169320Spiso 1044169320Spiso /* 1045169320Spiso * If any of the handlers for this ithread claim to be good 1046169320Spiso * sources of entropy, then gather some. 1047169320Spiso */ 1048169320Spiso if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 1049169320Spiso CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 1050173004Sjulian p->p_pid, td->td_name); 1051169320Spiso entropy.event = (uintptr_t)ie; 1052169320Spiso entropy.td = ctd; 1053169320Spiso random_harvest(&entropy, sizeof(entropy), 2, 0, 1054169320Spiso RANDOM_INTERRUPT); 1055169320Spiso } 1056169320Spiso 1057169320Spiso KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 1058169320Spiso 1059169320Spiso /* 1060169320Spiso * Set it_need to tell the thread to keep running if it is already 1061170307Sjeff * running. Then, lock the thread and see if we actually need to 1062170307Sjeff * put it on the runqueue. 1063169320Spiso */ 1064169320Spiso it->it_need = 1; 1065170307Sjeff thread_lock(td); 1066169320Spiso if (TD_AWAITING_INTR(td)) { 1067169320Spiso CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 1068173122Sjulian td->td_name); 1069169320Spiso TD_CLR_IWAIT(td); 1070169320Spiso sched_add(td, SRQ_INTR); 1071169320Spiso } else { 1072169320Spiso CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 1073173004Sjulian __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 1074169320Spiso } 1075170307Sjeff thread_unlock(td); 1076169320Spiso 1077169320Spiso return (0); 1078169320Spiso} 1079169320Spiso#endif 1080169320Spiso 1081151699Sjhb/* 1082192305Srwatson * Allow interrupt event binding for software interrupt handlers -- a no-op, 1083192305Srwatson * since interrupts are generated in software rather than being directed by 1084192305Srwatson * a PIC. 1085192305Srwatson */ 1086192305Srwatsonstatic int 1087192305Srwatsonswi_assign_cpu(void *arg, u_char cpu) 1088192305Srwatson{ 1089192305Srwatson 1090192305Srwatson return (0); 1091192305Srwatson} 1092192305Srwatson 1093192305Srwatson/* 1094151699Sjhb * Add a software interrupt handler to a specified event. If a given event 1095151699Sjhb * is not specified, then a new event is created. 1096151699Sjhb */ 109772759Sjhbint 1098151658Sjhbswi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, 109972237Sjhb void *arg, int pri, enum intr_type flags, void **cookiep) 110072237Sjhb{ 1101198854Sattilio struct thread *td; 1102151658Sjhb struct intr_event *ie; 110372237Sjhb int error; 110466698Sjhb 1105169320Spiso if (flags & INTR_ENTROPY) 110672759Sjhb return (EINVAL); 110772759Sjhb 1108151658Sjhb ie = (eventp != NULL) ? *eventp : NULL; 110966698Sjhb 1110151658Sjhb if (ie != NULL) { 1111151658Sjhb if (!(ie->ie_flags & IE_SOFT)) 1112151658Sjhb return (EINVAL); 111372759Sjhb } else { 1114178092Sjeff error = intr_event_create(&ie, NULL, IE_SOFT, 0, 1115192305Srwatson NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri); 111667551Sjhb if (error) 111772237Sjhb return (error); 1118151658Sjhb if (eventp != NULL) 1119151658Sjhb *eventp = ie; 112066698Sjhb } 1121177859Sjeff error = intr_event_add_handler(ie, name, NULL, handler, arg, 1122217292Sjhb PI_SWI(pri), flags, cookiep); 1123177859Sjeff if (error) 1124177859Sjeff return (error); 1125177859Sjeff if (pri == SWI_CLOCK) { 1126198854Sattilio td = ie->ie_thread->it_thread; 1127198854Sattilio thread_lock(td); 1128198854Sattilio td->td_flags |= TDF_NOLOAD; 1129198854Sattilio thread_unlock(td); 1130177859Sjeff } 1131177859Sjeff return (0); 113266698Sjhb} 113366698Sjhb 113466698Sjhb/* 1135151658Sjhb * Schedule a software interrupt thread. 113666698Sjhb */ 113767551Sjhbvoid 113872237Sjhbswi_sched(void *cookie, int flags) 113966698Sjhb{ 1140151658Sjhb struct intr_handler *ih = (struct intr_handler *)cookie; 1141151658Sjhb struct intr_event *ie = ih->ih_event; 114272759Sjhb int error; 114366698Sjhb 1144151658Sjhb CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, 1145151658Sjhb ih->ih_need); 1146151658Sjhb 114767551Sjhb /* 114872759Sjhb * Set ih_need for this handler so that if the ithread is already 114972759Sjhb * running it will execute this handler on the next pass. Otherwise, 115072759Sjhb * it will execute it the next time it runs. 115167551Sjhb */ 115272237Sjhb atomic_store_rel_int(&ih->ih_need, 1); 1153163474Sbde 115472237Sjhb if (!(flags & SWI_DELAY)) { 1155170291Sattilio PCPU_INC(cnt.v_soft); 1156169320Spiso#ifdef INTR_FILTER 1157169320Spiso error = intr_event_schedule_thread(ie, ie->ie_thread); 1158169320Spiso#else 1159151658Sjhb error = intr_event_schedule_thread(ie); 1160169320Spiso#endif 116172759Sjhb KASSERT(error == 0, ("stray software interrupt")); 116266698Sjhb } 116366698Sjhb} 116466698Sjhb 1165151699Sjhb/* 1166151699Sjhb * Remove a software interrupt handler. Currently this code does not 1167151699Sjhb * remove the associated interrupt event if it becomes empty. Calling code 1168151699Sjhb * may do so manually via intr_event_destroy(), but that's not really 1169151699Sjhb * an optimal interface. 1170151699Sjhb */ 1171151699Sjhbint 1172151699Sjhbswi_remove(void *cookie) 1173151699Sjhb{ 1174151699Sjhb 1175151699Sjhb return (intr_event_remove_handler(cookie)); 1176151699Sjhb} 1177151699Sjhb 1178169320Spiso#ifdef INTR_FILTER 1179151658Sjhbstatic void 1180169320Spisopriv_ithread_execute_handler(struct proc *p, struct intr_handler *ih) 1181169320Spiso{ 1182169320Spiso struct intr_event *ie; 1183169320Spiso 1184169320Spiso ie = ih->ih_event; 1185169320Spiso /* 1186169320Spiso * If this handler is marked for death, remove it from 1187169320Spiso * the list of handlers and wake up the sleeper. 1188169320Spiso */ 1189169320Spiso if (ih->ih_flags & IH_DEAD) { 1190169320Spiso mtx_lock(&ie->ie_lock); 1191169320Spiso TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1192169320Spiso ih->ih_flags &= ~IH_DEAD; 1193169320Spiso wakeup(ih); 1194169320Spiso mtx_unlock(&ie->ie_lock); 1195169320Spiso return; 1196169320Spiso } 1197169320Spiso 1198169320Spiso /* Execute this handler. */ 1199169320Spiso CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1200169320Spiso __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, 1201169320Spiso ih->ih_name, ih->ih_flags); 1202169320Spiso 1203169320Spiso if (!(ih->ih_flags & IH_MPSAFE)) 1204169320Spiso mtx_lock(&Giant); 1205169320Spiso ih->ih_handler(ih->ih_argument); 1206169320Spiso if (!(ih->ih_flags & IH_MPSAFE)) 1207169320Spiso mtx_unlock(&Giant); 1208169320Spiso} 1209169320Spiso#endif 1210169320Spiso 1211183052Sjhb/* 1212183052Sjhb * This is a public function for use by drivers that mux interrupt 1213183052Sjhb * handlers for child devices from their interrupt handler. 1214183052Sjhb */ 1215183052Sjhbvoid 1216183052Sjhbintr_event_execute_handlers(struct proc *p, struct intr_event *ie) 1217151658Sjhb{ 1218151658Sjhb struct intr_handler *ih, *ihn; 1219151658Sjhb 1220151658Sjhb TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { 1221151658Sjhb /* 1222151658Sjhb * If this handler is marked for death, remove it from 1223151658Sjhb * the list of handlers and wake up the sleeper. 1224151658Sjhb */ 1225151658Sjhb if (ih->ih_flags & IH_DEAD) { 1226151658Sjhb mtx_lock(&ie->ie_lock); 1227151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1228151658Sjhb ih->ih_flags &= ~IH_DEAD; 1229151658Sjhb wakeup(ih); 1230151658Sjhb mtx_unlock(&ie->ie_lock); 1231151658Sjhb continue; 1232151658Sjhb } 1233151658Sjhb 1234167080Spiso /* Skip filter only handlers */ 1235167080Spiso if (ih->ih_handler == NULL) 1236167080Spiso continue; 1237167080Spiso 1238151658Sjhb /* 1239151658Sjhb * For software interrupt threads, we only execute 1240151658Sjhb * handlers that have their need flag set. Hardware 1241151658Sjhb * interrupt threads always invoke all of their handlers. 1242151658Sjhb */ 1243151658Sjhb if (ie->ie_flags & IE_SOFT) { 1244151658Sjhb if (!ih->ih_need) 1245151658Sjhb continue; 1246151658Sjhb else 1247151658Sjhb atomic_store_rel_int(&ih->ih_need, 0); 1248151658Sjhb } 1249151658Sjhb 1250151658Sjhb /* Execute this handler. */ 1251151658Sjhb CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1252169320Spiso __func__, p->p_pid, (void *)ih->ih_handler, 1253169320Spiso ih->ih_argument, ih->ih_name, ih->ih_flags); 1254151658Sjhb 1255151658Sjhb if (!(ih->ih_flags & IH_MPSAFE)) 1256151658Sjhb mtx_lock(&Giant); 1257151658Sjhb ih->ih_handler(ih->ih_argument); 1258151658Sjhb if (!(ih->ih_flags & IH_MPSAFE)) 1259151658Sjhb mtx_unlock(&Giant); 1260151658Sjhb } 1261183052Sjhb} 1262183052Sjhb 1263183052Sjhbstatic void 1264183052Sjhbithread_execute_handlers(struct proc *p, struct intr_event *ie) 1265183052Sjhb{ 1266183052Sjhb 1267183052Sjhb /* Interrupt handlers should not sleep. */ 1268151658Sjhb if (!(ie->ie_flags & IE_SOFT)) 1269183052Sjhb THREAD_NO_SLEEPING(); 1270183052Sjhb intr_event_execute_handlers(p, ie); 1271183052Sjhb if (!(ie->ie_flags & IE_SOFT)) 1272151658Sjhb THREAD_SLEEPING_OK(); 1273151658Sjhb 1274151658Sjhb /* 1275151658Sjhb * Interrupt storm handling: 1276151658Sjhb * 1277151658Sjhb * If this interrupt source is currently storming, then throttle 1278151658Sjhb * it to only fire the handler once per clock tick. 1279151658Sjhb * 1280151658Sjhb * If this interrupt source is not currently storming, but the 1281151658Sjhb * number of back to back interrupts exceeds the storm threshold, 1282151658Sjhb * then enter storming mode. 1283151658Sjhb */ 1284167173Sjhb if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold && 1285167173Sjhb !(ie->ie_flags & IE_SOFT)) { 1286168850Snjl /* Report the message only once every second. */ 1287168850Snjl if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) { 1288151658Sjhb printf( 1289168850Snjl "interrupt storm detected on \"%s\"; throttling interrupt source\n", 1290151658Sjhb ie->ie_name); 1291151658Sjhb } 1292167173Sjhb pause("istorm", 1); 1293151658Sjhb } else 1294151658Sjhb ie->ie_count++; 1295151658Sjhb 1296151658Sjhb /* 1297151658Sjhb * Now that all the handlers have had a chance to run, reenable 1298151658Sjhb * the interrupt source. 1299151658Sjhb */ 1300177940Sjhb if (ie->ie_post_ithread != NULL) 1301177940Sjhb ie->ie_post_ithread(ie->ie_source); 1302151658Sjhb} 1303151658Sjhb 1304169320Spiso#ifndef INTR_FILTER 130566698Sjhb/* 130672237Sjhb * This is the main code for interrupt threads. 130766698Sjhb */ 1308104094Sphkstatic void 130972237Sjhbithread_loop(void *arg) 131066698Sjhb{ 1311151658Sjhb struct intr_thread *ithd; 1312151658Sjhb struct intr_event *ie; 131383366Sjulian struct thread *td; 131472237Sjhb struct proc *p; 1315219819Sjeff int wake; 1316151658Sjhb 131783366Sjulian td = curthread; 131883366Sjulian p = td->td_proc; 1319151658Sjhb ithd = (struct intr_thread *)arg; 1320151658Sjhb KASSERT(ithd->it_thread == td, 132187593Sobrien ("%s: ithread and proc linkage out of sync", __func__)); 1322151658Sjhb ie = ithd->it_event; 1323151658Sjhb ie->ie_count = 0; 1324219819Sjeff wake = 0; 132566698Sjhb 132667551Sjhb /* 132767551Sjhb * As long as we have interrupts outstanding, go through the 132867551Sjhb * list of handlers, giving each one a go at it. 132967551Sjhb */ 133066698Sjhb for (;;) { 133172237Sjhb /* 133272237Sjhb * If we are an orphaned thread, then just die. 133372237Sjhb */ 133472237Sjhb if (ithd->it_flags & IT_DEAD) { 1335151658Sjhb CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1336173004Sjulian p->p_pid, td->td_name); 133772237Sjhb free(ithd, M_ITHREAD); 1338173044Sjulian kthread_exit(); 133972237Sjhb } 134072237Sjhb 1341151658Sjhb /* 1342151658Sjhb * Service interrupts. If another interrupt arrives while 1343151658Sjhb * we are running, it will set it_need to note that we 1344151658Sjhb * should make another pass. 1345151658Sjhb */ 134672237Sjhb while (ithd->it_need) { 134767551Sjhb /* 1348151658Sjhb * This might need a full read and write barrier 1349151658Sjhb * to make sure that this write posts before any 1350151658Sjhb * of the memory or device accesses in the 1351151658Sjhb * handlers. 135267551Sjhb */ 135372237Sjhb atomic_store_rel_int(&ithd->it_need, 0); 1354151658Sjhb ithread_execute_handlers(p, ie); 135566698Sjhb } 1356128331Sjhb WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1357128331Sjhb mtx_assert(&Giant, MA_NOTOWNED); 135867551Sjhb 135966698Sjhb /* 136066698Sjhb * Processed all our interrupts. Now get the sched 136167551Sjhb * lock. This may take a while and it_need may get 136266698Sjhb * set again, so we have to check it again. 136366698Sjhb */ 1364170307Sjeff thread_lock(td); 1365219819Sjeff if (!ithd->it_need && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { 1366128331Sjhb TD_SET_IWAIT(td); 1367151658Sjhb ie->ie_count = 0; 1368178272Sjeff mi_switch(SW_VOL | SWT_IWAIT, NULL); 136966698Sjhb } 1370219819Sjeff if (ithd->it_flags & IT_WAIT) { 1371219819Sjeff wake = 1; 1372219819Sjeff ithd->it_flags &= ~IT_WAIT; 1373219819Sjeff } 1374170307Sjeff thread_unlock(td); 1375219819Sjeff if (wake) { 1376219819Sjeff wakeup(ithd); 1377219819Sjeff wake = 0; 1378219819Sjeff } 137966698Sjhb } 138066698Sjhb} 1381177940Sjhb 1382177940Sjhb/* 1383177940Sjhb * Main interrupt handling body. 1384177940Sjhb * 1385177940Sjhb * Input: 1386177940Sjhb * o ie: the event connected to this interrupt. 1387177940Sjhb * o frame: some archs (i.e. i386) pass a frame to some. 1388177940Sjhb * handlers as their main argument. 1389177940Sjhb * Return value: 1390177940Sjhb * o 0: everything ok. 1391177940Sjhb * o EINVAL: stray interrupt. 1392177940Sjhb */ 1393177940Sjhbint 1394177940Sjhbintr_event_handle(struct intr_event *ie, struct trapframe *frame) 1395177940Sjhb{ 1396177940Sjhb struct intr_handler *ih; 1397208988Smav struct trapframe *oldframe; 1398177940Sjhb struct thread *td; 1399177940Sjhb int error, ret, thread; 1400177940Sjhb 1401177940Sjhb td = curthread; 1402177940Sjhb 1403177940Sjhb /* An interrupt with no event or handlers is a stray interrupt. */ 1404177940Sjhb if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1405177940Sjhb return (EINVAL); 1406177940Sjhb 1407177940Sjhb /* 1408177940Sjhb * Execute fast interrupt handlers directly. 1409177940Sjhb * To support clock handlers, if a handler registers 1410177940Sjhb * with a NULL argument, then we pass it a pointer to 1411177940Sjhb * a trapframe as its argument. 1412177940Sjhb */ 1413177940Sjhb td->td_intr_nesting_level++; 1414177940Sjhb thread = 0; 1415177940Sjhb ret = 0; 1416177940Sjhb critical_enter(); 1417208988Smav oldframe = td->td_intr_frame; 1418208988Smav td->td_intr_frame = frame; 1419177940Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1420177940Sjhb if (ih->ih_filter == NULL) { 1421177940Sjhb thread = 1; 1422177940Sjhb continue; 1423177940Sjhb } 1424177940Sjhb CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, 1425177940Sjhb ih->ih_filter, ih->ih_argument == NULL ? frame : 1426177940Sjhb ih->ih_argument, ih->ih_name); 1427177940Sjhb if (ih->ih_argument == NULL) 1428177940Sjhb ret = ih->ih_filter(frame); 1429177940Sjhb else 1430177940Sjhb ret = ih->ih_filter(ih->ih_argument); 1431203061Savg KASSERT(ret == FILTER_STRAY || 1432203061Savg ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1433203061Savg (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1434203061Savg ("%s: incorrect return value %#x from %s", __func__, ret, 1435203061Savg ih->ih_name)); 1436203061Savg 1437177940Sjhb /* 1438177940Sjhb * Wrapper handler special handling: 1439177940Sjhb * 1440177940Sjhb * in some particular cases (like pccard and pccbb), 1441177940Sjhb * the _real_ device handler is wrapped in a couple of 1442177940Sjhb * functions - a filter wrapper and an ithread wrapper. 1443177940Sjhb * In this case (and just in this case), the filter wrapper 1444177940Sjhb * could ask the system to schedule the ithread and mask 1445177940Sjhb * the interrupt source if the wrapped handler is composed 1446177940Sjhb * of just an ithread handler. 1447177940Sjhb * 1448177940Sjhb * TODO: write a generic wrapper to avoid people rolling 1449177940Sjhb * their own 1450177940Sjhb */ 1451177940Sjhb if (!thread) { 1452177940Sjhb if (ret == FILTER_SCHEDULE_THREAD) 1453177940Sjhb thread = 1; 1454177940Sjhb } 1455177940Sjhb } 1456208988Smav td->td_intr_frame = oldframe; 1457177940Sjhb 1458177940Sjhb if (thread) { 1459177940Sjhb if (ie->ie_pre_ithread != NULL) 1460177940Sjhb ie->ie_pre_ithread(ie->ie_source); 1461177940Sjhb } else { 1462177940Sjhb if (ie->ie_post_filter != NULL) 1463177940Sjhb ie->ie_post_filter(ie->ie_source); 1464177940Sjhb } 1465177940Sjhb 1466177940Sjhb /* Schedule the ithread if needed. */ 1467177940Sjhb if (thread) { 1468177940Sjhb error = intr_event_schedule_thread(ie); 1469182024Skmacy#ifndef XEN 1470177940Sjhb KASSERT(error == 0, ("bad stray interrupt")); 1471182024Skmacy#else 1472182024Skmacy if (error != 0) 1473182024Skmacy log(LOG_WARNING, "bad stray interrupt"); 1474182024Skmacy#endif 1475177940Sjhb } 1476177940Sjhb critical_exit(); 1477177940Sjhb td->td_intr_nesting_level--; 1478177940Sjhb return (0); 1479177940Sjhb} 1480169320Spiso#else 1481169320Spiso/* 1482169320Spiso * This is the main code for interrupt threads. 1483169320Spiso */ 1484169320Spisostatic void 1485169320Spisoithread_loop(void *arg) 1486169320Spiso{ 1487169320Spiso struct intr_thread *ithd; 1488169320Spiso struct intr_handler *ih; 1489169320Spiso struct intr_event *ie; 1490169320Spiso struct thread *td; 1491169320Spiso struct proc *p; 1492169320Spiso int priv; 1493219819Sjeff int wake; 149466698Sjhb 1495169320Spiso td = curthread; 1496169320Spiso p = td->td_proc; 1497169320Spiso ih = (struct intr_handler *)arg; 1498169320Spiso priv = (ih->ih_thread != NULL) ? 1 : 0; 1499169320Spiso ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread; 1500169320Spiso KASSERT(ithd->it_thread == td, 1501169320Spiso ("%s: ithread and proc linkage out of sync", __func__)); 1502169320Spiso ie = ithd->it_event; 1503169320Spiso ie->ie_count = 0; 1504219819Sjeff wake = 0; 1505169320Spiso 1506169320Spiso /* 1507169320Spiso * As long as we have interrupts outstanding, go through the 1508169320Spiso * list of handlers, giving each one a go at it. 1509169320Spiso */ 1510169320Spiso for (;;) { 1511169320Spiso /* 1512169320Spiso * If we are an orphaned thread, then just die. 1513169320Spiso */ 1514169320Spiso if (ithd->it_flags & IT_DEAD) { 1515169320Spiso CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1516173004Sjulian p->p_pid, td->td_name); 1517169320Spiso free(ithd, M_ITHREAD); 1518173044Sjulian kthread_exit(); 1519169320Spiso } 1520169320Spiso 1521169320Spiso /* 1522169320Spiso * Service interrupts. If another interrupt arrives while 1523169320Spiso * we are running, it will set it_need to note that we 1524169320Spiso * should make another pass. 1525169320Spiso */ 1526169320Spiso while (ithd->it_need) { 1527169320Spiso /* 1528169320Spiso * This might need a full read and write barrier 1529169320Spiso * to make sure that this write posts before any 1530169320Spiso * of the memory or device accesses in the 1531169320Spiso * handlers. 1532169320Spiso */ 1533169320Spiso atomic_store_rel_int(&ithd->it_need, 0); 1534169320Spiso if (priv) 1535169320Spiso priv_ithread_execute_handler(p, ih); 1536169320Spiso else 1537169320Spiso ithread_execute_handlers(p, ie); 1538169320Spiso } 1539169320Spiso WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1540169320Spiso mtx_assert(&Giant, MA_NOTOWNED); 1541169320Spiso 1542169320Spiso /* 1543169320Spiso * Processed all our interrupts. Now get the sched 1544169320Spiso * lock. This may take a while and it_need may get 1545169320Spiso * set again, so we have to check it again. 1546169320Spiso */ 1547170307Sjeff thread_lock(td); 1548219819Sjeff if (!ithd->it_need && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { 1549169320Spiso TD_SET_IWAIT(td); 1550169320Spiso ie->ie_count = 0; 1551178272Sjeff mi_switch(SW_VOL | SWT_IWAIT, NULL); 1552169320Spiso } 1553219819Sjeff if (ithd->it_flags & IT_WAIT) { 1554219819Sjeff wake = 1; 1555219819Sjeff ithd->it_flags &= ~IT_WAIT; 1556219819Sjeff } 1557170307Sjeff thread_unlock(td); 1558219819Sjeff if (wake) { 1559219819Sjeff wakeup(ithd); 1560219819Sjeff wake = 0; 1561219819Sjeff } 1562169320Spiso } 1563169320Spiso} 1564169320Spiso 1565169320Spiso/* 1566169320Spiso * Main loop for interrupt filter. 1567169320Spiso * 1568169320Spiso * Some architectures (i386, amd64 and arm) require the optional frame 1569169320Spiso * parameter, and use it as the main argument for fast handler execution 1570169320Spiso * when ih_argument == NULL. 1571169320Spiso * 1572169320Spiso * Return value: 1573169320Spiso * o FILTER_STRAY: No filter recognized the event, and no 1574169320Spiso * filter-less handler is registered on this 1575169320Spiso * line. 1576169320Spiso * o FILTER_HANDLED: A filter claimed the event and served it. 1577169320Spiso * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at 1578169320Spiso * least one filter-less handler on this line. 1579169320Spiso * o FILTER_HANDLED | 1580169320Spiso * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for 1581169320Spiso * scheduling the per-handler ithread. 1582169320Spiso * 1583169320Spiso * In case an ithread has to be scheduled, in *ithd there will be a 1584169320Spiso * pointer to a struct intr_thread containing the thread to be 1585169320Spiso * scheduled. 1586169320Spiso */ 1587169320Spiso 1588177940Sjhbstatic int 1589169320Spisointr_filter_loop(struct intr_event *ie, struct trapframe *frame, 1590169320Spiso struct intr_thread **ithd) 1591169320Spiso{ 1592169320Spiso struct intr_handler *ih; 1593169320Spiso void *arg; 1594169320Spiso int ret, thread_only; 1595169320Spiso 1596169320Spiso ret = 0; 1597169320Spiso thread_only = 0; 1598169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1599169320Spiso /* 1600169320Spiso * Execute fast interrupt handlers directly. 1601169320Spiso * To support clock handlers, if a handler registers 1602169320Spiso * with a NULL argument, then we pass it a pointer to 1603169320Spiso * a trapframe as its argument. 1604169320Spiso */ 1605169320Spiso arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument); 1606169320Spiso 1607169320Spiso CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__, 1608169320Spiso ih->ih_filter, ih->ih_handler, arg, ih->ih_name); 1609169320Spiso 1610169320Spiso if (ih->ih_filter != NULL) 1611169320Spiso ret = ih->ih_filter(arg); 1612169320Spiso else { 1613169320Spiso thread_only = 1; 1614169320Spiso continue; 1615169320Spiso } 1616203061Savg KASSERT(ret == FILTER_STRAY || 1617203061Savg ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1618203061Savg (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1619203061Savg ("%s: incorrect return value %#x from %s", __func__, ret, 1620203061Savg ih->ih_name)); 1621169320Spiso if (ret & FILTER_STRAY) 1622169320Spiso continue; 1623169320Spiso else { 1624169320Spiso *ithd = ih->ih_thread; 1625169320Spiso return (ret); 1626169320Spiso } 1627169320Spiso } 1628169320Spiso 1629169320Spiso /* 1630169320Spiso * No filters handled the interrupt and we have at least 1631169320Spiso * one handler without a filter. In this case, we schedule 1632169320Spiso * all of the filter-less handlers to run in the ithread. 1633169320Spiso */ 1634169320Spiso if (thread_only) { 1635169320Spiso *ithd = ie->ie_thread; 1636169320Spiso return (FILTER_SCHEDULE_THREAD); 1637169320Spiso } 1638169320Spiso return (FILTER_STRAY); 1639169320Spiso} 1640169320Spiso 1641169320Spiso/* 1642169320Spiso * Main interrupt handling body. 1643169320Spiso * 1644169320Spiso * Input: 1645169320Spiso * o ie: the event connected to this interrupt. 1646169320Spiso * o frame: some archs (i.e. i386) pass a frame to some. 1647169320Spiso * handlers as their main argument. 1648169320Spiso * Return value: 1649169320Spiso * o 0: everything ok. 1650169320Spiso * o EINVAL: stray interrupt. 1651169320Spiso */ 1652169320Spisoint 1653169320Spisointr_event_handle(struct intr_event *ie, struct trapframe *frame) 1654169320Spiso{ 1655169320Spiso struct intr_thread *ithd; 1656208988Smav struct trapframe *oldframe; 1657169320Spiso struct thread *td; 1658169320Spiso int thread; 1659169320Spiso 1660169320Spiso ithd = NULL; 1661169320Spiso td = curthread; 1662169320Spiso 1663169320Spiso if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1664169320Spiso return (EINVAL); 1665169320Spiso 1666169320Spiso td->td_intr_nesting_level++; 1667169320Spiso thread = 0; 1668169320Spiso critical_enter(); 1669208988Smav oldframe = td->td_intr_frame; 1670208988Smav td->td_intr_frame = frame; 1671177940Sjhb thread = intr_filter_loop(ie, frame, &ithd); 1672169320Spiso if (thread & FILTER_HANDLED) { 1673177940Sjhb if (ie->ie_post_filter != NULL) 1674177940Sjhb ie->ie_post_filter(ie->ie_source); 1675169320Spiso } else { 1676177940Sjhb if (ie->ie_pre_ithread != NULL) 1677177940Sjhb ie->ie_pre_ithread(ie->ie_source); 1678169320Spiso } 1679208988Smav td->td_intr_frame = oldframe; 1680169320Spiso critical_exit(); 1681169320Spiso 1682169320Spiso /* Interrupt storm logic */ 1683169320Spiso if (thread & FILTER_STRAY) { 1684169320Spiso ie->ie_count++; 1685169320Spiso if (ie->ie_count < intr_storm_threshold) 1686169320Spiso printf("Interrupt stray detection not present\n"); 1687169320Spiso } 1688169320Spiso 1689169320Spiso /* Schedule an ithread if needed. */ 1690169320Spiso if (thread & FILTER_SCHEDULE_THREAD) { 1691169320Spiso if (intr_event_schedule_thread(ie, ithd) != 0) 1692169320Spiso panic("%s: impossible stray interrupt", __func__); 1693169320Spiso } 1694169320Spiso td->td_intr_nesting_level--; 1695169320Spiso return (0); 1696169320Spiso} 1697169320Spiso#endif 1698169320Spiso 1699121482Sjhb#ifdef DDB 170072237Sjhb/* 1701121482Sjhb * Dump details about an interrupt handler 1702121482Sjhb */ 1703121482Sjhbstatic void 1704151658Sjhbdb_dump_intrhand(struct intr_handler *ih) 1705121482Sjhb{ 1706121482Sjhb int comma; 1707121482Sjhb 1708121482Sjhb db_printf("\t%-10s ", ih->ih_name); 1709121482Sjhb switch (ih->ih_pri) { 1710121482Sjhb case PI_REALTIME: 1711121482Sjhb db_printf("CLK "); 1712121482Sjhb break; 1713121482Sjhb case PI_AV: 1714121482Sjhb db_printf("AV "); 1715121482Sjhb break; 1716217292Sjhb case PI_TTY: 1717121482Sjhb db_printf("TTY "); 1718121482Sjhb break; 1719121482Sjhb case PI_NET: 1720121482Sjhb db_printf("NET "); 1721121482Sjhb break; 1722121482Sjhb case PI_DISK: 1723121482Sjhb db_printf("DISK"); 1724121482Sjhb break; 1725121482Sjhb case PI_DULL: 1726121482Sjhb db_printf("DULL"); 1727121482Sjhb break; 1728121482Sjhb default: 1729121482Sjhb if (ih->ih_pri >= PI_SOFT) 1730121482Sjhb db_printf("SWI "); 1731121482Sjhb else 1732121482Sjhb db_printf("%4u", ih->ih_pri); 1733121482Sjhb break; 1734121482Sjhb } 1735121482Sjhb db_printf(" "); 1736121482Sjhb db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); 1737121482Sjhb db_printf("(%p)", ih->ih_argument); 1738121482Sjhb if (ih->ih_need || 1739166901Spiso (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | 1740121482Sjhb IH_MPSAFE)) != 0) { 1741121482Sjhb db_printf(" {"); 1742121482Sjhb comma = 0; 1743121482Sjhb if (ih->ih_flags & IH_EXCLUSIVE) { 1744121482Sjhb if (comma) 1745121482Sjhb db_printf(", "); 1746121482Sjhb db_printf("EXCL"); 1747121482Sjhb comma = 1; 1748121482Sjhb } 1749121482Sjhb if (ih->ih_flags & IH_ENTROPY) { 1750121482Sjhb if (comma) 1751121482Sjhb db_printf(", "); 1752121482Sjhb db_printf("ENTROPY"); 1753121482Sjhb comma = 1; 1754121482Sjhb } 1755121482Sjhb if (ih->ih_flags & IH_DEAD) { 1756121482Sjhb if (comma) 1757121482Sjhb db_printf(", "); 1758121482Sjhb db_printf("DEAD"); 1759121482Sjhb comma = 1; 1760121482Sjhb } 1761121482Sjhb if (ih->ih_flags & IH_MPSAFE) { 1762121482Sjhb if (comma) 1763121482Sjhb db_printf(", "); 1764121482Sjhb db_printf("MPSAFE"); 1765121482Sjhb comma = 1; 1766121482Sjhb } 1767121482Sjhb if (ih->ih_need) { 1768121482Sjhb if (comma) 1769121482Sjhb db_printf(", "); 1770121482Sjhb db_printf("NEED"); 1771121482Sjhb } 1772121482Sjhb db_printf("}"); 1773121482Sjhb } 1774121482Sjhb db_printf("\n"); 1775121482Sjhb} 1776121482Sjhb 1777121482Sjhb/* 1778151658Sjhb * Dump details about a event. 1779121482Sjhb */ 1780121482Sjhbvoid 1781151658Sjhbdb_dump_intr_event(struct intr_event *ie, int handlers) 1782121482Sjhb{ 1783151658Sjhb struct intr_handler *ih; 1784151658Sjhb struct intr_thread *it; 1785121482Sjhb int comma; 1786121482Sjhb 1787151658Sjhb db_printf("%s ", ie->ie_fullname); 1788151658Sjhb it = ie->ie_thread; 1789151658Sjhb if (it != NULL) 1790151658Sjhb db_printf("(pid %d)", it->it_thread->td_proc->p_pid); 1791151658Sjhb else 1792151658Sjhb db_printf("(no thread)"); 1793151658Sjhb if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || 1794151658Sjhb (it != NULL && it->it_need)) { 1795121482Sjhb db_printf(" {"); 1796121482Sjhb comma = 0; 1797151658Sjhb if (ie->ie_flags & IE_SOFT) { 1798121482Sjhb db_printf("SOFT"); 1799121482Sjhb comma = 1; 1800121482Sjhb } 1801151658Sjhb if (ie->ie_flags & IE_ENTROPY) { 1802121482Sjhb if (comma) 1803121482Sjhb db_printf(", "); 1804121482Sjhb db_printf("ENTROPY"); 1805121482Sjhb comma = 1; 1806121482Sjhb } 1807151658Sjhb if (ie->ie_flags & IE_ADDING_THREAD) { 1808121482Sjhb if (comma) 1809121482Sjhb db_printf(", "); 1810151658Sjhb db_printf("ADDING_THREAD"); 1811121482Sjhb comma = 1; 1812121482Sjhb } 1813151658Sjhb if (it != NULL && it->it_need) { 1814121482Sjhb if (comma) 1815121482Sjhb db_printf(", "); 1816121482Sjhb db_printf("NEED"); 1817121482Sjhb } 1818121482Sjhb db_printf("}"); 1819121482Sjhb } 1820121482Sjhb db_printf("\n"); 1821121482Sjhb 1822121482Sjhb if (handlers) 1823151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 1824121482Sjhb db_dump_intrhand(ih); 1825121482Sjhb} 1826151658Sjhb 1827151658Sjhb/* 1828151658Sjhb * Dump data about interrupt handlers 1829151658Sjhb */ 1830151658SjhbDB_SHOW_COMMAND(intr, db_show_intr) 1831151658Sjhb{ 1832151658Sjhb struct intr_event *ie; 1833160312Sjhb int all, verbose; 1834151658Sjhb 1835151658Sjhb verbose = index(modif, 'v') != NULL; 1836151658Sjhb all = index(modif, 'a') != NULL; 1837151658Sjhb TAILQ_FOREACH(ie, &event_list, ie_list) { 1838151658Sjhb if (!all && TAILQ_EMPTY(&ie->ie_handlers)) 1839151658Sjhb continue; 1840151658Sjhb db_dump_intr_event(ie, verbose); 1841160312Sjhb if (db_pager_quit) 1842160312Sjhb break; 1843151658Sjhb } 1844151658Sjhb} 1845121482Sjhb#endif /* DDB */ 1846121482Sjhb 1847121482Sjhb/* 184867551Sjhb * Start standard software interrupt threads 184966698Sjhb */ 185067551Sjhbstatic void 185172237Sjhbstart_softintr(void *dummy) 185267551Sjhb{ 185372237Sjhb 1854177859Sjeff if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) 1855177859Sjeff panic("died while creating vm swi ithread"); 185666698Sjhb} 1857177253SrwatsonSYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, 1858177253Srwatson NULL); 185966698Sjhb 1860151658Sjhb/* 186177582Stmm * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. 186277582Stmm * The data for this machine dependent, and the declarations are in machine 186377582Stmm * dependent code. The layout of intrnames and intrcnt however is machine 186477582Stmm * independent. 186577582Stmm * 186677582Stmm * We do not know the length of intrcnt and intrnames at compile time, so 186777582Stmm * calculate things at run time. 186877582Stmm */ 186977582Stmmstatic int 187077582Stmmsysctl_intrnames(SYSCTL_HANDLER_ARGS) 187177582Stmm{ 1872224187Sattilio return (sysctl_handle_opaque(oidp, intrnames, sintrnames, req)); 187377582Stmm} 187477582Stmm 187577582StmmSYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, 187677582Stmm NULL, 0, sysctl_intrnames, "", "Interrupt Names"); 187777582Stmm 187877582Stmmstatic int 187977582Stmmsysctl_intrcnt(SYSCTL_HANDLER_ARGS) 188077582Stmm{ 1881224187Sattilio return (sysctl_handle_opaque(oidp, intrcnt, sintrcnt, req)); 188277582Stmm} 188377582Stmm 188477582StmmSYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, 188577582Stmm NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); 1886121482Sjhb 1887121482Sjhb#ifdef DDB 1888121482Sjhb/* 1889121482Sjhb * DDB command to dump the interrupt statistics. 1890121482Sjhb */ 1891121482SjhbDB_SHOW_COMMAND(intrcnt, db_show_intrcnt) 1892121482Sjhb{ 1893121482Sjhb u_long *i; 1894121482Sjhb char *cp; 1895224187Sattilio u_int j; 1896121482Sjhb 1897121482Sjhb cp = intrnames; 1898224187Sattilio j = 0; 1899224187Sattilio for (i = intrcnt; j < (sintrcnt / sizeof(u_long)) && !db_pager_quit; 1900224187Sattilio i++, j++) { 1901121482Sjhb if (*cp == '\0') 1902121482Sjhb break; 1903121482Sjhb if (*i != 0) 1904121482Sjhb db_printf("%s\t%lu\n", cp, *i); 1905121482Sjhb cp += strlen(cp) + 1; 1906121482Sjhb } 1907121482Sjhb} 1908121482Sjhb#endif 1909