1139804Simp/*- 226156Sse * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 326156Sse * All rights reserved. 426156Sse * 526156Sse * Redistribution and use in source and binary forms, with or without 626156Sse * modification, are permitted provided that the following conditions 726156Sse * are met: 826156Sse * 1. Redistributions of source code must retain the above copyright 926156Sse * notice unmodified, this list of conditions, and the following 1026156Sse * disclaimer. 1126156Sse * 2. Redistributions in binary form must reproduce the above copyright 1226156Sse * notice, this list of conditions and the following disclaimer in the 1326156Sse * documentation and/or other materials provided with the distribution. 1426156Sse * 1526156Sse * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1626156Sse * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1726156Sse * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 1826156Sse * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 1926156Sse * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2026156Sse * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2126156Sse * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2226156Sse * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2326156Sse * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2426156Sse * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2526156Sse */ 2626156Sse 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD$"); 2936887Sdfr 30121482Sjhb#include "opt_ddb.h" 31121482Sjhb 3241059Speter#include <sys/param.h> 3365822Sjhb#include <sys/bus.h> 34110860Salfred#include <sys/conf.h> 35178092Sjeff#include <sys/cpuset.h> 3665822Sjhb#include <sys/rtprio.h> 3741059Speter#include <sys/systm.h> 3866698Sjhb#include <sys/interrupt.h> 3966698Sjhb#include <sys/kernel.h> 4066698Sjhb#include <sys/kthread.h> 4166698Sjhb#include <sys/ktr.h> 42130128Sbde#include <sys/limits.h> 4374914Sjhb#include <sys/lock.h> 4426156Sse#include <sys/malloc.h> 4567365Sjhb#include <sys/mutex.h> 46195249Sjhb#include <sys/priv.h> 4766698Sjhb#include <sys/proc.h> 4872759Sjhb#include <sys/random.h> 4972237Sjhb#include <sys/resourcevar.h> 50139451Sjhb#include <sys/sched.h> 51177181Sjhb#include <sys/smp.h> 5277582Stmm#include <sys/sysctl.h> 53182024Skmacy#include <sys/syslog.h> 5466698Sjhb#include <sys/unistd.h> 5566698Sjhb#include <sys/vmmeter.h> 5666698Sjhb#include <machine/atomic.h> 5766698Sjhb#include <machine/cpu.h> 5867551Sjhb#include <machine/md_var.h> 5972237Sjhb#include <machine/stdarg.h> 60121482Sjhb#ifdef DDB 61121482Sjhb#include <ddb/ddb.h> 62121482Sjhb#include <ddb/db_sym.h> 63121482Sjhb#endif 6426156Sse 65151658Sjhb/* 66151658Sjhb * Describe an interrupt thread. There is one of these per interrupt event. 67151658Sjhb */ 68151658Sjhbstruct intr_thread { 69151658Sjhb struct intr_event *it_event; 70151658Sjhb struct thread *it_thread; /* Kernel thread. */ 71151658Sjhb int it_flags; /* (j) IT_* flags. */ 72151658Sjhb int it_need; /* Needs service. */ 7372759Sjhb}; 7472759Sjhb 75151658Sjhb/* Interrupt thread flags kept in it_flags */ 76151658Sjhb#define IT_DEAD 0x000001 /* Thread is waiting to exit. */ 77219819Sjeff#define IT_WAIT 0x000002 /* Thread is waiting for completion. */ 78151658Sjhb 79151658Sjhbstruct intr_entropy { 80151658Sjhb struct thread *td; 81151658Sjhb uintptr_t event; 82151658Sjhb}; 83151658Sjhb 84151658Sjhbstruct intr_event *clk_intr_event; 85151658Sjhbstruct intr_event *tty_intr_event; 86128339Sbdevoid *vm_ih; 87173004Sjulianstruct proc *intrproc; 8838244Sbde 8972237Sjhbstatic MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 9072237Sjhb 91168850Snjlstatic int intr_storm_threshold = 1000; 92128331SjhbTUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold); 93128331SjhbSYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW, 94128331Sjhb &intr_storm_threshold, 0, 95128339Sbde "Number of consecutive interrupts before storm protection is enabled"); 96151658Sjhbstatic TAILQ_HEAD(, intr_event) event_list = 97151658Sjhb TAILQ_HEAD_INITIALIZER(event_list); 98178092Sjeffstatic struct mtx event_lock; 99178092SjeffMTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); 100128331Sjhb 101151658Sjhbstatic void intr_event_update(struct intr_event *ie); 102169320Spiso#ifdef INTR_FILTER 103177940Sjhbstatic int intr_event_schedule_thread(struct intr_event *ie, 104177940Sjhb struct intr_thread *ithd); 105177940Sjhbstatic int intr_filter_loop(struct intr_event *ie, 106177940Sjhb struct trapframe *frame, struct intr_thread **ithd); 107169320Spisostatic struct intr_thread *ithread_create(const char *name, 108169320Spiso struct intr_handler *ih); 109169320Spiso#else 110177940Sjhbstatic int intr_event_schedule_thread(struct intr_event *ie); 111151658Sjhbstatic struct intr_thread *ithread_create(const char *name); 112169320Spiso#endif 113151658Sjhbstatic void ithread_destroy(struct intr_thread *ithread); 114169320Spisostatic void ithread_execute_handlers(struct proc *p, 115169320Spiso struct intr_event *ie); 116169320Spiso#ifdef INTR_FILTER 117169320Spisostatic void priv_ithread_execute_handler(struct proc *p, 118169320Spiso struct intr_handler *ih); 119169320Spiso#endif 120128339Sbdestatic void ithread_loop(void *); 121151658Sjhbstatic void ithread_update(struct intr_thread *ithd); 122128339Sbdestatic void start_softintr(void *); 123128339Sbde 124165124Sjhb/* Map an interrupt type to an ithread priority. */ 12572237Sjhbu_char 126151658Sjhbintr_priority(enum intr_type flags) 12765822Sjhb{ 12872237Sjhb u_char pri; 12965822Sjhb 13072237Sjhb flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 13178365Speter INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); 13265822Sjhb switch (flags) { 13372237Sjhb case INTR_TYPE_TTY: 134217292Sjhb pri = PI_TTY; 13565822Sjhb break; 13665822Sjhb case INTR_TYPE_BIO: 13765822Sjhb pri = PI_DISK; 13865822Sjhb break; 13965822Sjhb case INTR_TYPE_NET: 14065822Sjhb pri = PI_NET; 14165822Sjhb break; 14265822Sjhb case INTR_TYPE_CAM: 143217292Sjhb pri = PI_DISK; 14465822Sjhb break; 145217292Sjhb case INTR_TYPE_AV: 14678365Speter pri = PI_AV; 14778365Speter break; 14872237Sjhb case INTR_TYPE_CLK: 14972237Sjhb pri = PI_REALTIME; 15072237Sjhb break; 15165822Sjhb case INTR_TYPE_MISC: 15265822Sjhb pri = PI_DULL; /* don't care */ 15365822Sjhb break; 15465822Sjhb default: 15572237Sjhb /* We didn't specify an interrupt level. */ 156151658Sjhb panic("intr_priority: no interrupt type in flags"); 15765822Sjhb } 15865822Sjhb 15965822Sjhb return pri; 16065822Sjhb} 16165822Sjhb 16272237Sjhb/* 163151658Sjhb * Update an ithread based on the associated intr_event. 16472237Sjhb */ 16572237Sjhbstatic void 166151658Sjhbithread_update(struct intr_thread *ithd) 16772237Sjhb{ 168151658Sjhb struct intr_event *ie; 16983366Sjulian struct thread *td; 170151658Sjhb u_char pri; 17167551Sjhb 172151658Sjhb ie = ithd->it_event; 173151658Sjhb td = ithd->it_thread; 17472237Sjhb 175151658Sjhb /* Determine the overall priority of this event. */ 176151658Sjhb if (TAILQ_EMPTY(&ie->ie_handlers)) 177151658Sjhb pri = PRI_MAX_ITHD; 178151658Sjhb else 179151658Sjhb pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri; 180105354Srobert 181151658Sjhb /* Update name and priority. */ 182173004Sjulian strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name)); 183232700Sjhb#ifdef KTR 184232700Sjhb sched_clear_tdname(td); 185232700Sjhb#endif 186170307Sjeff thread_lock(td); 187151658Sjhb sched_prio(td, pri); 188170307Sjeff thread_unlock(td); 189151658Sjhb} 190151658Sjhb 191151658Sjhb/* 192151658Sjhb * Regenerate the full name of an interrupt event and update its priority. 193151658Sjhb */ 194151658Sjhbstatic void 195151658Sjhbintr_event_update(struct intr_event *ie) 196151658Sjhb{ 197151658Sjhb struct intr_handler *ih; 198151658Sjhb char *last; 199151658Sjhb int missed, space; 200151658Sjhb 201151658Sjhb /* Start off with no entropy and just the name of the event. */ 202151658Sjhb mtx_assert(&ie->ie_lock, MA_OWNED); 203151658Sjhb strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 204151658Sjhb ie->ie_flags &= ~IE_ENTROPY; 205137267Sjhb missed = 0; 206151658Sjhb space = 1; 207151658Sjhb 208151658Sjhb /* Run through all the handlers updating values. */ 209151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 210151658Sjhb if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 < 211151658Sjhb sizeof(ie->ie_fullname)) { 212151658Sjhb strcat(ie->ie_fullname, " "); 213151658Sjhb strcat(ie->ie_fullname, ih->ih_name); 214151658Sjhb space = 0; 215137267Sjhb } else 216137267Sjhb missed++; 217137267Sjhb if (ih->ih_flags & IH_ENTROPY) 218151658Sjhb ie->ie_flags |= IE_ENTROPY; 219137267Sjhb } 220151658Sjhb 221151658Sjhb /* 222151658Sjhb * If the handler names were too long, add +'s to indicate missing 223151658Sjhb * names. If we run out of room and still have +'s to add, change 224151658Sjhb * the last character from a + to a *. 225151658Sjhb */ 226151658Sjhb last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2]; 227137267Sjhb while (missed-- > 0) { 228151658Sjhb if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) { 229151658Sjhb if (*last == '+') { 230151658Sjhb *last = '*'; 231151658Sjhb break; 232151658Sjhb } else 233151658Sjhb *last = '+'; 234151658Sjhb } else if (space) { 235151658Sjhb strcat(ie->ie_fullname, " +"); 236151658Sjhb space = 0; 23772237Sjhb } else 238151658Sjhb strcat(ie->ie_fullname, "+"); 23972237Sjhb } 240151658Sjhb 241151658Sjhb /* 242151658Sjhb * If this event has an ithread, update it's priority and 243151658Sjhb * name. 244151658Sjhb */ 245151658Sjhb if (ie->ie_thread != NULL) 246151658Sjhb ithread_update(ie->ie_thread); 247151658Sjhb CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname); 24872237Sjhb} 24972237Sjhb 25072237Sjhbint 251183298Sobrienintr_event_create(struct intr_event **event, void *source, int flags, int irq, 252177940Sjhb void (*pre_ithread)(void *), void (*post_ithread)(void *), 253177940Sjhb void (*post_filter)(void *), int (*assign_cpu)(void *, u_char), 254177940Sjhb const char *fmt, ...) 255169320Spiso{ 256169320Spiso struct intr_event *ie; 257169320Spiso va_list ap; 25872237Sjhb 259169320Spiso /* The only valid flag during creation is IE_SOFT. */ 260169320Spiso if ((flags & ~IE_SOFT) != 0) 261169320Spiso return (EINVAL); 262169320Spiso ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); 263169320Spiso ie->ie_source = source; 264177940Sjhb ie->ie_pre_ithread = pre_ithread; 265177940Sjhb ie->ie_post_ithread = post_ithread; 266177940Sjhb ie->ie_post_filter = post_filter; 267177181Sjhb ie->ie_assign_cpu = assign_cpu; 268169320Spiso ie->ie_flags = flags; 269178092Sjeff ie->ie_irq = irq; 270177181Sjhb ie->ie_cpu = NOCPU; 271169320Spiso TAILQ_INIT(&ie->ie_handlers); 272169320Spiso mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); 273169320Spiso 274169320Spiso va_start(ap, fmt); 275169320Spiso vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); 276169320Spiso va_end(ap); 277169320Spiso strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); 278178092Sjeff mtx_lock(&event_lock); 279169320Spiso TAILQ_INSERT_TAIL(&event_list, ie, ie_list); 280178092Sjeff mtx_unlock(&event_lock); 281169320Spiso if (event != NULL) 282169320Spiso *event = ie; 283169320Spiso CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); 284169320Spiso return (0); 285169320Spiso} 286169320Spiso 287177181Sjhb/* 288177181Sjhb * Bind an interrupt event to the specified CPU. Note that not all 289177181Sjhb * platforms support binding an interrupt to a CPU. For those 290177181Sjhb * platforms this request will fail. For supported platforms, any 291177181Sjhb * associated ithreads as well as the primary interrupt context will 292177181Sjhb * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds 293177181Sjhb * the interrupt event. 294177181Sjhb */ 295151658Sjhbint 296177181Sjhbintr_event_bind(struct intr_event *ie, u_char cpu) 297177181Sjhb{ 298178092Sjeff cpuset_t mask; 299178092Sjeff lwpid_t id; 300177181Sjhb int error; 301177181Sjhb 302177181Sjhb /* Need a CPU to bind to. */ 303177181Sjhb if (cpu != NOCPU && CPU_ABSENT(cpu)) 304177181Sjhb return (EINVAL); 305177181Sjhb 306177181Sjhb if (ie->ie_assign_cpu == NULL) 307177181Sjhb return (EOPNOTSUPP); 308195249Sjhb 309195249Sjhb error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR); 310195249Sjhb if (error) 311195249Sjhb return (error); 312195249Sjhb 313178092Sjeff /* 314195249Sjhb * If we have any ithreads try to set their mask first to verify 315195249Sjhb * permissions, etc. 316178092Sjeff */ 317177181Sjhb mtx_lock(&ie->ie_lock); 318178092Sjeff if (ie->ie_thread != NULL) { 319178092Sjeff CPU_ZERO(&mask); 320178092Sjeff if (cpu == NOCPU) 321178092Sjeff CPU_COPY(cpuset_root, &mask); 322178092Sjeff else 323178092Sjeff CPU_SET(cpu, &mask); 324178092Sjeff id = ie->ie_thread->it_thread->td_tid; 325177181Sjhb mtx_unlock(&ie->ie_lock); 326178092Sjeff error = cpuset_setthread(id, &mask); 327178092Sjeff if (error) 328178092Sjeff return (error); 329178092Sjeff } else 330178092Sjeff mtx_unlock(&ie->ie_lock); 331177181Sjhb error = ie->ie_assign_cpu(ie->ie_source, cpu); 332195249Sjhb if (error) { 333195249Sjhb mtx_lock(&ie->ie_lock); 334195249Sjhb if (ie->ie_thread != NULL) { 335195249Sjhb CPU_ZERO(&mask); 336195249Sjhb if (ie->ie_cpu == NOCPU) 337195249Sjhb CPU_COPY(cpuset_root, &mask); 338195249Sjhb else 339246452Sneel CPU_SET(ie->ie_cpu, &mask); 340195249Sjhb id = ie->ie_thread->it_thread->td_tid; 341195249Sjhb mtx_unlock(&ie->ie_lock); 342195249Sjhb (void)cpuset_setthread(id, &mask); 343195249Sjhb } else 344195249Sjhb mtx_unlock(&ie->ie_lock); 345177181Sjhb return (error); 346195249Sjhb } 347195249Sjhb 348177181Sjhb mtx_lock(&ie->ie_lock); 349177181Sjhb ie->ie_cpu = cpu; 350177181Sjhb mtx_unlock(&ie->ie_lock); 351178092Sjeff 352178092Sjeff return (error); 353178092Sjeff} 354178092Sjeff 355178092Sjeffstatic struct intr_event * 356178092Sjeffintr_lookup(int irq) 357178092Sjeff{ 358178092Sjeff struct intr_event *ie; 359178092Sjeff 360178092Sjeff mtx_lock(&event_lock); 361178092Sjeff TAILQ_FOREACH(ie, &event_list, ie_list) 362178092Sjeff if (ie->ie_irq == irq && 363178092Sjeff (ie->ie_flags & IE_SOFT) == 0 && 364178092Sjeff TAILQ_FIRST(&ie->ie_handlers) != NULL) 365178092Sjeff break; 366178092Sjeff mtx_unlock(&event_lock); 367178092Sjeff return (ie); 368178092Sjeff} 369178092Sjeff 370178092Sjeffint 371178092Sjeffintr_setaffinity(int irq, void *m) 372178092Sjeff{ 373178092Sjeff struct intr_event *ie; 374178092Sjeff cpuset_t *mask; 375178092Sjeff u_char cpu; 376178092Sjeff int n; 377178092Sjeff 378178092Sjeff mask = m; 379178092Sjeff cpu = NOCPU; 380178092Sjeff /* 381178092Sjeff * If we're setting all cpus we can unbind. Otherwise make sure 382178092Sjeff * only one cpu is in the set. 383178092Sjeff */ 384178092Sjeff if (CPU_CMP(cpuset_root, mask)) { 385178092Sjeff for (n = 0; n < CPU_SETSIZE; n++) { 386178092Sjeff if (!CPU_ISSET(n, mask)) 387178092Sjeff continue; 388178092Sjeff if (cpu != NOCPU) 389178092Sjeff return (EINVAL); 390178092Sjeff cpu = (u_char)n; 391178092Sjeff } 392178092Sjeff } 393178092Sjeff ie = intr_lookup(irq); 394178092Sjeff if (ie == NULL) 395178092Sjeff return (ESRCH); 396194987Sjhb return (intr_event_bind(ie, cpu)); 397178092Sjeff} 398178092Sjeff 399178092Sjeffint 400178092Sjeffintr_getaffinity(int irq, void *m) 401178092Sjeff{ 402178092Sjeff struct intr_event *ie; 403178092Sjeff cpuset_t *mask; 404178092Sjeff 405178092Sjeff mask = m; 406178092Sjeff ie = intr_lookup(irq); 407178092Sjeff if (ie == NULL) 408178092Sjeff return (ESRCH); 409178092Sjeff CPU_ZERO(mask); 410178092Sjeff mtx_lock(&ie->ie_lock); 411178092Sjeff if (ie->ie_cpu == NOCPU) 412178092Sjeff CPU_COPY(cpuset_root, mask); 413178092Sjeff else 414178092Sjeff CPU_SET(ie->ie_cpu, mask); 415178092Sjeff mtx_unlock(&ie->ie_lock); 416177181Sjhb return (0); 417177181Sjhb} 418177181Sjhb 419177181Sjhbint 420151658Sjhbintr_event_destroy(struct intr_event *ie) 421151658Sjhb{ 422151658Sjhb 423178092Sjeff mtx_lock(&event_lock); 424151658Sjhb mtx_lock(&ie->ie_lock); 425151658Sjhb if (!TAILQ_EMPTY(&ie->ie_handlers)) { 426151658Sjhb mtx_unlock(&ie->ie_lock); 427178092Sjeff mtx_unlock(&event_lock); 428151658Sjhb return (EBUSY); 429151658Sjhb } 430151658Sjhb TAILQ_REMOVE(&event_list, ie, ie_list); 431157728Sjhb#ifndef notyet 432157728Sjhb if (ie->ie_thread != NULL) { 433157728Sjhb ithread_destroy(ie->ie_thread); 434157728Sjhb ie->ie_thread = NULL; 435157728Sjhb } 436157728Sjhb#endif 437151658Sjhb mtx_unlock(&ie->ie_lock); 438178092Sjeff mtx_unlock(&event_lock); 439151658Sjhb mtx_destroy(&ie->ie_lock); 440151658Sjhb free(ie, M_ITHREAD); 441151658Sjhb return (0); 442151658Sjhb} 443151658Sjhb 444169320Spiso#ifndef INTR_FILTER 445151658Sjhbstatic struct intr_thread * 446151658Sjhbithread_create(const char *name) 447151658Sjhb{ 448151658Sjhb struct intr_thread *ithd; 449151658Sjhb struct thread *td; 450151658Sjhb int error; 451151658Sjhb 452151658Sjhb ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 453151658Sjhb 454173004Sjulian error = kproc_kthread_add(ithread_loop, ithd, &intrproc, 455173004Sjulian &td, RFSTOPPED | RFHIGHPID, 456173051Sjulian 0, "intr", "%s", name); 457151658Sjhb if (error) 458172836Sjulian panic("kproc_create() failed with %d", error); 459170307Sjeff thread_lock(td); 460164936Sjulian sched_class(td, PRI_ITHD); 461103216Sjulian TD_SET_IWAIT(td); 462170307Sjeff thread_unlock(td); 463151658Sjhb td->td_pflags |= TDP_ITHREAD; 464151658Sjhb ithd->it_thread = td; 465151658Sjhb CTR2(KTR_INTR, "%s: created %s", __func__, name); 466151658Sjhb return (ithd); 46772237Sjhb} 468169320Spiso#else 469169320Spisostatic struct intr_thread * 470169320Spisoithread_create(const char *name, struct intr_handler *ih) 471169320Spiso{ 472169320Spiso struct intr_thread *ithd; 473169320Spiso struct thread *td; 474169320Spiso int error; 47572237Sjhb 476169320Spiso ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); 477169320Spiso 478173153Sjulian error = kproc_kthread_add(ithread_loop, ih, &intrproc, 479173004Sjulian &td, RFSTOPPED | RFHIGHPID, 480173051Sjulian 0, "intr", "%s", name); 481169320Spiso if (error) 482172836Sjulian panic("kproc_create() failed with %d", error); 483170307Sjeff thread_lock(td); 484169320Spiso sched_class(td, PRI_ITHD); 485169320Spiso TD_SET_IWAIT(td); 486170307Sjeff thread_unlock(td); 487169320Spiso td->td_pflags |= TDP_ITHREAD; 488169320Spiso ithd->it_thread = td; 489169320Spiso CTR2(KTR_INTR, "%s: created %s", __func__, name); 490169320Spiso return (ithd); 491169320Spiso} 492169320Spiso#endif 493169320Spiso 494151658Sjhbstatic void 495151658Sjhbithread_destroy(struct intr_thread *ithread) 49672237Sjhb{ 49783366Sjulian struct thread *td; 49872237Sjhb 499157784Sscottl CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name); 500151658Sjhb td = ithread->it_thread; 501170307Sjeff thread_lock(td); 50276771Sjhb ithread->it_flags |= IT_DEAD; 503103216Sjulian if (TD_AWAITING_INTR(td)) { 504103216Sjulian TD_CLR_IWAIT(td); 505166188Sjeff sched_add(td, SRQ_INTR); 50672237Sjhb } 507170307Sjeff thread_unlock(td); 50872237Sjhb} 50972237Sjhb 510169320Spiso#ifndef INTR_FILTER 51172237Sjhbint 512151658Sjhbintr_event_add_handler(struct intr_event *ie, const char *name, 513166901Spiso driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 514166901Spiso enum intr_type flags, void **cookiep) 51572237Sjhb{ 516151658Sjhb struct intr_handler *ih, *temp_ih; 517151658Sjhb struct intr_thread *it; 51872237Sjhb 519166901Spiso if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 52072237Sjhb return (EINVAL); 52172237Sjhb 522151658Sjhb /* Allocate and populate an interrupt handler structure. */ 523151658Sjhb ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 524166901Spiso ih->ih_filter = filter; 52572237Sjhb ih->ih_handler = handler; 52672237Sjhb ih->ih_argument = arg; 527198134Sjhb strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 528151658Sjhb ih->ih_event = ie; 52972237Sjhb ih->ih_pri = pri; 530166901Spiso if (flags & INTR_EXCL) 53172237Sjhb ih->ih_flags = IH_EXCLUSIVE; 53272237Sjhb if (flags & INTR_MPSAFE) 53372237Sjhb ih->ih_flags |= IH_MPSAFE; 53472237Sjhb if (flags & INTR_ENTROPY) 53572237Sjhb ih->ih_flags |= IH_ENTROPY; 53672237Sjhb 537151658Sjhb /* We can only have one exclusive handler in a event. */ 538151658Sjhb mtx_lock(&ie->ie_lock); 539151658Sjhb if (!TAILQ_EMPTY(&ie->ie_handlers)) { 540151658Sjhb if ((flags & INTR_EXCL) || 541151658Sjhb (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 542151658Sjhb mtx_unlock(&ie->ie_lock); 543151658Sjhb free(ih, M_ITHREAD); 544151658Sjhb return (EINVAL); 545151658Sjhb } 546122002Sjhb } 54772237Sjhb 548151658Sjhb /* Create a thread if we need one. */ 549166901Spiso while (ie->ie_thread == NULL && handler != NULL) { 550151658Sjhb if (ie->ie_flags & IE_ADDING_THREAD) 551157815Sjhb msleep(ie, &ie->ie_lock, 0, "ithread", 0); 552151658Sjhb else { 553151658Sjhb ie->ie_flags |= IE_ADDING_THREAD; 554151658Sjhb mtx_unlock(&ie->ie_lock); 555151658Sjhb it = ithread_create("intr: newborn"); 556151658Sjhb mtx_lock(&ie->ie_lock); 557151658Sjhb ie->ie_flags &= ~IE_ADDING_THREAD; 558151658Sjhb ie->ie_thread = it; 559151658Sjhb it->it_event = ie; 560151658Sjhb ithread_update(it); 561151658Sjhb wakeup(ie); 562151658Sjhb } 563151658Sjhb } 564239095Skan 565239095Skan /* Add the new handler to the event in priority order. */ 566239095Skan TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 567239095Skan if (temp_ih->ih_pri > ih->ih_pri) 568239095Skan break; 569239095Skan } 570239095Skan if (temp_ih == NULL) 571239095Skan TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 572239095Skan else 573239095Skan TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 574239095Skan intr_event_update(ie); 575239095Skan 576151658Sjhb CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 577151658Sjhb ie->ie_name); 578151658Sjhb mtx_unlock(&ie->ie_lock); 579151658Sjhb 58072237Sjhb if (cookiep != NULL) 58172237Sjhb *cookiep = ih; 58272237Sjhb return (0); 58372237Sjhb} 584169320Spiso#else 585169320Spisoint 586169320Spisointr_event_add_handler(struct intr_event *ie, const char *name, 587169320Spiso driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, 588169320Spiso enum intr_type flags, void **cookiep) 589169320Spiso{ 590169320Spiso struct intr_handler *ih, *temp_ih; 591169320Spiso struct intr_thread *it; 59272237Sjhb 593169320Spiso if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) 594169320Spiso return (EINVAL); 595169320Spiso 596169320Spiso /* Allocate and populate an interrupt handler structure. */ 597169320Spiso ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); 598169320Spiso ih->ih_filter = filter; 599169320Spiso ih->ih_handler = handler; 600169320Spiso ih->ih_argument = arg; 601198134Sjhb strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); 602169320Spiso ih->ih_event = ie; 603169320Spiso ih->ih_pri = pri; 604169320Spiso if (flags & INTR_EXCL) 605169320Spiso ih->ih_flags = IH_EXCLUSIVE; 606169320Spiso if (flags & INTR_MPSAFE) 607169320Spiso ih->ih_flags |= IH_MPSAFE; 608169320Spiso if (flags & INTR_ENTROPY) 609169320Spiso ih->ih_flags |= IH_ENTROPY; 610169320Spiso 611169320Spiso /* We can only have one exclusive handler in a event. */ 612169320Spiso mtx_lock(&ie->ie_lock); 613169320Spiso if (!TAILQ_EMPTY(&ie->ie_handlers)) { 614169320Spiso if ((flags & INTR_EXCL) || 615169320Spiso (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { 616169320Spiso mtx_unlock(&ie->ie_lock); 617169320Spiso free(ih, M_ITHREAD); 618169320Spiso return (EINVAL); 619169320Spiso } 620169320Spiso } 621169320Spiso 622169320Spiso /* For filtered handlers, create a private ithread to run on. */ 623239095Skan if (filter != NULL && handler != NULL) { 624169320Spiso mtx_unlock(&ie->ie_lock); 625239095Skan it = ithread_create("intr: newborn", ih); 626169320Spiso mtx_lock(&ie->ie_lock); 627239095Skan it->it_event = ie; 628169320Spiso ih->ih_thread = it; 629230231Spluknet ithread_update(it); /* XXX - do we really need this?!?!? */ 630169320Spiso } else { /* Create the global per-event thread if we need one. */ 631169320Spiso while (ie->ie_thread == NULL && handler != NULL) { 632169320Spiso if (ie->ie_flags & IE_ADDING_THREAD) 633169320Spiso msleep(ie, &ie->ie_lock, 0, "ithread", 0); 634169320Spiso else { 635169320Spiso ie->ie_flags |= IE_ADDING_THREAD; 636169320Spiso mtx_unlock(&ie->ie_lock); 637169320Spiso it = ithread_create("intr: newborn", ih); 638169320Spiso mtx_lock(&ie->ie_lock); 639169320Spiso ie->ie_flags &= ~IE_ADDING_THREAD; 640169320Spiso ie->ie_thread = it; 641169320Spiso it->it_event = ie; 642169320Spiso ithread_update(it); 643169320Spiso wakeup(ie); 644169320Spiso } 645169320Spiso } 646169320Spiso } 647239095Skan 648239095Skan /* Add the new handler to the event in priority order. */ 649239095Skan TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { 650239095Skan if (temp_ih->ih_pri > ih->ih_pri) 651239095Skan break; 652239095Skan } 653239095Skan if (temp_ih == NULL) 654239095Skan TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); 655239095Skan else 656239095Skan TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 657239095Skan intr_event_update(ie); 658239095Skan 659169320Spiso CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 660169320Spiso ie->ie_name); 661169320Spiso mtx_unlock(&ie->ie_lock); 662169320Spiso 663169320Spiso if (cookiep != NULL) 664169320Spiso *cookiep = ih; 665169320Spiso return (0); 666169320Spiso} 667169320Spiso#endif 668169320Spiso 669165125Sjhb/* 670198134Sjhb * Append a description preceded by a ':' to the name of the specified 671198134Sjhb * interrupt handler. 672198134Sjhb */ 673198134Sjhbint 674198134Sjhbintr_event_describe_handler(struct intr_event *ie, void *cookie, 675198134Sjhb const char *descr) 676198134Sjhb{ 677198134Sjhb struct intr_handler *ih; 678198134Sjhb size_t space; 679198134Sjhb char *start; 680198134Sjhb 681198134Sjhb mtx_lock(&ie->ie_lock); 682198134Sjhb#ifdef INVARIANTS 683198134Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 684198134Sjhb if (ih == cookie) 685198134Sjhb break; 686198134Sjhb } 687198134Sjhb if (ih == NULL) { 688198134Sjhb mtx_unlock(&ie->ie_lock); 689198149Sjhb panic("handler %p not found in interrupt event %p", cookie, ie); 690198134Sjhb } 691198134Sjhb#endif 692198134Sjhb ih = cookie; 693198134Sjhb 694198134Sjhb /* 695198134Sjhb * Look for an existing description by checking for an 696198134Sjhb * existing ":". This assumes device names do not include 697198134Sjhb * colons. If one is found, prepare to insert the new 698198134Sjhb * description at that point. If one is not found, find the 699198134Sjhb * end of the name to use as the insertion point. 700198134Sjhb */ 701229272Sed start = strchr(ih->ih_name, ':'); 702198134Sjhb if (start == NULL) 703229272Sed start = strchr(ih->ih_name, 0); 704198134Sjhb 705198134Sjhb /* 706198134Sjhb * See if there is enough remaining room in the string for the 707198134Sjhb * description + ":". The "- 1" leaves room for the trailing 708198134Sjhb * '\0'. The "+ 1" accounts for the colon. 709198134Sjhb */ 710198134Sjhb space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1; 711198134Sjhb if (strlen(descr) + 1 > space) { 712198134Sjhb mtx_unlock(&ie->ie_lock); 713198134Sjhb return (ENOSPC); 714198134Sjhb } 715198134Sjhb 716198134Sjhb /* Append a colon followed by the description. */ 717198134Sjhb *start = ':'; 718198134Sjhb strcpy(start + 1, descr); 719198134Sjhb intr_event_update(ie); 720198134Sjhb mtx_unlock(&ie->ie_lock); 721198134Sjhb return (0); 722198134Sjhb} 723198134Sjhb 724198134Sjhb/* 725165125Sjhb * Return the ie_source field from the intr_event an intr_handler is 726165125Sjhb * associated with. 727165125Sjhb */ 728165125Sjhbvoid * 729165125Sjhbintr_handler_source(void *cookie) 730165125Sjhb{ 731165125Sjhb struct intr_handler *ih; 732165125Sjhb struct intr_event *ie; 733165125Sjhb 734165125Sjhb ih = (struct intr_handler *)cookie; 735165125Sjhb if (ih == NULL) 736165125Sjhb return (NULL); 737165125Sjhb ie = ih->ih_event; 738165125Sjhb KASSERT(ie != NULL, 739165125Sjhb ("interrupt handler \"%s\" has a NULL interrupt event", 740165125Sjhb ih->ih_name)); 741165125Sjhb return (ie->ie_source); 742165125Sjhb} 743165125Sjhb 744219819Sjeff/* 745219819Sjeff * Sleep until an ithread finishes executing an interrupt handler. 746219819Sjeff * 747219819Sjeff * XXX Doesn't currently handle interrupt filters or fast interrupt 748219819Sjeff * handlers. This is intended for compatibility with linux drivers 749219819Sjeff * only. Do not use in BSD code. 750219819Sjeff */ 751219819Sjeffvoid 752219819Sjeff_intr_drain(int irq) 753219819Sjeff{ 754219819Sjeff struct intr_event *ie; 755219819Sjeff struct intr_thread *ithd; 756219819Sjeff struct thread *td; 757219819Sjeff 758219819Sjeff ie = intr_lookup(irq); 759219819Sjeff if (ie == NULL) 760219819Sjeff return; 761219819Sjeff if (ie->ie_thread == NULL) 762219819Sjeff return; 763219819Sjeff ithd = ie->ie_thread; 764219819Sjeff td = ithd->it_thread; 765221055Sjeff /* 766221055Sjeff * We set the flag and wait for it to be cleared to avoid 767221055Sjeff * long delays with potentially busy interrupt handlers 768221055Sjeff * were we to only sample TD_AWAITING_INTR() every tick. 769221055Sjeff */ 770219819Sjeff thread_lock(td); 771219819Sjeff if (!TD_AWAITING_INTR(td)) { 772219819Sjeff ithd->it_flags |= IT_WAIT; 773221055Sjeff while (ithd->it_flags & IT_WAIT) { 774221055Sjeff thread_unlock(td); 775221055Sjeff pause("idrain", 1); 776221055Sjeff thread_lock(td); 777221055Sjeff } 778219819Sjeff } 779221055Sjeff thread_unlock(td); 780219819Sjeff return; 781219819Sjeff} 782219819Sjeff 783219819Sjeff 784169320Spiso#ifndef INTR_FILTER 78572237Sjhbint 786151658Sjhbintr_event_remove_handler(void *cookie) 78772237Sjhb{ 788151658Sjhb struct intr_handler *handler = (struct intr_handler *)cookie; 789151658Sjhb struct intr_event *ie; 79072237Sjhb#ifdef INVARIANTS 791151658Sjhb struct intr_handler *ih; 79272237Sjhb#endif 793151658Sjhb#ifdef notyet 794151658Sjhb int dead; 795151658Sjhb#endif 79672237Sjhb 79772759Sjhb if (handler == NULL) 79872237Sjhb return (EINVAL); 799151658Sjhb ie = handler->ih_event; 800151658Sjhb KASSERT(ie != NULL, 801151658Sjhb ("interrupt handler \"%s\" has a NULL interrupt event", 802165124Sjhb handler->ih_name)); 803151658Sjhb mtx_lock(&ie->ie_lock); 80487593Sobrien CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 805151658Sjhb ie->ie_name); 80672237Sjhb#ifdef INVARIANTS 807151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 80872759Sjhb if (ih == handler) 80972759Sjhb goto ok; 810151658Sjhb mtx_unlock(&ie->ie_lock); 811151658Sjhb panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 812151658Sjhb ih->ih_name, ie->ie_name); 81372759Sjhbok: 81472237Sjhb#endif 81572839Sjhb /* 816151658Sjhb * If there is no ithread, then just remove the handler and return. 817151658Sjhb * XXX: Note that an INTR_FAST handler might be running on another 818151658Sjhb * CPU! 819151658Sjhb */ 820151658Sjhb if (ie->ie_thread == NULL) { 821151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 822151658Sjhb mtx_unlock(&ie->ie_lock); 823151658Sjhb free(handler, M_ITHREAD); 824151658Sjhb return (0); 825151658Sjhb } 826151658Sjhb 827151658Sjhb /* 82872839Sjhb * If the interrupt thread is already running, then just mark this 82972839Sjhb * handler as being dead and let the ithread do the actual removal. 830124505Struckman * 831124505Struckman * During a cold boot while cold is set, msleep() does not sleep, 832124505Struckman * so we have to remove the handler here rather than letting the 833124505Struckman * thread do it. 83472839Sjhb */ 835170307Sjeff thread_lock(ie->ie_thread->it_thread); 836151658Sjhb if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) { 83772839Sjhb handler->ih_flags |= IH_DEAD; 83872839Sjhb 83972839Sjhb /* 84072839Sjhb * Ensure that the thread will process the handler list 84172839Sjhb * again and remove this handler if it has already passed 84272839Sjhb * it on the list. 84372839Sjhb */ 844252683Salfred atomic_store_rel_int(&ie->ie_thread->it_need, 1); 845151658Sjhb } else 846151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 847170307Sjeff thread_unlock(ie->ie_thread->it_thread); 848151658Sjhb while (handler->ih_flags & IH_DEAD) 849157815Sjhb msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 850151658Sjhb intr_event_update(ie); 851151658Sjhb#ifdef notyet 852151658Sjhb /* 853151658Sjhb * XXX: This could be bad in the case of ppbus(8). Also, I think 854151658Sjhb * this could lead to races of stale data when servicing an 855151658Sjhb * interrupt. 856151658Sjhb */ 857151658Sjhb dead = 1; 858151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 859151658Sjhb if (!(ih->ih_flags & IH_FAST)) { 860151658Sjhb dead = 0; 861151658Sjhb break; 862151658Sjhb } 863151658Sjhb } 864151658Sjhb if (dead) { 865151658Sjhb ithread_destroy(ie->ie_thread); 866151658Sjhb ie->ie_thread = NULL; 867151658Sjhb } 868151658Sjhb#endif 869151658Sjhb mtx_unlock(&ie->ie_lock); 87076771Sjhb free(handler, M_ITHREAD); 87172237Sjhb return (0); 87272237Sjhb} 87372237Sjhb 874177940Sjhbstatic int 875151658Sjhbintr_event_schedule_thread(struct intr_event *ie) 87672759Sjhb{ 877151658Sjhb struct intr_entropy entropy; 878151658Sjhb struct intr_thread *it; 87983366Sjulian struct thread *td; 880101176Sjulian struct thread *ctd; 88172759Sjhb struct proc *p; 88272759Sjhb 88372759Sjhb /* 88472759Sjhb * If no ithread or no handlers, then we have a stray interrupt. 88572759Sjhb */ 886151658Sjhb if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || 887151658Sjhb ie->ie_thread == NULL) 88872759Sjhb return (EINVAL); 88972759Sjhb 890101176Sjulian ctd = curthread; 891151658Sjhb it = ie->ie_thread; 892151658Sjhb td = it->it_thread; 893133191Srwatson p = td->td_proc; 894151658Sjhb 89572759Sjhb /* 89672759Sjhb * If any of the handlers for this ithread claim to be good 89772759Sjhb * sources of entropy, then gather some. 89872759Sjhb */ 899151658Sjhb if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 900133191Srwatson CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 901173004Sjulian p->p_pid, td->td_name); 902151658Sjhb entropy.event = (uintptr_t)ie; 903151658Sjhb entropy.td = ctd; 904256381Smarkm random_harvest(&entropy, sizeof(entropy), 2, 90572759Sjhb RANDOM_INTERRUPT); 90672759Sjhb } 90772759Sjhb 908151658Sjhb KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 90972759Sjhb 91072759Sjhb /* 91172759Sjhb * Set it_need to tell the thread to keep running if it is already 912170307Sjeff * running. Then, lock the thread and see if we actually need to 913170307Sjeff * put it on the runqueue. 91472759Sjhb */ 915252683Salfred atomic_store_rel_int(&it->it_need, 1); 916170307Sjeff thread_lock(td); 917103216Sjulian if (TD_AWAITING_INTR(td)) { 918151658Sjhb CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 919173004Sjulian td->td_name); 920103216Sjulian TD_CLR_IWAIT(td); 921166188Sjeff sched_add(td, SRQ_INTR); 92272759Sjhb } else { 923151658Sjhb CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 924173004Sjulian __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 92572759Sjhb } 926170307Sjeff thread_unlock(td); 92772759Sjhb 92872759Sjhb return (0); 92972759Sjhb} 930169320Spiso#else 931169320Spisoint 932169320Spisointr_event_remove_handler(void *cookie) 933169320Spiso{ 934169320Spiso struct intr_handler *handler = (struct intr_handler *)cookie; 935169320Spiso struct intr_event *ie; 936169320Spiso struct intr_thread *it; 937169320Spiso#ifdef INVARIANTS 938169320Spiso struct intr_handler *ih; 939169320Spiso#endif 940169320Spiso#ifdef notyet 941169320Spiso int dead; 942169320Spiso#endif 94372759Sjhb 944169320Spiso if (handler == NULL) 945169320Spiso return (EINVAL); 946169320Spiso ie = handler->ih_event; 947169320Spiso KASSERT(ie != NULL, 948169320Spiso ("interrupt handler \"%s\" has a NULL interrupt event", 949169320Spiso handler->ih_name)); 950169320Spiso mtx_lock(&ie->ie_lock); 951169320Spiso CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 952169320Spiso ie->ie_name); 953169320Spiso#ifdef INVARIANTS 954169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 955169320Spiso if (ih == handler) 956169320Spiso goto ok; 957169320Spiso mtx_unlock(&ie->ie_lock); 958169320Spiso panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", 959169320Spiso ih->ih_name, ie->ie_name); 960169320Spisook: 961169320Spiso#endif 962169320Spiso /* 963169320Spiso * If there are no ithreads (per event and per handler), then 964169320Spiso * just remove the handler and return. 965169320Spiso * XXX: Note that an INTR_FAST handler might be running on another CPU! 966169320Spiso */ 967169320Spiso if (ie->ie_thread == NULL && handler->ih_thread == NULL) { 968169320Spiso TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 969169320Spiso mtx_unlock(&ie->ie_lock); 970169320Spiso free(handler, M_ITHREAD); 971169320Spiso return (0); 972169320Spiso } 973169320Spiso 974169320Spiso /* Private or global ithread? */ 975169320Spiso it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread; 976169320Spiso /* 977169320Spiso * If the interrupt thread is already running, then just mark this 978169320Spiso * handler as being dead and let the ithread do the actual removal. 979169320Spiso * 980169320Spiso * During a cold boot while cold is set, msleep() does not sleep, 981169320Spiso * so we have to remove the handler here rather than letting the 982169320Spiso * thread do it. 983169320Spiso */ 984170307Sjeff thread_lock(it->it_thread); 985169320Spiso if (!TD_AWAITING_INTR(it->it_thread) && !cold) { 986169320Spiso handler->ih_flags |= IH_DEAD; 987169320Spiso 988169320Spiso /* 989169320Spiso * Ensure that the thread will process the handler list 990169320Spiso * again and remove this handler if it has already passed 991169320Spiso * it on the list. 992169320Spiso */ 993252683Salfred atomic_store_rel_int(&it->it_need, 1); 994169320Spiso } else 995169320Spiso TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); 996170307Sjeff thread_unlock(it->it_thread); 997169320Spiso while (handler->ih_flags & IH_DEAD) 998169320Spiso msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); 999169320Spiso /* 1000169320Spiso * At this point, the handler has been disconnected from the event, 1001169320Spiso * so we can kill the private ithread if any. 1002169320Spiso */ 1003169320Spiso if (handler->ih_thread) { 1004169320Spiso ithread_destroy(handler->ih_thread); 1005169320Spiso handler->ih_thread = NULL; 1006169320Spiso } 1007169320Spiso intr_event_update(ie); 1008169320Spiso#ifdef notyet 1009169320Spiso /* 1010169320Spiso * XXX: This could be bad in the case of ppbus(8). Also, I think 1011169320Spiso * this could lead to races of stale data when servicing an 1012169320Spiso * interrupt. 1013169320Spiso */ 1014169320Spiso dead = 1; 1015169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1016169320Spiso if (handler != NULL) { 1017169320Spiso dead = 0; 1018169320Spiso break; 1019169320Spiso } 1020169320Spiso } 1021169320Spiso if (dead) { 1022169320Spiso ithread_destroy(ie->ie_thread); 1023169320Spiso ie->ie_thread = NULL; 1024169320Spiso } 1025169320Spiso#endif 1026169320Spiso mtx_unlock(&ie->ie_lock); 1027169320Spiso free(handler, M_ITHREAD); 1028169320Spiso return (0); 1029169320Spiso} 1030169320Spiso 1031177940Sjhbstatic int 1032169320Spisointr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it) 1033169320Spiso{ 1034169320Spiso struct intr_entropy entropy; 1035169320Spiso struct thread *td; 1036169320Spiso struct thread *ctd; 1037169320Spiso struct proc *p; 1038169320Spiso 1039169320Spiso /* 1040169320Spiso * If no ithread or no handlers, then we have a stray interrupt. 1041169320Spiso */ 1042169320Spiso if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL) 1043169320Spiso return (EINVAL); 1044169320Spiso 1045169320Spiso ctd = curthread; 1046169320Spiso td = it->it_thread; 1047169320Spiso p = td->td_proc; 1048169320Spiso 1049169320Spiso /* 1050169320Spiso * If any of the handlers for this ithread claim to be good 1051169320Spiso * sources of entropy, then gather some. 1052169320Spiso */ 1053169320Spiso if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { 1054169320Spiso CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 1055173004Sjulian p->p_pid, td->td_name); 1056169320Spiso entropy.event = (uintptr_t)ie; 1057169320Spiso entropy.td = ctd; 1058256381Smarkm random_harvest(&entropy, sizeof(entropy), 2, 1059169320Spiso RANDOM_INTERRUPT); 1060169320Spiso } 1061169320Spiso 1062169320Spiso KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); 1063169320Spiso 1064169320Spiso /* 1065169320Spiso * Set it_need to tell the thread to keep running if it is already 1066170307Sjeff * running. Then, lock the thread and see if we actually need to 1067170307Sjeff * put it on the runqueue. 1068169320Spiso */ 1069252683Salfred atomic_store_rel_int(&it->it_need, 1); 1070170307Sjeff thread_lock(td); 1071169320Spiso if (TD_AWAITING_INTR(td)) { 1072169320Spiso CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, 1073173122Sjulian td->td_name); 1074169320Spiso TD_CLR_IWAIT(td); 1075169320Spiso sched_add(td, SRQ_INTR); 1076169320Spiso } else { 1077169320Spiso CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", 1078173004Sjulian __func__, p->p_pid, td->td_name, it->it_need, td->td_state); 1079169320Spiso } 1080170307Sjeff thread_unlock(td); 1081169320Spiso 1082169320Spiso return (0); 1083169320Spiso} 1084169320Spiso#endif 1085169320Spiso 1086151699Sjhb/* 1087192305Srwatson * Allow interrupt event binding for software interrupt handlers -- a no-op, 1088192305Srwatson * since interrupts are generated in software rather than being directed by 1089192305Srwatson * a PIC. 1090192305Srwatson */ 1091192305Srwatsonstatic int 1092192305Srwatsonswi_assign_cpu(void *arg, u_char cpu) 1093192305Srwatson{ 1094192305Srwatson 1095192305Srwatson return (0); 1096192305Srwatson} 1097192305Srwatson 1098192305Srwatson/* 1099151699Sjhb * Add a software interrupt handler to a specified event. If a given event 1100151699Sjhb * is not specified, then a new event is created. 1101151699Sjhb */ 110272759Sjhbint 1103151658Sjhbswi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, 110472237Sjhb void *arg, int pri, enum intr_type flags, void **cookiep) 110572237Sjhb{ 1106151658Sjhb struct intr_event *ie; 110772237Sjhb int error; 110866698Sjhb 1109169320Spiso if (flags & INTR_ENTROPY) 111072759Sjhb return (EINVAL); 111172759Sjhb 1112151658Sjhb ie = (eventp != NULL) ? *eventp : NULL; 111366698Sjhb 1114151658Sjhb if (ie != NULL) { 1115151658Sjhb if (!(ie->ie_flags & IE_SOFT)) 1116151658Sjhb return (EINVAL); 111772759Sjhb } else { 1118178092Sjeff error = intr_event_create(&ie, NULL, IE_SOFT, 0, 1119192305Srwatson NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri); 112067551Sjhb if (error) 112172237Sjhb return (error); 1122151658Sjhb if (eventp != NULL) 1123151658Sjhb *eventp = ie; 112466698Sjhb } 1125177859Sjeff error = intr_event_add_handler(ie, name, NULL, handler, arg, 1126217292Sjhb PI_SWI(pri), flags, cookiep); 1127247778Sdavide return (error); 112866698Sjhb} 112966698Sjhb 113066698Sjhb/* 1131151658Sjhb * Schedule a software interrupt thread. 113266698Sjhb */ 113367551Sjhbvoid 113472237Sjhbswi_sched(void *cookie, int flags) 113566698Sjhb{ 1136151658Sjhb struct intr_handler *ih = (struct intr_handler *)cookie; 1137151658Sjhb struct intr_event *ie = ih->ih_event; 1138240921Sjhb struct intr_entropy entropy; 113972759Sjhb int error; 114066698Sjhb 1141151658Sjhb CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, 1142151658Sjhb ih->ih_need); 1143151658Sjhb 1144240921Sjhb if (harvest.swi) { 1145240921Sjhb CTR2(KTR_INTR, "swi_sched: pid %d (%s) gathering entropy", 1146240921Sjhb curproc->p_pid, curthread->td_name); 1147240921Sjhb entropy.event = (uintptr_t)ih; 1148240921Sjhb entropy.td = curthread; 1149256381Smarkm random_harvest(&entropy, sizeof(entropy), 1, 1150255362Smarkm RANDOM_SWI); 1151240921Sjhb } 1152240921Sjhb 115367551Sjhb /* 115472759Sjhb * Set ih_need for this handler so that if the ithread is already 115572759Sjhb * running it will execute this handler on the next pass. Otherwise, 115672759Sjhb * it will execute it the next time it runs. 115767551Sjhb */ 115872237Sjhb atomic_store_rel_int(&ih->ih_need, 1); 1159163474Sbde 116072237Sjhb if (!(flags & SWI_DELAY)) { 1161170291Sattilio PCPU_INC(cnt.v_soft); 1162169320Spiso#ifdef INTR_FILTER 1163169320Spiso error = intr_event_schedule_thread(ie, ie->ie_thread); 1164169320Spiso#else 1165151658Sjhb error = intr_event_schedule_thread(ie); 1166169320Spiso#endif 116772759Sjhb KASSERT(error == 0, ("stray software interrupt")); 116866698Sjhb } 116966698Sjhb} 117066698Sjhb 1171151699Sjhb/* 1172151699Sjhb * Remove a software interrupt handler. Currently this code does not 1173151699Sjhb * remove the associated interrupt event if it becomes empty. Calling code 1174151699Sjhb * may do so manually via intr_event_destroy(), but that's not really 1175151699Sjhb * an optimal interface. 1176151699Sjhb */ 1177151699Sjhbint 1178151699Sjhbswi_remove(void *cookie) 1179151699Sjhb{ 1180151699Sjhb 1181151699Sjhb return (intr_event_remove_handler(cookie)); 1182151699Sjhb} 1183151699Sjhb 1184169320Spiso#ifdef INTR_FILTER 1185151658Sjhbstatic void 1186169320Spisopriv_ithread_execute_handler(struct proc *p, struct intr_handler *ih) 1187169320Spiso{ 1188169320Spiso struct intr_event *ie; 1189169320Spiso 1190169320Spiso ie = ih->ih_event; 1191169320Spiso /* 1192169320Spiso * If this handler is marked for death, remove it from 1193169320Spiso * the list of handlers and wake up the sleeper. 1194169320Spiso */ 1195169320Spiso if (ih->ih_flags & IH_DEAD) { 1196169320Spiso mtx_lock(&ie->ie_lock); 1197169320Spiso TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1198169320Spiso ih->ih_flags &= ~IH_DEAD; 1199169320Spiso wakeup(ih); 1200169320Spiso mtx_unlock(&ie->ie_lock); 1201169320Spiso return; 1202169320Spiso } 1203169320Spiso 1204169320Spiso /* Execute this handler. */ 1205169320Spiso CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1206169320Spiso __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, 1207169320Spiso ih->ih_name, ih->ih_flags); 1208169320Spiso 1209169320Spiso if (!(ih->ih_flags & IH_MPSAFE)) 1210169320Spiso mtx_lock(&Giant); 1211169320Spiso ih->ih_handler(ih->ih_argument); 1212169320Spiso if (!(ih->ih_flags & IH_MPSAFE)) 1213169320Spiso mtx_unlock(&Giant); 1214169320Spiso} 1215169320Spiso#endif 1216169320Spiso 1217183052Sjhb/* 1218183052Sjhb * This is a public function for use by drivers that mux interrupt 1219183052Sjhb * handlers for child devices from their interrupt handler. 1220183052Sjhb */ 1221183052Sjhbvoid 1222183052Sjhbintr_event_execute_handlers(struct proc *p, struct intr_event *ie) 1223151658Sjhb{ 1224151658Sjhb struct intr_handler *ih, *ihn; 1225151658Sjhb 1226151658Sjhb TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { 1227151658Sjhb /* 1228151658Sjhb * If this handler is marked for death, remove it from 1229151658Sjhb * the list of handlers and wake up the sleeper. 1230151658Sjhb */ 1231151658Sjhb if (ih->ih_flags & IH_DEAD) { 1232151658Sjhb mtx_lock(&ie->ie_lock); 1233151658Sjhb TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); 1234151658Sjhb ih->ih_flags &= ~IH_DEAD; 1235151658Sjhb wakeup(ih); 1236151658Sjhb mtx_unlock(&ie->ie_lock); 1237151658Sjhb continue; 1238151658Sjhb } 1239151658Sjhb 1240167080Spiso /* Skip filter only handlers */ 1241167080Spiso if (ih->ih_handler == NULL) 1242167080Spiso continue; 1243167080Spiso 1244151658Sjhb /* 1245151658Sjhb * For software interrupt threads, we only execute 1246151658Sjhb * handlers that have their need flag set. Hardware 1247151658Sjhb * interrupt threads always invoke all of their handlers. 1248151658Sjhb */ 1249151658Sjhb if (ie->ie_flags & IE_SOFT) { 1250252683Salfred if (atomic_load_acq_int(&ih->ih_need) == 0) 1251151658Sjhb continue; 1252151658Sjhb else 1253151658Sjhb atomic_store_rel_int(&ih->ih_need, 0); 1254151658Sjhb } 1255151658Sjhb 1256151658Sjhb /* Execute this handler. */ 1257151658Sjhb CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", 1258169320Spiso __func__, p->p_pid, (void *)ih->ih_handler, 1259169320Spiso ih->ih_argument, ih->ih_name, ih->ih_flags); 1260151658Sjhb 1261151658Sjhb if (!(ih->ih_flags & IH_MPSAFE)) 1262151658Sjhb mtx_lock(&Giant); 1263151658Sjhb ih->ih_handler(ih->ih_argument); 1264151658Sjhb if (!(ih->ih_flags & IH_MPSAFE)) 1265151658Sjhb mtx_unlock(&Giant); 1266151658Sjhb } 1267183052Sjhb} 1268183052Sjhb 1269183052Sjhbstatic void 1270183052Sjhbithread_execute_handlers(struct proc *p, struct intr_event *ie) 1271183052Sjhb{ 1272183052Sjhb 1273183052Sjhb /* Interrupt handlers should not sleep. */ 1274151658Sjhb if (!(ie->ie_flags & IE_SOFT)) 1275183052Sjhb THREAD_NO_SLEEPING(); 1276183052Sjhb intr_event_execute_handlers(p, ie); 1277183052Sjhb if (!(ie->ie_flags & IE_SOFT)) 1278151658Sjhb THREAD_SLEEPING_OK(); 1279151658Sjhb 1280151658Sjhb /* 1281151658Sjhb * Interrupt storm handling: 1282151658Sjhb * 1283151658Sjhb * If this interrupt source is currently storming, then throttle 1284151658Sjhb * it to only fire the handler once per clock tick. 1285151658Sjhb * 1286151658Sjhb * If this interrupt source is not currently storming, but the 1287151658Sjhb * number of back to back interrupts exceeds the storm threshold, 1288151658Sjhb * then enter storming mode. 1289151658Sjhb */ 1290167173Sjhb if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold && 1291167173Sjhb !(ie->ie_flags & IE_SOFT)) { 1292168850Snjl /* Report the message only once every second. */ 1293168850Snjl if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) { 1294151658Sjhb printf( 1295168850Snjl "interrupt storm detected on \"%s\"; throttling interrupt source\n", 1296151658Sjhb ie->ie_name); 1297151658Sjhb } 1298167173Sjhb pause("istorm", 1); 1299151658Sjhb } else 1300151658Sjhb ie->ie_count++; 1301151658Sjhb 1302151658Sjhb /* 1303151658Sjhb * Now that all the handlers have had a chance to run, reenable 1304151658Sjhb * the interrupt source. 1305151658Sjhb */ 1306177940Sjhb if (ie->ie_post_ithread != NULL) 1307177940Sjhb ie->ie_post_ithread(ie->ie_source); 1308151658Sjhb} 1309151658Sjhb 1310169320Spiso#ifndef INTR_FILTER 131166698Sjhb/* 131272237Sjhb * This is the main code for interrupt threads. 131366698Sjhb */ 1314104094Sphkstatic void 131572237Sjhbithread_loop(void *arg) 131666698Sjhb{ 1317151658Sjhb struct intr_thread *ithd; 1318151658Sjhb struct intr_event *ie; 131983366Sjulian struct thread *td; 132072237Sjhb struct proc *p; 1321219819Sjeff int wake; 1322151658Sjhb 132383366Sjulian td = curthread; 132483366Sjulian p = td->td_proc; 1325151658Sjhb ithd = (struct intr_thread *)arg; 1326151658Sjhb KASSERT(ithd->it_thread == td, 132787593Sobrien ("%s: ithread and proc linkage out of sync", __func__)); 1328151658Sjhb ie = ithd->it_event; 1329151658Sjhb ie->ie_count = 0; 1330219819Sjeff wake = 0; 133166698Sjhb 133267551Sjhb /* 133367551Sjhb * As long as we have interrupts outstanding, go through the 133467551Sjhb * list of handlers, giving each one a go at it. 133567551Sjhb */ 133666698Sjhb for (;;) { 133772237Sjhb /* 133872237Sjhb * If we are an orphaned thread, then just die. 133972237Sjhb */ 134072237Sjhb if (ithd->it_flags & IT_DEAD) { 1341151658Sjhb CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1342173004Sjulian p->p_pid, td->td_name); 134372237Sjhb free(ithd, M_ITHREAD); 1344173044Sjulian kthread_exit(); 134572237Sjhb } 134672237Sjhb 1347151658Sjhb /* 1348151658Sjhb * Service interrupts. If another interrupt arrives while 1349151658Sjhb * we are running, it will set it_need to note that we 1350151658Sjhb * should make another pass. 1351151658Sjhb */ 1352252683Salfred while (atomic_load_acq_int(&ithd->it_need) != 0) { 135367551Sjhb /* 1354151658Sjhb * This might need a full read and write barrier 1355151658Sjhb * to make sure that this write posts before any 1356151658Sjhb * of the memory or device accesses in the 1357151658Sjhb * handlers. 135867551Sjhb */ 135972237Sjhb atomic_store_rel_int(&ithd->it_need, 0); 1360151658Sjhb ithread_execute_handlers(p, ie); 136166698Sjhb } 1362128331Sjhb WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1363128331Sjhb mtx_assert(&Giant, MA_NOTOWNED); 136467551Sjhb 136566698Sjhb /* 136666698Sjhb * Processed all our interrupts. Now get the sched 136767551Sjhb * lock. This may take a while and it_need may get 136866698Sjhb * set again, so we have to check it again. 136966698Sjhb */ 1370170307Sjeff thread_lock(td); 1371252683Salfred if ((atomic_load_acq_int(&ithd->it_need) == 0) && 1372252683Salfred !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { 1373128331Sjhb TD_SET_IWAIT(td); 1374151658Sjhb ie->ie_count = 0; 1375178272Sjeff mi_switch(SW_VOL | SWT_IWAIT, NULL); 137666698Sjhb } 1377219819Sjeff if (ithd->it_flags & IT_WAIT) { 1378219819Sjeff wake = 1; 1379219819Sjeff ithd->it_flags &= ~IT_WAIT; 1380219819Sjeff } 1381170307Sjeff thread_unlock(td); 1382219819Sjeff if (wake) { 1383219819Sjeff wakeup(ithd); 1384219819Sjeff wake = 0; 1385219819Sjeff } 138666698Sjhb } 138766698Sjhb} 1388177940Sjhb 1389177940Sjhb/* 1390177940Sjhb * Main interrupt handling body. 1391177940Sjhb * 1392177940Sjhb * Input: 1393177940Sjhb * o ie: the event connected to this interrupt. 1394177940Sjhb * o frame: some archs (i.e. i386) pass a frame to some. 1395177940Sjhb * handlers as their main argument. 1396177940Sjhb * Return value: 1397177940Sjhb * o 0: everything ok. 1398177940Sjhb * o EINVAL: stray interrupt. 1399177940Sjhb */ 1400177940Sjhbint 1401177940Sjhbintr_event_handle(struct intr_event *ie, struct trapframe *frame) 1402177940Sjhb{ 1403177940Sjhb struct intr_handler *ih; 1404208988Smav struct trapframe *oldframe; 1405177940Sjhb struct thread *td; 1406177940Sjhb int error, ret, thread; 1407177940Sjhb 1408177940Sjhb td = curthread; 1409177940Sjhb 1410177940Sjhb /* An interrupt with no event or handlers is a stray interrupt. */ 1411177940Sjhb if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1412177940Sjhb return (EINVAL); 1413177940Sjhb 1414177940Sjhb /* 1415177940Sjhb * Execute fast interrupt handlers directly. 1416177940Sjhb * To support clock handlers, if a handler registers 1417177940Sjhb * with a NULL argument, then we pass it a pointer to 1418177940Sjhb * a trapframe as its argument. 1419177940Sjhb */ 1420177940Sjhb td->td_intr_nesting_level++; 1421177940Sjhb thread = 0; 1422177940Sjhb ret = 0; 1423177940Sjhb critical_enter(); 1424208988Smav oldframe = td->td_intr_frame; 1425208988Smav td->td_intr_frame = frame; 1426177940Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1427177940Sjhb if (ih->ih_filter == NULL) { 1428177940Sjhb thread = 1; 1429177940Sjhb continue; 1430177940Sjhb } 1431177940Sjhb CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, 1432177940Sjhb ih->ih_filter, ih->ih_argument == NULL ? frame : 1433177940Sjhb ih->ih_argument, ih->ih_name); 1434177940Sjhb if (ih->ih_argument == NULL) 1435177940Sjhb ret = ih->ih_filter(frame); 1436177940Sjhb else 1437177940Sjhb ret = ih->ih_filter(ih->ih_argument); 1438203061Savg KASSERT(ret == FILTER_STRAY || 1439203061Savg ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1440203061Savg (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1441203061Savg ("%s: incorrect return value %#x from %s", __func__, ret, 1442203061Savg ih->ih_name)); 1443203061Savg 1444177940Sjhb /* 1445177940Sjhb * Wrapper handler special handling: 1446177940Sjhb * 1447177940Sjhb * in some particular cases (like pccard and pccbb), 1448177940Sjhb * the _real_ device handler is wrapped in a couple of 1449177940Sjhb * functions - a filter wrapper and an ithread wrapper. 1450177940Sjhb * In this case (and just in this case), the filter wrapper 1451177940Sjhb * could ask the system to schedule the ithread and mask 1452177940Sjhb * the interrupt source if the wrapped handler is composed 1453177940Sjhb * of just an ithread handler. 1454177940Sjhb * 1455177940Sjhb * TODO: write a generic wrapper to avoid people rolling 1456177940Sjhb * their own 1457177940Sjhb */ 1458177940Sjhb if (!thread) { 1459177940Sjhb if (ret == FILTER_SCHEDULE_THREAD) 1460177940Sjhb thread = 1; 1461177940Sjhb } 1462177940Sjhb } 1463208988Smav td->td_intr_frame = oldframe; 1464177940Sjhb 1465177940Sjhb if (thread) { 1466177940Sjhb if (ie->ie_pre_ithread != NULL) 1467177940Sjhb ie->ie_pre_ithread(ie->ie_source); 1468177940Sjhb } else { 1469177940Sjhb if (ie->ie_post_filter != NULL) 1470177940Sjhb ie->ie_post_filter(ie->ie_source); 1471177940Sjhb } 1472177940Sjhb 1473177940Sjhb /* Schedule the ithread if needed. */ 1474177940Sjhb if (thread) { 1475177940Sjhb error = intr_event_schedule_thread(ie); 1476182024Skmacy#ifndef XEN 1477177940Sjhb KASSERT(error == 0, ("bad stray interrupt")); 1478182024Skmacy#else 1479182024Skmacy if (error != 0) 1480182024Skmacy log(LOG_WARNING, "bad stray interrupt"); 1481182024Skmacy#endif 1482177940Sjhb } 1483177940Sjhb critical_exit(); 1484177940Sjhb td->td_intr_nesting_level--; 1485177940Sjhb return (0); 1486177940Sjhb} 1487169320Spiso#else 1488169320Spiso/* 1489169320Spiso * This is the main code for interrupt threads. 1490169320Spiso */ 1491169320Spisostatic void 1492169320Spisoithread_loop(void *arg) 1493169320Spiso{ 1494169320Spiso struct intr_thread *ithd; 1495169320Spiso struct intr_handler *ih; 1496169320Spiso struct intr_event *ie; 1497169320Spiso struct thread *td; 1498169320Spiso struct proc *p; 1499169320Spiso int priv; 1500219819Sjeff int wake; 150166698Sjhb 1502169320Spiso td = curthread; 1503169320Spiso p = td->td_proc; 1504169320Spiso ih = (struct intr_handler *)arg; 1505169320Spiso priv = (ih->ih_thread != NULL) ? 1 : 0; 1506169320Spiso ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread; 1507169320Spiso KASSERT(ithd->it_thread == td, 1508169320Spiso ("%s: ithread and proc linkage out of sync", __func__)); 1509169320Spiso ie = ithd->it_event; 1510169320Spiso ie->ie_count = 0; 1511219819Sjeff wake = 0; 1512169320Spiso 1513169320Spiso /* 1514169320Spiso * As long as we have interrupts outstanding, go through the 1515169320Spiso * list of handlers, giving each one a go at it. 1516169320Spiso */ 1517169320Spiso for (;;) { 1518169320Spiso /* 1519169320Spiso * If we are an orphaned thread, then just die. 1520169320Spiso */ 1521169320Spiso if (ithd->it_flags & IT_DEAD) { 1522169320Spiso CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, 1523173004Sjulian p->p_pid, td->td_name); 1524169320Spiso free(ithd, M_ITHREAD); 1525173044Sjulian kthread_exit(); 1526169320Spiso } 1527169320Spiso 1528169320Spiso /* 1529169320Spiso * Service interrupts. If another interrupt arrives while 1530169320Spiso * we are running, it will set it_need to note that we 1531169320Spiso * should make another pass. 1532169320Spiso */ 1533252683Salfred while (atomic_load_acq_int(&ithd->it_need) != 0) { 1534169320Spiso /* 1535169320Spiso * This might need a full read and write barrier 1536169320Spiso * to make sure that this write posts before any 1537169320Spiso * of the memory or device accesses in the 1538169320Spiso * handlers. 1539169320Spiso */ 1540169320Spiso atomic_store_rel_int(&ithd->it_need, 0); 1541169320Spiso if (priv) 1542169320Spiso priv_ithread_execute_handler(p, ih); 1543169320Spiso else 1544169320Spiso ithread_execute_handlers(p, ie); 1545169320Spiso } 1546169320Spiso WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 1547169320Spiso mtx_assert(&Giant, MA_NOTOWNED); 1548169320Spiso 1549169320Spiso /* 1550169320Spiso * Processed all our interrupts. Now get the sched 1551169320Spiso * lock. This may take a while and it_need may get 1552169320Spiso * set again, so we have to check it again. 1553169320Spiso */ 1554170307Sjeff thread_lock(td); 1555252683Salfred if ((atomic_load_acq_int(&ithd->it_need) == 0) && 1556252683Salfred !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { 1557169320Spiso TD_SET_IWAIT(td); 1558169320Spiso ie->ie_count = 0; 1559178272Sjeff mi_switch(SW_VOL | SWT_IWAIT, NULL); 1560169320Spiso } 1561219819Sjeff if (ithd->it_flags & IT_WAIT) { 1562219819Sjeff wake = 1; 1563219819Sjeff ithd->it_flags &= ~IT_WAIT; 1564219819Sjeff } 1565170307Sjeff thread_unlock(td); 1566219819Sjeff if (wake) { 1567219819Sjeff wakeup(ithd); 1568219819Sjeff wake = 0; 1569219819Sjeff } 1570169320Spiso } 1571169320Spiso} 1572169320Spiso 1573169320Spiso/* 1574169320Spiso * Main loop for interrupt filter. 1575169320Spiso * 1576169320Spiso * Some architectures (i386, amd64 and arm) require the optional frame 1577169320Spiso * parameter, and use it as the main argument for fast handler execution 1578169320Spiso * when ih_argument == NULL. 1579169320Spiso * 1580169320Spiso * Return value: 1581169320Spiso * o FILTER_STRAY: No filter recognized the event, and no 1582169320Spiso * filter-less handler is registered on this 1583169320Spiso * line. 1584169320Spiso * o FILTER_HANDLED: A filter claimed the event and served it. 1585169320Spiso * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at 1586169320Spiso * least one filter-less handler on this line. 1587169320Spiso * o FILTER_HANDLED | 1588169320Spiso * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for 1589169320Spiso * scheduling the per-handler ithread. 1590169320Spiso * 1591169320Spiso * In case an ithread has to be scheduled, in *ithd there will be a 1592169320Spiso * pointer to a struct intr_thread containing the thread to be 1593169320Spiso * scheduled. 1594169320Spiso */ 1595169320Spiso 1596177940Sjhbstatic int 1597169320Spisointr_filter_loop(struct intr_event *ie, struct trapframe *frame, 1598169320Spiso struct intr_thread **ithd) 1599169320Spiso{ 1600169320Spiso struct intr_handler *ih; 1601169320Spiso void *arg; 1602169320Spiso int ret, thread_only; 1603169320Spiso 1604169320Spiso ret = 0; 1605169320Spiso thread_only = 0; 1606169320Spiso TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { 1607169320Spiso /* 1608169320Spiso * Execute fast interrupt handlers directly. 1609169320Spiso * To support clock handlers, if a handler registers 1610169320Spiso * with a NULL argument, then we pass it a pointer to 1611169320Spiso * a trapframe as its argument. 1612169320Spiso */ 1613169320Spiso arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument); 1614169320Spiso 1615169320Spiso CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__, 1616169320Spiso ih->ih_filter, ih->ih_handler, arg, ih->ih_name); 1617169320Spiso 1618169320Spiso if (ih->ih_filter != NULL) 1619169320Spiso ret = ih->ih_filter(arg); 1620169320Spiso else { 1621169320Spiso thread_only = 1; 1622169320Spiso continue; 1623169320Spiso } 1624203061Savg KASSERT(ret == FILTER_STRAY || 1625203061Savg ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && 1626203061Savg (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), 1627203061Savg ("%s: incorrect return value %#x from %s", __func__, ret, 1628203061Savg ih->ih_name)); 1629169320Spiso if (ret & FILTER_STRAY) 1630169320Spiso continue; 1631169320Spiso else { 1632169320Spiso *ithd = ih->ih_thread; 1633169320Spiso return (ret); 1634169320Spiso } 1635169320Spiso } 1636169320Spiso 1637169320Spiso /* 1638169320Spiso * No filters handled the interrupt and we have at least 1639169320Spiso * one handler without a filter. In this case, we schedule 1640169320Spiso * all of the filter-less handlers to run in the ithread. 1641169320Spiso */ 1642169320Spiso if (thread_only) { 1643169320Spiso *ithd = ie->ie_thread; 1644169320Spiso return (FILTER_SCHEDULE_THREAD); 1645169320Spiso } 1646169320Spiso return (FILTER_STRAY); 1647169320Spiso} 1648169320Spiso 1649169320Spiso/* 1650169320Spiso * Main interrupt handling body. 1651169320Spiso * 1652169320Spiso * Input: 1653169320Spiso * o ie: the event connected to this interrupt. 1654169320Spiso * o frame: some archs (i.e. i386) pass a frame to some. 1655169320Spiso * handlers as their main argument. 1656169320Spiso * Return value: 1657169320Spiso * o 0: everything ok. 1658169320Spiso * o EINVAL: stray interrupt. 1659169320Spiso */ 1660169320Spisoint 1661169320Spisointr_event_handle(struct intr_event *ie, struct trapframe *frame) 1662169320Spiso{ 1663169320Spiso struct intr_thread *ithd; 1664208988Smav struct trapframe *oldframe; 1665169320Spiso struct thread *td; 1666169320Spiso int thread; 1667169320Spiso 1668169320Spiso ithd = NULL; 1669169320Spiso td = curthread; 1670169320Spiso 1671169320Spiso if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) 1672169320Spiso return (EINVAL); 1673169320Spiso 1674169320Spiso td->td_intr_nesting_level++; 1675169320Spiso thread = 0; 1676169320Spiso critical_enter(); 1677208988Smav oldframe = td->td_intr_frame; 1678208988Smav td->td_intr_frame = frame; 1679177940Sjhb thread = intr_filter_loop(ie, frame, &ithd); 1680169320Spiso if (thread & FILTER_HANDLED) { 1681177940Sjhb if (ie->ie_post_filter != NULL) 1682177940Sjhb ie->ie_post_filter(ie->ie_source); 1683169320Spiso } else { 1684177940Sjhb if (ie->ie_pre_ithread != NULL) 1685177940Sjhb ie->ie_pre_ithread(ie->ie_source); 1686169320Spiso } 1687208988Smav td->td_intr_frame = oldframe; 1688169320Spiso critical_exit(); 1689169320Spiso 1690169320Spiso /* Interrupt storm logic */ 1691169320Spiso if (thread & FILTER_STRAY) { 1692169320Spiso ie->ie_count++; 1693169320Spiso if (ie->ie_count < intr_storm_threshold) 1694169320Spiso printf("Interrupt stray detection not present\n"); 1695169320Spiso } 1696169320Spiso 1697169320Spiso /* Schedule an ithread if needed. */ 1698169320Spiso if (thread & FILTER_SCHEDULE_THREAD) { 1699169320Spiso if (intr_event_schedule_thread(ie, ithd) != 0) 1700169320Spiso panic("%s: impossible stray interrupt", __func__); 1701169320Spiso } 1702169320Spiso td->td_intr_nesting_level--; 1703169320Spiso return (0); 1704169320Spiso} 1705169320Spiso#endif 1706169320Spiso 1707121482Sjhb#ifdef DDB 170872237Sjhb/* 1709121482Sjhb * Dump details about an interrupt handler 1710121482Sjhb */ 1711121482Sjhbstatic void 1712151658Sjhbdb_dump_intrhand(struct intr_handler *ih) 1713121482Sjhb{ 1714121482Sjhb int comma; 1715121482Sjhb 1716121482Sjhb db_printf("\t%-10s ", ih->ih_name); 1717121482Sjhb switch (ih->ih_pri) { 1718121482Sjhb case PI_REALTIME: 1719121482Sjhb db_printf("CLK "); 1720121482Sjhb break; 1721121482Sjhb case PI_AV: 1722121482Sjhb db_printf("AV "); 1723121482Sjhb break; 1724217292Sjhb case PI_TTY: 1725121482Sjhb db_printf("TTY "); 1726121482Sjhb break; 1727121482Sjhb case PI_NET: 1728121482Sjhb db_printf("NET "); 1729121482Sjhb break; 1730121482Sjhb case PI_DISK: 1731121482Sjhb db_printf("DISK"); 1732121482Sjhb break; 1733121482Sjhb case PI_DULL: 1734121482Sjhb db_printf("DULL"); 1735121482Sjhb break; 1736121482Sjhb default: 1737121482Sjhb if (ih->ih_pri >= PI_SOFT) 1738121482Sjhb db_printf("SWI "); 1739121482Sjhb else 1740121482Sjhb db_printf("%4u", ih->ih_pri); 1741121482Sjhb break; 1742121482Sjhb } 1743121482Sjhb db_printf(" "); 1744249163Skib if (ih->ih_filter != NULL) { 1745249163Skib db_printf("[F]"); 1746249163Skib db_printsym((uintptr_t)ih->ih_filter, DB_STGY_PROC); 1747249163Skib } 1748249163Skib if (ih->ih_handler != NULL) { 1749249163Skib if (ih->ih_filter != NULL) 1750249163Skib db_printf(","); 1751249163Skib db_printf("[H]"); 1752249163Skib db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); 1753249163Skib } 1754121482Sjhb db_printf("(%p)", ih->ih_argument); 1755121482Sjhb if (ih->ih_need || 1756166901Spiso (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | 1757121482Sjhb IH_MPSAFE)) != 0) { 1758121482Sjhb db_printf(" {"); 1759121482Sjhb comma = 0; 1760121482Sjhb if (ih->ih_flags & IH_EXCLUSIVE) { 1761121482Sjhb if (comma) 1762121482Sjhb db_printf(", "); 1763121482Sjhb db_printf("EXCL"); 1764121482Sjhb comma = 1; 1765121482Sjhb } 1766121482Sjhb if (ih->ih_flags & IH_ENTROPY) { 1767121482Sjhb if (comma) 1768121482Sjhb db_printf(", "); 1769121482Sjhb db_printf("ENTROPY"); 1770121482Sjhb comma = 1; 1771121482Sjhb } 1772121482Sjhb if (ih->ih_flags & IH_DEAD) { 1773121482Sjhb if (comma) 1774121482Sjhb db_printf(", "); 1775121482Sjhb db_printf("DEAD"); 1776121482Sjhb comma = 1; 1777121482Sjhb } 1778121482Sjhb if (ih->ih_flags & IH_MPSAFE) { 1779121482Sjhb if (comma) 1780121482Sjhb db_printf(", "); 1781121482Sjhb db_printf("MPSAFE"); 1782121482Sjhb comma = 1; 1783121482Sjhb } 1784121482Sjhb if (ih->ih_need) { 1785121482Sjhb if (comma) 1786121482Sjhb db_printf(", "); 1787121482Sjhb db_printf("NEED"); 1788121482Sjhb } 1789121482Sjhb db_printf("}"); 1790121482Sjhb } 1791121482Sjhb db_printf("\n"); 1792121482Sjhb} 1793121482Sjhb 1794121482Sjhb/* 1795151658Sjhb * Dump details about a event. 1796121482Sjhb */ 1797121482Sjhbvoid 1798151658Sjhbdb_dump_intr_event(struct intr_event *ie, int handlers) 1799121482Sjhb{ 1800151658Sjhb struct intr_handler *ih; 1801151658Sjhb struct intr_thread *it; 1802121482Sjhb int comma; 1803121482Sjhb 1804151658Sjhb db_printf("%s ", ie->ie_fullname); 1805151658Sjhb it = ie->ie_thread; 1806151658Sjhb if (it != NULL) 1807151658Sjhb db_printf("(pid %d)", it->it_thread->td_proc->p_pid); 1808151658Sjhb else 1809151658Sjhb db_printf("(no thread)"); 1810151658Sjhb if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || 1811151658Sjhb (it != NULL && it->it_need)) { 1812121482Sjhb db_printf(" {"); 1813121482Sjhb comma = 0; 1814151658Sjhb if (ie->ie_flags & IE_SOFT) { 1815121482Sjhb db_printf("SOFT"); 1816121482Sjhb comma = 1; 1817121482Sjhb } 1818151658Sjhb if (ie->ie_flags & IE_ENTROPY) { 1819121482Sjhb if (comma) 1820121482Sjhb db_printf(", "); 1821121482Sjhb db_printf("ENTROPY"); 1822121482Sjhb comma = 1; 1823121482Sjhb } 1824151658Sjhb if (ie->ie_flags & IE_ADDING_THREAD) { 1825121482Sjhb if (comma) 1826121482Sjhb db_printf(", "); 1827151658Sjhb db_printf("ADDING_THREAD"); 1828121482Sjhb comma = 1; 1829121482Sjhb } 1830151658Sjhb if (it != NULL && it->it_need) { 1831121482Sjhb if (comma) 1832121482Sjhb db_printf(", "); 1833121482Sjhb db_printf("NEED"); 1834121482Sjhb } 1835121482Sjhb db_printf("}"); 1836121482Sjhb } 1837121482Sjhb db_printf("\n"); 1838121482Sjhb 1839121482Sjhb if (handlers) 1840151658Sjhb TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) 1841121482Sjhb db_dump_intrhand(ih); 1842121482Sjhb} 1843151658Sjhb 1844151658Sjhb/* 1845151658Sjhb * Dump data about interrupt handlers 1846151658Sjhb */ 1847151658SjhbDB_SHOW_COMMAND(intr, db_show_intr) 1848151658Sjhb{ 1849151658Sjhb struct intr_event *ie; 1850160312Sjhb int all, verbose; 1851151658Sjhb 1852229272Sed verbose = strchr(modif, 'v') != NULL; 1853229272Sed all = strchr(modif, 'a') != NULL; 1854151658Sjhb TAILQ_FOREACH(ie, &event_list, ie_list) { 1855151658Sjhb if (!all && TAILQ_EMPTY(&ie->ie_handlers)) 1856151658Sjhb continue; 1857151658Sjhb db_dump_intr_event(ie, verbose); 1858160312Sjhb if (db_pager_quit) 1859160312Sjhb break; 1860151658Sjhb } 1861151658Sjhb} 1862121482Sjhb#endif /* DDB */ 1863121482Sjhb 1864121482Sjhb/* 186567551Sjhb * Start standard software interrupt threads 186666698Sjhb */ 186767551Sjhbstatic void 186872237Sjhbstart_softintr(void *dummy) 186967551Sjhb{ 187072237Sjhb 1871177859Sjeff if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) 1872177859Sjeff panic("died while creating vm swi ithread"); 187366698Sjhb} 1874177253SrwatsonSYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, 1875177253Srwatson NULL); 187666698Sjhb 1877151658Sjhb/* 187877582Stmm * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. 187977582Stmm * The data for this machine dependent, and the declarations are in machine 188077582Stmm * dependent code. The layout of intrnames and intrcnt however is machine 188177582Stmm * independent. 188277582Stmm * 188377582Stmm * We do not know the length of intrcnt and intrnames at compile time, so 188477582Stmm * calculate things at run time. 188577582Stmm */ 188677582Stmmstatic int 188777582Stmmsysctl_intrnames(SYSCTL_HANDLER_ARGS) 188877582Stmm{ 1889224187Sattilio return (sysctl_handle_opaque(oidp, intrnames, sintrnames, req)); 189077582Stmm} 189177582Stmm 189277582StmmSYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, 189377582Stmm NULL, 0, sysctl_intrnames, "", "Interrupt Names"); 189477582Stmm 189577582Stmmstatic int 189677582Stmmsysctl_intrcnt(SYSCTL_HANDLER_ARGS) 189777582Stmm{ 1898232751Sjmallett#ifdef SCTL_MASK32 1899232751Sjmallett uint32_t *intrcnt32; 1900232751Sjmallett unsigned i; 1901232751Sjmallett int error; 1902232751Sjmallett 1903232751Sjmallett if (req->flags & SCTL_MASK32) { 1904232751Sjmallett if (!req->oldptr) 1905232751Sjmallett return (sysctl_handle_opaque(oidp, NULL, sintrcnt / 2, req)); 1906232751Sjmallett intrcnt32 = malloc(sintrcnt / 2, M_TEMP, M_NOWAIT); 1907232751Sjmallett if (intrcnt32 == NULL) 1908232751Sjmallett return (ENOMEM); 1909232751Sjmallett for (i = 0; i < sintrcnt / sizeof (u_long); i++) 1910232751Sjmallett intrcnt32[i] = intrcnt[i]; 1911232751Sjmallett error = sysctl_handle_opaque(oidp, intrcnt32, sintrcnt / 2, req); 1912232751Sjmallett free(intrcnt32, M_TEMP); 1913232751Sjmallett return (error); 1914232751Sjmallett } 1915232751Sjmallett#endif 1916224187Sattilio return (sysctl_handle_opaque(oidp, intrcnt, sintrcnt, req)); 191777582Stmm} 191877582Stmm 191977582StmmSYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, 192077582Stmm NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); 1921121482Sjhb 1922121482Sjhb#ifdef DDB 1923121482Sjhb/* 1924121482Sjhb * DDB command to dump the interrupt statistics. 1925121482Sjhb */ 1926121482SjhbDB_SHOW_COMMAND(intrcnt, db_show_intrcnt) 1927121482Sjhb{ 1928121482Sjhb u_long *i; 1929121482Sjhb char *cp; 1930224187Sattilio u_int j; 1931121482Sjhb 1932121482Sjhb cp = intrnames; 1933224187Sattilio j = 0; 1934224187Sattilio for (i = intrcnt; j < (sintrcnt / sizeof(u_long)) && !db_pager_quit; 1935224187Sattilio i++, j++) { 1936121482Sjhb if (*cp == '\0') 1937121482Sjhb break; 1938121482Sjhb if (*i != 0) 1939121482Sjhb db_printf("%s\t%lu\n", cp, *i); 1940121482Sjhb cp += strlen(cp) + 1; 1941121482Sjhb } 1942121482Sjhb} 1943121482Sjhb#endif 1944