kern_intr.c revision 137267
126156Sse/* 226156Sse * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 326156Sse * All rights reserved. 426156Sse * 526156Sse * Redistribution and use in source and binary forms, with or without 626156Sse * modification, are permitted provided that the following conditions 726156Sse * are met: 826156Sse * 1. Redistributions of source code must retain the above copyright 926156Sse * notice unmodified, this list of conditions, and the following 1026156Sse * disclaimer. 1126156Sse * 2. Redistributions in binary form must reproduce the above copyright 1226156Sse * notice, this list of conditions and the following disclaimer in the 1326156Sse * documentation and/or other materials provided with the distribution. 1426156Sse * 1526156Sse * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1626156Sse * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1726156Sse * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 1826156Sse * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 1926156Sse * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2026156Sse * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2126156Sse * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2226156Sse * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2326156Sse * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2426156Sse * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2526156Sse */ 2626156Sse 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_intr.c 137267 2004-11-05 19:11:24Z jhb $"); 2936887Sdfr 30121482Sjhb#include "opt_ddb.h" 31121482Sjhb 3241059Speter#include <sys/param.h> 3365822Sjhb#include <sys/bus.h> 34110860Salfred#include <sys/conf.h> 3565822Sjhb#include <sys/rtprio.h> 3641059Speter#include <sys/systm.h> 3766698Sjhb#include <sys/interrupt.h> 3866698Sjhb#include <sys/kernel.h> 3966698Sjhb#include <sys/kthread.h> 4066698Sjhb#include <sys/ktr.h> 41130128Sbde#include <sys/limits.h> 4274914Sjhb#include <sys/lock.h> 4326156Sse#include <sys/malloc.h> 4467365Sjhb#include <sys/mutex.h> 4566698Sjhb#include <sys/proc.h> 4672759Sjhb#include <sys/random.h> 4772237Sjhb#include <sys/resourcevar.h> 4877582Stmm#include <sys/sysctl.h> 4966698Sjhb#include <sys/unistd.h> 5066698Sjhb#include <sys/vmmeter.h> 5166698Sjhb#include <machine/atomic.h> 5266698Sjhb#include <machine/cpu.h> 5367551Sjhb#include <machine/md_var.h> 5472237Sjhb#include <machine/stdarg.h> 55121482Sjhb#ifdef DDB 56121482Sjhb#include <ddb/ddb.h> 57121482Sjhb#include <ddb/db_sym.h> 58121482Sjhb#endif 5926156Sse 6072759Sjhbstruct int_entropy { 6172759Sjhb struct proc *proc; 62122840Speter uintptr_t vector; 6372759Sjhb}; 6472759Sjhb 6572237Sjhbstruct ithd *clk_ithd; 6672237Sjhbstruct ithd *tty_ithd; 67128339Sbdevoid *softclock_ih; 68128339Sbdevoid *vm_ih; 6938244Sbde 7072237Sjhbstatic MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); 7172237Sjhb 72128331Sjhbstatic int intr_storm_threshold = 500; 73128331SjhbTUNABLE_INT("hw.intr_storm_threshold", &intr_storm_threshold); 74128331SjhbSYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RW, 75128331Sjhb &intr_storm_threshold, 0, 76128339Sbde "Number of consecutive interrupts before storm protection is enabled"); 77128331Sjhb 78128339Sbdestatic void ithread_loop(void *); 79128339Sbdestatic void ithread_update(struct ithd *); 80128339Sbdestatic void start_softintr(void *); 81128339Sbde 8272237Sjhbu_char 8372237Sjhbithread_priority(enum intr_type flags) 8465822Sjhb{ 8572237Sjhb u_char pri; 8665822Sjhb 8772237Sjhb flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | 8878365Speter INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); 8965822Sjhb switch (flags) { 9072237Sjhb case INTR_TYPE_TTY: 9165822Sjhb pri = PI_TTYLOW; 9265822Sjhb break; 9365822Sjhb case INTR_TYPE_BIO: 9465822Sjhb /* 9565822Sjhb * XXX We need to refine this. BSD/OS distinguishes 9665822Sjhb * between tape and disk priorities. 9765822Sjhb */ 9865822Sjhb pri = PI_DISK; 9965822Sjhb break; 10065822Sjhb case INTR_TYPE_NET: 10165822Sjhb pri = PI_NET; 10265822Sjhb break; 10365822Sjhb case INTR_TYPE_CAM: 10465822Sjhb pri = PI_DISK; /* XXX or PI_CAM? */ 10565822Sjhb break; 10678365Speter case INTR_TYPE_AV: /* Audio/video */ 10778365Speter pri = PI_AV; 10878365Speter break; 10972237Sjhb case INTR_TYPE_CLK: 11072237Sjhb pri = PI_REALTIME; 11172237Sjhb break; 11265822Sjhb case INTR_TYPE_MISC: 11365822Sjhb pri = PI_DULL; /* don't care */ 11465822Sjhb break; 11565822Sjhb default: 11672237Sjhb /* We didn't specify an interrupt level. */ 11765822Sjhb panic("ithread_priority: no interrupt type in flags"); 11865822Sjhb } 11965822Sjhb 12065822Sjhb return pri; 12165822Sjhb} 12265822Sjhb 12372237Sjhb/* 12472237Sjhb * Regenerate the name (p_comm) and priority for a threaded interrupt thread. 12572237Sjhb */ 12672237Sjhbstatic void 12772237Sjhbithread_update(struct ithd *ithd) 12872237Sjhb{ 12972237Sjhb struct intrhand *ih; 13083366Sjulian struct thread *td; 13172237Sjhb struct proc *p; 132137267Sjhb int missed; 13367551Sjhb 13476771Sjhb mtx_assert(&ithd->it_lock, MA_OWNED); 13583366Sjulian td = ithd->it_td; 13683366Sjulian if (td == NULL) 13772237Sjhb return; 13883366Sjulian p = td->td_proc; 13972237Sjhb 140105357Srobert strlcpy(p->p_comm, ithd->it_name, sizeof(p->p_comm)); 141137267Sjhb ithd->it_flags &= ~IT_ENTROPY; 142105354Srobert 14372237Sjhb ih = TAILQ_FIRST(&ithd->it_handlers); 14472237Sjhb if (ih == NULL) { 14594457Sjhb mtx_lock_spin(&sched_lock); 14690538Sjulian td->td_priority = PRI_MAX_ITHD; 14794457Sjhb td->td_base_pri = PRI_MAX_ITHD; 14894457Sjhb mtx_unlock_spin(&sched_lock); 14972237Sjhb return; 15072237Sjhb } 15194457Sjhb mtx_lock_spin(&sched_lock); 15290538Sjulian td->td_priority = ih->ih_pri; 15390538Sjulian td->td_base_pri = ih->ih_pri; 15494457Sjhb mtx_unlock_spin(&sched_lock); 155137267Sjhb missed = 0; 15672237Sjhb TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) { 15772237Sjhb if (strlen(p->p_comm) + strlen(ih->ih_name) + 1 < 15872237Sjhb sizeof(p->p_comm)) { 15972237Sjhb strcat(p->p_comm, " "); 16072237Sjhb strcat(p->p_comm, ih->ih_name); 161137267Sjhb } else 162137267Sjhb missed++; 163137267Sjhb if (ih->ih_flags & IH_ENTROPY) 164137267Sjhb ithd->it_flags |= IT_ENTROPY; 165137267Sjhb } 166137267Sjhb while (missed-- > 0) { 167137267Sjhb if (strlen(p->p_comm) + 1 == sizeof(p->p_comm)) { 16872237Sjhb if (p->p_comm[sizeof(p->p_comm) - 2] == '+') 16972237Sjhb p->p_comm[sizeof(p->p_comm) - 2] = '*'; 17072237Sjhb else 17172237Sjhb p->p_comm[sizeof(p->p_comm) - 2] = '+'; 17272237Sjhb } else 17372237Sjhb strcat(p->p_comm, "+"); 17472237Sjhb } 175108372Sjake CTR2(KTR_INTR, "%s: updated %s", __func__, p->p_comm); 17672237Sjhb} 17772237Sjhb 17872237Sjhbint 179122840Speterithread_create(struct ithd **ithread, uintptr_t vector, int flags, 180122840Speter void (*disable)(uintptr_t), void (*enable)(uintptr_t), const char *fmt, ...) 18166698Sjhb{ 18272237Sjhb struct ithd *ithd; 18383366Sjulian struct thread *td; 18467551Sjhb struct proc *p; 18572237Sjhb int error; 18672237Sjhb va_list ap; 18772237Sjhb 18872759Sjhb /* The only valid flag during creation is IT_SOFT. */ 18972759Sjhb if ((flags & ~IT_SOFT) != 0) 19072759Sjhb return (EINVAL); 19172759Sjhb 192111119Simp ithd = malloc(sizeof(struct ithd), M_ITHREAD, M_WAITOK | M_ZERO); 19372237Sjhb ithd->it_vector = vector; 19472237Sjhb ithd->it_disable = disable; 19572237Sjhb ithd->it_enable = enable; 19672237Sjhb ithd->it_flags = flags; 19772237Sjhb TAILQ_INIT(&ithd->it_handlers); 19893818Sjhb mtx_init(&ithd->it_lock, "ithread", NULL, MTX_DEF); 19972237Sjhb 20072237Sjhb va_start(ap, fmt); 20172237Sjhb vsnprintf(ithd->it_name, sizeof(ithd->it_name), fmt, ap); 20272237Sjhb va_end(ap); 20372237Sjhb 20472237Sjhb error = kthread_create(ithread_loop, ithd, &p, RFSTOPPED | RFHIGHPID, 205104354Sscottl 0, "%s", ithd->it_name); 20672237Sjhb if (error) { 20776771Sjhb mtx_destroy(&ithd->it_lock); 20872237Sjhb free(ithd, M_ITHREAD); 20972237Sjhb return (error); 21072237Sjhb } 21190361Sjulian td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */ 212113629Sjhb mtx_lock_spin(&sched_lock); 21390538Sjulian td->td_ksegrp->kg_pri_class = PRI_ITHD; 21490538Sjulian td->td_priority = PRI_MAX_ITHD; 215103216Sjulian TD_SET_IWAIT(td); 216113629Sjhb mtx_unlock_spin(&sched_lock); 21783366Sjulian ithd->it_td = td; 21883366Sjulian td->td_ithd = ithd; 21972237Sjhb if (ithread != NULL) 22072237Sjhb *ithread = ithd; 22187593Sobrien CTR2(KTR_INTR, "%s: created %s", __func__, ithd->it_name); 22272237Sjhb return (0); 22372237Sjhb} 22472237Sjhb 22572237Sjhbint 22672237Sjhbithread_destroy(struct ithd *ithread) 22772237Sjhb{ 22872237Sjhb 22983366Sjulian struct thread *td; 23076771Sjhb if (ithread == NULL) 23172237Sjhb return (EINVAL); 23272237Sjhb 23383366Sjulian td = ithread->it_td; 23476771Sjhb mtx_lock(&ithread->it_lock); 23576771Sjhb if (!TAILQ_EMPTY(&ithread->it_handlers)) { 23676771Sjhb mtx_unlock(&ithread->it_lock); 23776771Sjhb return (EINVAL); 23876771Sjhb } 23976771Sjhb ithread->it_flags |= IT_DEAD; 24072237Sjhb mtx_lock_spin(&sched_lock); 241103216Sjulian if (TD_AWAITING_INTR(td)) { 242103216Sjulian TD_CLR_IWAIT(td); 243134586Sjulian setrunqueue(td, SRQ_INTR); 24472237Sjhb } 24572237Sjhb mtx_unlock_spin(&sched_lock); 24676771Sjhb mtx_unlock(&ithread->it_lock); 24787593Sobrien CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_name); 24872237Sjhb return (0); 24972237Sjhb} 25072237Sjhb 25172237Sjhbint 25272237Sjhbithread_add_handler(struct ithd* ithread, const char *name, 25372237Sjhb driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, 25472237Sjhb void **cookiep) 25572237Sjhb{ 25672237Sjhb struct intrhand *ih, *temp_ih; 25772237Sjhb 25872237Sjhb if (ithread == NULL || name == NULL || handler == NULL) 25972237Sjhb return (EINVAL); 26072237Sjhb 261111119Simp ih = malloc(sizeof(struct intrhand), M_ITHREAD, M_WAITOK | M_ZERO); 26272237Sjhb ih->ih_handler = handler; 26372237Sjhb ih->ih_argument = arg; 26472237Sjhb ih->ih_name = name; 26572237Sjhb ih->ih_ithread = ithread; 26672237Sjhb ih->ih_pri = pri; 26772237Sjhb if (flags & INTR_FAST) 268122002Sjhb ih->ih_flags = IH_FAST; 26972237Sjhb else if (flags & INTR_EXCL) 27072237Sjhb ih->ih_flags = IH_EXCLUSIVE; 27172237Sjhb if (flags & INTR_MPSAFE) 27272237Sjhb ih->ih_flags |= IH_MPSAFE; 27372237Sjhb if (flags & INTR_ENTROPY) 27472237Sjhb ih->ih_flags |= IH_ENTROPY; 27572237Sjhb 27676771Sjhb mtx_lock(&ithread->it_lock); 277122002Sjhb if ((flags & INTR_EXCL) != 0 && !TAILQ_EMPTY(&ithread->it_handlers)) 27872237Sjhb goto fail; 279122002Sjhb if (!TAILQ_EMPTY(&ithread->it_handlers)) { 280122002Sjhb temp_ih = TAILQ_FIRST(&ithread->it_handlers); 281122002Sjhb if (temp_ih->ih_flags & IH_EXCLUSIVE) 282122002Sjhb goto fail; 283122002Sjhb if ((ih->ih_flags & IH_FAST) && !(temp_ih->ih_flags & IH_FAST)) 284122002Sjhb goto fail; 285122002Sjhb if (!(ih->ih_flags & IH_FAST) && (temp_ih->ih_flags & IH_FAST)) 286122002Sjhb goto fail; 287122002Sjhb } 28872237Sjhb 28972237Sjhb TAILQ_FOREACH(temp_ih, &ithread->it_handlers, ih_next) 29072237Sjhb if (temp_ih->ih_pri > ih->ih_pri) 29172237Sjhb break; 29272237Sjhb if (temp_ih == NULL) 29372237Sjhb TAILQ_INSERT_TAIL(&ithread->it_handlers, ih, ih_next); 29472237Sjhb else 29572237Sjhb TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); 29672237Sjhb ithread_update(ithread); 29776771Sjhb mtx_unlock(&ithread->it_lock); 29872237Sjhb 29972237Sjhb if (cookiep != NULL) 30072237Sjhb *cookiep = ih; 30187593Sobrien CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, 30272837Sjhb ithread->it_name); 30372237Sjhb return (0); 30472237Sjhb 30572237Sjhbfail: 30676771Sjhb mtx_unlock(&ithread->it_lock); 30772237Sjhb free(ih, M_ITHREAD); 30872237Sjhb return (EINVAL); 30972237Sjhb} 31072237Sjhb 31172237Sjhbint 31272237Sjhbithread_remove_handler(void *cookie) 31372237Sjhb{ 31472237Sjhb struct intrhand *handler = (struct intrhand *)cookie; 31572237Sjhb struct ithd *ithread; 31672237Sjhb#ifdef INVARIANTS 31772237Sjhb struct intrhand *ih; 31872237Sjhb#endif 31972237Sjhb 32072759Sjhb if (handler == NULL) 32172237Sjhb return (EINVAL); 32272827Sjhb ithread = handler->ih_ithread; 32372827Sjhb KASSERT(ithread != NULL, 32472759Sjhb ("interrupt handler \"%s\" has a NULL interrupt thread", 32572759Sjhb handler->ih_name)); 32687593Sobrien CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, 32772837Sjhb ithread->it_name); 32876771Sjhb mtx_lock(&ithread->it_lock); 32972237Sjhb#ifdef INVARIANTS 33072237Sjhb TAILQ_FOREACH(ih, &ithread->it_handlers, ih_next) 33172759Sjhb if (ih == handler) 33272759Sjhb goto ok; 33376771Sjhb mtx_unlock(&ithread->it_lock); 33472759Sjhb panic("interrupt handler \"%s\" not found in interrupt thread \"%s\"", 33572759Sjhb ih->ih_name, ithread->it_name); 33672759Sjhbok: 33772237Sjhb#endif 33872839Sjhb /* 33972839Sjhb * If the interrupt thread is already running, then just mark this 34072839Sjhb * handler as being dead and let the ithread do the actual removal. 341124505Struckman * 342124505Struckman * During a cold boot while cold is set, msleep() does not sleep, 343124505Struckman * so we have to remove the handler here rather than letting the 344124505Struckman * thread do it. 34572839Sjhb */ 34672839Sjhb mtx_lock_spin(&sched_lock); 347124505Struckman if (!TD_AWAITING_INTR(ithread->it_td) && !cold) { 34872839Sjhb handler->ih_flags |= IH_DEAD; 34972839Sjhb 35072839Sjhb /* 35172839Sjhb * Ensure that the thread will process the handler list 35272839Sjhb * again and remove this handler if it has already passed 35372839Sjhb * it on the list. 35472839Sjhb */ 35572839Sjhb ithread->it_need = 1; 35676771Sjhb } else 35772839Sjhb TAILQ_REMOVE(&ithread->it_handlers, handler, ih_next); 35872839Sjhb mtx_unlock_spin(&sched_lock); 35976771Sjhb if ((handler->ih_flags & IH_DEAD) != 0) 36076771Sjhb msleep(handler, &ithread->it_lock, PUSER, "itrmh", 0); 36176771Sjhb ithread_update(ithread); 36276771Sjhb mtx_unlock(&ithread->it_lock); 36376771Sjhb free(handler, M_ITHREAD); 36472237Sjhb return (0); 36572237Sjhb} 36672237Sjhb 36772237Sjhbint 368131481Sjhbithread_schedule(struct ithd *ithread) 36972759Sjhb{ 37072759Sjhb struct int_entropy entropy; 37183366Sjulian struct thread *td; 372101176Sjulian struct thread *ctd; 37372759Sjhb struct proc *p; 37472759Sjhb 37572759Sjhb /* 37672759Sjhb * If no ithread or no handlers, then we have a stray interrupt. 37772759Sjhb */ 37872759Sjhb if ((ithread == NULL) || TAILQ_EMPTY(&ithread->it_handlers)) 37972759Sjhb return (EINVAL); 38072759Sjhb 381101176Sjulian ctd = curthread; 382133191Srwatson td = ithread->it_td; 383133191Srwatson p = td->td_proc; 38472759Sjhb /* 38572759Sjhb * If any of the handlers for this ithread claim to be good 38672759Sjhb * sources of entropy, then gather some. 38772759Sjhb */ 38872759Sjhb if (harvest.interrupt && ithread->it_flags & IT_ENTROPY) { 389133191Srwatson CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, 390133191Srwatson p->p_pid, p->p_comm); 39172759Sjhb entropy.vector = ithread->it_vector; 392102996Sdavidxu entropy.proc = ctd->td_proc; 39372759Sjhb random_harvest(&entropy, sizeof(entropy), 2, 0, 39472759Sjhb RANDOM_INTERRUPT); 39572759Sjhb } 39672759Sjhb 39773313Sjhb KASSERT(p != NULL, ("ithread %s has no process", ithread->it_name)); 39899072Sjulian CTR4(KTR_INTR, "%s: pid %d: (%s) need = %d", 39999072Sjulian __func__, p->p_pid, p->p_comm, ithread->it_need); 40072759Sjhb 40172759Sjhb /* 40272759Sjhb * Set it_need to tell the thread to keep running if it is already 40372759Sjhb * running. Then, grab sched_lock and see if we actually need to 404131481Sjhb * put this thread on the runqueue. 40572759Sjhb */ 40672759Sjhb ithread->it_need = 1; 40772759Sjhb mtx_lock_spin(&sched_lock); 408103216Sjulian if (TD_AWAITING_INTR(td)) { 40987593Sobrien CTR2(KTR_INTR, "%s: setrunqueue %d", __func__, p->p_pid); 410103216Sjulian TD_CLR_IWAIT(td); 411134586Sjulian setrunqueue(td, SRQ_INTR); 41272759Sjhb } else { 41387593Sobrien CTR4(KTR_INTR, "%s: pid %d: it_need %d, state %d", 414113612Sjhb __func__, p->p_pid, ithread->it_need, td->td_state); 41572759Sjhb } 41672759Sjhb mtx_unlock_spin(&sched_lock); 41772759Sjhb 41872759Sjhb return (0); 41972759Sjhb} 42072759Sjhb 42172759Sjhbint 42272237Sjhbswi_add(struct ithd **ithdp, const char *name, driver_intr_t handler, 42372237Sjhb void *arg, int pri, enum intr_type flags, void **cookiep) 42472237Sjhb{ 42567551Sjhb struct ithd *ithd; 42672237Sjhb int error; 42766698Sjhb 42872759Sjhb if (flags & (INTR_FAST | INTR_ENTROPY)) 42972759Sjhb return (EINVAL); 43072759Sjhb 43167551Sjhb ithd = (ithdp != NULL) ? *ithdp : NULL; 43266698Sjhb 43372759Sjhb if (ithd != NULL) { 43472759Sjhb if ((ithd->it_flags & IT_SOFT) == 0) 43572759Sjhb return(EINVAL); 43672759Sjhb } else { 43772237Sjhb error = ithread_create(&ithd, pri, IT_SOFT, NULL, NULL, 43872237Sjhb "swi%d:", pri); 43967551Sjhb if (error) 44072237Sjhb return (error); 44172237Sjhb 44267551Sjhb if (ithdp != NULL) 44367551Sjhb *ithdp = ithd; 44466698Sjhb } 44572376Sjake return (ithread_add_handler(ithd, name, handler, arg, 44672376Sjake (pri * RQ_PPQ) + PI_SOFT, flags, cookiep)); 447134791Sjulian /* XXKSE.. think of a better way to get separate queues */ 44866698Sjhb} 44966698Sjhb 45066698Sjhb 45166698Sjhb/* 45267551Sjhb * Schedule a heavyweight software interrupt process. 45366698Sjhb */ 45467551Sjhbvoid 45572237Sjhbswi_sched(void *cookie, int flags) 45666698Sjhb{ 45772237Sjhb struct intrhand *ih = (struct intrhand *)cookie; 45872237Sjhb struct ithd *it = ih->ih_ithread; 45972759Sjhb int error; 46066698Sjhb 46167551Sjhb atomic_add_int(&cnt.v_intr, 1); /* one more global interrupt */ 46267551Sjhb 46372237Sjhb CTR3(KTR_INTR, "swi_sched pid %d(%s) need=%d", 46483366Sjulian it->it_td->td_proc->p_pid, it->it_td->td_proc->p_comm, it->it_need); 46567551Sjhb 46667551Sjhb /* 46772759Sjhb * Set ih_need for this handler so that if the ithread is already 46872759Sjhb * running it will execute this handler on the next pass. Otherwise, 46972759Sjhb * it will execute it the next time it runs. 47067551Sjhb */ 47172237Sjhb atomic_store_rel_int(&ih->ih_need, 1); 47272237Sjhb if (!(flags & SWI_DELAY)) { 473131481Sjhb error = ithread_schedule(it); 47472759Sjhb KASSERT(error == 0, ("stray software interrupt")); 47566698Sjhb } 47666698Sjhb} 47766698Sjhb 47866698Sjhb/* 47972237Sjhb * This is the main code for interrupt threads. 48066698Sjhb */ 481104094Sphkstatic void 48272237Sjhbithread_loop(void *arg) 48366698Sjhb{ 48472237Sjhb struct ithd *ithd; /* our thread context */ 48567551Sjhb struct intrhand *ih; /* and our interrupt handler chain */ 48683366Sjulian struct thread *td; 48772237Sjhb struct proc *p; 488137174Sjhb int count, warned; 48967551Sjhb 49083366Sjulian td = curthread; 49183366Sjulian p = td->td_proc; 49272237Sjhb ithd = (struct ithd *)arg; /* point to myself */ 49383366Sjulian KASSERT(ithd->it_td == td && td->td_ithd == ithd, 49487593Sobrien ("%s: ithread and proc linkage out of sync", __func__)); 495137174Sjhb count = 0; 496128331Sjhb warned = 0; 49766698Sjhb 49867551Sjhb /* 49967551Sjhb * As long as we have interrupts outstanding, go through the 50067551Sjhb * list of handlers, giving each one a go at it. 50167551Sjhb */ 50266698Sjhb for (;;) { 50372237Sjhb /* 50472237Sjhb * If we are an orphaned thread, then just die. 50572237Sjhb */ 50672237Sjhb if (ithd->it_flags & IT_DEAD) { 50787593Sobrien CTR3(KTR_INTR, "%s: pid %d: (%s) exiting", __func__, 50872237Sjhb p->p_pid, p->p_comm); 50983366Sjulian td->td_ithd = NULL; 51076771Sjhb mtx_destroy(&ithd->it_lock); 51172237Sjhb free(ithd, M_ITHREAD); 51272237Sjhb kthread_exit(0); 51372237Sjhb } 51472237Sjhb 51587593Sobrien CTR4(KTR_INTR, "%s: pid %d: (%s) need=%d", __func__, 51672237Sjhb p->p_pid, p->p_comm, ithd->it_need); 51772237Sjhb while (ithd->it_need) { 51867551Sjhb /* 51967551Sjhb * Service interrupts. If another interrupt 52067551Sjhb * arrives while we are running, they will set 52167551Sjhb * it_need to denote that we should make 52267551Sjhb * another pass. 52367551Sjhb */ 52472237Sjhb atomic_store_rel_int(&ithd->it_need, 0); 52572839Sjhbrestart: 52672237Sjhb TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) { 52772237Sjhb if (ithd->it_flags & IT_SOFT && !ih->ih_need) 52867551Sjhb continue; 52972237Sjhb atomic_store_rel_int(&ih->ih_need, 0); 53087593Sobrien CTR6(KTR_INTR, 53187593Sobrien "%s: pid %d ih=%p: %p(%p) flg=%x", __func__, 53267551Sjhb p->p_pid, (void *)ih, 53367551Sjhb (void *)ih->ih_handler, ih->ih_argument, 53467551Sjhb ih->ih_flags); 53566698Sjhb 53672839Sjhb if ((ih->ih_flags & IH_DEAD) != 0) { 53776771Sjhb mtx_lock(&ithd->it_lock); 53872839Sjhb TAILQ_REMOVE(&ithd->it_handlers, ih, 53972839Sjhb ih_next); 54076771Sjhb wakeup(ih); 54176771Sjhb mtx_unlock(&ithd->it_lock); 54272839Sjhb goto restart; 54372839Sjhb } 54476771Sjhb if ((ih->ih_flags & IH_MPSAFE) == 0) 54576771Sjhb mtx_lock(&Giant); 54667551Sjhb ih->ih_handler(ih->ih_argument); 54772237Sjhb if ((ih->ih_flags & IH_MPSAFE) == 0) 54872200Sbmilekic mtx_unlock(&Giant); 54967551Sjhb } 550130128Sbde 551130128Sbde /* 552137174Sjhb * If we detect an interrupt storm, pause with the 553137174Sjhb * source masked until the next hardclock tick. 554130128Sbde */ 555137174Sjhb if (intr_storm_threshold != 0 && 556137174Sjhb count >= intr_storm_threshold) { 557130128Sbde if (!warned) { 558130128Sbde printf( 559130128Sbde "Interrupt storm detected on \"%s\"; throttling interrupt source\n", 560130128Sbde p->p_comm); 561130128Sbde warned = 1; 562130128Sbde } 563130128Sbde tsleep(&count, td->td_priority, "istorm", 1); 564137174Sjhb count = 0; 565137174Sjhb } else 566137174Sjhb count++; 567130128Sbde 568137174Sjhb if (ithd->it_enable != NULL) 569137174Sjhb ithd->it_enable(ithd->it_vector); 57066698Sjhb } 571128331Sjhb WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); 572128331Sjhb mtx_assert(&Giant, MA_NOTOWNED); 57367551Sjhb 57466698Sjhb /* 57566698Sjhb * Processed all our interrupts. Now get the sched 57667551Sjhb * lock. This may take a while and it_need may get 57766698Sjhb * set again, so we have to check it again. 57866698Sjhb */ 57972200Sbmilekic mtx_lock_spin(&sched_lock); 58072237Sjhb if (!ithd->it_need) { 581128331Sjhb TD_SET_IWAIT(td); 582137174Sjhb count = 0; 58387593Sobrien CTR2(KTR_INTR, "%s: pid %d: done", __func__, p->p_pid); 584131473Sjhb mi_switch(SW_VOL, NULL); 58587593Sobrien CTR2(KTR_INTR, "%s: pid %d: resumed", __func__, p->p_pid); 58666698Sjhb } 58772200Sbmilekic mtx_unlock_spin(&sched_lock); 58866698Sjhb } 58966698Sjhb} 59066698Sjhb 591121482Sjhb#ifdef DDB 59272237Sjhb/* 593121482Sjhb * Dump details about an interrupt handler 594121482Sjhb */ 595121482Sjhbstatic void 596121482Sjhbdb_dump_intrhand(struct intrhand *ih) 597121482Sjhb{ 598121482Sjhb int comma; 599121482Sjhb 600121482Sjhb db_printf("\t%-10s ", ih->ih_name); 601121482Sjhb switch (ih->ih_pri) { 602121482Sjhb case PI_REALTIME: 603121482Sjhb db_printf("CLK "); 604121482Sjhb break; 605121482Sjhb case PI_AV: 606121482Sjhb db_printf("AV "); 607121482Sjhb break; 608121482Sjhb case PI_TTYHIGH: 609121482Sjhb case PI_TTYLOW: 610121482Sjhb db_printf("TTY "); 611121482Sjhb break; 612121482Sjhb case PI_TAPE: 613121482Sjhb db_printf("TAPE"); 614121482Sjhb break; 615121482Sjhb case PI_NET: 616121482Sjhb db_printf("NET "); 617121482Sjhb break; 618121482Sjhb case PI_DISK: 619121482Sjhb case PI_DISKLOW: 620121482Sjhb db_printf("DISK"); 621121482Sjhb break; 622121482Sjhb case PI_DULL: 623121482Sjhb db_printf("DULL"); 624121482Sjhb break; 625121482Sjhb default: 626121482Sjhb if (ih->ih_pri >= PI_SOFT) 627121482Sjhb db_printf("SWI "); 628121482Sjhb else 629121482Sjhb db_printf("%4u", ih->ih_pri); 630121482Sjhb break; 631121482Sjhb } 632121482Sjhb db_printf(" "); 633121482Sjhb db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); 634121482Sjhb db_printf("(%p)", ih->ih_argument); 635121482Sjhb if (ih->ih_need || 636121482Sjhb (ih->ih_flags & (IH_FAST | IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | 637121482Sjhb IH_MPSAFE)) != 0) { 638121482Sjhb db_printf(" {"); 639121482Sjhb comma = 0; 640121482Sjhb if (ih->ih_flags & IH_FAST) { 641121482Sjhb db_printf("FAST"); 642121482Sjhb comma = 1; 643121482Sjhb } 644121482Sjhb if (ih->ih_flags & IH_EXCLUSIVE) { 645121482Sjhb if (comma) 646121482Sjhb db_printf(", "); 647121482Sjhb db_printf("EXCL"); 648121482Sjhb comma = 1; 649121482Sjhb } 650121482Sjhb if (ih->ih_flags & IH_ENTROPY) { 651121482Sjhb if (comma) 652121482Sjhb db_printf(", "); 653121482Sjhb db_printf("ENTROPY"); 654121482Sjhb comma = 1; 655121482Sjhb } 656121482Sjhb if (ih->ih_flags & IH_DEAD) { 657121482Sjhb if (comma) 658121482Sjhb db_printf(", "); 659121482Sjhb db_printf("DEAD"); 660121482Sjhb comma = 1; 661121482Sjhb } 662121482Sjhb if (ih->ih_flags & IH_MPSAFE) { 663121482Sjhb if (comma) 664121482Sjhb db_printf(", "); 665121482Sjhb db_printf("MPSAFE"); 666121482Sjhb comma = 1; 667121482Sjhb } 668121482Sjhb if (ih->ih_need) { 669121482Sjhb if (comma) 670121482Sjhb db_printf(", "); 671121482Sjhb db_printf("NEED"); 672121482Sjhb } 673121482Sjhb db_printf("}"); 674121482Sjhb } 675121482Sjhb db_printf("\n"); 676121482Sjhb} 677121482Sjhb 678121482Sjhb/* 679121482Sjhb * Dump details about an ithread 680121482Sjhb */ 681121482Sjhbvoid 682121482Sjhbdb_dump_ithread(struct ithd *ithd, int handlers) 683121482Sjhb{ 684121482Sjhb struct proc *p; 685121482Sjhb struct intrhand *ih; 686121482Sjhb int comma; 687121482Sjhb 688121482Sjhb if (ithd->it_td != NULL) { 689121482Sjhb p = ithd->it_td->td_proc; 690121482Sjhb db_printf("%s (pid %d)", p->p_comm, p->p_pid); 691121482Sjhb } else 692121482Sjhb db_printf("%s: (no thread)", ithd->it_name); 693121482Sjhb if ((ithd->it_flags & (IT_SOFT | IT_ENTROPY | IT_DEAD)) != 0 || 694121482Sjhb ithd->it_need) { 695121482Sjhb db_printf(" {"); 696121482Sjhb comma = 0; 697121482Sjhb if (ithd->it_flags & IT_SOFT) { 698121482Sjhb db_printf("SOFT"); 699121482Sjhb comma = 1; 700121482Sjhb } 701121482Sjhb if (ithd->it_flags & IT_ENTROPY) { 702121482Sjhb if (comma) 703121482Sjhb db_printf(", "); 704121482Sjhb db_printf("ENTROPY"); 705121482Sjhb comma = 1; 706121482Sjhb } 707121482Sjhb if (ithd->it_flags & IT_DEAD) { 708121482Sjhb if (comma) 709121482Sjhb db_printf(", "); 710121482Sjhb db_printf("DEAD"); 711121482Sjhb comma = 1; 712121482Sjhb } 713121482Sjhb if (ithd->it_need) { 714121482Sjhb if (comma) 715121482Sjhb db_printf(", "); 716121482Sjhb db_printf("NEED"); 717121482Sjhb } 718121482Sjhb db_printf("}"); 719121482Sjhb } 720121482Sjhb db_printf("\n"); 721121482Sjhb 722121482Sjhb if (handlers) 723121482Sjhb TAILQ_FOREACH(ih, &ithd->it_handlers, ih_next) 724121482Sjhb db_dump_intrhand(ih); 725121482Sjhb} 726121482Sjhb#endif /* DDB */ 727121482Sjhb 728121482Sjhb/* 72967551Sjhb * Start standard software interrupt threads 73066698Sjhb */ 73167551Sjhbstatic void 73272237Sjhbstart_softintr(void *dummy) 73367551Sjhb{ 734113613Sjhb struct proc *p; 73572237Sjhb 736103781Sjake if (swi_add(&clk_ithd, "clock", softclock, NULL, SWI_CLOCK, 73772237Sjhb INTR_MPSAFE, &softclock_ih) || 738117128Sscottl swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) 73972237Sjhb panic("died while creating standard software ithreads"); 74072759Sjhb 741113613Sjhb p = clk_ithd->it_td->td_proc; 742113613Sjhb PROC_LOCK(p); 743113613Sjhb p->p_flag |= P_NOLOAD; 744113613Sjhb PROC_UNLOCK(p); 74566698Sjhb} 74672237SjhbSYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, NULL) 74766698Sjhb 74877582Stmm/* 74977582Stmm * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. 75077582Stmm * The data for this machine dependent, and the declarations are in machine 75177582Stmm * dependent code. The layout of intrnames and intrcnt however is machine 75277582Stmm * independent. 75377582Stmm * 75477582Stmm * We do not know the length of intrcnt and intrnames at compile time, so 75577582Stmm * calculate things at run time. 75677582Stmm */ 75777582Stmmstatic int 75877582Stmmsysctl_intrnames(SYSCTL_HANDLER_ARGS) 75977582Stmm{ 76077582Stmm return (sysctl_handle_opaque(oidp, intrnames, eintrnames - intrnames, 76177582Stmm req)); 76277582Stmm} 76377582Stmm 76477582StmmSYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, 76577582Stmm NULL, 0, sysctl_intrnames, "", "Interrupt Names"); 76677582Stmm 76777582Stmmstatic int 76877582Stmmsysctl_intrcnt(SYSCTL_HANDLER_ARGS) 76977582Stmm{ 77077582Stmm return (sysctl_handle_opaque(oidp, intrcnt, 77177582Stmm (char *)eintrcnt - (char *)intrcnt, req)); 77277582Stmm} 77377582Stmm 77477582StmmSYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, 77577582Stmm NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); 776121482Sjhb 777121482Sjhb#ifdef DDB 778121482Sjhb/* 779121482Sjhb * DDB command to dump the interrupt statistics. 780121482Sjhb */ 781121482SjhbDB_SHOW_COMMAND(intrcnt, db_show_intrcnt) 782121482Sjhb{ 783121482Sjhb u_long *i; 784121482Sjhb char *cp; 785121482Sjhb int quit; 786121482Sjhb 787121482Sjhb cp = intrnames; 788137117Sjhb db_setup_paging(db_simple_pager, &quit, db_lines_per_page); 789121482Sjhb for (i = intrcnt, quit = 0; i != eintrcnt && !quit; i++) { 790121482Sjhb if (*cp == '\0') 791121482Sjhb break; 792121482Sjhb if (*i != 0) 793121482Sjhb db_printf("%s\t%lu\n", cp, *i); 794121482Sjhb cp += strlen(cp) + 1; 795121482Sjhb } 796121482Sjhb} 797121482Sjhb#endif 798