sys/kern/kern_intr.c

101 static void	intr_event_update(struct intr_event *ie);
103 static int	intr_event_schedule_thread(struct intr_event *ie,
105 static int	intr_filter_loop(struct intr_event *ie,
110 static int	intr_event_schedule_thread(struct intr_event *ie);
115 		    struct intr_event *ie);
168 	struct intr_event *ie;
172 	ie = ithd->it_event;
176 	if (TAILQ_EMPTY(&ie->ie_handlers))
179 		pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri;
182 	strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name));
195 intr_event_update(struct intr_event *ie)
202 	mtx_assert(&ie->ie_lock, MA_OWNED);
203 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
204 	ie->ie_flags &= ~IE_ENTROPY;
209 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
210 		if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 <
211 		    sizeof(ie->ie_fullname)) {
212 			strcat(ie->ie_fullname, " ");
213 			strcat(ie->ie_fullname, ih->ih_name);
218 			ie->ie_flags |= IE_ENTROPY;
226 	last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2];
228 		if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) {
235 			strcat(ie->ie_fullname, " +");
238 			strcat(ie->ie_fullname, "+");
245 	if (ie->ie_thread != NULL)
246 		ithread_update(ie->ie_thread);
247 	CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname);
256 	struct intr_event *ie;
262 	ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO);
263 	ie->ie_source = source;
264 	ie->ie_pre_ithread = pre_ithread;
265 	ie->ie_post_ithread = post_ithread;
266 	ie->ie_post_filter = post_filter;
267 	ie->ie_assign_cpu = assign_cpu;
268 	ie->ie_flags = flags;
269 	ie->ie_irq = irq;
270 	ie->ie_cpu = NOCPU;
271 	TAILQ_INIT(&ie->ie_handlers);
272 	mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF);
275 	vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap);
277 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
279 	TAILQ_INSERT_TAIL(&event_list, ie, ie_list);
282 		*event = ie;
283 	CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name);
296 intr_event_bind(struct intr_event *ie, u_char cpu)
306 	if (ie->ie_assign_cpu == NULL)
317 	mtx_lock(&ie->ie_lock);
318 	if (ie->ie_thread != NULL) {
324 		id = ie->ie_thread->it_thread->td_tid;
325 		mtx_unlock(&ie->ie_lock);
330 		mtx_unlock(&ie->ie_lock);
331 	error = ie->ie_assign_cpu(ie->ie_source, cpu);
333 		mtx_lock(&ie->ie_lock);
334 		if (ie->ie_thread != NULL) {
336 			if (ie->ie_cpu == NOCPU)
339 				CPU_SET(ie->ie_cpu, &mask);
340 			id = ie->ie_thread->it_thread->td_tid;
341 			mtx_unlock(&ie->ie_lock);
344 			mtx_unlock(&ie->ie_lock);
348 	mtx_lock(&ie->ie_lock);
349 	ie->ie_cpu = cpu;
350 	mtx_unlock(&ie->ie_lock);
358 	struct intr_event *ie;
361 	TAILQ_FOREACH(ie, &event_list, ie_list)
362 		if (ie->ie_irq == irq &&
363 		    (ie->ie_flags & IE_SOFT) == 0 &&
364 		    TAILQ_FIRST(&ie->ie_handlers) != NULL)
367 	return (ie);
373 	struct intr_event *ie;
393 	ie = intr_lookup(irq);
394 	if (ie == NULL)
396 	return (intr_event_bind(ie, cpu));
402 	struct intr_event *ie;
406 	ie = intr_lookup(irq);
407 	if (ie == NULL)
410 	mtx_lock(&ie->ie_lock);
411 	if (ie->ie_cpu == NOCPU)
414 		CPU_SET(ie->ie_cpu, mask);
415 	mtx_unlock(&ie->ie_lock);
420 intr_event_destroy(struct intr_event *ie)
424 	mtx_lock(&ie->ie_lock);
425 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
426 		mtx_unlock(&ie->ie_lock);
430 	TAILQ_REMOVE(&event_list, ie, ie_list);
432 	if (ie->ie_thread != NULL) {
433 		ithread_destroy(ie->ie_thread);
434 		ie->ie_thread = NULL;
437 	mtx_unlock(&ie->ie_lock);
439 	mtx_destroy(&ie->ie_lock);
440 	free(ie, M_ITHREAD);
512 intr_event_add_handler(struct intr_event *ie, const char *name,
519 	if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
528 	ih->ih_event = ie;
538 	mtx_lock(&ie->ie_lock);
539 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
541 		    (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
542 			mtx_unlock(&ie->ie_lock);
549 	while (ie->ie_thread == NULL && handler != NULL) {
550 		if (ie->ie_flags & IE_ADDING_THREAD)
551 			msleep(ie, &ie->ie_lock, 0, "ithread", 0);
553 			ie->ie_flags |= IE_ADDING_THREAD;
554 			mtx_unlock(&ie->ie_lock);
556 			mtx_lock(&ie->ie_lock);
557 			ie->ie_flags &= ~IE_ADDING_THREAD;
558 			ie->ie_thread = it;
559 			it->it_event = ie;
561 			wakeup(ie);
566 	TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
571 		TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
574 	intr_event_update(ie);
577 	    ie->ie_name);
578 	mtx_unlock(&ie->ie_lock);
586 intr_event_add_handler(struct intr_event *ie, const char *name,
593 	if (ie == NULL || name == NULL || (handler == NULL && filter == NULL))
602 	ih->ih_event = ie;
612 	mtx_lock(&ie->ie_lock);
613 	if (!TAILQ_EMPTY(&ie->ie_handlers)) {
615 		    (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) {
616 			mtx_unlock(&ie->ie_lock);
624 		mtx_unlock(&ie->ie_lock);
626 		mtx_lock(&ie->ie_lock);
627 		it->it_event = ie;
631 		while (ie->ie_thread == NULL && handler != NULL) {
632 			if (ie->ie_flags & IE_ADDING_THREAD)
633 				msleep(ie, &ie->ie_lock, 0, "ithread", 0);
635 				ie->ie_flags |= IE_ADDING_THREAD;
636 				mtx_unlock(&ie->ie_lock);
638 				mtx_lock(&ie->ie_lock);
639 				ie->ie_flags &= ~IE_ADDING_THREAD;
640 				ie->ie_thread = it;
641 				it->it_event = ie;
643 				wakeup(ie);
649 	TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) {
654 		TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next);
657 	intr_event_update(ie);
660 	    ie->ie_name);
661 	mtx_unlock(&ie->ie_lock);
674 intr_event_describe_handler(struct intr_event *ie, void *cookie,
681 	mtx_lock(&ie->ie_lock);
683 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
688 		mtx_unlock(&ie->ie_lock);
689 		panic("handler %p not found in interrupt event %p", cookie, ie);
712 		mtx_unlock(&ie->ie_lock);
719 	intr_event_update(ie);
720 	mtx_unlock(&ie->ie_lock);
732 	struct intr_event *ie;
737 	ie = ih->ih_event;
738 	KASSERT(ie != NULL,
741 	return (ie->ie_source);
754 	struct intr_event *ie;
758 	ie = intr_lookup(irq);
759 	if (ie == NULL)
761 	if (ie->ie_thread == NULL)
763 	ithd = ie->ie_thread;
789 	struct intr_event *ie;
799 	ie = handler->ih_event;
800 	KASSERT(ie != NULL,
803 	mtx_lock(&ie->ie_lock);
805 	    ie->ie_name);
807 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
810 	mtx_unlock(&ie->ie_lock);
812 	    ih->ih_name, ie->ie_name);
820 	if (ie->ie_thread == NULL) {
821 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
822 		mtx_unlock(&ie->ie_lock);
835 	thread_lock(ie->ie_thread->it_thread);
836 	if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) {
844 		atomic_store_rel_int(&ie->ie_thread->it_need, 1);
846 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
847 	thread_unlock(ie->ie_thread->it_thread);
849 		msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
850 	intr_event_update(ie);
858 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
865 		ithread_destroy(ie->ie_thread);
866 		ie->ie_thread = NULL;
869 	mtx_unlock(&ie->ie_lock);
875 intr_event_schedule_thread(struct intr_event *ie)
886 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) ||
887 	    ie->ie_thread == NULL)
891 	it = ie->ie_thread;
899 	if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
902 		entropy.event = (uintptr_t)ie;
908 	KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
935 	struct intr_event *ie;
946 	ie = handler->ih_event;
947 	KASSERT(ie != NULL,
950 	mtx_lock(&ie->ie_lock);
952 	    ie->ie_name);
954 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
957 	mtx_unlock(&ie->ie_lock);
959 	    ih->ih_name, ie->ie_name);
967 	if (ie->ie_thread == NULL && handler->ih_thread == NULL) {
968 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
969 		mtx_unlock(&ie->ie_lock);
975 	it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread;
995 		TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next);
998 		msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0);
1007 	intr_event_update(ie);
1015 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1022 		ithread_destroy(ie->ie_thread);
1023 		ie->ie_thread = NULL;
1026 	mtx_unlock(&ie->ie_lock);
1032 intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it)
1042 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL)
1053 	if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) {
1056 		entropy.event = (uintptr_t)ie;
1062 	KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name));
1106 	struct intr_event *ie;
1112 	ie = (eventp != NULL) ? *eventp : NULL;
1114 	if (ie != NULL) {
1115 		if (!(ie->ie_flags & IE_SOFT))
1118 		error = intr_event_create(&ie, NULL, IE_SOFT, 0,
1123 			*eventp = ie;
1125 	error = intr_event_add_handler(ie, name, NULL, handler, arg,
1137 	struct intr_event *ie = ih->ih_event;
1141 	CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
1163 		error = intr_event_schedule_thread(ie, ie->ie_thread);
1165 		error = intr_event_schedule_thread(ie);
1188 	struct intr_event *ie;
1190 	ie = ih->ih_event;
1196 		mtx_lock(&ie->ie_lock);
1197 		TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1200 		mtx_unlock(&ie->ie_lock);
1222 intr_event_execute_handlers(struct proc *p, struct intr_event *ie)
1226 	TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) {
1232 			mtx_lock(&ie->ie_lock);
1233 			TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next);
1236 			mtx_unlock(&ie->ie_lock);
1249 		if (ie->ie_flags & IE_SOFT) {
1270 ithread_execute_handlers(struct proc *p, struct intr_event *ie)
1274 	if (!(ie->ie_flags & IE_SOFT))
1276 	intr_event_execute_handlers(p, ie);
1277 	if (!(ie->ie_flags & IE_SOFT))
1290 	if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold &&
1291 	    !(ie->ie_flags & IE_SOFT)) {
1293 		if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) {
1296 			    ie->ie_name);
1300 		ie->ie_count++;
1306 	if (ie->ie_post_ithread != NULL)
1307 		ie->ie_post_ithread(ie->ie_source);
1318 	struct intr_event *ie;
1328 	ie = ithd->it_event;
1329 	ie->ie_count = 0;
1360 			ithread_execute_handlers(p, ie);
1374 			ie->ie_count = 0;
1393  * o ie:                        the event connected to this interrupt.
1401 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
1411 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
1426 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1466 		if (ie->ie_pre_ithread != NULL)
1467 			ie->ie_pre_ithread(ie->ie_source);
1469 		if (ie->ie_post_filter != NULL)
1470 			ie->ie_post_filter(ie->ie_source);
1475 		error = intr_event_schedule_thread(ie);
1496 	struct intr_event *ie;
1509 	ie = ithd->it_event;
1510 	ie->ie_count = 0;
1544 				ithread_execute_handlers(p, ie);
1558 			ie->ie_count = 0;
1597 intr_filter_loop(struct intr_event *ie, struct trapframe *frame,
1606 	TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
1643 		*ithd = ie->ie_thread;
1653  * o ie:                        the event connected to this interrupt.
1661 intr_event_handle(struct intr_event *ie, struct trapframe *frame)
1671 	if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers))
1679 	thread = intr_filter_loop(ie, frame, &ithd);
1681 		if (ie->ie_post_filter != NULL)
1682 			ie->ie_post_filter(ie->ie_source);
1684 		if (ie->ie_pre_ithread != NULL)
1685 			ie->ie_pre_ithread(ie->ie_source);
1692 		ie->ie_count++;
1693 		if (ie->ie_count < intr_storm_threshold)
1699 		if (intr_event_schedule_thread(ie, ithd) != 0)
1798 db_dump_intr_event(struct intr_event *ie, int handlers)
1804 	db_printf("%s ", ie->ie_fullname);
1805 	it = ie->ie_thread;
1810 	if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 ||
1814 		if (ie->ie_flags & IE_SOFT) {
1818 		if (ie->ie_flags & IE_ENTROPY) {
1824 		if (ie->ie_flags & IE_ADDING_THREAD) {
1840 		TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next)
1849 	struct intr_event *ie;
1854 	TAILQ_FOREACH(ie, &event_list, ie_list) {
1855 		if (!all && TAILQ_EMPTY(&ie->ie_handlers))
1857 		db_dump_intr_event(ie, verbose);