1/* $NetBSD: pic.c,v 1.85 2022/10/30 10:20:45 riastradh Exp $ */ 2 3/*- 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#define _INTR_PRIVATE 33#include "opt_ddb.h" 34#include "opt_multiprocessor.h" 35 36#include <sys/cdefs.h> 37__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.85 2022/10/30 10:20:45 riastradh Exp $"); 38 39#include <sys/param.h> 40#include <sys/atomic.h> 41#include <sys/cpu.h> 42#include <sys/evcnt.h> 43#include <sys/interrupt.h> 44#include <sys/intr.h> 45#include <sys/ipi.h> 46#include <sys/kernel.h> 47#include <sys/kmem.h> 48#include <sys/mutex.h> 49#include <sys/once.h> 50#include <sys/sdt.h> 51#include <sys/xcall.h> 52 53#include <arm/armreg.h> 54#include <arm/cpufunc.h> 55#include <arm/locore.h> /* for compat aarch64 */ 56 57#ifdef DDB 58#include <arm/db_machdep.h> 59#endif 60 61#include <arm/pic/picvar.h> 62 63#if defined(__HAVE_PIC_PENDING_INTRS) 64/* 65 * This implementation of pending interrupts on a MULTIPROCESSOR system makes 66 * the assumption that a PIC (pic_softc) shall only have all its interrupts 67 * come from the same CPU. In other words, interrupts from a single PIC will 68 * not be distributed among multiple CPUs. 69 */ 70static uint32_t 71 pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int); 72static struct pic_softc * 73 pic_list_find_pic_by_pending_ipl(struct cpu_info *, uint32_t); 74static void 75 pic_deliver_irqs(struct cpu_info *, struct pic_softc *, int, void *); 76static void 77 pic_list_deliver_irqs(struct cpu_info *, register_t, int, void *); 78 79#endif /* __HAVE_PIC_PENDING_INTRS */ 80 81struct pic_softc *pic_list[PIC_MAXPICS]; 82#if PIC_MAXPICS > 32 83#error PIC_MAXPICS > 32 not supported 84#endif 85struct intrsource *pic_sources[PIC_MAXMAXSOURCES]; 86struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES]; 87size_t pic_ipl_offset[NIPL + 1]; 88 89static kmutex_t pic_lock; 90static size_t pic_sourcebase; 91static int pic_lastbase; 92static struct evcnt pic_deferral_ev = 93 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "deferred", "intr"); 94EVCNT_ATTACH_STATIC(pic_deferral_ev); 95 96static int pic_init(void); 97 98SDT_PROBE_DEFINE3(sdt, kernel, intr, entry, 99 "void (*)(void *)"/*func*/, 100 "void *"/*arg*/, 101 "struct intrsource *"/*is*/); 102SDT_PROBE_DEFINE4(sdt, kernel, intr, return, 103 "void (*)(void *)"/*func*/, 104 "void *"/*arg*/, 105 "struct intrsource *"/*is*/, 106 "int"/*handled*/); 107 108#ifdef __HAVE_PIC_SET_PRIORITY 109void 110pic_set_priority(struct cpu_info *ci, int newipl) 111{ 112 if (__predict_false(pic_list[0] == NULL)) { 113 ci->ci_cpl = newipl; 114 return; 115 } 116 117 pic_list[0]->pic_ops->pic_set_priority(pic_list[0], newipl); 118} 119#endif 120 121#ifdef MULTIPROCESSOR 122int 123pic_ipi_ast(void *arg) 124{ 125 setsoftast(curcpu()); 126 return 1; 127} 128 129int 130pic_ipi_nop(void *arg) 131{ 132 /* do nothing */ 133 return 1; 134} 135 136int 137pic_ipi_xcall(void *arg) 138{ 139 xc_ipi_handler(); 140 return 1; 141} 142 143int 144pic_ipi_generic(void *arg) 145{ 146 ipi_cpu_handler(); 147 return 1; 148} 149 150#ifdef DDB 151int 152pic_ipi_ddb(void *arg) 153{ 154// printf("%s: %s: tf=%p\n", __func__, curcpu()->ci_cpuname, arg); 155 kdb_trap(-1, arg); 156 return 1; 157} 158#endif /* DDB */ 159 160#ifdef __HAVE_PREEMPTION 161int 162pic_ipi_kpreempt(void *arg) 163{ 164 struct lwp * const l = curlwp; 165 166 l->l_md.md_astpending |= __BIT(1); 167 return 1; 168} 169#endif /* __HAVE_PREEMPTION */ 170 171void 172intr_cpu_init(struct cpu_info *ci) 173{ 174 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 175 struct pic_softc * const pic = pic_list[slot]; 176 if (pic != NULL && pic->pic_ops->pic_cpu_init != NULL) { 177 (*pic->pic_ops->pic_cpu_init)(pic, ci); 178 } 179 } 180} 181 182typedef void (*pic_ipi_send_func_t)(struct pic_softc *, u_long); 183 184void 185intr_ipi_send(const kcpuset_t *kcp, u_long ipi) 186{ 187 struct cpu_info * const ci = curcpu(); 188 KASSERT(ipi < NIPI); 189 KASSERT(kcp == NULL || kcpuset_countset(kcp) == 1); 190 bool __diagused sent_p = false; 191 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 192 struct pic_softc * const pic = pic_list[slot]; 193 if (pic == NULL || pic->pic_cpus == NULL) 194 continue; 195 if (kcp == NULL || kcpuset_intersecting_p(kcp, pic->pic_cpus)) { 196 /* 197 * Never send to ourself. 198 * 199 * This test uses pointer comparison for systems 200 * that have a pic per cpu, e.g. RPI[23]. GIC sets 201 * pic_cpus to kcpuset_running and handles "not for 202 * self" internally. 203 */ 204 if (pic->pic_cpus == ci->ci_kcpuset) 205 continue; 206 207 (*pic->pic_ops->pic_ipi_send)(pic, kcp, ipi); 208 209 /* 210 * If we were targeting a single CPU or this pic 211 * handles all cpus, we're done. 212 */ 213 if (kcp != NULL || pic->pic_cpus == kcpuset_running) 214 return; 215 sent_p = true; 216 } 217 } 218 KASSERTMSG(cold || sent_p || ncpu <= 1, "cold %d sent_p %d ncpu %d", 219 cold, sent_p, ncpu); 220} 221#endif /* MULTIPROCESSOR */ 222 223#ifdef __HAVE_PIC_FAST_SOFTINTS 224int 225pic_handle_softint(void *arg) 226{ 227 void softint_switch(lwp_t *, int); 228 struct cpu_info * const ci = curcpu(); 229 const size_t softint = (size_t) arg; 230 int s = splhigh(); 231 ci->ci_intr_depth--; // don't count these as interrupts 232 softint_switch(ci->ci_softlwps[softint], s); 233 ci->ci_intr_depth++; 234 splx(s); 235 return 1; 236} 237#endif 238 239int 240pic_handle_intr(void *arg) 241{ 242 struct pic_softc * const pic = arg; 243 int rv; 244 245 rv = (*pic->pic_ops->pic_find_pending_irqs)(pic); 246 247 return rv > 0; 248} 249 250#if defined(__HAVE_PIC_PENDING_INTRS) 251void 252pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is) 253{ 254 const uint32_t ipl_mask = __BIT(is->is_ipl); 255 struct cpu_info * const ci = curcpu(); 256 257 atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5], 258 __BIT(is->is_irq & 0x1f)); 259 260 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 261 ci->ci_pending_ipls |= ipl_mask; 262 ci->ci_pending_pics |= __BIT(pic->pic_id); 263} 264 265void 266pic_mark_pending(struct pic_softc *pic, int irq) 267{ 268 struct intrsource * const is = pic->pic_sources[irq]; 269 270 KASSERT(irq < pic->pic_maxsources); 271 KASSERT(is != NULL); 272 273 pic_mark_pending_source(pic, is); 274} 275 276uint32_t 277pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base, 278 uint32_t pending) 279{ 280 struct intrsource ** const isbase = &pic->pic_sources[irq_base]; 281 struct cpu_info * const ci = curcpu(); 282 struct intrsource *is; 283 volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5]; 284 uint32_t ipl_mask = 0; 285 286 if (pending == 0) 287 return ipl_mask; 288 289 KASSERT((irq_base & 31) == 0); 290 291 (*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending); 292 293 atomic_or_32(ipending, pending); 294 while (pending != 0) { 295 int n = ffs(pending); 296 if (n-- == 0) 297 break; 298 is = isbase[n]; 299 KASSERT(is != NULL); 300 KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32); 301 pending &= ~__BIT(n); 302 ipl_mask |= __BIT(is->is_ipl); 303 } 304 305 atomic_or_32(&pic->pic_pending_ipls, ipl_mask); 306 ci->ci_pending_ipls |= ipl_mask; 307 ci->ci_pending_pics |= __BIT(pic->pic_id); 308 309 return ipl_mask; 310} 311 312static uint32_t 313pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base, 314 uint32_t pending, int ipl) 315{ 316 uint32_t ipl_irq_mask = 0; 317 uint32_t irq_mask; 318 319 for (;;) { 320 int irq = ffs(pending); 321 if (irq-- == 0) 322 return ipl_irq_mask; 323 324 irq_mask = __BIT(irq); 325#if 1 326 KASSERTMSG(pic->pic_sources[irq_base + irq] != NULL, 327 "%s: irq_base %zu irq %d\n", __func__, irq_base, irq); 328#else 329 if (pic->pic_sources[irq_base + irq] == NULL) { 330 aprint_error("stray interrupt? irq_base=%zu irq=%d\n", 331 irq_base, irq); 332 } else 333#endif 334 if (pic->pic_sources[irq_base + irq]->is_ipl == ipl) 335 ipl_irq_mask |= irq_mask; 336 337 pending &= ~irq_mask; 338 } 339} 340#endif /* __HAVE_PIC_PENDING_INTRS */ 341 342void 343pic_dispatch(struct intrsource *is, void *frame) 344{ 345 int (*func)(void *) = is->is_func; 346 void *arg = is->is_arg; 347 int ocpl, ncpl, handled __unused; 348 349 if (__predict_false(arg == NULL)) { 350 if (__predict_false(frame == NULL)) { 351 pic_deferral_ev.ev_count++; 352 return; 353 } 354 arg = frame; 355 } 356 357 ocpl = curcpu()->ci_cpl; 358#ifdef MULTIPROCESSOR 359 const bool mpsafe = is->is_mpsafe; 360#else 361 const bool mpsafe = true; 362#endif 363 if (!mpsafe) { 364 KERNEL_LOCK(1, NULL); 365 const u_int ci_blcnt __diagused = curcpu()->ci_biglock_count; 366 const u_int l_blcnt __diagused = curlwp->l_blcnt; 367 SDT_PROBE3(sdt, kernel, intr, entry, func, arg, is); 368 handled = (*func)(arg); 369 SDT_PROBE4(sdt, kernel, intr, return, func, arg, is, handled); 370 KASSERT(ci_blcnt == curcpu()->ci_biglock_count); 371 KASSERT(l_blcnt == curlwp->l_blcnt); 372 KERNEL_UNLOCK_ONE(NULL); 373 } else { 374 SDT_PROBE3(sdt, kernel, intr, entry, func, arg, is); 375 handled = (*func)(arg); 376 SDT_PROBE4(sdt, kernel, intr, return, func, arg, is, handled); 377 } 378 ncpl = curcpu()->ci_cpl; 379 KASSERTMSG(ocpl <= ncpl, "pic %s irq %u intrsource %s:" 380 " cpl slipped %d -> %d", 381 is->is_pic->pic_name, is->is_irq, is->is_source, 382 ocpl, ncpl); 383 384 struct pic_percpu * const pcpu = percpu_getref(is->is_pic->pic_percpu); 385 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 386 pcpu->pcpu_evs[is->is_irq].ev_count++; 387 percpu_putref(is->is_pic->pic_percpu); 388} 389 390#if defined(__HAVE_PIC_PENDING_INTRS) 391static void 392pic_deliver_irqs(struct cpu_info *ci, struct pic_softc *pic, int ipl, 393 void *frame) 394{ 395 const uint32_t ipl_mask = __BIT(ipl); 396 struct intrsource *is; 397 volatile uint32_t *ipending = pic->pic_pending_irqs; 398 volatile uint32_t *iblocked = pic->pic_blocked_irqs; 399 size_t irq_base; 400#if PIC_MAXSOURCES > 32 401 size_t irq_count; 402 int poi = 0; /* Possibility of interrupting */ 403#endif 404 uint32_t pending_irqs; 405 uint32_t blocked_irqs; 406 int irq; 407 bool progress __diagused = false; 408 409 KASSERT(pic->pic_pending_ipls & ipl_mask); 410 411 irq_base = 0; 412#if PIC_MAXSOURCES > 32 413 irq_count = 0; 414#endif 415 416 for (;;) { 417 pending_irqs = pic_find_pending_irqs_by_ipl(pic, irq_base, 418 *ipending, ipl); 419 KASSERT((pending_irqs & *ipending) == pending_irqs); 420 KASSERT((pending_irqs & ~(*ipending)) == 0); 421 if (pending_irqs == 0) { 422#if PIC_MAXSOURCES > 32 423 irq_count += 32; 424 if (__predict_true(irq_count >= pic->pic_maxsources)) { 425 if (!poi) 426 /*Interrupt at this level was handled.*/ 427 break; 428 irq_base = 0; 429 irq_count = 0; 430 poi = 0; 431 ipending = pic->pic_pending_irqs; 432 iblocked = pic->pic_blocked_irqs; 433 } else { 434 irq_base += 32; 435 ipending++; 436 iblocked++; 437 KASSERT(irq_base <= pic->pic_maxsources); 438 } 439 continue; 440#else 441 break; 442#endif 443 } 444 progress = true; 445 blocked_irqs = 0; 446 do { 447 irq = ffs(pending_irqs) - 1; 448 KASSERT(irq >= 0); 449 450 atomic_and_32(ipending, ~__BIT(irq)); 451 is = pic->pic_sources[irq_base + irq]; 452 if (is != NULL) { 453 ENABLE_INTERRUPT(); 454 pic_dispatch(is, frame); 455 DISABLE_INTERRUPT(); 456#if PIC_MAXSOURCES > 32 457 /* 458 * There is a possibility of interrupting 459 * from ENABLE_INTERRUPT() to 460 * DISABLE_INTERRUPT(). 461 */ 462 poi = 1; 463#endif 464 blocked_irqs |= __BIT(irq); 465 } else { 466 KASSERT(0); 467 } 468 pending_irqs = pic_find_pending_irqs_by_ipl(pic, 469 irq_base, *ipending, ipl); 470 } while (pending_irqs); 471 if (blocked_irqs) { 472 atomic_or_32(iblocked, blocked_irqs); 473 ci->ci_blocked_pics |= __BIT(pic->pic_id); 474 } 475 } 476 477 KASSERT(progress); 478 /* 479 * Since interrupts are disabled, we don't have to be too careful 480 * about these. 481 */ 482 if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0) 483 ci->ci_pending_pics &= ~__BIT(pic->pic_id); 484} 485 486static void 487pic_list_unblock_irqs(struct cpu_info *ci) 488{ 489 uint32_t blocked_pics = ci->ci_blocked_pics; 490 491 ci->ci_blocked_pics = 0; 492 493 for (;;) { 494 struct pic_softc *pic; 495#if PIC_MAXSOURCES > 32 496 volatile uint32_t *iblocked; 497 uint32_t blocked; 498 size_t irq_base; 499#endif 500 501 int pic_id = ffs(blocked_pics); 502 if (pic_id-- == 0) 503 return; 504 505 pic = pic_list[pic_id]; 506 KASSERT(pic != NULL); 507#if PIC_MAXSOURCES > 32 508 for (irq_base = 0, iblocked = pic->pic_blocked_irqs; 509 irq_base < pic->pic_maxsources; 510 irq_base += 32, iblocked++) { 511 if ((blocked = *iblocked) != 0) { 512 (*pic->pic_ops->pic_unblock_irqs)(pic, 513 irq_base, blocked); 514 atomic_and_32(iblocked, ~blocked); 515 } 516 } 517#else 518 KASSERT(pic->pic_blocked_irqs[0] != 0); 519 (*pic->pic_ops->pic_unblock_irqs)(pic, 520 0, pic->pic_blocked_irqs[0]); 521 pic->pic_blocked_irqs[0] = 0; 522#endif 523 blocked_pics &= ~__BIT(pic_id); 524 } 525} 526 527static struct pic_softc * 528pic_list_find_pic_by_pending_ipl(struct cpu_info *ci, uint32_t ipl_mask) 529{ 530 uint32_t pending_pics = ci->ci_pending_pics; 531 struct pic_softc *pic; 532 533 for (;;) { 534 int pic_id = ffs(pending_pics); 535 if (pic_id-- == 0) 536 return NULL; 537 538 pic = pic_list[pic_id]; 539 KASSERT(pic != NULL); 540 if (pic->pic_pending_ipls & ipl_mask) 541 return pic; 542 pending_pics &= ~__BIT(pic_id); 543 } 544} 545 546static void 547pic_list_deliver_irqs(struct cpu_info *ci, register_t psw, int ipl, 548 void *frame) 549{ 550 const uint32_t ipl_mask = __BIT(ipl); 551 struct pic_softc *pic; 552 553 while ((pic = pic_list_find_pic_by_pending_ipl(ci, ipl_mask)) != NULL) { 554 pic_deliver_irqs(ci, pic, ipl, frame); 555 KASSERT((pic->pic_pending_ipls & ipl_mask) == 0); 556 } 557 ci->ci_pending_ipls &= ~ipl_mask; 558} 559#endif /* __HAVE_PIC_PENDING_INTRS */ 560 561void 562pic_do_pending_ints(register_t psw, int newipl, void *frame) 563{ 564 struct cpu_info * const ci = curcpu(); 565 if (__predict_false(newipl == IPL_HIGH)) { 566 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "cpl %d", ci->ci_cpl); 567 return; 568 } 569#if defined(__HAVE_PIC_PENDING_INTRS) 570 while ((ci->ci_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) { 571 KASSERT(ci->ci_pending_ipls < __BIT(NIPL)); 572 for (;;) { 573 int ipl = 31 - __builtin_clz(ci->ci_pending_ipls); 574 KASSERT(ipl < NIPL); 575 if (ipl <= newipl) 576 break; 577 578 pic_set_priority(ci, ipl); 579 pic_list_deliver_irqs(ci, psw, ipl, frame); 580 pic_list_unblock_irqs(ci); 581 } 582 } 583#endif /* __HAVE_PIC_PENDING_INTRS */ 584#ifdef __HAVE_PREEMPTION 585 struct lwp * const l = curlwp; 586 if (newipl == IPL_NONE && (l->l_md.md_astpending & __BIT(1))) { 587 pic_set_priority(ci, IPL_SCHED); 588 kpreempt(0); 589 } 590#endif 591 if (ci->ci_cpl != newipl) 592 pic_set_priority(ci, newipl); 593} 594 595static void 596pic_percpu_allocate(void *v0, void *v1, struct cpu_info *ci) 597{ 598 struct pic_percpu * const pcpu = v0; 599 struct pic_softc * const pic = v1; 600 601 pcpu->pcpu_evs = kmem_zalloc(pic->pic_maxsources * sizeof(pcpu->pcpu_evs[0]), 602 KM_SLEEP); 603 KASSERT(pcpu->pcpu_evs != NULL); 604 605#define PCPU_NAMELEN 32 606 const size_t namelen = strlen(pic->pic_name) + 4 + strlen(ci->ci_data.cpu_name); 607 608 KASSERT(namelen < PCPU_NAMELEN); 609 pcpu->pcpu_name = kmem_alloc(PCPU_NAMELEN, KM_SLEEP); 610#ifdef MULTIPROCESSOR 611 snprintf(pcpu->pcpu_name, PCPU_NAMELEN, 612 "%s (%s)", pic->pic_name, ci->ci_data.cpu_name); 613#else 614 strlcpy(pcpu->pcpu_name, pic->pic_name, PCPU_NAMELEN); 615#endif 616 pcpu->pcpu_magic = PICPERCPU_MAGIC; 617#if 0 618 printf("%s: %s %s: <%s>\n", 619 __func__, ci->ci_data.cpu_name, pic->pic_name, 620 pcpu->pcpu_name); 621#endif 622} 623 624static int 625pic_init(void) 626{ 627 628 mutex_init(&pic_lock, MUTEX_DEFAULT, IPL_HIGH); 629 630 return 0; 631} 632 633int 634pic_add(struct pic_softc *pic, int irqbase) 635{ 636 int slot, maybe_slot = -1; 637 size_t sourcebase; 638 static ONCE_DECL(pic_once); 639 640 ASSERT_SLEEPABLE(); 641 642 RUN_ONCE(&pic_once, pic_init); 643 644 KASSERT(strlen(pic->pic_name) > 0); 645 646 mutex_enter(&pic_lock); 647 if (irqbase == PIC_IRQBASE_ALLOC) { 648 irqbase = pic_lastbase; 649 } 650 for (slot = 0; slot < PIC_MAXPICS; slot++) { 651 struct pic_softc * const xpic = pic_list[slot]; 652 if (xpic == NULL) { 653 if (maybe_slot < 0) 654 maybe_slot = slot; 655 if (irqbase < 0) 656 break; 657 continue; 658 } 659 if (irqbase < 0 || xpic->pic_irqbase < 0) 660 continue; 661 if (irqbase >= xpic->pic_irqbase + xpic->pic_maxsources) 662 continue; 663 if (irqbase + pic->pic_maxsources <= xpic->pic_irqbase) 664 continue; 665 panic("pic_add: pic %s (%zu sources @ irq %u) conflicts" 666 " with pic %s (%zu sources @ irq %u)", 667 pic->pic_name, pic->pic_maxsources, irqbase, 668 xpic->pic_name, xpic->pic_maxsources, xpic->pic_irqbase); 669 } 670 slot = maybe_slot; 671#if 0 672 printf("%s: pic_sourcebase=%zu pic_maxsources=%zu\n", 673 pic->pic_name, pic_sourcebase, pic->pic_maxsources); 674#endif 675 KASSERTMSG(pic->pic_maxsources <= PIC_MAXSOURCES, "%zu", 676 pic->pic_maxsources); 677 KASSERT(pic_sourcebase + pic->pic_maxsources <= PIC_MAXMAXSOURCES); 678 sourcebase = pic_sourcebase; 679 pic_sourcebase += pic->pic_maxsources; 680 if (pic_lastbase < irqbase + pic->pic_maxsources) 681 pic_lastbase = irqbase + pic->pic_maxsources; 682 mutex_exit(&pic_lock); 683 684 /* 685 * Allocate a pointer to each cpu's evcnts and then, for each cpu, 686 * allocate its evcnts and then attach an evcnt for each pin. 687 * We can't allocate the evcnt structures directly since 688 * percpu will move the contents of percpu memory around and 689 * corrupt the pointers in the evcnts themselves. Remember, any 690 * problem can be solved with sufficient indirection. 691 */ 692 pic->pic_percpu = percpu_create(sizeof(struct pic_percpu), 693 pic_percpu_allocate, NULL, pic); 694 695 pic->pic_sources = &pic_sources[sourcebase]; 696 pic->pic_irqbase = irqbase; 697 pic->pic_id = slot; 698#ifdef __HAVE_PIC_SET_PRIORITY 699 KASSERT((slot == 0) == (pic->pic_ops->pic_set_priority != NULL)); 700#endif 701#ifdef MULTIPROCESSOR 702 KASSERT((pic->pic_cpus != NULL) == (pic->pic_ops->pic_ipi_send != NULL)); 703#endif 704 pic_list[slot] = pic; 705 706 return irqbase; 707} 708 709int 710pic_alloc_irq(struct pic_softc *pic) 711{ 712 int irq; 713 714 for (irq = 0; irq < pic->pic_maxsources; irq++) { 715 if (pic->pic_sources[irq] == NULL) 716 return irq; 717 } 718 719 return -1; 720} 721 722static void 723pic_percpu_evcnt_attach(void *v0, void *v1, struct cpu_info *ci) 724{ 725 struct pic_percpu * const pcpu = v0; 726 struct intrsource * const is = v1; 727 728 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 729 evcnt_attach_dynamic(&pcpu->pcpu_evs[is->is_irq], EVCNT_TYPE_INTR, NULL, 730 pcpu->pcpu_name, is->is_source); 731} 732 733static void 734pic_unblock_percpu(void *arg1, void *arg2) 735{ 736 struct pic_softc *pic = arg1; 737 struct intrsource *is = arg2; 738 739 (*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f, 740 __BIT(is->is_irq & 0x1f)); 741} 742 743void * 744pic_establish_intr(struct pic_softc *pic, int irq, int ipl, int type, 745 int (*func)(void *), void *arg, const char *xname) 746{ 747 struct intrsource *is; 748 int off, nipl; 749 750 if (pic->pic_sources[irq]) { 751 printf("pic_establish_intr: pic %s irq %d already present\n", 752 pic->pic_name, irq); 753 return NULL; 754 } 755 756 is = kmem_zalloc(sizeof(*is), KM_SLEEP); 757 is->is_pic = pic; 758 is->is_irq = irq; 759 is->is_ipl = ipl; 760 is->is_type = type & 0xff; 761 is->is_func = func; 762 is->is_arg = arg; 763#ifdef MULTIPROCESSOR 764 is->is_mpsafe = (type & IST_MPSAFE) || ipl != IPL_VM; 765#endif 766 767 if (pic->pic_ops->pic_source_name) 768 (*pic->pic_ops->pic_source_name)(pic, irq, is->is_source, 769 sizeof(is->is_source)); 770 else 771 snprintf(is->is_source, sizeof(is->is_source), "irq %d", irq); 772 773 /* 774 * Now attach the per-cpu evcnts. 775 */ 776 percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_attach, is); 777 778 pic->pic_sources[irq] = is; 779 780 /* 781 * First try to use an existing slot which is empty. 782 */ 783 bool found = false; 784 for (off = pic_ipl_offset[ipl]; off < pic_ipl_offset[ipl + 1]; off++) { 785 if (pic__iplsources[off] == NULL) { 786 found = true; 787 break; 788 } 789 } 790 791 if (!found) { 792 /* 793 * Move up all the sources by one. 794 */ 795 if (ipl < NIPL) { 796 off = pic_ipl_offset[ipl + 1]; 797 memmove(&pic__iplsources[off + 1], &pic__iplsources[off], 798 sizeof(pic__iplsources[0]) * (pic_ipl_offset[NIPL] - off)); 799 } 800 801 /* 802 * Advance the offset of all IPLs higher than this. Include an 803 * extra one as well. Thus the number of sources per ipl is 804 * pic_ipl_offset[ipl + 1] - pic_ipl_offset[ipl]. 805 */ 806 for (nipl = ipl + 1; nipl <= NIPL; nipl++) 807 pic_ipl_offset[nipl]++; 808 809 off = pic_ipl_offset[ipl + 1] - 1; 810 } 811 812 /* 813 * Insert into the 'found' or the just made slot position at the end 814 * of this IPL's sources. 815 */ 816 is->is_iplidx = off - pic_ipl_offset[ipl]; 817 pic__iplsources[off] = is; 818 819 (*pic->pic_ops->pic_establish_irq)(pic, is); 820 821 if (!mp_online || !is->is_mpsafe || !is->is_percpu) { 822 (*pic->pic_ops->pic_unblock_irqs)(pic, is->is_irq & ~0x1f, 823 __BIT(is->is_irq & 0x1f)); 824 } else { 825 uint64_t xc = xc_broadcast(0, pic_unblock_percpu, pic, is); 826 xc_wait(xc); 827 } 828 829 if (xname) { 830 if (is->is_xname == NULL) 831 is->is_xname = kmem_zalloc(INTRDEVNAMEBUF, KM_SLEEP); 832 if (is->is_xname[0] != '\0') 833 strlcat(is->is_xname, ", ", INTRDEVNAMEBUF); 834 strlcat(is->is_xname, xname, INTRDEVNAMEBUF); 835 } 836 837 /* We're done. */ 838 return is; 839} 840 841static void 842pic_percpu_evcnt_deattach(void *v0, void *v1, struct cpu_info *ci) 843{ 844 struct pic_percpu * const pcpu = v0; 845 struct intrsource * const is = v1; 846 847 KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC); 848 evcnt_detach(&pcpu->pcpu_evs[is->is_irq]); 849} 850 851void 852pic_disestablish_source(struct intrsource *is) 853{ 854 struct pic_softc * const pic = is->is_pic; 855 const int irq = is->is_irq; 856 857 KASSERT(is == pic->pic_sources[irq]); 858 859 (*pic->pic_ops->pic_block_irqs)(pic, irq & ~0x1f, __BIT(irq & 0x1f)); 860 pic->pic_sources[irq] = NULL; 861 pic__iplsources[pic_ipl_offset[is->is_ipl] + is->is_iplidx] = NULL; 862 if (is->is_xname != NULL) { 863 kmem_free(is->is_xname, INTRDEVNAMEBUF); 864 is->is_xname = NULL; 865 } 866 /* 867 * Now detach the per-cpu evcnts. 868 */ 869 percpu_foreach(pic->pic_percpu, pic_percpu_evcnt_deattach, is); 870 871 kmem_free(is, sizeof(*is)); 872} 873 874void * 875intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg) 876{ 877 return intr_establish_xname(irq, ipl, type, func, arg, NULL); 878} 879 880void * 881intr_establish_xname(int irq, int ipl, int type, int (*func)(void *), void *arg, 882 const char *xname) 883{ 884 KASSERT(!cpu_intr_p()); 885 KASSERT(!cpu_softintr_p()); 886 887 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 888 struct pic_softc * const pic = pic_list[slot]; 889 if (pic == NULL || pic->pic_irqbase < 0) 890 continue; 891 if (pic->pic_irqbase <= irq 892 && irq < pic->pic_irqbase + pic->pic_maxsources) { 893 return pic_establish_intr(pic, irq - pic->pic_irqbase, 894 ipl, type, func, arg, xname); 895 } 896 } 897 898 return NULL; 899} 900 901void 902intr_disestablish(void *ih) 903{ 904 struct intrsource * const is = ih; 905 906 KASSERT(!cpu_intr_p()); 907 KASSERT(!cpu_softintr_p()); 908 909 pic_disestablish_source(is); 910} 911 912void 913intr_mask(void *ih) 914{ 915 struct intrsource * const is = ih; 916 struct pic_softc * const pic = is->is_pic; 917 const int irq = is->is_irq; 918 919 if (atomic_inc_32_nv(&is->is_mask_count) == 1) 920 (*pic->pic_ops->pic_block_irqs)(pic, irq & ~0x1f, __BIT(irq & 0x1f)); 921} 922 923void 924intr_unmask(void *ih) 925{ 926 struct intrsource * const is = ih; 927 struct pic_softc * const pic = is->is_pic; 928 const int irq = is->is_irq; 929 930 if (atomic_dec_32_nv(&is->is_mask_count) == 0) 931 (*pic->pic_ops->pic_unblock_irqs)(pic, irq & ~0x1f, __BIT(irq & 0x1f)); 932} 933 934const char * 935intr_string(intr_handle_t irq, char *buf, size_t len) 936{ 937 for (size_t slot = 0; slot < PIC_MAXPICS; slot++) { 938 struct pic_softc * const pic = pic_list[slot]; 939 if (pic == NULL || pic->pic_irqbase < 0) 940 continue; 941 if (pic->pic_irqbase <= irq 942 && irq < pic->pic_irqbase + pic->pic_maxsources) { 943 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 944 snprintf(buf, len, "%s %s", pic->pic_name, is->is_source); 945 return buf; 946 } 947 } 948 949 return NULL; 950} 951 952static struct intrsource * 953intr_get_source(const char *intrid) 954{ 955 struct intrsource *is; 956 intrid_t buf; 957 size_t slot; 958 int irq; 959 960 KASSERT(mutex_owned(&cpu_lock)); 961 962 for (slot = 0; slot < PIC_MAXPICS; slot++) { 963 struct pic_softc * const pic = pic_list[slot]; 964 if (pic == NULL || pic->pic_irqbase < 0) 965 continue; 966 for (irq = 0; irq < pic->pic_maxsources; irq++) { 967 is = pic->pic_sources[irq]; 968 if (is == NULL || is->is_source[0] == '\0') 969 continue; 970 971 snprintf(buf, sizeof(buf), "%s %s", pic->pic_name, is->is_source); 972 if (strcmp(buf, intrid) == 0) 973 return is; 974 } 975 } 976 977 return NULL; 978} 979 980struct intrids_handler * 981interrupt_construct_intrids(const kcpuset_t *cpuset) 982{ 983 struct intrids_handler *iih; 984 struct intrsource *is; 985 int count, irq, n; 986 size_t slot; 987 988 if (kcpuset_iszero(cpuset)) 989 return NULL; 990 991 count = 0; 992 for (slot = 0; slot < PIC_MAXPICS; slot++) { 993 struct pic_softc * const pic = pic_list[slot]; 994 if (pic != NULL && pic->pic_irqbase >= 0) { 995 for (irq = 0; irq < pic->pic_maxsources; irq++) { 996 is = pic->pic_sources[irq]; 997 if (is && is->is_source[0] != '\0') 998 count++; 999 } 1000 } 1001 } 1002 1003 iih = kmem_zalloc(sizeof(int) + sizeof(intrid_t) * count, KM_SLEEP); 1004 iih->iih_nids = count; 1005 1006 for (n = 0, slot = 0; n < count && slot < PIC_MAXPICS; slot++) { 1007 struct pic_softc * const pic = pic_list[slot]; 1008 if (pic == NULL || pic->pic_irqbase < 0) 1009 continue; 1010 for (irq = 0; irq < pic->pic_maxsources; irq++) { 1011 is = pic->pic_sources[irq]; 1012 if (is == NULL || is->is_source[0] == '\0') 1013 continue; 1014 1015 snprintf(iih->iih_intrids[n++], sizeof(intrid_t), "%s %s", 1016 pic->pic_name, is->is_source); 1017 } 1018 } 1019 1020 return iih; 1021} 1022 1023void 1024interrupt_destruct_intrids(struct intrids_handler *iih) 1025{ 1026 if (iih == NULL) 1027 return; 1028 1029 kmem_free(iih, sizeof(int) + sizeof(intrid_t) * iih->iih_nids); 1030} 1031 1032void 1033interrupt_get_available(kcpuset_t *cpuset) 1034{ 1035 CPU_INFO_ITERATOR cii; 1036 struct cpu_info *ci; 1037 1038 kcpuset_zero(cpuset); 1039 1040 mutex_enter(&cpu_lock); 1041 for (CPU_INFO_FOREACH(cii, ci)) { 1042 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) == 0) 1043 kcpuset_set(cpuset, cpu_index(ci)); 1044 } 1045 mutex_exit(&cpu_lock); 1046} 1047 1048void 1049interrupt_get_devname(const char *intrid, char *buf, size_t len) 1050{ 1051 struct intrsource *is; 1052 1053 mutex_enter(&cpu_lock); 1054 is = intr_get_source(intrid); 1055 if (is == NULL || is->is_xname == NULL) 1056 buf[0] = '\0'; 1057 else 1058 strlcpy(buf, is->is_xname, len); 1059 mutex_exit(&cpu_lock); 1060} 1061 1062struct interrupt_get_count_arg { 1063 struct intrsource *is; 1064 uint64_t count; 1065 u_int cpu_idx; 1066}; 1067 1068static void 1069interrupt_get_count_cb(void *v0, void *v1, struct cpu_info *ci) 1070{ 1071 struct pic_percpu * const pcpu = v0; 1072 struct interrupt_get_count_arg * const arg = v1; 1073 1074 if (arg->cpu_idx != cpu_index(ci)) 1075 return; 1076 1077 arg->count = pcpu->pcpu_evs[arg->is->is_irq].ev_count; 1078} 1079 1080uint64_t 1081interrupt_get_count(const char *intrid, u_int cpu_idx) 1082{ 1083 struct interrupt_get_count_arg arg; 1084 struct intrsource *is; 1085 uint64_t count; 1086 1087 count = 0; 1088 1089 mutex_enter(&cpu_lock); 1090 is = intr_get_source(intrid); 1091 if (is != NULL && is->is_pic != NULL) { 1092 arg.is = is; 1093 arg.count = 0; 1094 arg.cpu_idx = cpu_idx; 1095 percpu_foreach(is->is_pic->pic_percpu, interrupt_get_count_cb, &arg); 1096 count = arg.count; 1097 } 1098 mutex_exit(&cpu_lock); 1099 1100 return count; 1101} 1102 1103#ifdef MULTIPROCESSOR 1104void 1105interrupt_get_assigned(const char *intrid, kcpuset_t *cpuset) 1106{ 1107 struct intrsource *is; 1108 struct pic_softc *pic; 1109 1110 kcpuset_zero(cpuset); 1111 1112 mutex_enter(&cpu_lock); 1113 is = intr_get_source(intrid); 1114 if (is != NULL) { 1115 pic = is->is_pic; 1116 if (pic && pic->pic_ops->pic_get_affinity) 1117 pic->pic_ops->pic_get_affinity(pic, is->is_irq, cpuset); 1118 } 1119 mutex_exit(&cpu_lock); 1120} 1121 1122int 1123interrupt_distribute_handler(const char *intrid, const kcpuset_t *newset, 1124 kcpuset_t *oldset) 1125{ 1126 struct intrsource *is; 1127 int error; 1128 1129 mutex_enter(&cpu_lock); 1130 is = intr_get_source(intrid); 1131 if (is == NULL) { 1132 error = ENOENT; 1133 } else { 1134 error = interrupt_distribute(is, newset, oldset); 1135 } 1136 mutex_exit(&cpu_lock); 1137 1138 return error; 1139} 1140 1141int 1142interrupt_distribute(void *ih, const kcpuset_t *newset, kcpuset_t *oldset) 1143{ 1144 struct intrsource * const is = ih; 1145 struct pic_softc * const pic = is->is_pic; 1146 1147 if (pic == NULL) 1148 return EOPNOTSUPP; 1149 if (pic->pic_ops->pic_set_affinity == NULL || 1150 pic->pic_ops->pic_get_affinity == NULL) 1151 return EOPNOTSUPP; 1152 1153 if (!is->is_mpsafe) 1154 return EINVAL; 1155 1156 if (oldset != NULL) 1157 pic->pic_ops->pic_get_affinity(pic, is->is_irq, oldset); 1158 1159 return pic->pic_ops->pic_set_affinity(pic, is->is_irq, newset); 1160} 1161#endif 1162