1/*- 2 * Copyright (c) 2015-2016 Svatopluk Kraus 3 * Copyright (c) 2015-2016 Michal Meloun 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD$"); 30 31/* 32 * New-style Interrupt Framework 33 * 34 * TODO: - add support for disconnected PICs. 35 * - to support IPI (PPI) enabling on other CPUs if already started. 36 * - to complete things for removable PICs. 37 */ 38 39#include "opt_ddb.h" 40#include "opt_hwpmc_hooks.h" 41 42#include <sys/param.h> 43#include <sys/systm.h> 44#include <sys/kernel.h> 45#include <sys/syslog.h> 46#include <sys/malloc.h> 47#include <sys/proc.h> 48#include <sys/queue.h> 49#include <sys/bus.h> 50#include <sys/interrupt.h> 51#include <sys/conf.h> 52#include <sys/cpuset.h> 53#include <sys/rman.h> 54#include <sys/sched.h> 55#include <sys/smp.h> 56#include <sys/vmmeter.h> 57#ifdef HWPMC_HOOKS 58#include <sys/pmckern.h> 59#endif 60 61#include <machine/atomic.h> 62#include <machine/intr.h> 63#include <machine/cpu.h> 64#include <machine/smp.h> 65#include <machine/stdarg.h> 66 67#ifdef DDB 68#include <ddb/ddb.h> 69#endif 70 71#include "pic_if.h" 72#include "msi_if.h" 73 74#define INTRNAME_LEN (2*MAXCOMLEN + 1) 75 76#ifdef DEBUG 77#define debugf(fmt, args...) do { printf("%s(): ", __func__); \ 78 printf(fmt,##args); } while (0) 79#else 80#define debugf(fmt, args...) 81#endif 82 83MALLOC_DECLARE(M_INTRNG); 84MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); 85 86/* Main interrupt handler called from assembler -> 'hidden' for C code. */ 87void intr_irq_handler(struct trapframe *tf); 88 89/* Root interrupt controller stuff. */ 90device_t intr_irq_root_dev; 91static intr_irq_filter_t *irq_root_filter; 92static void *irq_root_arg; 93static u_int irq_root_ipicount; 94 95struct intr_pic_child { 96 SLIST_ENTRY(intr_pic_child) pc_next; 97 struct intr_pic *pc_pic; 98 intr_child_irq_filter_t *pc_filter; 99 void *pc_filter_arg; 100 uintptr_t pc_start; 101 uintptr_t pc_length; 102}; 103 104/* Interrupt controller definition. */ 105struct intr_pic { 106 SLIST_ENTRY(intr_pic) pic_next; 107 intptr_t pic_xref; /* hardware identification */ 108 device_t pic_dev; 109/* Only one of FLAG_PIC or FLAG_MSI may be set */ 110#define FLAG_PIC (1 << 0) 111#define FLAG_MSI (1 << 1) 112#define FLAG_TYPE_MASK (FLAG_PIC | FLAG_MSI) 113 u_int pic_flags; 114 struct mtx pic_child_lock; 115 SLIST_HEAD(, intr_pic_child) pic_children; 116}; 117 118static struct mtx pic_list_lock; 119static SLIST_HEAD(, intr_pic) pic_list; 120 121static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, int flags); 122 123/* Interrupt source definition. */ 124static struct mtx isrc_table_lock; 125static struct intr_irqsrc *irq_sources[NIRQ]; 126u_int irq_next_free; 127 128#ifdef SMP 129static boolean_t irq_assign_cpu = FALSE; 130#endif 131 132/* 133 * - 2 counters for each I/O interrupt. 134 * - MAXCPU counters for each IPI counters for SMP. 135 */ 136#ifdef SMP 137#define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) 138#else 139#define INTRCNT_COUNT (NIRQ * 2) 140#endif 141 142/* Data for MI statistics reporting. */ 143u_long intrcnt[INTRCNT_COUNT]; 144char intrnames[INTRCNT_COUNT * INTRNAME_LEN]; 145size_t sintrcnt = sizeof(intrcnt); 146size_t sintrnames = sizeof(intrnames); 147static u_int intrcnt_index; 148 149static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); 150static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); 151static struct intr_map_data * intr_map_get_map_data(u_int res_id); 152static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, 153 struct intr_map_data **data); 154 155/* 156 * Interrupt framework initialization routine. 157 */ 158static void 159intr_irq_init(void *dummy __unused) 160{ 161 162 SLIST_INIT(&pic_list); 163 mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); 164 165 mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); 166} 167SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); 168 169static void 170intrcnt_setname(const char *name, int index) 171{ 172 173 snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 174 INTRNAME_LEN - 1, name); 175} 176 177/* 178 * Update name for interrupt source with interrupt event. 179 */ 180static void 181intrcnt_updatename(struct intr_irqsrc *isrc) 182{ 183 184 /* QQQ: What about stray counter name? */ 185 mtx_assert(&isrc_table_lock, MA_OWNED); 186 intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); 187} 188 189/* 190 * Virtualization for interrupt source interrupt counter increment. 191 */ 192static inline void 193isrc_increment_count(struct intr_irqsrc *isrc) 194{ 195 196 if (isrc->isrc_flags & INTR_ISRCF_PPI) 197 atomic_add_long(&isrc->isrc_count[0], 1); 198 else 199 isrc->isrc_count[0]++; 200} 201 202/* 203 * Virtualization for interrupt source interrupt stray counter increment. 204 */ 205static inline void 206isrc_increment_straycount(struct intr_irqsrc *isrc) 207{ 208 209 isrc->isrc_count[1]++; 210} 211 212/* 213 * Virtualization for interrupt source interrupt name update. 214 */ 215static void 216isrc_update_name(struct intr_irqsrc *isrc, const char *name) 217{ 218 char str[INTRNAME_LEN]; 219 220 mtx_assert(&isrc_table_lock, MA_OWNED); 221 222 if (name != NULL) { 223 snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); 224 intrcnt_setname(str, isrc->isrc_index); 225 snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, 226 name); 227 intrcnt_setname(str, isrc->isrc_index + 1); 228 } else { 229 snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); 230 intrcnt_setname(str, isrc->isrc_index); 231 snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); 232 intrcnt_setname(str, isrc->isrc_index + 1); 233 } 234} 235 236/* 237 * Virtualization for interrupt source interrupt counters setup. 238 */ 239static void 240isrc_setup_counters(struct intr_irqsrc *isrc) 241{ 242 u_int index; 243 244 /* 245 * XXX - it does not work well with removable controllers and 246 * interrupt sources !!! 247 */ 248 index = atomic_fetchadd_int(&intrcnt_index, 2); 249 isrc->isrc_index = index; 250 isrc->isrc_count = &intrcnt[index]; 251 isrc_update_name(isrc, NULL); 252} 253 254/* 255 * Virtualization for interrupt source interrupt counters release. 256 */ 257static void 258isrc_release_counters(struct intr_irqsrc *isrc) 259{ 260 261 panic("%s: not implemented", __func__); 262} 263 264#ifdef SMP 265/* 266 * Virtualization for interrupt source IPI counters setup. 267 */ 268u_long * 269intr_ipi_setup_counters(const char *name) 270{ 271 u_int index, i; 272 char str[INTRNAME_LEN]; 273 274 index = atomic_fetchadd_int(&intrcnt_index, MAXCPU); 275 for (i = 0; i < MAXCPU; i++) { 276 snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); 277 intrcnt_setname(str, index + i); 278 } 279 return (&intrcnt[index]); 280} 281#endif 282 283/* 284 * Main interrupt dispatch handler. It's called straight 285 * from the assembler, where CPU interrupt is served. 286 */ 287void 288intr_irq_handler(struct trapframe *tf) 289{ 290 struct trapframe * oldframe; 291 struct thread * td; 292 293 KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); 294 295 VM_CNT_INC(v_intr); 296 critical_enter(); 297 td = curthread; 298 oldframe = td->td_intr_frame; 299 td->td_intr_frame = tf; 300 irq_root_filter(irq_root_arg); 301 td->td_intr_frame = oldframe; 302 critical_exit(); 303#ifdef HWPMC_HOOKS 304 if (pmc_hook && TRAPF_USERMODE(tf) && 305 (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) 306 pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); 307#endif 308} 309 310int 311intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) 312{ 313 struct intr_pic_child *child; 314 bool found; 315 316 found = false; 317 mtx_lock_spin(&parent->pic_child_lock); 318 SLIST_FOREACH(child, &parent->pic_children, pc_next) { 319 if (child->pc_start <= irq && 320 irq < (child->pc_start + child->pc_length)) { 321 found = true; 322 break; 323 } 324 } 325 mtx_unlock_spin(&parent->pic_child_lock); 326 327 if (found) 328 return (child->pc_filter(child->pc_filter_arg, irq)); 329 330 return (FILTER_STRAY); 331} 332 333/* 334 * interrupt controller dispatch function for interrupts. It should 335 * be called straight from the interrupt controller, when associated interrupt 336 * source is learned. 337 */ 338int 339intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) 340{ 341 342 KASSERT(isrc != NULL, ("%s: no source", __func__)); 343 344 isrc_increment_count(isrc); 345 346#ifdef INTR_SOLO 347 if (isrc->isrc_filter != NULL) { 348 int error; 349 error = isrc->isrc_filter(isrc->isrc_arg, tf); 350 PIC_POST_FILTER(isrc->isrc_dev, isrc); 351 if (error == FILTER_HANDLED) 352 return (0); 353 } else 354#endif 355 if (isrc->isrc_event != NULL) { 356 if (intr_event_handle(isrc->isrc_event, tf) == 0) 357 return (0); 358 } 359 360 isrc_increment_straycount(isrc); 361 return (EINVAL); 362} 363 364/* 365 * Alloc unique interrupt number (resource handle) for interrupt source. 366 * 367 * There could be various strategies how to allocate free interrupt number 368 * (resource handle) for new interrupt source. 369 * 370 * 1. Handles are always allocated forward, so handles are not recycled 371 * immediately. However, if only one free handle left which is reused 372 * constantly... 373 */ 374static inline int 375isrc_alloc_irq(struct intr_irqsrc *isrc) 376{ 377 u_int maxirqs, irq; 378 379 mtx_assert(&isrc_table_lock, MA_OWNED); 380 381 maxirqs = nitems(irq_sources); 382 if (irq_next_free >= maxirqs) 383 return (ENOSPC); 384 385 for (irq = irq_next_free; irq < maxirqs; irq++) { 386 if (irq_sources[irq] == NULL) 387 goto found; 388 } 389 for (irq = 0; irq < irq_next_free; irq++) { 390 if (irq_sources[irq] == NULL) 391 goto found; 392 } 393 394 irq_next_free = maxirqs; 395 return (ENOSPC); 396 397found: 398 isrc->isrc_irq = irq; 399 irq_sources[irq] = isrc; 400 401 irq_next_free = irq + 1; 402 if (irq_next_free >= maxirqs) 403 irq_next_free = 0; 404 return (0); 405} 406 407/* 408 * Free unique interrupt number (resource handle) from interrupt source. 409 */ 410static inline int 411isrc_free_irq(struct intr_irqsrc *isrc) 412{ 413 414 mtx_assert(&isrc_table_lock, MA_OWNED); 415 416 if (isrc->isrc_irq >= nitems(irq_sources)) 417 return (EINVAL); 418 if (irq_sources[isrc->isrc_irq] != isrc) 419 return (EINVAL); 420 421 irq_sources[isrc->isrc_irq] = NULL; 422 isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 423 return (0); 424} 425 426/* 427 * Initialize interrupt source and register it into global interrupt table. 428 */ 429int 430intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, 431 const char *fmt, ...) 432{ 433 int error; 434 va_list ap; 435 436 bzero(isrc, sizeof(struct intr_irqsrc)); 437 isrc->isrc_dev = dev; 438 isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ 439 isrc->isrc_flags = flags; 440 441 va_start(ap, fmt); 442 vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); 443 va_end(ap); 444 445 mtx_lock(&isrc_table_lock); 446 error = isrc_alloc_irq(isrc); 447 if (error != 0) { 448 mtx_unlock(&isrc_table_lock); 449 return (error); 450 } 451 /* 452 * Setup interrupt counters, but not for IPI sources. Those are setup 453 * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust 454 * our counter pool. 455 */ 456 if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 457 isrc_setup_counters(isrc); 458 mtx_unlock(&isrc_table_lock); 459 return (0); 460} 461 462/* 463 * Deregister interrupt source from global interrupt table. 464 */ 465int 466intr_isrc_deregister(struct intr_irqsrc *isrc) 467{ 468 int error; 469 470 mtx_lock(&isrc_table_lock); 471 if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) 472 isrc_release_counters(isrc); 473 error = isrc_free_irq(isrc); 474 mtx_unlock(&isrc_table_lock); 475 return (error); 476} 477 478#ifdef SMP 479/* 480 * A support function for a PIC to decide if provided ISRC should be inited 481 * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of 482 * struct intr_irqsrc is the following: 483 * 484 * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus 485 * set in isrc_cpu. If not, the ISRC should be inited on every cpu and 486 * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. 487 */ 488bool 489intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) 490{ 491 492 if (isrc->isrc_handlers == 0) 493 return (false); 494 if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) 495 return (false); 496 if (isrc->isrc_flags & INTR_ISRCF_BOUND) 497 return (CPU_ISSET(cpu, &isrc->isrc_cpu)); 498 499 CPU_SET(cpu, &isrc->isrc_cpu); 500 return (true); 501} 502#endif 503 504#ifdef INTR_SOLO 505/* 506 * Setup filter into interrupt source. 507 */ 508static int 509iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, 510 intr_irq_filter_t *filter, void *arg, void **cookiep) 511{ 512 513 if (filter == NULL) 514 return (EINVAL); 515 516 mtx_lock(&isrc_table_lock); 517 /* 518 * Make sure that we do not mix the two ways 519 * how we handle interrupt sources. 520 */ 521 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 522 mtx_unlock(&isrc_table_lock); 523 return (EBUSY); 524 } 525 isrc->isrc_filter = filter; 526 isrc->isrc_arg = arg; 527 isrc_update_name(isrc, name); 528 mtx_unlock(&isrc_table_lock); 529 530 *cookiep = isrc; 531 return (0); 532} 533#endif 534 535/* 536 * Interrupt source pre_ithread method for MI interrupt framework. 537 */ 538static void 539intr_isrc_pre_ithread(void *arg) 540{ 541 struct intr_irqsrc *isrc = arg; 542 543 PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); 544} 545 546/* 547 * Interrupt source post_ithread method for MI interrupt framework. 548 */ 549static void 550intr_isrc_post_ithread(void *arg) 551{ 552 struct intr_irqsrc *isrc = arg; 553 554 PIC_POST_ITHREAD(isrc->isrc_dev, isrc); 555} 556 557/* 558 * Interrupt source post_filter method for MI interrupt framework. 559 */ 560static void 561intr_isrc_post_filter(void *arg) 562{ 563 struct intr_irqsrc *isrc = arg; 564 565 PIC_POST_FILTER(isrc->isrc_dev, isrc); 566} 567 568/* 569 * Interrupt source assign_cpu method for MI interrupt framework. 570 */ 571static int 572intr_isrc_assign_cpu(void *arg, int cpu) 573{ 574#ifdef SMP 575 struct intr_irqsrc *isrc = arg; 576 int error; 577 578 mtx_lock(&isrc_table_lock); 579 if (cpu == NOCPU) { 580 CPU_ZERO(&isrc->isrc_cpu); 581 isrc->isrc_flags &= ~INTR_ISRCF_BOUND; 582 } else { 583 CPU_SETOF(cpu, &isrc->isrc_cpu); 584 isrc->isrc_flags |= INTR_ISRCF_BOUND; 585 } 586 587 /* 588 * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or 589 * re-balance it to another CPU or enable it on more CPUs. However, 590 * PIC is expected to change isrc_cpu appropriately to keep us well 591 * informed if the call is successful. 592 */ 593 if (irq_assign_cpu) { 594 error = PIC_BIND_INTR(isrc->isrc_dev, isrc); 595 if (error) { 596 CPU_ZERO(&isrc->isrc_cpu); 597 mtx_unlock(&isrc_table_lock); 598 return (error); 599 } 600 } 601 mtx_unlock(&isrc_table_lock); 602 return (0); 603#else 604 return (EOPNOTSUPP); 605#endif 606} 607 608/* 609 * Create interrupt event for interrupt source. 610 */ 611static int 612isrc_event_create(struct intr_irqsrc *isrc) 613{ 614 struct intr_event *ie; 615 int error; 616 617 error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, 618 intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, 619 intr_isrc_assign_cpu, "%s:", isrc->isrc_name); 620 if (error) 621 return (error); 622 623 mtx_lock(&isrc_table_lock); 624 /* 625 * Make sure that we do not mix the two ways 626 * how we handle interrupt sources. Let contested event wins. 627 */ 628#ifdef INTR_SOLO 629 if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { 630#else 631 if (isrc->isrc_event != NULL) { 632#endif 633 mtx_unlock(&isrc_table_lock); 634 intr_event_destroy(ie); 635 return (isrc->isrc_event != NULL ? EBUSY : 0); 636 } 637 isrc->isrc_event = ie; 638 mtx_unlock(&isrc_table_lock); 639 640 return (0); 641} 642#ifdef notyet 643/* 644 * Destroy interrupt event for interrupt source. 645 */ 646static void 647isrc_event_destroy(struct intr_irqsrc *isrc) 648{ 649 struct intr_event *ie; 650 651 mtx_lock(&isrc_table_lock); 652 ie = isrc->isrc_event; 653 isrc->isrc_event = NULL; 654 mtx_unlock(&isrc_table_lock); 655 656 if (ie != NULL) 657 intr_event_destroy(ie); 658} 659#endif 660/* 661 * Add handler to interrupt source. 662 */ 663static int 664isrc_add_handler(struct intr_irqsrc *isrc, const char *name, 665 driver_filter_t filter, driver_intr_t handler, void *arg, 666 enum intr_type flags, void **cookiep) 667{ 668 int error; 669 670 if (isrc->isrc_event == NULL) { 671 error = isrc_event_create(isrc); 672 if (error) 673 return (error); 674 } 675 676 error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, 677 arg, intr_priority(flags), flags, cookiep); 678 if (error == 0) { 679 mtx_lock(&isrc_table_lock); 680 intrcnt_updatename(isrc); 681 mtx_unlock(&isrc_table_lock); 682 } 683 684 return (error); 685} 686 687/* 688 * Lookup interrupt controller locked. 689 */ 690static inline struct intr_pic * 691pic_lookup_locked(device_t dev, intptr_t xref, int flags) 692{ 693 struct intr_pic *pic; 694 695 mtx_assert(&pic_list_lock, MA_OWNED); 696 697 if (dev == NULL && xref == 0) 698 return (NULL); 699 700 /* Note that pic->pic_dev is never NULL on registered PIC. */ 701 SLIST_FOREACH(pic, &pic_list, pic_next) { 702 if ((pic->pic_flags & FLAG_TYPE_MASK) != 703 (flags & FLAG_TYPE_MASK)) 704 continue; 705 706 if (dev == NULL) { 707 if (xref == pic->pic_xref) 708 return (pic); 709 } else if (xref == 0 || pic->pic_xref == 0) { 710 if (dev == pic->pic_dev) 711 return (pic); 712 } else if (xref == pic->pic_xref && dev == pic->pic_dev) 713 return (pic); 714 } 715 return (NULL); 716} 717 718/* 719 * Lookup interrupt controller. 720 */ 721static struct intr_pic * 722pic_lookup(device_t dev, intptr_t xref, int flags) 723{ 724 struct intr_pic *pic; 725 726 mtx_lock(&pic_list_lock); 727 pic = pic_lookup_locked(dev, xref, flags); 728 mtx_unlock(&pic_list_lock); 729 return (pic); 730} 731 732/* 733 * Create interrupt controller. 734 */ 735static struct intr_pic * 736pic_create(device_t dev, intptr_t xref, int flags) 737{ 738 struct intr_pic *pic; 739 740 mtx_lock(&pic_list_lock); 741 pic = pic_lookup_locked(dev, xref, flags); 742 if (pic != NULL) { 743 mtx_unlock(&pic_list_lock); 744 return (pic); 745 } 746 pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); 747 if (pic == NULL) { 748 mtx_unlock(&pic_list_lock); 749 return (NULL); 750 } 751 pic->pic_xref = xref; 752 pic->pic_dev = dev; 753 pic->pic_flags = flags; 754 mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); 755 SLIST_INSERT_HEAD(&pic_list, pic, pic_next); 756 mtx_unlock(&pic_list_lock); 757 758 return (pic); 759} 760#ifdef notyet 761/* 762 * Destroy interrupt controller. 763 */ 764static void 765pic_destroy(device_t dev, intptr_t xref, int flags) 766{ 767 struct intr_pic *pic; 768 769 mtx_lock(&pic_list_lock); 770 pic = pic_lookup_locked(dev, xref, flags); 771 if (pic == NULL) { 772 mtx_unlock(&pic_list_lock); 773 return; 774 } 775 SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); 776 mtx_unlock(&pic_list_lock); 777 778 free(pic, M_INTRNG); 779} 780#endif 781/* 782 * Register interrupt controller. 783 */ 784struct intr_pic * 785intr_pic_register(device_t dev, intptr_t xref) 786{ 787 struct intr_pic *pic; 788 789 if (dev == NULL) 790 return (NULL); 791 pic = pic_create(dev, xref, FLAG_PIC); 792 if (pic == NULL) 793 return (NULL); 794 795 debugf("PIC %p registered for %s <dev %p, xref %x>\n", pic, 796 device_get_nameunit(dev), dev, xref); 797 return (pic); 798} 799 800/* 801 * Unregister interrupt controller. 802 */ 803int 804intr_pic_deregister(device_t dev, intptr_t xref) 805{ 806 807 panic("%s: not implemented", __func__); 808} 809 810/* 811 * Mark interrupt controller (itself) as a root one. 812 * 813 * Note that only an interrupt controller can really know its position 814 * in interrupt controller's tree. So root PIC must claim itself as a root. 815 * 816 * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, 817 * page 30: 818 * "The root of the interrupt tree is determined when traversal 819 * of the interrupt tree reaches an interrupt controller node without 820 * an interrupts property and thus no explicit interrupt parent." 821 */ 822int 823intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, 824 void *arg, u_int ipicount) 825{ 826 struct intr_pic *pic; 827 828 pic = pic_lookup(dev, xref, FLAG_PIC); 829 if (pic == NULL) { 830 device_printf(dev, "not registered\n"); 831 return (EINVAL); 832 } 833 834 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 835 ("%s: Found a non-PIC controller: %s", __func__, 836 device_get_name(pic->pic_dev))); 837 838 if (filter == NULL) { 839 device_printf(dev, "filter missing\n"); 840 return (EINVAL); 841 } 842 843 /* 844 * Only one interrupt controllers could be on the root for now. 845 * Note that we further suppose that there is not threaded interrupt 846 * routine (handler) on the root. See intr_irq_handler(). 847 */ 848 if (intr_irq_root_dev != NULL) { 849 device_printf(dev, "another root already set\n"); 850 return (EBUSY); 851 } 852 853 intr_irq_root_dev = dev; 854 irq_root_filter = filter; 855 irq_root_arg = arg; 856 irq_root_ipicount = ipicount; 857 858 debugf("irq root set to %s\n", device_get_nameunit(dev)); 859 return (0); 860} 861 862/* 863 * Add a handler to manage a sub range of a parents interrupts. 864 */ 865struct intr_pic * 866intr_pic_add_handler(device_t parent, struct intr_pic *pic, 867 intr_child_irq_filter_t *filter, void *arg, uintptr_t start, 868 uintptr_t length) 869{ 870 struct intr_pic *parent_pic; 871 struct intr_pic_child *newchild; 872#ifdef INVARIANTS 873 struct intr_pic_child *child; 874#endif 875 876 /* Find the parent PIC */ 877 parent_pic = pic_lookup(parent, 0, FLAG_PIC); 878 if (parent_pic == NULL) 879 return (NULL); 880 881 newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO); 882 newchild->pc_pic = pic; 883 newchild->pc_filter = filter; 884 newchild->pc_filter_arg = arg; 885 newchild->pc_start = start; 886 newchild->pc_length = length; 887 888 mtx_lock_spin(&parent_pic->pic_child_lock); 889#ifdef INVARIANTS 890 SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { 891 KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", 892 __func__)); 893 } 894#endif 895 SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); 896 mtx_unlock_spin(&parent_pic->pic_child_lock); 897 898 return (pic); 899} 900 901static int 902intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, 903 struct intr_irqsrc **isrc) 904{ 905 struct intr_pic *pic; 906 struct intr_map_data_msi *msi; 907 908 if (data == NULL) 909 return (EINVAL); 910 911 pic = pic_lookup(dev, xref, 912 (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC); 913 if (pic == NULL) 914 return (ESRCH); 915 916 switch (data->type) { 917 case INTR_MAP_DATA_MSI: 918 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 919 ("%s: Found a non-MSI controller: %s", __func__, 920 device_get_name(pic->pic_dev))); 921 msi = (struct intr_map_data_msi *)data; 922 *isrc = msi->isrc; 923 return (0); 924 925 default: 926 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, 927 ("%s: Found a non-PIC controller: %s", __func__, 928 device_get_name(pic->pic_dev))); 929 return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); 930 931 } 932} 933 934bool 935intr_is_per_cpu(struct resource *res) 936{ 937 u_int res_id; 938 struct intr_irqsrc *isrc; 939 940 res_id = (u_int)rman_get_start(res); 941 isrc = intr_map_get_isrc(res_id); 942 943 if (isrc == NULL) 944 panic("Attempt to get isrc for non-active resource id: %u\n", 945 res_id); 946 return ((isrc->isrc_flags & INTR_ISRCF_PPI) != 0); 947} 948 949int 950intr_activate_irq(device_t dev, struct resource *res) 951{ 952 device_t map_dev; 953 intptr_t map_xref; 954 struct intr_map_data *data; 955 struct intr_irqsrc *isrc; 956 u_int res_id; 957 int error; 958 959 KASSERT(rman_get_start(res) == rman_get_end(res), 960 ("%s: more interrupts in resource", __func__)); 961 962 res_id = (u_int)rman_get_start(res); 963 if (intr_map_get_isrc(res_id) != NULL) 964 panic("Attempt to double activation of resource id: %u\n", 965 res_id); 966 intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); 967 error = intr_resolve_irq(map_dev, map_xref, data, &isrc); 968 if (error != 0) { 969 free(data, M_INTRNG); 970 /* XXX TODO DISCONECTED PICs */ 971 /* if (error == EINVAL) return(0); */ 972 return (error); 973 } 974 intr_map_set_isrc(res_id, isrc); 975 rman_set_virtual(res, data); 976 return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); 977} 978 979int 980intr_deactivate_irq(device_t dev, struct resource *res) 981{ 982 struct intr_map_data *data; 983 struct intr_irqsrc *isrc; 984 u_int res_id; 985 int error; 986 987 KASSERT(rman_get_start(res) == rman_get_end(res), 988 ("%s: more interrupts in resource", __func__)); 989 990 res_id = (u_int)rman_get_start(res); 991 isrc = intr_map_get_isrc(res_id); 992 if (isrc == NULL) 993 panic("Attempt to deactivate non-active resource id: %u\n", 994 res_id); 995 996 data = rman_get_virtual(res); 997 error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); 998 intr_map_set_isrc(res_id, NULL); 999 rman_set_virtual(res, NULL); 1000 free(data, M_INTRNG); 1001 return (error); 1002} 1003 1004int 1005intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, 1006 driver_intr_t hand, void *arg, int flags, void **cookiep) 1007{ 1008 int error; 1009 struct intr_map_data *data; 1010 struct intr_irqsrc *isrc; 1011 const char *name; 1012 u_int res_id; 1013 1014 KASSERT(rman_get_start(res) == rman_get_end(res), 1015 ("%s: more interrupts in resource", __func__)); 1016 1017 res_id = (u_int)rman_get_start(res); 1018 isrc = intr_map_get_isrc(res_id); 1019 if (isrc == NULL) { 1020 /* XXX TODO DISCONECTED PICs */ 1021 return (EINVAL); 1022 } 1023 1024 data = rman_get_virtual(res); 1025 name = device_get_nameunit(dev); 1026 1027#ifdef INTR_SOLO 1028 /* 1029 * Standard handling is done through MI interrupt framework. However, 1030 * some interrupts could request solely own special handling. This 1031 * non standard handling can be used for interrupt controllers without 1032 * handler (filter only), so in case that interrupt controllers are 1033 * chained, MI interrupt framework is called only in leaf controller. 1034 * 1035 * Note that root interrupt controller routine is served as well, 1036 * however in intr_irq_handler(), i.e. main system dispatch routine. 1037 */ 1038 if (flags & INTR_SOLO && hand != NULL) { 1039 debugf("irq %u cannot solo on %s\n", irq, name); 1040 return (EINVAL); 1041 } 1042 1043 if (flags & INTR_SOLO) { 1044 error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, 1045 arg, cookiep); 1046 debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error, 1047 name); 1048 } else 1049#endif 1050 { 1051 error = isrc_add_handler(isrc, name, filt, hand, arg, flags, 1052 cookiep); 1053 debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name); 1054 } 1055 if (error != 0) 1056 return (error); 1057 1058 mtx_lock(&isrc_table_lock); 1059 error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); 1060 if (error == 0) { 1061 isrc->isrc_handlers++; 1062 if (isrc->isrc_handlers == 1) 1063 PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 1064 } 1065 mtx_unlock(&isrc_table_lock); 1066 if (error != 0) 1067 intr_event_remove_handler(*cookiep); 1068 return (error); 1069} 1070 1071int 1072intr_teardown_irq(device_t dev, struct resource *res, void *cookie) 1073{ 1074 int error; 1075 struct intr_map_data *data; 1076 struct intr_irqsrc *isrc; 1077 u_int res_id; 1078 1079 KASSERT(rman_get_start(res) == rman_get_end(res), 1080 ("%s: more interrupts in resource", __func__)); 1081 1082 res_id = (u_int)rman_get_start(res); 1083 isrc = intr_map_get_isrc(res_id); 1084 if (isrc == NULL || isrc->isrc_handlers == 0) 1085 return (EINVAL); 1086 1087 data = rman_get_virtual(res); 1088 1089#ifdef INTR_SOLO 1090 if (isrc->isrc_filter != NULL) { 1091 if (isrc != cookie) 1092 return (EINVAL); 1093 1094 mtx_lock(&isrc_table_lock); 1095 isrc->isrc_filter = NULL; 1096 isrc->isrc_arg = NULL; 1097 isrc->isrc_handlers = 0; 1098 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1099 PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 1100 isrc_update_name(isrc, NULL); 1101 mtx_unlock(&isrc_table_lock); 1102 return (0); 1103 } 1104#endif 1105 if (isrc != intr_handler_source(cookie)) 1106 return (EINVAL); 1107 1108 error = intr_event_remove_handler(cookie); 1109 if (error == 0) { 1110 mtx_lock(&isrc_table_lock); 1111 isrc->isrc_handlers--; 1112 if (isrc->isrc_handlers == 0) 1113 PIC_DISABLE_INTR(isrc->isrc_dev, isrc); 1114 PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); 1115 intrcnt_updatename(isrc); 1116 mtx_unlock(&isrc_table_lock); 1117 } 1118 return (error); 1119} 1120 1121int 1122intr_describe_irq(device_t dev, struct resource *res, void *cookie, 1123 const char *descr) 1124{ 1125 int error; 1126 struct intr_irqsrc *isrc; 1127 u_int res_id; 1128 1129 KASSERT(rman_get_start(res) == rman_get_end(res), 1130 ("%s: more interrupts in resource", __func__)); 1131 1132 res_id = (u_int)rman_get_start(res); 1133 isrc = intr_map_get_isrc(res_id); 1134 if (isrc == NULL || isrc->isrc_handlers == 0) 1135 return (EINVAL); 1136#ifdef INTR_SOLO 1137 if (isrc->isrc_filter != NULL) { 1138 if (isrc != cookie) 1139 return (EINVAL); 1140 1141 mtx_lock(&isrc_table_lock); 1142 isrc_update_name(isrc, descr); 1143 mtx_unlock(&isrc_table_lock); 1144 return (0); 1145 } 1146#endif 1147 error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); 1148 if (error == 0) { 1149 mtx_lock(&isrc_table_lock); 1150 intrcnt_updatename(isrc); 1151 mtx_unlock(&isrc_table_lock); 1152 } 1153 return (error); 1154} 1155 1156#ifdef SMP 1157int 1158intr_bind_irq(device_t dev, struct resource *res, int cpu) 1159{ 1160 struct intr_irqsrc *isrc; 1161 u_int res_id; 1162 1163 KASSERT(rman_get_start(res) == rman_get_end(res), 1164 ("%s: more interrupts in resource", __func__)); 1165 1166 res_id = (u_int)rman_get_start(res); 1167 isrc = intr_map_get_isrc(res_id); 1168 if (isrc == NULL || isrc->isrc_handlers == 0) 1169 return (EINVAL); 1170#ifdef INTR_SOLO 1171 if (isrc->isrc_filter != NULL) 1172 return (intr_isrc_assign_cpu(isrc, cpu)); 1173#endif 1174 return (intr_event_bind(isrc->isrc_event, cpu)); 1175} 1176 1177/* 1178 * Return the CPU that the next interrupt source should use. 1179 * For now just returns the next CPU according to round-robin. 1180 */ 1181u_int 1182intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) 1183{ 1184 u_int cpu; 1185 1186 KASSERT(!CPU_EMPTY(cpumask), ("%s: Empty CPU mask", __func__)); 1187 if (!irq_assign_cpu || mp_ncpus == 1) { 1188 cpu = PCPU_GET(cpuid); 1189 1190 if (CPU_ISSET(cpu, cpumask)) 1191 return (curcpu); 1192 1193 return (CPU_FFS(cpumask) - 1); 1194 } 1195 1196 do { 1197 last_cpu++; 1198 if (last_cpu > mp_maxid) 1199 last_cpu = 0; 1200 } while (!CPU_ISSET(last_cpu, cpumask)); 1201 return (last_cpu); 1202} 1203 1204/* 1205 * Distribute all the interrupt sources among the available 1206 * CPUs once the AP's have been launched. 1207 */ 1208static void 1209intr_irq_shuffle(void *arg __unused) 1210{ 1211 struct intr_irqsrc *isrc; 1212 u_int i; 1213 1214 if (mp_ncpus == 1) 1215 return; 1216 1217 mtx_lock(&isrc_table_lock); 1218 irq_assign_cpu = TRUE; 1219 for (i = 0; i < NIRQ; i++) { 1220 isrc = irq_sources[i]; 1221 if (isrc == NULL || isrc->isrc_handlers == 0 || 1222 isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) 1223 continue; 1224 1225 if (isrc->isrc_event != NULL && 1226 isrc->isrc_flags & INTR_ISRCF_BOUND && 1227 isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) 1228 panic("%s: CPU inconsistency", __func__); 1229 1230 if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) 1231 CPU_ZERO(&isrc->isrc_cpu); /* start again */ 1232 1233 /* 1234 * We are in wicked position here if the following call fails 1235 * for bound ISRC. The best thing we can do is to clear 1236 * isrc_cpu so inconsistency with ie_cpu will be detectable. 1237 */ 1238 if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) 1239 CPU_ZERO(&isrc->isrc_cpu); 1240 } 1241 mtx_unlock(&isrc_table_lock); 1242} 1243SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); 1244 1245#else 1246u_int 1247intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) 1248{ 1249 1250 return (PCPU_GET(cpuid)); 1251} 1252#endif 1253 1254/* 1255 * Allocate memory for new intr_map_data structure. 1256 * Initialize common fields. 1257 */ 1258struct intr_map_data * 1259intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) 1260{ 1261 struct intr_map_data *data; 1262 1263 data = malloc(len, M_INTRNG, flags); 1264 data->type = type; 1265 data->len = len; 1266 return (data); 1267} 1268 1269void intr_free_intr_map_data(struct intr_map_data *data) 1270{ 1271 1272 free(data, M_INTRNG); 1273} 1274 1275 1276/* 1277 * Register a MSI/MSI-X interrupt controller 1278 */ 1279int 1280intr_msi_register(device_t dev, intptr_t xref) 1281{ 1282 struct intr_pic *pic; 1283 1284 if (dev == NULL) 1285 return (EINVAL); 1286 pic = pic_create(dev, xref, FLAG_MSI); 1287 if (pic == NULL) 1288 return (ENOMEM); 1289 1290 debugf("PIC %p registered for %s <dev %p, xref %jx>\n", pic, 1291 device_get_nameunit(dev), dev, (uintmax_t)xref); 1292 return (0); 1293} 1294 1295int 1296intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, 1297 int maxcount, int *irqs) 1298{ 1299 struct intr_irqsrc **isrc; 1300 struct intr_pic *pic; 1301 device_t pdev; 1302 struct intr_map_data_msi *msi; 1303 int err, i; 1304 1305 pic = pic_lookup(NULL, xref, FLAG_MSI); 1306 if (pic == NULL) 1307 return (ESRCH); 1308 1309 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1310 ("%s: Found a non-MSI controller: %s", __func__, 1311 device_get_name(pic->pic_dev))); 1312 1313 isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 1314 err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); 1315 if (err != 0) { 1316 free(isrc, M_INTRNG); 1317 return (err); 1318 } 1319 1320 for (i = 0; i < count; i++) { 1321 msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1322 INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1323 msi-> isrc = isrc[i]; 1324 irqs[i] = intr_map_irq(pic->pic_dev, xref, 1325 (struct intr_map_data *)msi); 1326 1327 } 1328 free(isrc, M_INTRNG); 1329 1330 return (err); 1331} 1332 1333int 1334intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, 1335 int *irqs) 1336{ 1337 struct intr_irqsrc **isrc; 1338 struct intr_pic *pic; 1339 struct intr_map_data_msi *msi; 1340 int i, err; 1341 1342 pic = pic_lookup(NULL, xref, FLAG_MSI); 1343 if (pic == NULL) 1344 return (ESRCH); 1345 1346 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1347 ("%s: Found a non-MSI controller: %s", __func__, 1348 device_get_name(pic->pic_dev))); 1349 1350 isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); 1351 1352 for (i = 0; i < count; i++) { 1353 msi = (struct intr_map_data_msi *) 1354 intr_map_get_map_data(irqs[i]); 1355 KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1356 ("%s: irq %d map data is not MSI", __func__, 1357 irqs[i])); 1358 isrc[i] = msi->isrc; 1359 } 1360 1361 err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); 1362 1363 for (i = 0; i < count; i++) { 1364 if (isrc[i] != NULL) 1365 intr_unmap_irq(irqs[i]); 1366 } 1367 1368 free(isrc, M_INTRNG); 1369 return (err); 1370} 1371 1372int 1373intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) 1374{ 1375 struct intr_irqsrc *isrc; 1376 struct intr_pic *pic; 1377 device_t pdev; 1378 struct intr_map_data_msi *msi; 1379 int err; 1380 1381 pic = pic_lookup(NULL, xref, FLAG_MSI); 1382 if (pic == NULL) 1383 return (ESRCH); 1384 1385 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1386 ("%s: Found a non-MSI controller: %s", __func__, 1387 device_get_name(pic->pic_dev))); 1388 1389 1390 err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); 1391 if (err != 0) 1392 return (err); 1393 1394 msi = (struct intr_map_data_msi *)intr_alloc_map_data( 1395 INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); 1396 msi->isrc = isrc; 1397 *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); 1398 return (0); 1399} 1400 1401int 1402intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) 1403{ 1404 struct intr_irqsrc *isrc; 1405 struct intr_pic *pic; 1406 struct intr_map_data_msi *msi; 1407 int err; 1408 1409 pic = pic_lookup(NULL, xref, FLAG_MSI); 1410 if (pic == NULL) 1411 return (ESRCH); 1412 1413 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1414 ("%s: Found a non-MSI controller: %s", __func__, 1415 device_get_name(pic->pic_dev))); 1416 1417 msi = (struct intr_map_data_msi *) 1418 intr_map_get_map_data(irq); 1419 KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, 1420 ("%s: irq %d map data is not MSI", __func__, 1421 irq)); 1422 isrc = msi->isrc; 1423 if (isrc == NULL) { 1424 intr_unmap_irq(irq); 1425 return (EINVAL); 1426 } 1427 1428 err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); 1429 intr_unmap_irq(irq); 1430 1431 return (err); 1432} 1433 1434int 1435intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, 1436 uint64_t *addr, uint32_t *data) 1437{ 1438 struct intr_irqsrc *isrc; 1439 struct intr_pic *pic; 1440 int err; 1441 1442 pic = pic_lookup(NULL, xref, FLAG_MSI); 1443 if (pic == NULL) 1444 return (ESRCH); 1445 1446 KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, 1447 ("%s: Found a non-MSI controller: %s", __func__, 1448 device_get_name(pic->pic_dev))); 1449 1450 isrc = intr_map_get_isrc(irq); 1451 if (isrc == NULL) 1452 return (EINVAL); 1453 1454 err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); 1455 return (err); 1456} 1457 1458 1459void dosoftints(void); 1460void 1461dosoftints(void) 1462{ 1463} 1464 1465#ifdef SMP 1466/* 1467 * Init interrupt controller on another CPU. 1468 */ 1469void 1470intr_pic_init_secondary(void) 1471{ 1472 1473 /* 1474 * QQQ: Only root PIC is aware of other CPUs ??? 1475 */ 1476 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 1477 1478 //mtx_lock(&isrc_table_lock); 1479 PIC_INIT_SECONDARY(intr_irq_root_dev); 1480 //mtx_unlock(&isrc_table_lock); 1481} 1482#endif 1483 1484#ifdef DDB 1485DB_SHOW_COMMAND(irqs, db_show_irqs) 1486{ 1487 u_int i, irqsum; 1488 u_long num; 1489 struct intr_irqsrc *isrc; 1490 1491 for (irqsum = 0, i = 0; i < NIRQ; i++) { 1492 isrc = irq_sources[i]; 1493 if (isrc == NULL) 1494 continue; 1495 1496 num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; 1497 db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, 1498 isrc->isrc_name, isrc->isrc_cpu.__bits[0], 1499 isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); 1500 irqsum += num; 1501 } 1502 db_printf("irq total %u\n", irqsum); 1503} 1504#endif 1505 1506/* 1507 * Interrupt mapping table functions. 1508 * 1509 * Please, keep this part separately, it can be transformed to 1510 * extension of standard resources. 1511 */ 1512struct intr_map_entry 1513{ 1514 device_t dev; 1515 intptr_t xref; 1516 struct intr_map_data *map_data; 1517 struct intr_irqsrc *isrc; 1518 /* XXX TODO DISCONECTED PICs */ 1519 /*int flags */ 1520}; 1521 1522/* XXX Convert irq_map[] to dynamicaly expandable one. */ 1523static struct intr_map_entry *irq_map[2 * NIRQ]; 1524static int irq_map_count = nitems(irq_map); 1525static int irq_map_first_free_idx; 1526static struct mtx irq_map_lock; 1527 1528static struct intr_irqsrc * 1529intr_map_get_isrc(u_int res_id) 1530{ 1531 struct intr_irqsrc *isrc; 1532 1533 mtx_lock(&irq_map_lock); 1534 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) { 1535 mtx_unlock(&irq_map_lock); 1536 return (NULL); 1537 } 1538 isrc = irq_map[res_id]->isrc; 1539 mtx_unlock(&irq_map_lock); 1540 return (isrc); 1541} 1542 1543static void 1544intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) 1545{ 1546 1547 mtx_lock(&irq_map_lock); 1548 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) { 1549 mtx_unlock(&irq_map_lock); 1550 return; 1551 } 1552 irq_map[res_id]->isrc = isrc; 1553 mtx_unlock(&irq_map_lock); 1554} 1555 1556/* 1557 * Get a copy of intr_map_entry data 1558 */ 1559static struct intr_map_data * 1560intr_map_get_map_data(u_int res_id) 1561{ 1562 struct intr_map_data *data; 1563 1564 data = NULL; 1565 mtx_lock(&irq_map_lock); 1566 if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1567 panic("Attempt to copy invalid resource id: %u\n", res_id); 1568 data = irq_map[res_id]->map_data; 1569 mtx_unlock(&irq_map_lock); 1570 1571 return (data); 1572} 1573 1574/* 1575 * Get a copy of intr_map_entry data 1576 */ 1577static void 1578intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, 1579 struct intr_map_data **data) 1580{ 1581 size_t len; 1582 1583 len = 0; 1584 mtx_lock(&irq_map_lock); 1585 if (res_id >= irq_map_count || irq_map[res_id] == NULL) 1586 panic("Attempt to copy invalid resource id: %u\n", res_id); 1587 if (irq_map[res_id]->map_data != NULL) 1588 len = irq_map[res_id]->map_data->len; 1589 mtx_unlock(&irq_map_lock); 1590 1591 if (len == 0) 1592 *data = NULL; 1593 else 1594 *data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO); 1595 mtx_lock(&irq_map_lock); 1596 if (irq_map[res_id] == NULL) 1597 panic("Attempt to copy invalid resource id: %u\n", res_id); 1598 if (len != 0) { 1599 if (len != irq_map[res_id]->map_data->len) 1600 panic("Resource id: %u has changed.\n", res_id); 1601 memcpy(*data, irq_map[res_id]->map_data, len); 1602 } 1603 *map_dev = irq_map[res_id]->dev; 1604 *map_xref = irq_map[res_id]->xref; 1605 mtx_unlock(&irq_map_lock); 1606} 1607 1608 1609/* 1610 * Allocate and fill new entry in irq_map table. 1611 */ 1612u_int 1613intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) 1614{ 1615 u_int i; 1616 struct intr_map_entry *entry; 1617 1618 /* Prepare new entry first. */ 1619 entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO); 1620 1621 entry->dev = dev; 1622 entry->xref = xref; 1623 entry->map_data = data; 1624 entry->isrc = NULL; 1625 1626 mtx_lock(&irq_map_lock); 1627 for (i = irq_map_first_free_idx; i < irq_map_count; i++) { 1628 if (irq_map[i] == NULL) { 1629 irq_map[i] = entry; 1630 irq_map_first_free_idx = i + 1; 1631 mtx_unlock(&irq_map_lock); 1632 return (i); 1633 } 1634 } 1635 mtx_unlock(&irq_map_lock); 1636 1637 /* XXX Expand irq_map table */ 1638 panic("IRQ mapping table is full."); 1639} 1640 1641/* 1642 * Remove and free mapping entry. 1643 */ 1644void 1645intr_unmap_irq(u_int res_id) 1646{ 1647 struct intr_map_entry *entry; 1648 1649 mtx_lock(&irq_map_lock); 1650 if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) 1651 panic("Attempt to unmap invalid resource id: %u\n", res_id); 1652 entry = irq_map[res_id]; 1653 irq_map[res_id] = NULL; 1654 irq_map_first_free_idx = res_id; 1655 mtx_unlock(&irq_map_lock); 1656 intr_free_intr_map_data(entry->map_data); 1657 free(entry, M_INTRNG); 1658} 1659 1660/* 1661 * Clone mapping entry. 1662 */ 1663u_int 1664intr_map_clone_irq(u_int old_res_id) 1665{ 1666 device_t map_dev; 1667 intptr_t map_xref; 1668 struct intr_map_data *data; 1669 1670 intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); 1671 return (intr_map_irq(map_dev, map_xref, data)); 1672} 1673 1674static void 1675intr_map_init(void *dummy __unused) 1676{ 1677 1678 mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); 1679} 1680SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); 1681