1/*- 2 * Copyright (c) 2011 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * Developed by Damjan Marion <damjan.marion@gmail.com> 6 * 7 * Based on OMAP4 GIC code by Ben Gray 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. The name of the company nor the name of the author may be used to 18 * endorse or promote products derived from this software without specific 19 * prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34#include <sys/cdefs.h> 35__FBSDID("$FreeBSD: stable/11/sys/arm/arm/gic.c 329280 2018-02-14 21:39:10Z gonzo $"); 36 37#include "opt_platform.h" 38 39#include <sys/param.h> 40#include <sys/systm.h> 41#include <sys/bus.h> 42#include <sys/kernel.h> 43#include <sys/ktr.h> 44#include <sys/module.h> 45#include <sys/malloc.h> 46#include <sys/rman.h> 47#include <sys/pcpu.h> 48#include <sys/proc.h> 49#include <sys/cpuset.h> 50#include <sys/lock.h> 51#include <sys/mutex.h> 52#include <sys/smp.h> 53#ifdef INTRNG 54#include <sys/sched.h> 55#endif 56 57#include <vm/vm.h> 58#include <vm/pmap.h> 59 60#include <machine/bus.h> 61#include <machine/intr.h> 62#include <machine/smp.h> 63 64#include <dev/fdt/fdt_common.h> 65#include <dev/ofw/openfirm.h> 66#include <dev/ofw/ofw_bus.h> 67#include <dev/ofw/ofw_bus_subr.h> 68 69#ifdef INTRNG 70#include "pic_if.h" 71#include "msi_if.h" 72#endif 73 74#define GIC_DEBUG_SPURIOUS 75 76/* We are using GICv2 register naming */ 77 78/* Distributor Registers */ 79#define GICD_CTLR 0x000 /* v1 ICDDCR */ 80#define GICD_TYPER 0x004 /* v1 ICDICTR */ 81#define GICD_IIDR 0x008 /* v1 ICDIIDR */ 82#define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ 83#define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ 84#define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ 85#define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ 86#define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ 87#define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ 88#define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ 89#define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ 90#define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ 91#define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ 92#define GICD_SGI_TARGET_SHIFT 16 93 94/* CPU Registers */ 95#define GICC_CTLR 0x0000 /* v1 ICCICR */ 96#define GICC_PMR 0x0004 /* v1 ICCPMR */ 97#define GICC_BPR 0x0008 /* v1 ICCBPR */ 98#define GICC_IAR 0x000C /* v1 ICCIAR */ 99#define GICC_EOIR 0x0010 /* v1 ICCEOIR */ 100#define GICC_RPR 0x0014 /* v1 ICCRPR */ 101#define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ 102#define GICC_ABPR 0x001C /* v1 ICCABPR */ 103#define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ 104 105#define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ 106#define GIC_LAST_SGI 15 107#define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ 108#define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ 109#define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ 110 111/* TYPER Registers */ 112#define GICD_TYPER_SECURITYEXT 0x400 113#define GIC_SUPPORT_SECEXT(_sc) \ 114 ((_sc->typer & GICD_TYPER_SECURITYEXT) == GICD_TYPER_SECURITYEXT) 115 116/* First bit is a polarity bit (0 - low, 1 - high) */ 117#define GICD_ICFGR_POL_LOW (0 << 0) 118#define GICD_ICFGR_POL_HIGH (1 << 0) 119#define GICD_ICFGR_POL_MASK 0x1 120/* Second bit is a trigger bit (0 - level, 1 - edge) */ 121#define GICD_ICFGR_TRIG_LVL (0 << 1) 122#define GICD_ICFGR_TRIG_EDGE (1 << 1) 123#define GICD_ICFGR_TRIG_MASK 0x2 124 125#ifndef GIC_DEFAULT_ICFGR_INIT 126#define GIC_DEFAULT_ICFGR_INIT 0x00000000 127#endif 128 129#ifdef INTRNG 130struct gic_irqsrc { 131 struct intr_irqsrc gi_isrc; 132 uint32_t gi_irq; 133 enum intr_polarity gi_pol; 134 enum intr_trigger gi_trig; 135#define GI_FLAG_EARLY_EOI (1 << 0) 136#define GI_FLAG_MSI (1 << 1) /* This interrupt source should only */ 137 /* be used for MSI/MSI-X interrupts */ 138#define GI_FLAG_MSI_USED (1 << 2) /* This irq is already allocated */ 139 /* for a MSI/MSI-X interrupt */ 140 u_int gi_flags; 141}; 142 143static u_int gic_irq_cpu; 144static int arm_gic_intr(void *); 145static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc); 146 147#ifdef SMP 148static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1]; 149static u_int sgi_first_unused = GIC_FIRST_SGI; 150#endif 151#endif 152 153#ifdef INTRNG 154struct arm_gic_range { 155 uint64_t bus; 156 uint64_t host; 157 uint64_t size; 158}; 159 160struct arm_gic_devinfo { 161 struct ofw_bus_devinfo obdinfo; 162 struct resource_list rl; 163}; 164#endif 165 166struct arm_gic_softc { 167 device_t gic_dev; 168#ifdef INTRNG 169 void * gic_intrhand; 170 struct gic_irqsrc * gic_irqs; 171#endif 172 struct resource * gic_res[3]; 173 bus_space_tag_t gic_c_bst; 174 bus_space_tag_t gic_d_bst; 175 bus_space_handle_t gic_c_bsh; 176 bus_space_handle_t gic_d_bsh; 177 uint8_t ver; 178 struct mtx mutex; 179 uint32_t nirqs; 180 uint32_t typer; 181#ifdef GIC_DEBUG_SPURIOUS 182 uint32_t last_irq[MAXCPU]; 183#endif 184 185#ifdef INTRNG 186 /* FDT child data */ 187 pcell_t addr_cells; 188 pcell_t size_cells; 189 int nranges; 190 struct arm_gic_range * ranges; 191#endif 192}; 193 194#ifdef INTRNG 195#define GIC_INTR_ISRC(sc, irq) (&sc->gic_irqs[irq].gi_isrc) 196#endif 197 198static struct resource_spec arm_gic_spec[] = { 199 { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ 200 { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ 201#ifdef INTRNG 202 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ 203#endif 204 { -1, 0 } 205}; 206 207static u_int arm_gic_map[MAXCPU]; 208 209static struct arm_gic_softc *gic_sc = NULL; 210 211#define gic_c_read_4(_sc, _reg) \ 212 bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) 213#define gic_c_write_4(_sc, _reg, _val) \ 214 bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) 215#define gic_d_read_4(_sc, _reg) \ 216 bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) 217#define gic_d_write_1(_sc, _reg, _val) \ 218 bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 219#define gic_d_write_4(_sc, _reg, _val) \ 220 bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 221 222#ifndef INTRNG 223static int gic_config_irq(int irq, enum intr_trigger trig, 224 enum intr_polarity pol); 225static void gic_post_filter(void *); 226#endif 227 228static struct ofw_compat_data compat_data[] = { 229 {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ 230 {"arm,gic-400", true}, 231 {"arm,cortex-a15-gic", true}, 232 {"arm,cortex-a9-gic", true}, 233 {"arm,cortex-a7-gic", true}, 234 {"arm,arm11mp-gic", true}, 235 {"brcm,brahma-b15-gic", true}, 236 {"qcom,msm-qgic2", true}, 237 {NULL, false} 238}; 239 240static int 241arm_gic_probe(device_t dev) 242{ 243 244 if (!ofw_bus_status_okay(dev)) 245 return (ENXIO); 246 247 if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) 248 return (ENXIO); 249 device_set_desc(dev, "ARM Generic Interrupt Controller"); 250 return (BUS_PROBE_DEFAULT); 251} 252 253#ifdef INTRNG 254static inline void 255gic_irq_unmask(struct arm_gic_softc *sc, u_int irq) 256{ 257 258 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 259} 260 261static inline void 262gic_irq_mask(struct arm_gic_softc *sc, u_int irq) 263{ 264 265 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 266} 267#endif 268 269static uint8_t 270gic_cpu_mask(struct arm_gic_softc *sc) 271{ 272 uint32_t mask; 273 int i; 274 275 /* Read the current cpuid mask by reading ITARGETSR{0..7} */ 276 for (i = 0; i < 8; i++) { 277 mask = gic_d_read_4(sc, GICD_ITARGETSR(i)); 278 if (mask != 0) 279 break; 280 } 281 /* No mask found, assume we are on CPU interface 0 */ 282 if (mask == 0) 283 return (1); 284 285 /* Collect the mask in the lower byte */ 286 mask |= mask >> 16; 287 mask |= mask >> 8; 288 289 return (mask); 290} 291 292#ifdef SMP 293#ifdef INTRNG 294static void 295arm_gic_init_secondary(device_t dev) 296{ 297 struct arm_gic_softc *sc = device_get_softc(dev); 298 u_int irq, cpu; 299 300 /* Set the mask so we can find this CPU to send it IPIs */ 301 cpu = PCPU_GET(cpuid); 302 arm_gic_map[cpu] = gic_cpu_mask(sc); 303 304 for (irq = 0; irq < sc->nirqs; irq += 4) 305 gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0); 306 307 /* Set all the interrupts to be in Group 0 (secure) */ 308 for (irq = 0; GIC_SUPPORT_SECEXT(sc) && irq < sc->nirqs; irq += 32) { 309 gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0); 310 } 311 312 /* Enable CPU interface */ 313 gic_c_write_4(sc, GICC_CTLR, 1); 314 315 /* Set priority mask register. */ 316 gic_c_write_4(sc, GICC_PMR, 0xff); 317 318 /* Enable interrupt distribution */ 319 gic_d_write_4(sc, GICD_CTLR, 0x01); 320 321 /* Unmask attached SGI interrupts. */ 322 for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) 323 if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) 324 gic_irq_unmask(sc, irq); 325 326 /* Unmask attached PPI interrupts. */ 327 for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) 328 if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) 329 gic_irq_unmask(sc, irq); 330} 331#else 332static void 333arm_gic_init_secondary(device_t dev) 334{ 335 struct arm_gic_softc *sc = device_get_softc(dev); 336 int i; 337 338 /* Set the mask so we can find this CPU to send it IPIs */ 339 arm_gic_map[PCPU_GET(cpuid)] = gic_cpu_mask(sc); 340 341 for (i = 0; i < sc->nirqs; i += 4) 342 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 343 344 /* Set all the interrupts to be in Group 0 (secure) */ 345 for (i = 0; GIC_SUPPORT_SECEXT(sc) && i < sc->nirqs; i += 32) { 346 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 347 } 348 349 /* Enable CPU interface */ 350 gic_c_write_4(sc, GICC_CTLR, 1); 351 352 /* Set priority mask register. */ 353 gic_c_write_4(sc, GICC_PMR, 0xff); 354 355 /* Enable interrupt distribution */ 356 gic_d_write_4(sc, GICD_CTLR, 0x01); 357 358 /* 359 * Activate the timer interrupts: virtual, secure, and non-secure. 360 */ 361 gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); 362 gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); 363 gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); 364} 365#endif /* INTRNG */ 366#endif /* SMP */ 367 368#ifndef INTRNG 369int 370gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, 371 int *trig, int *pol) 372{ 373 static u_int num_intr_cells; 374 static phandle_t self; 375 struct ofw_compat_data *ocd; 376 377 if (self == 0) { 378 for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) { 379 if (fdt_is_compatible(iparent, ocd->ocd_str)) { 380 self = iparent; 381 break; 382 } 383 } 384 } 385 if (self != iparent) 386 return (ENXIO); 387 388 if (num_intr_cells == 0) { 389 if (OF_searchencprop(OF_node_from_xref(iparent), 390 "#interrupt-cells", &num_intr_cells, 391 sizeof(num_intr_cells)) == -1) { 392 num_intr_cells = 1; 393 } 394 } 395 396 if (num_intr_cells == 1) { 397 *interrupt = fdt32_to_cpu(intr[0]); 398 *trig = INTR_TRIGGER_CONFORM; 399 *pol = INTR_POLARITY_CONFORM; 400 } else { 401 if (fdt32_to_cpu(intr[0]) == 0) 402 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; 403 else 404 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; 405 /* 406 * In intr[2], bits[3:0] are trigger type and level flags. 407 * 1 = low-to-high edge triggered 408 * 2 = high-to-low edge triggered 409 * 4 = active high level-sensitive 410 * 8 = active low level-sensitive 411 * The hardware only supports active-high-level or rising-edge 412 * for SPIs 413 */ 414 if (*interrupt >= GIC_FIRST_SPI && 415 fdt32_to_cpu(intr[2]) & 0x0a) { 416 printf("unsupported trigger/polarity configuration " 417 "0x%02x\n", fdt32_to_cpu(intr[2]) & 0x0f); 418 } 419 *pol = INTR_POLARITY_CONFORM; 420 if (fdt32_to_cpu(intr[2]) & 0x03) 421 *trig = INTR_TRIGGER_EDGE; 422 else 423 *trig = INTR_TRIGGER_LEVEL; 424 } 425 return (0); 426} 427#endif 428 429#ifdef INTRNG 430static inline intptr_t 431gic_xref(device_t dev) 432{ 433#ifdef FDT 434 return (OF_xref_from_node(ofw_bus_get_node(dev))); 435#else 436 return (0); 437#endif 438} 439 440static int 441arm_gic_register_isrcs(struct arm_gic_softc *sc, uint32_t num) 442{ 443 int error; 444 uint32_t irq; 445 struct gic_irqsrc *irqs; 446 struct intr_irqsrc *isrc; 447 const char *name; 448 449 irqs = malloc(num * sizeof(struct gic_irqsrc), M_DEVBUF, 450 M_WAITOK | M_ZERO); 451 452 name = device_get_nameunit(sc->gic_dev); 453 for (irq = 0; irq < num; irq++) { 454 irqs[irq].gi_irq = irq; 455 irqs[irq].gi_pol = INTR_POLARITY_CONFORM; 456 irqs[irq].gi_trig = INTR_TRIGGER_CONFORM; 457 458 isrc = &irqs[irq].gi_isrc; 459 if (irq <= GIC_LAST_SGI) { 460 error = intr_isrc_register(isrc, sc->gic_dev, 461 INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI); 462 } else if (irq <= GIC_LAST_PPI) { 463 error = intr_isrc_register(isrc, sc->gic_dev, 464 INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI); 465 } else { 466 error = intr_isrc_register(isrc, sc->gic_dev, 0, 467 "%s,s%u", name, irq - GIC_FIRST_SPI); 468 } 469 if (error != 0) { 470 /* XXX call intr_isrc_deregister() */ 471 free(irqs, M_DEVBUF); 472 return (error); 473 } 474 } 475 sc->gic_irqs = irqs; 476 sc->nirqs = num; 477 return (0); 478} 479 480static int 481arm_gic_fill_ranges(phandle_t node, struct arm_gic_softc *sc) 482{ 483 pcell_t host_cells; 484 cell_t *base_ranges; 485 ssize_t nbase_ranges; 486 int i, j, k; 487 488 host_cells = 1; 489 OF_getencprop(OF_parent(node), "#address-cells", &host_cells, 490 sizeof(host_cells)); 491 sc->addr_cells = 2; 492 OF_getencprop(node, "#address-cells", &sc->addr_cells, 493 sizeof(sc->addr_cells)); 494 sc->size_cells = 2; 495 OF_getencprop(node, "#size-cells", &sc->size_cells, 496 sizeof(sc->size_cells)); 497 498 nbase_ranges = OF_getproplen(node, "ranges"); 499 if (nbase_ranges < 0) 500 return (-1); 501 sc->nranges = nbase_ranges / sizeof(cell_t) / 502 (sc->addr_cells + host_cells + sc->size_cells); 503 if (sc->nranges == 0) 504 return (0); 505 506 sc->ranges = malloc(sc->nranges * sizeof(sc->ranges[0]), 507 M_DEVBUF, M_WAITOK); 508 base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); 509 OF_getencprop(node, "ranges", base_ranges, nbase_ranges); 510 511 for (i = 0, j = 0; i < sc->nranges; i++) { 512 sc->ranges[i].bus = 0; 513 for (k = 0; k < sc->addr_cells; k++) { 514 sc->ranges[i].bus <<= 32; 515 sc->ranges[i].bus |= base_ranges[j++]; 516 } 517 sc->ranges[i].host = 0; 518 for (k = 0; k < host_cells; k++) { 519 sc->ranges[i].host <<= 32; 520 sc->ranges[i].host |= base_ranges[j++]; 521 } 522 sc->ranges[i].size = 0; 523 for (k = 0; k < sc->size_cells; k++) { 524 sc->ranges[i].size <<= 32; 525 sc->ranges[i].size |= base_ranges[j++]; 526 } 527 } 528 529 free(base_ranges, M_DEVBUF); 530 return (sc->nranges); 531} 532 533static bool 534arm_gic_add_children(device_t dev) 535{ 536 struct arm_gic_softc *sc; 537 struct arm_gic_devinfo *dinfo; 538 phandle_t child, node; 539 device_t cdev; 540 541 sc = device_get_softc(dev); 542 node = ofw_bus_get_node(dev); 543 544 /* If we have no children don't probe for them */ 545 child = OF_child(node); 546 if (child == 0) 547 return (false); 548 549 if (arm_gic_fill_ranges(node, sc) < 0) { 550 device_printf(dev, "Have a child, but no ranges\n"); 551 return (false); 552 } 553 554 for (; child != 0; child = OF_peer(child)) { 555 dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); 556 557 if (ofw_bus_gen_setup_devinfo(&dinfo->obdinfo, child) != 0) { 558 free(dinfo, M_DEVBUF); 559 continue; 560 } 561 562 resource_list_init(&dinfo->rl); 563 ofw_bus_reg_to_rl(dev, child, sc->addr_cells, 564 sc->size_cells, &dinfo->rl); 565 566 cdev = device_add_child(dev, NULL, -1); 567 if (cdev == NULL) { 568 device_printf(dev, "<%s>: device_add_child failed\n", 569 dinfo->obdinfo.obd_name); 570 resource_list_free(&dinfo->rl); 571 ofw_bus_gen_destroy_devinfo(&dinfo->obdinfo); 572 free(dinfo, M_DEVBUF); 573 continue; 574 } 575 device_set_ivars(cdev, dinfo); 576 } 577 578 return (true); 579} 580 581static void 582arm_gic_reserve_msi_range(device_t dev, u_int start, u_int count) 583{ 584 struct arm_gic_softc *sc; 585 int i; 586 587 sc = device_get_softc(dev); 588 589 KASSERT((start + count) < sc->nirqs, 590 ("%s: Trying to allocate too many MSI IRQs: %d + %d > %d", __func__, 591 start, count, sc->nirqs)); 592 for (i = 0; i < count; i++) { 593 KASSERT(sc->gic_irqs[start + i].gi_isrc.isrc_handlers == 0, 594 ("%s: MSI interrupt %d already has a handler", __func__, 595 count + i)); 596 KASSERT(sc->gic_irqs[start + i].gi_pol == INTR_POLARITY_CONFORM, 597 ("%s: MSI interrupt %d already has a polarity", __func__, 598 count + i)); 599 KASSERT(sc->gic_irqs[start + i].gi_trig == INTR_TRIGGER_CONFORM, 600 ("%s: MSI interrupt %d already has a trigger", __func__, 601 count + i)); 602 sc->gic_irqs[start + i].gi_pol = INTR_POLARITY_HIGH; 603 sc->gic_irqs[start + i].gi_trig = INTR_TRIGGER_EDGE; 604 sc->gic_irqs[start + i].gi_flags |= GI_FLAG_MSI; 605 } 606} 607#endif 608 609static int 610arm_gic_attach(device_t dev) 611{ 612 struct arm_gic_softc *sc; 613 int i; 614 uint32_t icciidr, mask, nirqs; 615#ifdef INTRNG 616 phandle_t pxref; 617 intptr_t xref = gic_xref(dev); 618#endif 619 620 if (gic_sc) 621 return (ENXIO); 622 623 sc = device_get_softc(dev); 624 625 if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { 626 device_printf(dev, "could not allocate resources\n"); 627 return (ENXIO); 628 } 629 630 sc->gic_dev = dev; 631 gic_sc = sc; 632 633 /* Initialize mutex */ 634 mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); 635 636 /* Distributor Interface */ 637 sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); 638 sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); 639 640 /* CPU Interface */ 641 sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); 642 sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); 643 644 /* Disable interrupt forwarding to the CPU interface */ 645 gic_d_write_4(sc, GICD_CTLR, 0x00); 646 647 /* Get the number of interrupts */ 648 sc->typer = gic_d_read_4(sc, GICD_TYPER); 649 nirqs = 32 * ((sc->typer & 0x1f) + 1); 650 651#ifdef INTRNG 652 if (arm_gic_register_isrcs(sc, nirqs)) { 653 device_printf(dev, "could not register irqs\n"); 654 goto cleanup; 655 } 656#else 657 sc->nirqs = nirqs; 658 659 /* Set up function pointers */ 660 arm_post_filter = gic_post_filter; 661 arm_config_irq = gic_config_irq; 662#endif 663 664 icciidr = gic_c_read_4(sc, GICC_IIDR); 665 device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", 666 icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, 667 (icciidr & 0xfff), sc->nirqs); 668 669 /* Set all global interrupts to be level triggered, active low. */ 670 for (i = 32; i < sc->nirqs; i += 16) { 671 gic_d_write_4(sc, GICD_ICFGR(i >> 4), GIC_DEFAULT_ICFGR_INIT); 672 } 673 674 /* Disable all interrupts. */ 675 for (i = 32; i < sc->nirqs; i += 32) { 676 gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); 677 } 678 679 /* Find the current cpu mask */ 680 mask = gic_cpu_mask(sc); 681 /* Set the mask so we can find this CPU to send it IPIs */ 682 arm_gic_map[PCPU_GET(cpuid)] = mask; 683 /* Set all four targets to this cpu */ 684 mask |= mask << 8; 685 mask |= mask << 16; 686 687 for (i = 0; i < sc->nirqs; i += 4) { 688 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 689 if (i > 32) { 690 gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), mask); 691 } 692 } 693 694 /* Set all the interrupts to be in Group 0 (secure) */ 695 for (i = 0; GIC_SUPPORT_SECEXT(sc) && i < sc->nirqs; i += 32) { 696 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 697 } 698 699 /* Enable CPU interface */ 700 gic_c_write_4(sc, GICC_CTLR, 1); 701 702 /* Set priority mask register. */ 703 gic_c_write_4(sc, GICC_PMR, 0xff); 704 705 /* Enable interrupt distribution */ 706 gic_d_write_4(sc, GICD_CTLR, 0x01); 707#ifndef INTRNG 708 return (0); 709#else 710 /* 711 * Now, when everything is initialized, it's right time to 712 * register interrupt controller to interrupt framefork. 713 */ 714 if (intr_pic_register(dev, xref) == NULL) { 715 device_printf(dev, "could not register PIC\n"); 716 goto cleanup; 717 } 718 719 /* 720 * Controller is root if: 721 * - doesn't have interrupt parent 722 * - his interrupt parent is this controller 723 */ 724 pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); 725 if (pxref == 0 || xref == pxref) { 726 if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, 727 GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { 728 device_printf(dev, "could not set PIC as a root\n"); 729 intr_pic_deregister(dev, xref); 730 goto cleanup; 731 } 732 } else { 733 if (sc->gic_res[2] == NULL) { 734 device_printf(dev, 735 "not root PIC must have defined interrupt\n"); 736 intr_pic_deregister(dev, xref); 737 goto cleanup; 738 } 739 if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, 740 arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { 741 device_printf(dev, "could not setup irq handler\n"); 742 intr_pic_deregister(dev, xref); 743 goto cleanup; 744 } 745 } 746 747 OF_device_register_xref(xref, dev); 748 749 /* If we have children probe and attach them */ 750 if (arm_gic_add_children(dev)) { 751 bus_generic_probe(dev); 752 return (bus_generic_attach(dev)); 753 } 754 755 return (0); 756 757cleanup: 758 /* 759 * XXX - not implemented arm_gic_detach() should be called ! 760 */ 761 if (sc->gic_irqs != NULL) 762 free(sc->gic_irqs, M_DEVBUF); 763 bus_release_resources(dev, arm_gic_spec, sc->gic_res); 764 return(ENXIO); 765#endif 766} 767 768#ifdef INTRNG 769static struct resource * 770arm_gic_alloc_resource(device_t bus, device_t child, int type, int *rid, 771 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) 772{ 773 struct arm_gic_softc *sc; 774 struct arm_gic_devinfo *di; 775 struct resource_list_entry *rle; 776 int j; 777 778 KASSERT(type == SYS_RES_MEMORY, ("Invalid resoure type %x", type)); 779 780 sc = device_get_softc(bus); 781 782 /* 783 * Request for the default allocation with a given rid: use resource 784 * list stored in the local device info. 785 */ 786 if (RMAN_IS_DEFAULT_RANGE(start, end)) { 787 if ((di = device_get_ivars(child)) == NULL) 788 return (NULL); 789 790 if (type == SYS_RES_IOPORT) 791 type = SYS_RES_MEMORY; 792 793 rle = resource_list_find(&di->rl, type, *rid); 794 if (rle == NULL) { 795 if (bootverbose) 796 device_printf(bus, "no default resources for " 797 "rid = %d, type = %d\n", *rid, type); 798 return (NULL); 799 } 800 start = rle->start; 801 end = rle->end; 802 count = rle->count; 803 } 804 805 /* Remap through ranges property */ 806 for (j = 0; j < sc->nranges; j++) { 807 if (start >= sc->ranges[j].bus && end < 808 sc->ranges[j].bus + sc->ranges[j].size) { 809 start -= sc->ranges[j].bus; 810 start += sc->ranges[j].host; 811 end -= sc->ranges[j].bus; 812 end += sc->ranges[j].host; 813 break; 814 } 815 } 816 if (j == sc->nranges && sc->nranges != 0) { 817 if (bootverbose) 818 device_printf(bus, "Could not map resource " 819 "%#jx-%#jx\n", (uintmax_t)start, (uintmax_t)end); 820 821 return (NULL); 822 } 823 824 return (bus_generic_alloc_resource(bus, child, type, rid, start, end, 825 count, flags)); 826} 827 828static const struct ofw_bus_devinfo * 829arm_gic_ofw_get_devinfo(device_t bus __unused, device_t child) 830{ 831 struct arm_gic_devinfo *di; 832 833 di = device_get_ivars(child); 834 835 return (&di->obdinfo); 836} 837 838static int 839arm_gic_intr(void *arg) 840{ 841 struct arm_gic_softc *sc = arg; 842 struct gic_irqsrc *gi; 843 uint32_t irq_active_reg, irq; 844 struct trapframe *tf; 845 846 irq_active_reg = gic_c_read_4(sc, GICC_IAR); 847 irq = irq_active_reg & 0x3FF; 848 849 /* 850 * 1. We do EOI here because recent read value from active interrupt 851 * register must be used for it. Another approach is to save this 852 * value into associated interrupt source. 853 * 2. EOI must be done on same CPU where interrupt has fired. Thus 854 * we must ensure that interrupted thread does not migrate to 855 * another CPU. 856 * 3. EOI cannot be delayed by any preemption which could happen on 857 * critical_exit() used in MI intr code, when interrupt thread is 858 * scheduled. See next point. 859 * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during 860 * an action and any use of critical_exit() could break this 861 * assumption. See comments within smp_rendezvous_action(). 862 * 5. We always return FILTER_HANDLED as this is an interrupt 863 * controller dispatch function. Otherwise, in cascaded interrupt 864 * case, the whole interrupt subtree would be masked. 865 */ 866 867 if (irq >= sc->nirqs) { 868#ifdef GIC_DEBUG_SPURIOUS 869 device_printf(sc->gic_dev, 870 "Spurious interrupt detected: last irq: %d on CPU%d\n", 871 sc->last_irq[PCPU_GET(cpuid)], PCPU_GET(cpuid)); 872#endif 873 return (FILTER_HANDLED); 874 } 875 876 tf = curthread->td_intr_frame; 877dispatch_irq: 878 gi = sc->gic_irqs + irq; 879 /* 880 * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement 881 * as compiler complains that comparing u_int >= 0 is always true. 882 */ 883 if (irq <= GIC_LAST_SGI) { 884#ifdef SMP 885 /* Call EOI for all IPI before dispatch. */ 886 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 887 intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf); 888 goto next_irq; 889#else 890 device_printf(sc->gic_dev, "SGI %u on UP system detected\n", 891 irq - GIC_FIRST_SGI); 892 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 893 goto next_irq; 894#endif 895 } 896 897#ifdef GIC_DEBUG_SPURIOUS 898 sc->last_irq[PCPU_GET(cpuid)] = irq; 899#endif 900 if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) 901 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 902 903 if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) { 904 gic_irq_mask(sc, irq); 905 if ((gi->gi_flags & GI_FLAG_EARLY_EOI) != GI_FLAG_EARLY_EOI) 906 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 907 device_printf(sc->gic_dev, "Stray irq %u disabled\n", irq); 908 } 909 910next_irq: 911 arm_irq_memory_barrier(irq); 912 irq_active_reg = gic_c_read_4(sc, GICC_IAR); 913 irq = irq_active_reg & 0x3FF; 914 if (irq < sc->nirqs) 915 goto dispatch_irq; 916 917 return (FILTER_HANDLED); 918} 919 920static void 921gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig, 922 enum intr_polarity pol) 923{ 924 uint32_t reg; 925 uint32_t mask; 926 927 if (irq < GIC_FIRST_SPI) 928 return; 929 930 mtx_lock_spin(&sc->mutex); 931 932 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 933 mask = (reg >> 2*(irq % 16)) & 0x3; 934 935 if (pol == INTR_POLARITY_LOW) { 936 mask &= ~GICD_ICFGR_POL_MASK; 937 mask |= GICD_ICFGR_POL_LOW; 938 } else if (pol == INTR_POLARITY_HIGH) { 939 mask &= ~GICD_ICFGR_POL_MASK; 940 mask |= GICD_ICFGR_POL_HIGH; 941 } 942 943 if (trig == INTR_TRIGGER_LEVEL) { 944 mask &= ~GICD_ICFGR_TRIG_MASK; 945 mask |= GICD_ICFGR_TRIG_LVL; 946 } else if (trig == INTR_TRIGGER_EDGE) { 947 mask &= ~GICD_ICFGR_TRIG_MASK; 948 mask |= GICD_ICFGR_TRIG_EDGE; 949 } 950 951 /* Set mask */ 952 reg = reg & ~(0x3 << 2*(irq % 16)); 953 reg = reg | (mask << 2*(irq % 16)); 954 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 955 956 mtx_unlock_spin(&sc->mutex); 957} 958 959static int 960gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus) 961{ 962 uint32_t cpu, end, mask; 963 964 end = min(mp_ncpus, 8); 965 for (cpu = end; cpu < MAXCPU; cpu++) 966 if (CPU_ISSET(cpu, cpus)) 967 return (EINVAL); 968 969 for (mask = 0, cpu = 0; cpu < end; cpu++) 970 if (CPU_ISSET(cpu, cpus)) 971 mask |= arm_gic_map[cpu]; 972 973 gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask); 974 return (0); 975} 976 977#ifdef FDT 978static int 979gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp, 980 enum intr_polarity *polp, enum intr_trigger *trigp) 981{ 982 983 if (ncells == 1) { 984 *irqp = cells[0]; 985 *polp = INTR_POLARITY_CONFORM; 986 *trigp = INTR_TRIGGER_CONFORM; 987 return (0); 988 } 989 if (ncells == 3) { 990 u_int irq, tripol; 991 992 /* 993 * The 1st cell is the interrupt type: 994 * 0 = SPI 995 * 1 = PPI 996 * The 2nd cell contains the interrupt number: 997 * [0 - 987] for SPI 998 * [0 - 15] for PPI 999 * The 3rd cell is the flags, encoded as follows: 1000 * bits[3:0] trigger type and level flags 1001 * 1 = low-to-high edge triggered 1002 * 2 = high-to-low edge triggered 1003 * 4 = active high level-sensitive 1004 * 8 = active low level-sensitive 1005 * bits[15:8] PPI interrupt cpu mask 1006 * Each bit corresponds to each of the 8 possible cpus 1007 * attached to the GIC. A bit set to '1' indicated 1008 * the interrupt is wired to that CPU. 1009 */ 1010 switch (cells[0]) { 1011 case 0: 1012 irq = GIC_FIRST_SPI + cells[1]; 1013 /* SPI irq is checked later. */ 1014 break; 1015 case 1: 1016 irq = GIC_FIRST_PPI + cells[1]; 1017 if (irq > GIC_LAST_PPI) { 1018 device_printf(dev, "unsupported PPI interrupt " 1019 "number %u\n", cells[1]); 1020 return (EINVAL); 1021 } 1022 break; 1023 default: 1024 device_printf(dev, "unsupported interrupt type " 1025 "configuration %u\n", cells[0]); 1026 return (EINVAL); 1027 } 1028 1029 tripol = cells[2] & 0xff; 1030 if (tripol & 0xf0 || (tripol & 0x0a && cells[0] == 0)) 1031 device_printf(dev, "unsupported trigger/polarity " 1032 "configuration 0x%02x\n", tripol); 1033 1034 *irqp = irq; 1035 *polp = INTR_POLARITY_CONFORM; 1036 *trigp = tripol & 0x03 ? INTR_TRIGGER_EDGE : INTR_TRIGGER_LEVEL; 1037 return (0); 1038 } 1039 return (EINVAL); 1040} 1041#endif 1042 1043static int 1044gic_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp, 1045 enum intr_polarity *polp, enum intr_trigger *trigp) 1046{ 1047 u_int irq; 1048 enum intr_polarity pol; 1049 enum intr_trigger trig; 1050 struct arm_gic_softc *sc; 1051#ifdef FDT 1052 struct intr_map_data_fdt *daf; 1053#endif 1054 1055 sc = device_get_softc(dev); 1056 switch (data->type) { 1057#ifdef FDT 1058 case INTR_MAP_DATA_FDT: 1059 daf = (struct intr_map_data_fdt *)data; 1060 if (gic_map_fdt(dev, daf->ncells, daf->cells, &irq, &pol, 1061 &trig) != 0) 1062 return (EINVAL); 1063 KASSERT(irq >= sc->nirqs || 1064 (sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) == 0, 1065 ("%s: Attempting to map a MSI interrupt from FDT", 1066 __func__)); 1067 break; 1068#endif 1069 default: 1070 return (ENOTSUP); 1071 } 1072 1073 if (irq >= sc->nirqs) 1074 return (EINVAL); 1075 if (pol != INTR_POLARITY_CONFORM && pol != INTR_POLARITY_LOW && 1076 pol != INTR_POLARITY_HIGH) 1077 return (EINVAL); 1078 if (trig != INTR_TRIGGER_CONFORM && trig != INTR_TRIGGER_EDGE && 1079 trig != INTR_TRIGGER_LEVEL) 1080 return (EINVAL); 1081 1082 *irqp = irq; 1083 if (polp != NULL) 1084 *polp = pol; 1085 if (trigp != NULL) 1086 *trigp = trig; 1087 return (0); 1088} 1089 1090static int 1091arm_gic_map_intr(device_t dev, struct intr_map_data *data, 1092 struct intr_irqsrc **isrcp) 1093{ 1094 int error; 1095 u_int irq; 1096 struct arm_gic_softc *sc; 1097 1098 error = gic_map_intr(dev, data, &irq, NULL, NULL); 1099 if (error == 0) { 1100 sc = device_get_softc(dev); 1101 *isrcp = GIC_INTR_ISRC(sc, irq); 1102 } 1103 return (error); 1104} 1105 1106static int 1107arm_gic_setup_intr(device_t dev, struct intr_irqsrc *isrc, 1108 struct resource *res, struct intr_map_data *data) 1109{ 1110 struct arm_gic_softc *sc = device_get_softc(dev); 1111 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1112 enum intr_trigger trig; 1113 enum intr_polarity pol; 1114 1115 if ((gi->gi_flags & GI_FLAG_MSI) == GI_FLAG_MSI) { 1116 pol = gi->gi_pol; 1117 trig = gi->gi_trig; 1118 KASSERT(pol == INTR_POLARITY_HIGH, 1119 ("%s: MSI interrupts must be active-high", __func__)); 1120 KASSERT(trig == INTR_TRIGGER_EDGE, 1121 ("%s: MSI interrupts must be edge triggered", __func__)); 1122 } else if (data != NULL) { 1123 u_int irq; 1124 1125 /* Get config for resource. */ 1126 if (gic_map_intr(dev, data, &irq, &pol, &trig) || 1127 gi->gi_irq != irq) 1128 return (EINVAL); 1129 } else { 1130 pol = INTR_POLARITY_CONFORM; 1131 trig = INTR_TRIGGER_CONFORM; 1132 } 1133 1134 /* Compare config if this is not first setup. */ 1135 if (isrc->isrc_handlers != 0) { 1136 if ((pol != INTR_POLARITY_CONFORM && pol != gi->gi_pol) || 1137 (trig != INTR_TRIGGER_CONFORM && trig != gi->gi_trig)) 1138 return (EINVAL); 1139 else 1140 return (0); 1141 } 1142 1143 /* For MSI/MSI-X we should have already configured these */ 1144 if ((gi->gi_flags & GI_FLAG_MSI) == 0) { 1145 if (pol == INTR_POLARITY_CONFORM) 1146 pol = INTR_POLARITY_LOW; /* just pick some */ 1147 if (trig == INTR_TRIGGER_CONFORM) 1148 trig = INTR_TRIGGER_EDGE; /* just pick some */ 1149 1150 gi->gi_pol = pol; 1151 gi->gi_trig = trig; 1152 1153 /* Edge triggered interrupts need an early EOI sent */ 1154 if (gi->gi_pol == INTR_TRIGGER_EDGE) 1155 gi->gi_flags |= GI_FLAG_EARLY_EOI; 1156 } 1157 1158 /* 1159 * XXX - In case that per CPU interrupt is going to be enabled in time 1160 * when SMP is already started, we need some IPI call which 1161 * enables it on others CPUs. Further, it's more complicated as 1162 * pic_enable_source() and pic_disable_source() should act on 1163 * per CPU basis only. Thus, it should be solved here somehow. 1164 */ 1165 if (isrc->isrc_flags & INTR_ISRCF_PPI) 1166 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 1167 1168 gic_config(sc, gi->gi_irq, gi->gi_trig, gi->gi_pol); 1169 arm_gic_bind_intr(dev, isrc); 1170 return (0); 1171} 1172 1173static int 1174arm_gic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, 1175 struct resource *res, struct intr_map_data *data) 1176{ 1177 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1178 1179 if (isrc->isrc_handlers == 0 && (gi->gi_flags & GI_FLAG_MSI) == 0) { 1180 gi->gi_pol = INTR_POLARITY_CONFORM; 1181 gi->gi_trig = INTR_TRIGGER_CONFORM; 1182 } 1183 return (0); 1184} 1185 1186static void 1187arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc) 1188{ 1189 struct arm_gic_softc *sc = device_get_softc(dev); 1190 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1191 1192 arm_irq_memory_barrier(gi->gi_irq); 1193 gic_irq_unmask(sc, gi->gi_irq); 1194} 1195 1196static void 1197arm_gic_disable_intr(device_t dev, struct intr_irqsrc *isrc) 1198{ 1199 struct arm_gic_softc *sc = device_get_softc(dev); 1200 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1201 1202 gic_irq_mask(sc, gi->gi_irq); 1203} 1204 1205static void 1206arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 1207{ 1208 struct arm_gic_softc *sc = device_get_softc(dev); 1209 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1210 1211 arm_gic_disable_intr(dev, isrc); 1212 gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); 1213} 1214 1215static void 1216arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc) 1217{ 1218 1219 arm_irq_memory_barrier(0); 1220 arm_gic_enable_intr(dev, isrc); 1221} 1222 1223static void 1224arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc) 1225{ 1226 struct arm_gic_softc *sc = device_get_softc(dev); 1227 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1228 1229 /* EOI for edge-triggered done earlier. */ 1230 if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) 1231 return; 1232 1233 arm_irq_memory_barrier(0); 1234 gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); 1235} 1236 1237static int 1238arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc) 1239{ 1240 struct arm_gic_softc *sc = device_get_softc(dev); 1241 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1242 1243 if (gi->gi_irq < GIC_FIRST_SPI) 1244 return (EINVAL); 1245 1246 if (CPU_EMPTY(&isrc->isrc_cpu)) { 1247 gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); 1248 CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); 1249 } 1250 return (gic_bind(sc, gi->gi_irq, &isrc->isrc_cpu)); 1251} 1252 1253#ifdef SMP 1254static void 1255arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, 1256 u_int ipi) 1257{ 1258 struct arm_gic_softc *sc = device_get_softc(dev); 1259 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1260 uint32_t val = 0, i; 1261 1262 for (i = 0; i < MAXCPU; i++) 1263 if (CPU_ISSET(i, &cpus)) 1264 val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; 1265 1266 gic_d_write_4(sc, GICD_SGIR(0), val | gi->gi_irq); 1267} 1268 1269static int 1270arm_gic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) 1271{ 1272 struct intr_irqsrc *isrc; 1273 struct arm_gic_softc *sc = device_get_softc(dev); 1274 1275 if (sgi_first_unused > GIC_LAST_SGI) 1276 return (ENOSPC); 1277 1278 isrc = GIC_INTR_ISRC(sc, sgi_first_unused); 1279 sgi_to_ipi[sgi_first_unused++] = ipi; 1280 1281 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 1282 1283 *isrcp = isrc; 1284 return (0); 1285} 1286#endif 1287#else 1288static int 1289arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq) 1290{ 1291 uint32_t active_irq; 1292 1293 active_irq = gic_c_read_4(sc, GICC_IAR); 1294 1295 /* 1296 * Immediately EOIR the SGIs, because doing so requires the other 1297 * bits (ie CPU number), not just the IRQ number, and we do not 1298 * have this information later. 1299 */ 1300 if ((active_irq & 0x3ff) <= GIC_LAST_SGI) 1301 gic_c_write_4(sc, GICC_EOIR, active_irq); 1302 active_irq &= 0x3FF; 1303 1304 if (active_irq == 0x3FF) { 1305 if (last_irq == -1) 1306 device_printf(sc->gic_dev, 1307 "Spurious interrupt detected\n"); 1308 return -1; 1309 } 1310 1311 return active_irq; 1312} 1313 1314static int 1315arm_gic_config(device_t dev, int irq, enum intr_trigger trig, 1316 enum intr_polarity pol) 1317{ 1318 struct arm_gic_softc *sc = device_get_softc(dev); 1319 uint32_t reg; 1320 uint32_t mask; 1321 1322 /* Function is public-accessible, so validate input arguments */ 1323 if ((irq < 0) || (irq >= sc->nirqs)) 1324 goto invalid_args; 1325 if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && 1326 (trig != INTR_TRIGGER_CONFORM)) 1327 goto invalid_args; 1328 if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && 1329 (pol != INTR_POLARITY_CONFORM)) 1330 goto invalid_args; 1331 1332 mtx_lock_spin(&sc->mutex); 1333 1334 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 1335 mask = (reg >> 2*(irq % 16)) & 0x3; 1336 1337 if (pol == INTR_POLARITY_LOW) { 1338 mask &= ~GICD_ICFGR_POL_MASK; 1339 mask |= GICD_ICFGR_POL_LOW; 1340 } else if (pol == INTR_POLARITY_HIGH) { 1341 mask &= ~GICD_ICFGR_POL_MASK; 1342 mask |= GICD_ICFGR_POL_HIGH; 1343 } 1344 1345 if (trig == INTR_TRIGGER_LEVEL) { 1346 mask &= ~GICD_ICFGR_TRIG_MASK; 1347 mask |= GICD_ICFGR_TRIG_LVL; 1348 } else if (trig == INTR_TRIGGER_EDGE) { 1349 mask &= ~GICD_ICFGR_TRIG_MASK; 1350 mask |= GICD_ICFGR_TRIG_EDGE; 1351 } 1352 1353 /* Set mask */ 1354 reg = reg & ~(0x3 << 2*(irq % 16)); 1355 reg = reg | (mask << 2*(irq % 16)); 1356 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 1357 1358 mtx_unlock_spin(&sc->mutex); 1359 1360 return (0); 1361 1362invalid_args: 1363 device_printf(dev, "gic_config_irg, invalid parameters\n"); 1364 return (EINVAL); 1365} 1366 1367 1368static void 1369arm_gic_mask(device_t dev, int irq) 1370{ 1371 struct arm_gic_softc *sc = device_get_softc(dev); 1372 1373 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 1374 gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */ 1375} 1376 1377static void 1378arm_gic_unmask(device_t dev, int irq) 1379{ 1380 struct arm_gic_softc *sc = device_get_softc(dev); 1381 1382 if (irq > GIC_LAST_SGI) 1383 arm_irq_memory_barrier(irq); 1384 1385 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 1386} 1387 1388#ifdef SMP 1389static void 1390arm_gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) 1391{ 1392 struct arm_gic_softc *sc = device_get_softc(dev); 1393 uint32_t val = 0, i; 1394 1395 for (i = 0; i < MAXCPU; i++) 1396 if (CPU_ISSET(i, &cpus)) 1397 val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; 1398 1399 gic_d_write_4(sc, GICD_SGIR(0), val | ipi); 1400} 1401 1402static int 1403arm_gic_ipi_read(device_t dev, int i) 1404{ 1405 1406 if (i != -1) { 1407 /* 1408 * The intr code will automagically give the frame pointer 1409 * if the interrupt argument is 0. 1410 */ 1411 if ((unsigned int)i > 16) 1412 return (0); 1413 return (i); 1414 } 1415 1416 return (0x3ff); 1417} 1418 1419static void 1420arm_gic_ipi_clear(device_t dev, int ipi) 1421{ 1422 /* no-op */ 1423} 1424#endif 1425 1426static void 1427gic_post_filter(void *arg) 1428{ 1429 struct arm_gic_softc *sc = gic_sc; 1430 uintptr_t irq = (uintptr_t) arg; 1431 1432 if (irq > GIC_LAST_SGI) 1433 arm_irq_memory_barrier(irq); 1434 gic_c_write_4(sc, GICC_EOIR, irq); 1435} 1436 1437static int 1438gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) 1439{ 1440 1441 return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol)); 1442} 1443 1444void 1445arm_mask_irq(uintptr_t nb) 1446{ 1447 1448 arm_gic_mask(gic_sc->gic_dev, nb); 1449} 1450 1451void 1452arm_unmask_irq(uintptr_t nb) 1453{ 1454 1455 arm_gic_unmask(gic_sc->gic_dev, nb); 1456} 1457 1458int 1459arm_get_next_irq(int last_irq) 1460{ 1461 1462 return (arm_gic_next_irq(gic_sc, last_irq)); 1463} 1464 1465#ifdef SMP 1466void 1467intr_pic_init_secondary(void) 1468{ 1469 1470 arm_gic_init_secondary(gic_sc->gic_dev); 1471} 1472 1473void 1474pic_ipi_send(cpuset_t cpus, u_int ipi) 1475{ 1476 1477 arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi); 1478} 1479 1480int 1481pic_ipi_read(int i) 1482{ 1483 1484 return (arm_gic_ipi_read(gic_sc->gic_dev, i)); 1485} 1486 1487void 1488pic_ipi_clear(int ipi) 1489{ 1490 1491 arm_gic_ipi_clear(gic_sc->gic_dev, ipi); 1492} 1493#endif 1494#endif /* INTRNG */ 1495 1496static device_method_t arm_gic_methods[] = { 1497 /* Device interface */ 1498 DEVMETHOD(device_probe, arm_gic_probe), 1499 DEVMETHOD(device_attach, arm_gic_attach), 1500 1501#ifdef INTRNG 1502 /* Bus interface */ 1503 DEVMETHOD(bus_add_child, bus_generic_add_child), 1504 DEVMETHOD(bus_alloc_resource, arm_gic_alloc_resource), 1505 DEVMETHOD(bus_release_resource, bus_generic_release_resource), 1506 DEVMETHOD(bus_activate_resource,bus_generic_activate_resource), 1507 1508 /* ofw_bus interface */ 1509 DEVMETHOD(ofw_bus_get_devinfo, arm_gic_ofw_get_devinfo), 1510 DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), 1511 DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), 1512 DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), 1513 DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), 1514 DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), 1515 1516 /* Interrupt controller interface */ 1517 DEVMETHOD(pic_disable_intr, arm_gic_disable_intr), 1518 DEVMETHOD(pic_enable_intr, arm_gic_enable_intr), 1519 DEVMETHOD(pic_map_intr, arm_gic_map_intr), 1520 DEVMETHOD(pic_setup_intr, arm_gic_setup_intr), 1521 DEVMETHOD(pic_teardown_intr, arm_gic_teardown_intr), 1522 DEVMETHOD(pic_post_filter, arm_gic_post_filter), 1523 DEVMETHOD(pic_post_ithread, arm_gic_post_ithread), 1524 DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread), 1525#ifdef SMP 1526 DEVMETHOD(pic_bind_intr, arm_gic_bind_intr), 1527 DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), 1528 DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), 1529 DEVMETHOD(pic_ipi_setup, arm_gic_ipi_setup), 1530#endif 1531#endif 1532 { 0, 0 } 1533}; 1534 1535static driver_t arm_gic_driver = { 1536 "gic", 1537 arm_gic_methods, 1538 sizeof(struct arm_gic_softc), 1539}; 1540 1541static devclass_t arm_gic_devclass; 1542 1543EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, 1544 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1545EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, 1546 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1547 1548#ifdef INTRNG 1549/* 1550 * GICv2m support -- the GICv2 MSI/MSI-X controller. 1551 */ 1552 1553#define GICV2M_MSI_TYPER 0x008 1554#define MSI_TYPER_SPI_BASE(x) (((x) >> 16) & 0x3ff) 1555#define MSI_TYPER_SPI_COUNT(x) (((x) >> 0) & 0x3ff) 1556#define GICv2M_MSI_SETSPI_NS 0x040 1557#define GICV2M_MSI_IIDR 0xFCC 1558 1559struct arm_gicv2m_softc { 1560 struct resource *sc_mem; 1561 struct mtx sc_mutex; 1562 u_int sc_spi_start; 1563 u_int sc_spi_end; 1564 u_int sc_spi_count; 1565}; 1566 1567static struct ofw_compat_data gicv2m_compat_data[] = { 1568 {"arm,gic-v2m-frame", true}, 1569 {NULL, false} 1570}; 1571 1572static int 1573arm_gicv2m_probe(device_t dev) 1574{ 1575 1576 if (!ofw_bus_status_okay(dev)) 1577 return (ENXIO); 1578 1579 if (!ofw_bus_search_compatible(dev, gicv2m_compat_data)->ocd_data) 1580 return (ENXIO); 1581 1582 device_set_desc(dev, "ARM Generic Interrupt Controller MSI/MSIX"); 1583 return (BUS_PROBE_DEFAULT); 1584} 1585 1586static int 1587arm_gicv2m_attach(device_t dev) 1588{ 1589 struct arm_gicv2m_softc *sc; 1590 struct arm_gic_softc *psc; 1591 uint32_t typer; 1592 int rid; 1593 1594 psc = device_get_softc(device_get_parent(dev)); 1595 sc = device_get_softc(dev); 1596 1597 rid = 0; 1598 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 1599 RF_ACTIVE); 1600 if (sc->sc_mem == NULL) { 1601 device_printf(dev, "Unable to allocate resources\n"); 1602 return (ENXIO); 1603 } 1604 1605 typer = bus_read_4(sc->sc_mem, GICV2M_MSI_TYPER); 1606 sc->sc_spi_start = MSI_TYPER_SPI_BASE(typer); 1607 sc->sc_spi_count = MSI_TYPER_SPI_COUNT(typer); 1608 sc->sc_spi_end = sc->sc_spi_start + sc->sc_spi_count; 1609 1610 /* Reserve these interrupts for MSI/MSI-X use */ 1611 arm_gic_reserve_msi_range(device_get_parent(dev), sc->sc_spi_start, 1612 sc->sc_spi_count); 1613 1614 mtx_init(&sc->sc_mutex, "GICv2m lock", "", MTX_DEF); 1615 1616 intr_msi_register(dev, gic_xref(dev)); 1617 1618 if (bootverbose) 1619 device_printf(dev, "using spi %u to %u\n", sc->sc_spi_start, 1620 sc->sc_spi_start + sc->sc_spi_count - 1); 1621 1622 return (0); 1623} 1624 1625static int 1626arm_gicv2m_alloc_msi(device_t dev, device_t child, int count, int maxcount, 1627 device_t *pic, struct intr_irqsrc **srcs) 1628{ 1629 struct arm_gic_softc *psc; 1630 struct arm_gicv2m_softc *sc; 1631 int i, irq, end_irq; 1632 bool found; 1633 1634 KASSERT(powerof2(count), ("%s: bad count", __func__)); 1635 KASSERT(powerof2(maxcount), ("%s: bad maxcount", __func__)); 1636 1637 psc = device_get_softc(device_get_parent(dev)); 1638 sc = device_get_softc(dev); 1639 1640 mtx_lock(&sc->sc_mutex); 1641 1642 found = false; 1643 for (irq = sc->sc_spi_start; irq < sc->sc_spi_end; irq++) { 1644 /* Start on an aligned interrupt */ 1645 if ((irq & (maxcount - 1)) != 0) 1646 continue; 1647 1648 /* Assume we found a valid range until shown otherwise */ 1649 found = true; 1650 1651 /* Check this range is valid */ 1652 for (end_irq = irq; end_irq != irq + count; end_irq++) { 1653 /* No free interrupts */ 1654 if (end_irq == sc->sc_spi_end) { 1655 found = false; 1656 break; 1657 } 1658 1659 KASSERT((psc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI)!= 0, 1660 ("%s: Non-MSI interrupt found", __func__)); 1661 1662 /* This is already used */ 1663 if ((psc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI_USED) == 1664 GI_FLAG_MSI_USED) { 1665 found = false; 1666 break; 1667 } 1668 } 1669 if (found) 1670 break; 1671 } 1672 1673 /* Not enough interrupts were found */ 1674 if (!found || irq == sc->sc_spi_end) { 1675 mtx_unlock(&sc->sc_mutex); 1676 return (ENXIO); 1677 } 1678 1679 for (i = 0; i < count; i++) { 1680 /* Mark the interrupt as used */ 1681 psc->gic_irqs[irq + i].gi_flags |= GI_FLAG_MSI_USED; 1682 1683 } 1684 mtx_unlock(&sc->sc_mutex); 1685 1686 for (i = 0; i < count; i++) 1687 srcs[i] = (struct intr_irqsrc *)&psc->gic_irqs[irq + i]; 1688 *pic = device_get_parent(dev); 1689 1690 return (0); 1691} 1692 1693static int 1694arm_gicv2m_release_msi(device_t dev, device_t child, int count, 1695 struct intr_irqsrc **isrc) 1696{ 1697 struct arm_gicv2m_softc *sc; 1698 struct gic_irqsrc *gi; 1699 int i; 1700 1701 sc = device_get_softc(dev); 1702 1703 mtx_lock(&sc->sc_mutex); 1704 for (i = 0; i < count; i++) { 1705 gi = (struct gic_irqsrc *)isrc[i]; 1706 1707 KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED, 1708 ("%s: Trying to release an unused MSI-X interrupt", 1709 __func__)); 1710 1711 gi->gi_flags &= ~GI_FLAG_MSI_USED; 1712 } 1713 mtx_unlock(&sc->sc_mutex); 1714 1715 return (0); 1716} 1717 1718static int 1719arm_gicv2m_alloc_msix(device_t dev, device_t child, device_t *pic, 1720 struct intr_irqsrc **isrcp) 1721{ 1722 struct arm_gicv2m_softc *sc; 1723 struct arm_gic_softc *psc; 1724 int irq; 1725 1726 psc = device_get_softc(device_get_parent(dev)); 1727 sc = device_get_softc(dev); 1728 1729 mtx_lock(&sc->sc_mutex); 1730 /* Find an unused interrupt */ 1731 for (irq = sc->sc_spi_start; irq < sc->sc_spi_end; irq++) { 1732 KASSERT((psc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) != 0, 1733 ("%s: Non-MSI interrupt found", __func__)); 1734 if ((psc->gic_irqs[irq].gi_flags & GI_FLAG_MSI_USED) == 0) 1735 break; 1736 } 1737 /* No free interrupt was found */ 1738 if (irq == sc->sc_spi_end) { 1739 mtx_unlock(&sc->sc_mutex); 1740 return (ENXIO); 1741 } 1742 1743 /* Mark the interrupt as used */ 1744 psc->gic_irqs[irq].gi_flags |= GI_FLAG_MSI_USED; 1745 mtx_unlock(&sc->sc_mutex); 1746 1747 *isrcp = (struct intr_irqsrc *)&psc->gic_irqs[irq]; 1748 *pic = device_get_parent(dev); 1749 1750 return (0); 1751} 1752 1753static int 1754arm_gicv2m_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1755{ 1756 struct arm_gicv2m_softc *sc; 1757 struct gic_irqsrc *gi; 1758 1759 sc = device_get_softc(dev); 1760 gi = (struct gic_irqsrc *)isrc; 1761 1762 KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED, 1763 ("%s: Trying to release an unused MSI-X interrupt", __func__)); 1764 1765 mtx_lock(&sc->sc_mutex); 1766 gi->gi_flags &= ~GI_FLAG_MSI_USED; 1767 mtx_unlock(&sc->sc_mutex); 1768 1769 return (0); 1770} 1771 1772static int 1773arm_gicv2m_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1774 uint64_t *addr, uint32_t *data) 1775{ 1776 struct arm_gicv2m_softc *sc = device_get_softc(dev); 1777 struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; 1778 1779 *addr = vtophys(rman_get_virtual(sc->sc_mem)) + GICv2M_MSI_SETSPI_NS; 1780 *data = gi->gi_irq; 1781 1782 return (0); 1783} 1784 1785static device_method_t arm_gicv2m_methods[] = { 1786 /* Device interface */ 1787 DEVMETHOD(device_probe, arm_gicv2m_probe), 1788 DEVMETHOD(device_attach, arm_gicv2m_attach), 1789 1790 /* MSI/MSI-X */ 1791 DEVMETHOD(msi_alloc_msi, arm_gicv2m_alloc_msi), 1792 DEVMETHOD(msi_release_msi, arm_gicv2m_release_msi), 1793 DEVMETHOD(msi_alloc_msix, arm_gicv2m_alloc_msix), 1794 DEVMETHOD(msi_release_msix, arm_gicv2m_release_msix), 1795 DEVMETHOD(msi_map_msi, arm_gicv2m_map_msi), 1796 1797 /* End */ 1798 DEVMETHOD_END 1799}; 1800 1801DEFINE_CLASS_0(gicv2m, arm_gicv2m_driver, arm_gicv2m_methods, 1802 sizeof(struct arm_gicv2m_softc)); 1803 1804static devclass_t arm_gicv2m_devclass; 1805 1806EARLY_DRIVER_MODULE(gicv2m, gic, arm_gicv2m_driver, 1807 arm_gicv2m_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1808#endif 1809