gic.c revision 292426
1/*- 2 * Copyright (c) 2011 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * Developed by Damjan Marion <damjan.marion@gmail.com> 6 * 7 * Based on OMAP4 GIC code by Ben Gray 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. The name of the company nor the name of the author may be used to 18 * endorse or promote products derived from this software without specific 19 * prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34#include <sys/cdefs.h> 35__FBSDID("$FreeBSD: head/sys/arm/arm/gic.c 292426 2015-12-18 05:43:59Z adrian $"); 36 37#include "opt_platform.h" 38 39#include "opt_platform.h" 40 41#include <sys/param.h> 42#include <sys/systm.h> 43#include <sys/bus.h> 44#include <sys/kernel.h> 45#include <sys/ktr.h> 46#include <sys/module.h> 47#include <sys/malloc.h> 48#include <sys/rman.h> 49#include <sys/pcpu.h> 50#include <sys/proc.h> 51#include <sys/cpuset.h> 52#include <sys/lock.h> 53#include <sys/mutex.h> 54#include <sys/smp.h> 55#ifdef ARM_INTRNG 56#include <sys/sched.h> 57#endif 58#include <machine/bus.h> 59#include <machine/intr.h> 60#include <machine/smp.h> 61 62#include <dev/fdt/fdt_common.h> 63#include <dev/ofw/openfirm.h> 64#include <dev/ofw/ofw_bus.h> 65#include <dev/ofw/ofw_bus_subr.h> 66 67#ifdef ARM_INTRNG 68#include "pic_if.h" 69#endif 70 71#define GIC_DEBUG_SPURIOUS 72 73/* We are using GICv2 register naming */ 74 75/* Distributor Registers */ 76#define GICD_CTLR 0x000 /* v1 ICDDCR */ 77#define GICD_TYPER 0x004 /* v1 ICDICTR */ 78#define GICD_IIDR 0x008 /* v1 ICDIIDR */ 79#define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ 80#define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ 81#define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ 82#define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ 83#define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ 84#define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ 85#define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ 86#define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ 87#define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ 88#define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ 89 90/* CPU Registers */ 91#define GICC_CTLR 0x0000 /* v1 ICCICR */ 92#define GICC_PMR 0x0004 /* v1 ICCPMR */ 93#define GICC_BPR 0x0008 /* v1 ICCBPR */ 94#define GICC_IAR 0x000C /* v1 ICCIAR */ 95#define GICC_EOIR 0x0010 /* v1 ICCEOIR */ 96#define GICC_RPR 0x0014 /* v1 ICCRPR */ 97#define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ 98#define GICC_ABPR 0x001C /* v1 ICCABPR */ 99#define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ 100 101#define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ 102#define GIC_LAST_SGI 15 103#define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ 104#define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ 105#define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ 106 107/* First bit is a polarity bit (0 - low, 1 - high) */ 108#define GICD_ICFGR_POL_LOW (0 << 0) 109#define GICD_ICFGR_POL_HIGH (1 << 0) 110#define GICD_ICFGR_POL_MASK 0x1 111/* Second bit is a trigger bit (0 - level, 1 - edge) */ 112#define GICD_ICFGR_TRIG_LVL (0 << 1) 113#define GICD_ICFGR_TRIG_EDGE (1 << 1) 114#define GICD_ICFGR_TRIG_MASK 0x2 115 116#ifndef GIC_DEFAULT_ICFGR_INIT 117#define GIC_DEFAULT_ICFGR_INIT 0x00000000 118#endif 119 120#ifdef ARM_INTRNG 121static u_int gic_irq_cpu; 122static int arm_gic_intr(void *); 123static int arm_gic_bind(device_t dev, struct intr_irqsrc *isrc); 124#endif 125 126struct arm_gic_softc { 127 device_t gic_dev; 128#ifdef ARM_INTRNG 129 void * gic_intrhand; 130 struct intr_irqsrc ** gic_irqs; 131#endif 132 struct resource * gic_res[3]; 133 bus_space_tag_t gic_c_bst; 134 bus_space_tag_t gic_d_bst; 135 bus_space_handle_t gic_c_bsh; 136 bus_space_handle_t gic_d_bsh; 137 uint8_t ver; 138 struct mtx mutex; 139 uint32_t nirqs; 140#ifdef GIC_DEBUG_SPURIOUS 141 uint32_t last_irq[MAXCPU]; 142#endif 143}; 144 145static struct resource_spec arm_gic_spec[] = { 146 { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ 147 { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ 148#ifdef ARM_INTRNG 149 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ 150#endif 151 { -1, 0 } 152}; 153 154static struct arm_gic_softc *gic_sc = NULL; 155 156#define gic_c_read_4(_sc, _reg) \ 157 bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) 158#define gic_c_write_4(_sc, _reg, _val) \ 159 bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) 160#define gic_d_read_4(_sc, _reg) \ 161 bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) 162#define gic_d_write_1(_sc, _reg, _val) \ 163 bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 164#define gic_d_write_4(_sc, _reg, _val) \ 165 bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 166 167#ifndef ARM_INTRNG 168static int gic_config_irq(int irq, enum intr_trigger trig, 169 enum intr_polarity pol); 170static void gic_post_filter(void *); 171#endif 172 173static struct ofw_compat_data compat_data[] = { 174 {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ 175 {"arm,gic-400", true}, 176 {"arm,cortex-a15-gic", true}, 177 {"arm,cortex-a9-gic", true}, 178 {"arm,cortex-a7-gic", true}, 179 {"arm,arm11mp-gic", true}, 180 {"brcm,brahma-b15-gic", true}, 181 {NULL, false} 182}; 183 184static int 185arm_gic_probe(device_t dev) 186{ 187 188 if (!ofw_bus_status_okay(dev)) 189 return (ENXIO); 190 191 if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) 192 return (ENXIO); 193 device_set_desc(dev, "ARM Generic Interrupt Controller"); 194 return (BUS_PROBE_DEFAULT); 195} 196 197#ifdef ARM_INTRNG 198static inline void 199gic_irq_unmask(struct arm_gic_softc *sc, u_int irq) 200{ 201 202 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 203} 204 205static inline void 206gic_irq_mask(struct arm_gic_softc *sc, u_int irq) 207{ 208 209 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 210} 211#endif 212 213#ifdef SMP 214#ifdef ARM_INTRNG 215static void 216arm_gic_init_secondary(device_t dev) 217{ 218 struct arm_gic_softc *sc = device_get_softc(dev); 219 struct intr_irqsrc *isrc; 220 u_int irq; 221 222 for (irq = 0; irq < sc->nirqs; irq += 4) 223 gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0); 224 225 /* Set all the interrupts to be in Group 0 (secure) */ 226 for (irq = 0; irq < sc->nirqs; irq += 32) { 227 gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0); 228 } 229 230 /* Enable CPU interface */ 231 gic_c_write_4(sc, GICC_CTLR, 1); 232 233 /* Set priority mask register. */ 234 gic_c_write_4(sc, GICC_PMR, 0xff); 235 236 /* Enable interrupt distribution */ 237 gic_d_write_4(sc, GICD_CTLR, 0x01); 238 239 /* Unmask attached SGI interrupts. */ 240 for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) { 241 isrc = sc->gic_irqs[irq]; 242 if (isrc != NULL && isrc->isrc_handlers != 0) { 243 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 244 gic_irq_unmask(sc, irq); 245 } 246 } 247 248 /* Unmask attached PPI interrupts. */ 249 for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) { 250 isrc = sc->gic_irqs[irq]; 251 if (isrc == NULL || isrc->isrc_handlers == 0) 252 continue; 253 if (isrc->isrc_flags & INTR_ISRCF_BOUND) { 254 if (CPU_ISSET(PCPU_GET(cpuid), &isrc->isrc_cpu)) 255 gic_irq_unmask(sc, irq); 256 } else { 257 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 258 gic_irq_unmask(sc, irq); 259 } 260 } 261} 262#else 263static void 264arm_gic_init_secondary(device_t dev) 265{ 266 struct arm_gic_softc *sc = device_get_softc(dev); 267 int i; 268 269 for (i = 0; i < sc->nirqs; i += 4) 270 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 271 272 /* Set all the interrupts to be in Group 0 (secure) */ 273 for (i = 0; i < sc->nirqs; i += 32) { 274 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 275 } 276 277 /* Enable CPU interface */ 278 gic_c_write_4(sc, GICC_CTLR, 1); 279 280 /* Set priority mask register. */ 281 gic_c_write_4(sc, GICC_PMR, 0xff); 282 283 /* Enable interrupt distribution */ 284 gic_d_write_4(sc, GICD_CTLR, 0x01); 285 286 /* 287 * Activate the timer interrupts: virtual, secure, and non-secure. 288 */ 289 gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); 290 gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); 291 gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); 292} 293#endif /* ARM_INTRNG */ 294#endif /* SMP */ 295 296#ifndef ARM_INTRNG 297int 298gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, 299 int *trig, int *pol) 300{ 301 static u_int num_intr_cells; 302 static phandle_t self; 303 struct ofw_compat_data *ocd; 304 305 if (self == 0) { 306 for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) { 307 if (fdt_is_compatible(iparent, ocd->ocd_str)) { 308 self = iparent; 309 break; 310 } 311 } 312 } 313 if (self != iparent) 314 return (ENXIO); 315 316 if (num_intr_cells == 0) { 317 if (OF_searchencprop(OF_node_from_xref(iparent), 318 "#interrupt-cells", &num_intr_cells, 319 sizeof(num_intr_cells)) == -1) { 320 num_intr_cells = 1; 321 } 322 } 323 324 if (num_intr_cells == 1) { 325 *interrupt = fdt32_to_cpu(intr[0]); 326 *trig = INTR_TRIGGER_CONFORM; 327 *pol = INTR_POLARITY_CONFORM; 328 } else { 329 if (fdt32_to_cpu(intr[0]) == 0) 330 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; 331 else 332 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; 333 /* 334 * In intr[2], bits[3:0] are trigger type and level flags. 335 * 1 = low-to-high edge triggered 336 * 2 = high-to-low edge triggered 337 * 4 = active high level-sensitive 338 * 8 = active low level-sensitive 339 * The hardware only supports active-high-level or rising-edge. 340 */ 341 if (fdt32_to_cpu(intr[2]) & 0x0a) { 342 printf("unsupported trigger/polarity configuration " 343 "0x%02x\n", fdt32_to_cpu(intr[2]) & 0x0f); 344 } 345 *pol = INTR_POLARITY_CONFORM; 346 if (fdt32_to_cpu(intr[2]) & 0x03) 347 *trig = INTR_TRIGGER_EDGE; 348 else 349 *trig = INTR_TRIGGER_LEVEL; 350 } 351 return (0); 352} 353#endif 354 355#ifdef ARM_INTRNG 356static inline intptr_t 357gic_xref(device_t dev) 358{ 359#ifdef FDT 360 return (OF_xref_from_node(ofw_bus_get_node(dev))); 361#else 362 return (0); 363#endif 364} 365#endif 366 367static int 368arm_gic_attach(device_t dev) 369{ 370 struct arm_gic_softc *sc; 371 int i; 372 uint32_t icciidr; 373#ifdef ARM_INTRNG 374 phandle_t pxref; 375 intptr_t xref = gic_xref(dev); 376#endif 377 378 if (gic_sc) 379 return (ENXIO); 380 381 sc = device_get_softc(dev); 382 383 if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { 384 device_printf(dev, "could not allocate resources\n"); 385 return (ENXIO); 386 } 387 388 sc->gic_dev = dev; 389 gic_sc = sc; 390 391 /* Initialize mutex */ 392 mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); 393 394 /* Distributor Interface */ 395 sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); 396 sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); 397 398 /* CPU Interface */ 399 sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); 400 sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); 401 402 /* Disable interrupt forwarding to the CPU interface */ 403 gic_d_write_4(sc, GICD_CTLR, 0x00); 404 405 /* Get the number of interrupts */ 406 sc->nirqs = gic_d_read_4(sc, GICD_TYPER); 407 sc->nirqs = 32 * ((sc->nirqs & 0x1f) + 1); 408 409#ifdef ARM_INTRNG 410 sc->gic_irqs = malloc(sc->nirqs * sizeof (*sc->gic_irqs), M_DEVBUF, 411 M_WAITOK | M_ZERO); 412#else 413 /* Set up function pointers */ 414 arm_post_filter = gic_post_filter; 415 arm_config_irq = gic_config_irq; 416#endif 417 418 icciidr = gic_c_read_4(sc, GICC_IIDR); 419 device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", 420 icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, 421 (icciidr & 0xfff), sc->nirqs); 422 423 /* Set all global interrupts to be level triggered, active low. */ 424 for (i = 32; i < sc->nirqs; i += 16) { 425 gic_d_write_4(sc, GICD_ICFGR(i >> 4), GIC_DEFAULT_ICFGR_INIT); 426 } 427 428 /* Disable all interrupts. */ 429 for (i = 32; i < sc->nirqs; i += 32) { 430 gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); 431 } 432 433 for (i = 0; i < sc->nirqs; i += 4) { 434 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 435 gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), 436 1 << 0 | 1 << 8 | 1 << 16 | 1 << 24); 437 } 438 439 /* Set all the interrupts to be in Group 0 (secure) */ 440 for (i = 0; i < sc->nirqs; i += 32) { 441 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 442 } 443 444 /* Enable CPU interface */ 445 gic_c_write_4(sc, GICC_CTLR, 1); 446 447 /* Set priority mask register. */ 448 gic_c_write_4(sc, GICC_PMR, 0xff); 449 450 /* Enable interrupt distribution */ 451 gic_d_write_4(sc, GICD_CTLR, 0x01); 452#ifndef ARM_INTRNG 453 return (0); 454#else 455 /* 456 * Now, when everything is initialized, it's right time to 457 * register interrupt controller to interrupt framefork. 458 */ 459 if (intr_pic_register(dev, xref) != 0) { 460 device_printf(dev, "could not register PIC\n"); 461 goto cleanup; 462 } 463 464 /* 465 * Controller is root if: 466 * - doesn't have interrupt parent 467 * - his interrupt parent is this controller 468 */ 469 pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); 470 if (pxref == 0 || xref == pxref) { 471 if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, 472 GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { 473 device_printf(dev, "could not set PIC as a root\n"); 474 intr_pic_unregister(dev, xref); 475 goto cleanup; 476 } 477 } else { 478 if (sc->gic_res[2] == NULL) { 479 device_printf(dev, 480 "not root PIC must have defined interrupt\n"); 481 intr_pic_unregister(dev, xref); 482 goto cleanup; 483 } 484 if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, 485 arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { 486 device_printf(dev, "could not setup irq handler\n"); 487 intr_pic_unregister(dev, xref); 488 goto cleanup; 489 } 490 } 491 492 OF_device_register_xref(xref, dev); 493 return (0); 494 495cleanup: 496 /* 497 * XXX - not implemented arm_gic_detach() should be called ! 498 */ 499 if (sc->gic_irqs != NULL) 500 free(sc->gic_irqs, M_DEVBUF); 501 bus_release_resources(dev, arm_gic_spec, sc->gic_res); 502 return(ENXIO); 503#endif 504} 505 506#ifdef ARM_INTRNG 507static int 508arm_gic_intr(void *arg) 509{ 510 struct arm_gic_softc *sc = arg; 511 struct intr_irqsrc *isrc; 512 uint32_t irq_active_reg, irq; 513 struct trapframe *tf; 514 515 irq_active_reg = gic_c_read_4(sc, GICC_IAR); 516 irq = irq_active_reg & 0x3FF; 517 518 /* 519 * 1. We do EOI here because recent read value from active interrupt 520 * register must be used for it. Another approach is to save this 521 * value into associated interrupt source. 522 * 2. EOI must be done on same CPU where interrupt has fired. Thus 523 * we must ensure that interrupted thread does not migrate to 524 * another CPU. 525 * 3. EOI cannot be delayed by any preemption which could happen on 526 * critical_exit() used in MI intr code, when interrupt thread is 527 * scheduled. See next point. 528 * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during 529 * an action and any use of critical_exit() could break this 530 * assumption. See comments within smp_rendezvous_action(). 531 * 5. We always return FILTER_HANDLED as this is an interrupt 532 * controller dispatch function. Otherwise, in cascaded interrupt 533 * case, the whole interrupt subtree would be masked. 534 */ 535 536 if (irq >= sc->nirqs) { 537#ifdef GIC_DEBUG_SPURIOUS 538 device_printf(sc->gic_dev, 539 "Spurious interrupt detected: last irq: %d on CPU%d\n", 540 sc->last_irq[PCPU_GET(cpuid)], PCPU_GET(cpuid)); 541#endif 542 return (FILTER_HANDLED); 543 } 544 545 tf = curthread->td_intr_frame; 546dispatch_irq: 547 isrc = sc->gic_irqs[irq]; 548 if (isrc == NULL) { 549 device_printf(sc->gic_dev, "Stray interrupt %u detected\n", irq); 550 gic_irq_mask(sc, irq); 551 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 552 goto next_irq; 553 } 554 555 /* 556 * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement 557 * as compiler complains that comparing u_int >= 0 is always true. 558 */ 559 if (irq <= GIC_LAST_SGI) { 560#ifdef SMP 561 /* Call EOI for all IPI before dispatch. */ 562 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 563 intr_ipi_dispatch(isrc, tf); 564 goto next_irq; 565#else 566 device_printf(sc->gic_dev, "SGI %u on UP system detected\n", 567 irq - GIC_FIRST_SGI); 568 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 569 goto next_irq; 570#endif 571 } 572 573#ifdef GIC_DEBUG_SPURIOUS 574 sc->last_irq[PCPU_GET(cpuid)] = irq; 575#endif 576 if (isrc->isrc_trig == INTR_TRIGGER_EDGE) 577 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 578 579 intr_irq_dispatch(isrc, tf); 580 581next_irq: 582 arm_irq_memory_barrier(irq); 583 irq_active_reg = gic_c_read_4(sc, GICC_IAR); 584 irq = irq_active_reg & 0x3FF; 585 if (irq < sc->nirqs) 586 goto dispatch_irq; 587 588 return (FILTER_HANDLED); 589} 590 591static int 592gic_attach_isrc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int irq) 593{ 594 const char *name; 595 596 /* 597 * 1. The link between ISRC and controller must be set atomically. 598 * 2. Just do things only once in rare case when consumers 599 * of shared interrupt came here at the same moment. 600 */ 601 mtx_lock_spin(&sc->mutex); 602 if (sc->gic_irqs[irq] != NULL) { 603 mtx_unlock_spin(&sc->mutex); 604 return (sc->gic_irqs[irq] == isrc ? 0 : EEXIST); 605 } 606 sc->gic_irqs[irq] = isrc; 607 isrc->isrc_data = irq; 608 mtx_unlock_spin(&sc->mutex); 609 610 name = device_get_nameunit(sc->gic_dev); 611 if (irq <= GIC_LAST_SGI) 612 intr_irq_set_name(isrc, "%s,i%u", name, irq - GIC_FIRST_SGI); 613 else if (irq <= GIC_LAST_PPI) 614 intr_irq_set_name(isrc, "%s,p%u", name, irq - GIC_FIRST_PPI); 615 else 616 intr_irq_set_name(isrc, "%s,s%u", name, irq - GIC_FIRST_SPI); 617 return (0); 618} 619 620static int 621gic_detach_isrc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int irq) 622{ 623 624 mtx_lock_spin(&sc->mutex); 625 if (sc->gic_irqs[irq] != isrc) { 626 mtx_unlock_spin(&sc->mutex); 627 return (sc->gic_irqs[irq] == NULL ? 0 : EINVAL); 628 } 629 sc->gic_irqs[irq] = NULL; 630 isrc->isrc_data = 0; 631 mtx_unlock_spin(&sc->mutex); 632 633 intr_irq_set_name(isrc, ""); 634 return (0); 635} 636 637static void 638gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig, 639 enum intr_polarity pol) 640{ 641 uint32_t reg; 642 uint32_t mask; 643 644 if (irq < GIC_FIRST_SPI) 645 return; 646 647 mtx_lock_spin(&sc->mutex); 648 649 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 650 mask = (reg >> 2*(irq % 16)) & 0x3; 651 652 if (pol == INTR_POLARITY_LOW) { 653 mask &= ~GICD_ICFGR_POL_MASK; 654 mask |= GICD_ICFGR_POL_LOW; 655 } else if (pol == INTR_POLARITY_HIGH) { 656 mask &= ~GICD_ICFGR_POL_MASK; 657 mask |= GICD_ICFGR_POL_HIGH; 658 } 659 660 if (trig == INTR_TRIGGER_LEVEL) { 661 mask &= ~GICD_ICFGR_TRIG_MASK; 662 mask |= GICD_ICFGR_TRIG_LVL; 663 } else if (trig == INTR_TRIGGER_EDGE) { 664 mask &= ~GICD_ICFGR_TRIG_MASK; 665 mask |= GICD_ICFGR_TRIG_EDGE; 666 } 667 668 /* Set mask */ 669 reg = reg & ~(0x3 << 2*(irq % 16)); 670 reg = reg | (mask << 2*(irq % 16)); 671 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 672 673 mtx_unlock_spin(&sc->mutex); 674} 675 676static int 677gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus) 678{ 679 uint32_t cpu, end, mask; 680 681 end = min(mp_ncpus, 8); 682 for (cpu = end; cpu < MAXCPU; cpu++) 683 if (CPU_ISSET(cpu, cpus)) 684 return (EINVAL); 685 686 for (mask = 0, cpu = 0; cpu < end; cpu++) 687 if (CPU_ISSET(cpu, cpus)) 688 mask |= 1 << cpu; 689 690 gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask); 691 return (0); 692} 693 694static int 695gic_irq_from_nspc(struct arm_gic_softc *sc, u_int type, u_int num, u_int *irqp) 696{ 697 698 switch (type) { 699 case INTR_IRQ_NSPC_PLAIN: 700 *irqp = num; 701 return (*irqp < sc->nirqs ? 0 : EINVAL); 702 703 case INTR_IRQ_NSPC_IRQ: 704 *irqp = num + GIC_FIRST_PPI; 705 return (*irqp < sc->nirqs ? 0 : EINVAL); 706 707 case INTR_IRQ_NSPC_IPI: 708 *irqp = num + GIC_FIRST_SGI; 709 return (*irqp < GIC_LAST_SGI ? 0 : EINVAL); 710 711 default: 712 return (EINVAL); 713 } 714} 715 716static int 717gic_map_nspc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int *irqp) 718{ 719 int error; 720 721 error = gic_irq_from_nspc(sc, isrc->isrc_nspc_type, isrc->isrc_nspc_num, 722 irqp); 723 if (error != 0) 724 return (error); 725 return (gic_attach_isrc(sc, isrc, *irqp)); 726} 727 728#ifdef FDT 729static int 730gic_map_fdt(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int *irqp) 731{ 732 u_int irq, tripol; 733 enum intr_trigger trig; 734 enum intr_polarity pol; 735 int error; 736 737 if (isrc->isrc_ncells == 1) { 738 irq = isrc->isrc_cells[0]; 739 pol = INTR_POLARITY_CONFORM; 740 trig = INTR_TRIGGER_CONFORM; 741 } else if (isrc->isrc_ncells == 3) { 742 if (isrc->isrc_cells[0] == 0) 743 irq = isrc->isrc_cells[1] + GIC_FIRST_SPI; 744 else 745 irq = isrc->isrc_cells[1] + GIC_FIRST_PPI; 746 747 /* 748 * In intr[2], bits[3:0] are trigger type and level flags. 749 * 1 = low-to-high edge triggered 750 * 2 = high-to-low edge triggered 751 * 4 = active high level-sensitive 752 * 8 = active low level-sensitive 753 * The hardware only supports active-high-level or rising-edge. 754 */ 755 tripol = isrc->isrc_cells[2]; 756 if (tripol & 0x0a) { 757 device_printf(sc->gic_dev, 758 "unsupported trigger/polarity configuration " 759 "0x%02x\n", tripol & 0x0f); 760 } 761 pol = INTR_POLARITY_CONFORM; 762 if (tripol & 0x03) 763 trig = INTR_TRIGGER_EDGE; 764 else 765 trig = INTR_TRIGGER_LEVEL; 766 } else 767 return (EINVAL); 768 769 if (irq >= sc->nirqs) 770 return (EINVAL); 771 772 error = gic_attach_isrc(sc, isrc, irq); 773 if (error != 0) 774 return (error); 775 776 isrc->isrc_nspc_type = INTR_IRQ_NSPC_PLAIN; 777 isrc->isrc_nspc_num = irq; 778 isrc->isrc_trig = trig; 779 isrc->isrc_pol = pol; 780 781 *irqp = irq; 782 return (0); 783} 784#endif 785 786static int 787arm_gic_register(device_t dev, struct intr_irqsrc *isrc, boolean_t *is_percpu) 788{ 789 struct arm_gic_softc *sc = device_get_softc(dev); 790 u_int irq; 791 int error; 792 793 if (isrc->isrc_type == INTR_ISRCT_NAMESPACE) 794 error = gic_map_nspc(sc, isrc, &irq); 795#ifdef FDT 796 else if (isrc->isrc_type == INTR_ISRCT_FDT) 797 error = gic_map_fdt(sc, isrc, &irq); 798#endif 799 else 800 return (EINVAL); 801 802 if (error == 0) 803 *is_percpu = irq < GIC_FIRST_SPI ? TRUE : FALSE; 804 return (error); 805} 806 807static void 808arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc) 809{ 810 struct arm_gic_softc *sc = device_get_softc(dev); 811 u_int irq = isrc->isrc_data; 812 813 if (isrc->isrc_trig == INTR_TRIGGER_CONFORM) 814 isrc->isrc_trig = INTR_TRIGGER_LEVEL; 815 816 /* 817 * XXX - In case that per CPU interrupt is going to be enabled in time 818 * when SMP is already started, we need some IPI call which 819 * enables it on others CPUs. Further, it's more complicated as 820 * pic_enable_source() and pic_disable_source() should act on 821 * per CPU basis only. Thus, it should be solved here somehow. 822 */ 823 if (isrc->isrc_flags & INTR_ISRCF_PERCPU) 824 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 825 826 gic_config(sc, irq, isrc->isrc_trig, isrc->isrc_pol); 827 arm_gic_bind(dev, isrc); 828} 829 830static void 831arm_gic_enable_source(device_t dev, struct intr_irqsrc *isrc) 832{ 833 struct arm_gic_softc *sc = device_get_softc(dev); 834 u_int irq = isrc->isrc_data; 835 836 arm_irq_memory_barrier(irq); 837 gic_irq_unmask(sc, irq); 838} 839 840static void 841arm_gic_disable_source(device_t dev, struct intr_irqsrc *isrc) 842{ 843 struct arm_gic_softc *sc = device_get_softc(dev); 844 u_int irq = isrc->isrc_data; 845 846 gic_irq_mask(sc, irq); 847} 848 849static int 850arm_gic_unregister(device_t dev, struct intr_irqsrc *isrc) 851{ 852 struct arm_gic_softc *sc = device_get_softc(dev); 853 u_int irq = isrc->isrc_data; 854 855 return (gic_detach_isrc(sc, isrc, irq)); 856} 857 858static void 859arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 860{ 861 struct arm_gic_softc *sc = device_get_softc(dev); 862 863 arm_gic_disable_source(dev, isrc); 864 gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data); 865} 866 867static void 868arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc) 869{ 870 871 arm_irq_memory_barrier(0); 872 arm_gic_enable_source(dev, isrc); 873} 874 875static void 876arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc) 877{ 878 struct arm_gic_softc *sc = device_get_softc(dev); 879 880 /* EOI for edge-triggered done earlier. */ 881 if (isrc->isrc_trig == INTR_TRIGGER_EDGE) 882 return; 883 884 arm_irq_memory_barrier(0); 885 gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data); 886} 887 888static int 889arm_gic_bind(device_t dev, struct intr_irqsrc *isrc) 890{ 891 struct arm_gic_softc *sc = device_get_softc(dev); 892 uint32_t irq = isrc->isrc_data; 893 894 if (irq < GIC_FIRST_SPI) 895 return (EINVAL); 896 897 if (CPU_EMPTY(&isrc->isrc_cpu)) { 898 gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); 899 CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); 900 } 901 return (gic_bind(sc, irq, &isrc->isrc_cpu)); 902} 903 904#ifdef SMP 905static void 906arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus) 907{ 908 struct arm_gic_softc *sc = device_get_softc(dev); 909 uint32_t irq, val = 0, i; 910 911 irq = isrc->isrc_data; 912 913 for (i = 0; i < MAXCPU; i++) 914 if (CPU_ISSET(i, &cpus)) 915 val |= 1 << (16 + i); 916 917 gic_d_write_4(sc, GICD_SGIR(0), val | irq); 918} 919#endif 920#else 921static int 922arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq) 923{ 924 uint32_t active_irq; 925 926 active_irq = gic_c_read_4(sc, GICC_IAR); 927 928 /* 929 * Immediatly EOIR the SGIs, because doing so requires the other 930 * bits (ie CPU number), not just the IRQ number, and we do not 931 * have this information later. 932 */ 933 if ((active_irq & 0x3ff) <= GIC_LAST_SGI) 934 gic_c_write_4(sc, GICC_EOIR, active_irq); 935 active_irq &= 0x3FF; 936 937 if (active_irq == 0x3FF) { 938 if (last_irq == -1) 939 device_printf(sc->gic_dev, 940 "Spurious interrupt detected\n"); 941 return -1; 942 } 943 944 return active_irq; 945} 946 947static int 948arm_gic_config(device_t dev, int irq, enum intr_trigger trig, 949 enum intr_polarity pol) 950{ 951 struct arm_gic_softc *sc = device_get_softc(dev); 952 uint32_t reg; 953 uint32_t mask; 954 955 /* Function is public-accessible, so validate input arguments */ 956 if ((irq < 0) || (irq >= sc->nirqs)) 957 goto invalid_args; 958 if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && 959 (trig != INTR_TRIGGER_CONFORM)) 960 goto invalid_args; 961 if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && 962 (pol != INTR_POLARITY_CONFORM)) 963 goto invalid_args; 964 965 mtx_lock_spin(&sc->mutex); 966 967 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 968 mask = (reg >> 2*(irq % 16)) & 0x3; 969 970 if (pol == INTR_POLARITY_LOW) { 971 mask &= ~GICD_ICFGR_POL_MASK; 972 mask |= GICD_ICFGR_POL_LOW; 973 } else if (pol == INTR_POLARITY_HIGH) { 974 mask &= ~GICD_ICFGR_POL_MASK; 975 mask |= GICD_ICFGR_POL_HIGH; 976 } 977 978 if (trig == INTR_TRIGGER_LEVEL) { 979 mask &= ~GICD_ICFGR_TRIG_MASK; 980 mask |= GICD_ICFGR_TRIG_LVL; 981 } else if (trig == INTR_TRIGGER_EDGE) { 982 mask &= ~GICD_ICFGR_TRIG_MASK; 983 mask |= GICD_ICFGR_TRIG_EDGE; 984 } 985 986 /* Set mask */ 987 reg = reg & ~(0x3 << 2*(irq % 16)); 988 reg = reg | (mask << 2*(irq % 16)); 989 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 990 991 mtx_unlock_spin(&sc->mutex); 992 993 return (0); 994 995invalid_args: 996 device_printf(dev, "gic_config_irg, invalid parameters\n"); 997 return (EINVAL); 998} 999 1000 1001static void 1002arm_gic_mask(device_t dev, int irq) 1003{ 1004 struct arm_gic_softc *sc = device_get_softc(dev); 1005 1006 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 1007 gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */ 1008} 1009 1010static void 1011arm_gic_unmask(device_t dev, int irq) 1012{ 1013 struct arm_gic_softc *sc = device_get_softc(dev); 1014 1015 if (irq > GIC_LAST_SGI) 1016 arm_irq_memory_barrier(irq); 1017 1018 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 1019} 1020 1021#ifdef SMP 1022static void 1023arm_gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) 1024{ 1025 struct arm_gic_softc *sc = device_get_softc(dev); 1026 uint32_t val = 0, i; 1027 1028 for (i = 0; i < MAXCPU; i++) 1029 if (CPU_ISSET(i, &cpus)) 1030 val |= 1 << (16 + i); 1031 1032 gic_d_write_4(sc, GICD_SGIR(0), val | ipi); 1033} 1034 1035static int 1036arm_gic_ipi_read(device_t dev, int i) 1037{ 1038 1039 if (i != -1) { 1040 /* 1041 * The intr code will automagically give the frame pointer 1042 * if the interrupt argument is 0. 1043 */ 1044 if ((unsigned int)i > 16) 1045 return (0); 1046 return (i); 1047 } 1048 1049 return (0x3ff); 1050} 1051 1052static void 1053arm_gic_ipi_clear(device_t dev, int ipi) 1054{ 1055 /* no-op */ 1056} 1057#endif 1058 1059static void 1060gic_post_filter(void *arg) 1061{ 1062 struct arm_gic_softc *sc = gic_sc; 1063 uintptr_t irq = (uintptr_t) arg; 1064 1065 if (irq > GIC_LAST_SGI) 1066 arm_irq_memory_barrier(irq); 1067 gic_c_write_4(sc, GICC_EOIR, irq); 1068} 1069 1070static int 1071gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) 1072{ 1073 1074 return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol)); 1075} 1076 1077void 1078arm_mask_irq(uintptr_t nb) 1079{ 1080 1081 arm_gic_mask(gic_sc->gic_dev, nb); 1082} 1083 1084void 1085arm_unmask_irq(uintptr_t nb) 1086{ 1087 1088 arm_gic_unmask(gic_sc->gic_dev, nb); 1089} 1090 1091int 1092arm_get_next_irq(int last_irq) 1093{ 1094 1095 return (arm_gic_next_irq(gic_sc, last_irq)); 1096} 1097 1098#ifdef SMP 1099void 1100intr_pic_init_secondary(void) 1101{ 1102 1103 arm_gic_init_secondary(gic_sc->gic_dev); 1104} 1105 1106void 1107pic_ipi_send(cpuset_t cpus, u_int ipi) 1108{ 1109 1110 arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi); 1111} 1112 1113int 1114pic_ipi_read(int i) 1115{ 1116 1117 return (arm_gic_ipi_read(gic_sc->gic_dev, i)); 1118} 1119 1120void 1121pic_ipi_clear(int ipi) 1122{ 1123 1124 arm_gic_ipi_clear(gic_sc->gic_dev, ipi); 1125} 1126#endif 1127#endif /* ARM_INTRNG */ 1128 1129static device_method_t arm_gic_methods[] = { 1130 /* Device interface */ 1131 DEVMETHOD(device_probe, arm_gic_probe), 1132 DEVMETHOD(device_attach, arm_gic_attach), 1133#ifdef ARM_INTRNG 1134 /* Interrupt controller interface */ 1135 DEVMETHOD(pic_disable_source, arm_gic_disable_source), 1136 DEVMETHOD(pic_enable_intr, arm_gic_enable_intr), 1137 DEVMETHOD(pic_enable_source, arm_gic_enable_source), 1138 DEVMETHOD(pic_post_filter, arm_gic_post_filter), 1139 DEVMETHOD(pic_post_ithread, arm_gic_post_ithread), 1140 DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread), 1141 DEVMETHOD(pic_register, arm_gic_register), 1142 DEVMETHOD(pic_unregister, arm_gic_unregister), 1143#ifdef SMP 1144 DEVMETHOD(pic_bind, arm_gic_bind), 1145 DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), 1146 DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), 1147#endif 1148#endif 1149 { 0, 0 } 1150}; 1151 1152static driver_t arm_gic_driver = { 1153 "gic", 1154 arm_gic_methods, 1155 sizeof(struct arm_gic_softc), 1156}; 1157 1158static devclass_t arm_gic_devclass; 1159 1160EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, 1161 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1162EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, 1163 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1164