gic.c revision 1.55
1/* $NetBSD: gic.c,v 1.55 2022/06/25 13:24:34 jmcneill Exp $ */ 2/*- 3 * Copyright (c) 2012 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas of 3am Software Foundry. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31#include "opt_ddb.h" 32#include "opt_multiprocessor.h" 33#include "opt_gic.h" 34 35#define _INTR_PRIVATE 36 37#include <sys/cdefs.h> 38__KERNEL_RCSID(0, "$NetBSD: gic.c,v 1.55 2022/06/25 13:24:34 jmcneill Exp $"); 39 40#include <sys/param.h> 41#include <sys/bus.h> 42#include <sys/cpu.h> 43#include <sys/device.h> 44#include <sys/evcnt.h> 45#include <sys/intr.h> 46#include <sys/proc.h> 47#include <sys/atomic.h> 48 49#include <arm/armreg.h> 50#include <arm/cpufunc.h> 51#include <arm/locore.h> 52 53#include <arm/cortex/gic_reg.h> 54#include <arm/cortex/mpcore_var.h> 55 56void armgic_irq_handler(void *); 57 58#define ARMGIC_SGI_IPIBASE 0 59 60/* 61 * SGIs 8-16 are reserved for use by ARM Trusted Firmware. 62 */ 63__CTASSERT(ARMGIC_SGI_IPIBASE + NIPI <= 8); 64 65static int armgic_match(device_t, cfdata_t, void *); 66static void armgic_attach(device_t, device_t, void *); 67 68static void armgic_set_priority(struct pic_softc *, int); 69static void armgic_unblock_irqs(struct pic_softc *, size_t, uint32_t); 70static void armgic_block_irqs(struct pic_softc *, size_t, uint32_t); 71static void armgic_establish_irq(struct pic_softc *, struct intrsource *); 72#if 0 73static void armgic_source_name(struct pic_softc *, int, char *, size_t); 74#endif 75 76#ifdef MULTIPROCESSOR 77static void armgic_cpu_init(struct pic_softc *, struct cpu_info *); 78static void armgic_ipi_send(struct pic_softc *, const kcpuset_t *, u_long); 79static void armgic_get_affinity(struct pic_softc *, size_t, kcpuset_t *); 80static int armgic_set_affinity(struct pic_softc *, size_t, const kcpuset_t *); 81#endif 82 83static const struct pic_ops armgic_picops = { 84 .pic_unblock_irqs = armgic_unblock_irqs, 85 .pic_block_irqs = armgic_block_irqs, 86 .pic_establish_irq = armgic_establish_irq, 87#if 0 88 .pic_source_name = armgic_source_name, 89#endif 90 .pic_set_priority = armgic_set_priority, 91#ifdef MULTIPROCESSOR 92 .pic_cpu_init = armgic_cpu_init, 93 .pic_ipi_send = armgic_ipi_send, 94 .pic_get_affinity = armgic_get_affinity, 95 .pic_set_affinity = armgic_set_affinity, 96#endif 97}; 98 99#define PICTOSOFTC(pic) ((struct armgic_softc *)(pic)) 100 101static struct armgic_softc { 102 struct pic_softc sc_pic; 103 device_t sc_dev; 104 bus_space_tag_t sc_memt; 105 bus_space_handle_t sc_gicch; 106 bus_space_handle_t sc_gicdh; 107 size_t sc_gic_lines; 108 uint32_t sc_gic_type; 109 uint32_t sc_gic_valid_lines[1024/32]; 110 uint32_t sc_enabled_local; 111#ifdef MULTIPROCESSOR 112 uint32_t sc_target[MAXCPUS]; 113 uint32_t sc_mptargets; 114#endif 115 uint32_t sc_bptargets; 116} armgic_softc = { 117 .sc_pic = { 118 .pic_ops = &armgic_picops, 119 .pic_name = "armgic", 120 }, 121}; 122 123static struct intrsource armgic_dummy_source; 124 125__CTASSERT(NIPL == 8); 126 127/* 128 * GIC register are always in little-endian. It is assumed the bus_space 129 * will do any endian conversion required. 130 */ 131static inline uint32_t 132gicc_read(struct armgic_softc *sc, bus_size_t o) 133{ 134 return bus_space_read_4(sc->sc_memt, sc->sc_gicch, o); 135} 136 137static inline void 138gicc_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 139{ 140 bus_space_write_4(sc->sc_memt, sc->sc_gicch, o, v); 141} 142 143static inline uint32_t 144gicd_read(struct armgic_softc *sc, bus_size_t o) 145{ 146 return bus_space_read_4(sc->sc_memt, sc->sc_gicdh, o); 147} 148 149static inline void 150gicd_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 151{ 152 bus_space_write_4(sc->sc_memt, sc->sc_gicdh, o, v); 153} 154 155static uint32_t 156gicd_find_targets(struct armgic_softc *sc) 157{ 158 uint32_t targets = 0; 159 160 /* 161 * GICD_ITARGETSR0 through 7 are read-only, and each field returns 162 * a value that corresponds only to the processor reading the 163 * register. Use this to determine the current processor's 164 * CPU interface number. 165 */ 166 for (int i = 0; i < 8; i++) { 167 targets = gicd_read(sc, GICD_ITARGETSRn(i)); 168 if (targets != 0) 169 break; 170 } 171 targets |= (targets >> 16); 172 targets |= (targets >> 8); 173 targets &= 0xff; 174 175 return targets ? targets : 1; 176} 177 178/* 179 * In the GIC prioritization scheme, lower numbers have higher priority. 180 * Only write priorities that could be non-secure. 181 */ 182static inline uint32_t 183armgic_ipl_to_priority(int ipl) 184{ 185 return GICC_PMR_NONSECURE 186 | ((IPL_HIGH - ipl) * GICC_PMR_NS_PRIORITIES / NIPL); 187} 188 189#if 0 190static inline int 191armgic_priority_to_ipl(uint32_t priority) 192{ 193 return IPL_HIGH 194 - (priority & ~GICC_PMR_NONSECURE) * NIPL / GICC_PMR_NS_PRIORITIES; 195} 196#endif 197 198static void 199armgic_unblock_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 200{ 201 struct armgic_softc * const sc = PICTOSOFTC(pic); 202 const size_t group = irq_base / 32; 203 204 if (group == 0) 205 sc->sc_enabled_local |= irq_mask; 206 207 gicd_write(sc, GICD_ISENABLERn(group), irq_mask); 208} 209 210static void 211armgic_block_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 212{ 213 struct armgic_softc * const sc = PICTOSOFTC(pic); 214 const size_t group = irq_base / 32; 215 216 if (group == 0) 217 sc->sc_enabled_local &= ~irq_mask; 218 219 gicd_write(sc, GICD_ICENABLERn(group), irq_mask); 220} 221 222static void 223armgic_set_priority(struct pic_softc *pic, int ipl) 224{ 225 struct armgic_softc * const sc = PICTOSOFTC(pic); 226 struct cpu_info * const ci = curcpu(); 227 228 while (ipl < ci->ci_hwpl) { 229 /* Lowering priority mask */ 230 ci->ci_hwpl = ipl; 231 __insn_barrier(); 232 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 233 } 234 __insn_barrier(); 235 ci->ci_cpl = ipl; 236} 237 238#ifdef MULTIPROCESSOR 239static void 240armgic_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 241{ 242 struct armgic_softc * const sc = PICTOSOFTC(pic); 243 const size_t group = irq / 32; 244 int n; 245 246 kcpuset_zero(affinity); 247 if (group == 0) { 248 /* All CPUs are targets for group 0 (SGI/PPI) */ 249 for (n = 0; n < MAXCPUS; n++) { 250 if (sc->sc_target[n] != 0) 251 kcpuset_set(affinity, n); 252 } 253 } else { 254 /* Find distributor targets (SPI) */ 255 const u_int byte_shift = 8 * (irq & 3); 256 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 257 const uint32_t targets = gicd_read(sc, targets_reg); 258 const uint32_t targets_val = (targets >> byte_shift) & 0xff; 259 260 for (n = 0; n < MAXCPUS; n++) { 261 if (sc->sc_target[n] & targets_val) 262 kcpuset_set(affinity, n); 263 } 264 } 265} 266 267static int 268armgic_set_affinity(struct pic_softc *pic, size_t irq, 269 const kcpuset_t *affinity) 270{ 271 struct armgic_softc * const sc = PICTOSOFTC(pic); 272 const size_t group = irq / 32; 273 if (group == 0) 274 return EINVAL; 275 276 const u_int byte_shift = 8 * (irq & 3); 277 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 278 uint32_t targets_val = 0; 279 int n; 280 281 for (n = 0; n < MAXCPUS; n++) { 282 if (kcpuset_isset(affinity, n)) 283 targets_val |= sc->sc_target[n]; 284 } 285 286 uint32_t targets = gicd_read(sc, targets_reg); 287 targets &= ~(0xff << byte_shift); 288 targets |= (targets_val << byte_shift); 289 gicd_write(sc, targets_reg, targets); 290 291 return 0; 292} 293#endif 294 295#ifdef __HAVE_PIC_FAST_SOFTINTS 296void 297softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep_p) 298{ 299 lwp_t **lp = &l->l_cpu->ci_softlwps[level]; 300 KASSERT(*lp == NULL || *lp == l); 301 *lp = l; 302 /* 303 * Really easy. Just tell it to trigger the local CPU. 304 */ 305 *machdep_p = GICD_SGIR_TargetListFilter_Me 306 | __SHIFTIN(level, GICD_SGIR_SGIINTID); 307} 308 309void 310softint_trigger(uintptr_t machdep) 311{ 312 313 gicd_write(&armgic_softc, GICD_SGIR, machdep); 314} 315#endif 316 317void 318armgic_irq_handler(void *tf) 319{ 320 struct cpu_info * const ci = curcpu(); 321 struct armgic_softc * const sc = &armgic_softc; 322 const int old_ipl = ci->ci_cpl; 323 const int old_mtx_count = ci->ci_mtx_count; 324 const int old_l_biglocks = ci->ci_curlwp->l_biglocks; 325#ifdef DEBUG 326 size_t n = 0; 327#endif 328 329 ci->ci_data.cpu_nintr++; 330 331 /* 332 * Raise ci_hwpl (and PMR) to ci_cpl and IAR will tell us if the 333 * interrupt that got us here can have its handler run or not. 334 */ 335 if (ci->ci_hwpl <= old_ipl) { 336 ci->ci_hwpl = old_ipl; 337 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(old_ipl)); 338 /* 339 * we'll get no interrupts when PMR is IPL_HIGH, so bail 340 * early. 341 */ 342 if (old_ipl == IPL_HIGH) { 343 return; 344 } 345 } 346 347 for (;;) { 348 uint32_t iar = gicc_read(sc, GICC_IAR); 349 uint32_t irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 350 351 if (irq == GICC_IAR_IRQ_SPURIOUS || 352 irq == GICC_IAR_IRQ_SSPURIOUS) { 353 iar = gicc_read(sc, GICC_IAR); 354 irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 355 if (irq == GICC_IAR_IRQ_SPURIOUS) 356 break; 357 if (irq == GICC_IAR_IRQ_SSPURIOUS) { 358 break; 359 } 360 } 361 362 KASSERTMSG(old_ipl != IPL_HIGH, "old_ipl %d pmr %#x hppir %#x", 363 old_ipl, gicc_read(sc, GICC_PMR), gicc_read(sc, GICC_HPPIR)); 364 365 //const uint32_t cpuid = __SHIFTOUT(iar, GICC_IAR_CPUID_MASK); 366 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 367 KASSERT(is != &armgic_dummy_source); 368 369 /* 370 * GIC has asserted IPL for us so we can just update ci_cpl. 371 * 372 * But it's not that simple. We may have already bumped ci_cpl 373 * due to a high priority interrupt and now we are about to 374 * dispatch one lower than the previous. It's possible for 375 * that previous interrupt to have deferred some interrupts 376 * so we need deal with those when lowering to the current 377 * interrupt's ipl. 378 * 379 * However, if are just raising ipl, we can just update ci_cpl. 380 */ 381 382 /* Surely we can KASSERT(ipl < ci->ci_cpl); */ 383 const int ipl = is->is_ipl; 384 if (__predict_false(ipl < ci->ci_cpl)) { 385 pic_do_pending_ints(I32_bit, ipl, tf); 386 KASSERT(ci->ci_cpl == ipl); 387 } else if (ci->ci_cpl != ipl) { 388 KASSERTMSG(ipl > ci->ci_cpl, "ipl %d cpl %d hw-ipl %#x", 389 ipl, ci->ci_cpl, 390 gicc_read(sc, GICC_PMR)); 391 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 392 ci->ci_hwpl = ci->ci_cpl = ipl; 393 } 394 ENABLE_INTERRUPT(); 395 pic_dispatch(is, tf); 396 DISABLE_INTERRUPT(); 397 gicc_write(sc, GICC_EOIR, iar); 398#ifdef DEBUG 399 n++; 400 KDASSERTMSG(n < 5, "%s: processed too many (%zu)", 401 ci->ci_data.cpu_name, n); 402#endif 403 } 404 405 /* 406 * Now handle any pending ints. 407 */ 408 pic_do_pending_ints(I32_bit, old_ipl, tf); 409 KASSERTMSG(ci->ci_cpl == old_ipl, "ci_cpl %d old_ipl %d", ci->ci_cpl, old_ipl); 410 KASSERT(old_mtx_count == ci->ci_mtx_count); 411 KASSERT(old_l_biglocks == ci->ci_curlwp->l_biglocks); 412} 413 414void 415armgic_establish_irq(struct pic_softc *pic, struct intrsource *is) 416{ 417 struct armgic_softc * const sc = PICTOSOFTC(pic); 418 const size_t group = is->is_irq / 32; 419 const u_int irq = is->is_irq & 31; 420 const u_int byte_shift = 8 * (irq & 3); 421 const u_int twopair_shift = 2 * (irq & 15); 422 423 KASSERTMSG(sc->sc_gic_valid_lines[group] & __BIT(irq), 424 "irq %u: not valid (group[%zu]=0x%08x [0x%08x])", 425 is->is_irq, group, sc->sc_gic_valid_lines[group], 426 (uint32_t)__BIT(irq)); 427 428 KASSERTMSG(is->is_type == IST_LEVEL || is->is_type == IST_EDGE, 429 "irq %u: type %u unsupported", is->is_irq, is->is_type); 430 431 const bus_size_t targets_reg = GICD_ITARGETSRn(is->is_irq / 4); 432 const bus_size_t cfg_reg = GICD_ICFGRn(is->is_irq / 16); 433 uint32_t targets = gicd_read(sc, targets_reg); 434 uint32_t cfg = gicd_read(sc, cfg_reg); 435 436 if (group > 0) { 437 /* 438 * There are 4 irqs per TARGETS register. For now bind 439 * to the primary cpu. 440 */ 441 targets &= ~(0xffU << byte_shift); 442#if 0 443#ifdef MULTIPROCESSOR 444 if (is->is_mpsafe) { 445 targets |= sc->sc_mptargets << byte_shift; 446 } else 447#endif 448#endif 449 targets |= sc->sc_bptargets << byte_shift; 450 gicd_write(sc, targets_reg, targets); 451 452 /* 453 * There are 16 irqs per CFG register. 10=EDGE 00=LEVEL 454 */ 455 uint32_t new_cfg = cfg; 456 uint32_t old_cfg = (cfg >> twopair_shift) & __BITS(1, 0); 457 if (is->is_type == IST_LEVEL && (old_cfg & __BIT(1)) != 0) { 458 new_cfg &= ~(__BITS(1, 0) << twopair_shift); 459 } else if (is->is_type == IST_EDGE && (old_cfg & 2) == 0) { 460 new_cfg |= __BIT(1) << twopair_shift; 461 } 462 if (new_cfg != cfg) { 463 gicd_write(sc, cfg_reg, new_cfg); 464 } 465#ifdef MULTIPROCESSOR 466 } else { 467 /* 468 * All group 0 interrupts are per processor and MPSAFE by 469 * default. 470 */ 471 is->is_mpsafe = true; 472 is->is_percpu = true; 473#endif 474 } 475 476 /* 477 * There are 4 irqs per PRIORITY register. Map the IPL 478 * to GIC priority. 479 */ 480 const bus_size_t priority_reg = GICD_IPRIORITYRn(is->is_irq / 4); 481 uint32_t priority = gicd_read(sc, priority_reg); 482 priority &= ~(0xffU << byte_shift); 483 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 484 gicd_write(sc, priority_reg, priority); 485} 486 487#ifdef MULTIPROCESSOR 488static void 489armgic_cpu_init_priorities(struct armgic_softc *sc) 490{ 491 /* Set lowest priority, i.e. disable interrupts */ 492 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4) { 493 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 494 gicd_write(sc, priority_reg, ~0); 495 } 496} 497 498static void 499armgic_cpu_update_priorities(struct armgic_softc *sc) 500{ 501 uint32_t enabled = sc->sc_enabled_local; 502 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4, enabled >>= 4) { 503 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 504 uint32_t priority = gicd_read(sc, priority_reg); 505 uint32_t byte_mask = 0xff; 506 size_t byte_shift = 0; 507 for (size_t j = 0; j < 4; j++, byte_mask <<= 8, byte_shift += 8) { 508 struct intrsource * const is = sc->sc_pic.pic_sources[i+j]; 509 priority |= byte_mask; 510 if (is == NULL || is == &armgic_dummy_source) 511 continue; 512 priority &= ~byte_mask; 513 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 514 } 515 gicd_write(sc, priority_reg, priority); 516 } 517} 518 519static void 520armgic_cpu_init_targets(struct armgic_softc *sc) 521{ 522 /* 523 * Update the mpsafe targets 524 */ 525 for (size_t irq = 32; irq < sc->sc_pic.pic_maxsources; irq++) { 526 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 527 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 528 if (is != NULL && is->is_mpsafe) { 529 const u_int byte_shift = 8 * (irq & 3); 530 uint32_t targets = gicd_read(sc, targets_reg); 531#if 0 532 targets |= sc->sc_mptargets << byte_shift; 533#else 534 targets |= sc->sc_bptargets << byte_shift; 535#endif 536 gicd_write(sc, targets_reg, targets); 537 } 538 } 539} 540 541void 542armgic_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 543{ 544 struct armgic_softc * const sc = PICTOSOFTC(pic); 545 sc->sc_target[cpu_index(ci)] = gicd_find_targets(sc); 546 atomic_or_32(&sc->sc_mptargets, sc->sc_target[cpu_index(ci)]); 547 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 548 armgic_cpu_init_priorities(sc); 549 if (!CPU_IS_PRIMARY(ci)) { 550 if (popcount(sc->sc_mptargets) != 1) { 551 armgic_cpu_init_targets(sc); 552 } 553 if (sc->sc_enabled_local) { 554 armgic_cpu_update_priorities(sc); 555 gicd_write(sc, GICD_ISENABLERn(0), 556 sc->sc_enabled_local); 557 } 558 } 559 ci->ci_hwpl = ci->ci_cpl; 560 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ci->ci_cpl)); // set PMR 561 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable interrupt 562 ENABLE_INTERRUPT(); // allow IRQ exceptions 563} 564 565void 566armgic_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 567{ 568 struct armgic_softc * const sc = PICTOSOFTC(pic); 569 570#if 0 571 if (ipi == IPI_NOP) { 572 sev(); 573 return; 574 } 575#endif 576 577 uint32_t sgir = __SHIFTIN(ARMGIC_SGI_IPIBASE + ipi, GICD_SGIR_SGIINTID); 578 if (kcp != NULL) { 579 uint32_t targets_val = 0; 580 for (int n = 0; n < MAXCPUS; n++) { 581 if (kcpuset_isset(kcp, n)) 582 targets_val |= sc->sc_target[n]; 583 } 584 sgir |= __SHIFTIN(targets_val, GICD_SGIR_TargetList); 585 sgir |= GICD_SGIR_TargetListFilter_List; 586 } else { 587 if (ncpu == 1) 588 return; 589 sgir |= GICD_SGIR_TargetListFilter_NotMe; 590 } 591 592 gicd_write(sc, GICD_SGIR, sgir); 593} 594#endif 595 596int 597armgic_match(device_t parent, cfdata_t cf, void *aux) 598{ 599 struct mpcore_attach_args * const mpcaa = aux; 600 601 if (strcmp(cf->cf_name, mpcaa->mpcaa_name) != 0) 602 return 0; 603 604 return 1; 605} 606 607void 608armgic_attach(device_t parent, device_t self, void *aux) 609{ 610 struct armgic_softc * const sc = &armgic_softc; 611 struct mpcore_attach_args * const mpcaa = aux; 612 613 sc->sc_dev = self; 614 device_set_private(self, sc); 615 616 sc->sc_memt = mpcaa->mpcaa_memt; /* provided for us */ 617 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off1, 618 4096, &sc->sc_gicdh); 619 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off2, 620 4096, &sc->sc_gicch); 621 622 sc->sc_gic_type = gicd_read(sc, GICD_TYPER); 623 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gic_type); 624 625 gicc_write(sc, GICC_CTRL, 0); /* disable all interrupts */ 626 gicd_write(sc, GICD_CTRL, 0); /* disable all interrupts */ 627 628 gicc_write(sc, GICC_PMR, 0xff); 629 uint32_t pmr = gicc_read(sc, GICC_PMR); 630 u_int priorities = 1 << popcount32(pmr); 631 632 const uint32_t iidr = gicc_read(sc, GICC_IIDR); 633 const int iidr_prod = __SHIFTOUT(iidr, GICC_IIDR_ProductID); 634 const int iidr_arch = __SHIFTOUT(iidr, GICC_IIDR_ArchVersion); 635 const int iidr_rev = __SHIFTOUT(iidr, GICC_IIDR_Revision); 636 const int iidr_imp = __SHIFTOUT(iidr, GICC_IIDR_Implementer); 637 638 /* 639 * Find the boot processor's CPU interface number. 640 */ 641 sc->sc_bptargets = gicd_find_targets(sc); 642 643 /* 644 * Let's find out how many real sources we have. 645 */ 646 for (size_t i = 0, group = 0; 647 i < sc->sc_pic.pic_maxsources; 648 i += 32, group++) { 649 /* 650 * To figure what sources are real, one enables all interrupts 651 * and then reads back the enable mask so which ones really 652 * got enabled. 653 */ 654 gicd_write(sc, GICD_ISENABLERn(group), 0xffffffff); 655 uint32_t valid = gicd_read(sc, GICD_ISENABLERn(group)); 656 657 /* 658 * Now disable (clear enable) them again. 659 */ 660 gicd_write(sc, GICD_ICENABLERn(group), valid); 661 662 /* 663 * Count how many are valid. 664 */ 665 sc->sc_gic_lines += popcount32(valid); 666 sc->sc_gic_valid_lines[group] = valid; 667 } 668 669 aprint_normal(": Generic Interrupt Controller, " 670 "%zu sources (%zu valid)\n", 671 sc->sc_pic.pic_maxsources, sc->sc_gic_lines); 672 aprint_debug_dev(sc->sc_dev, "Architecture version %d" 673 " (0x%x:%d rev %d)\n", iidr_arch, iidr_imp, iidr_prod, 674 iidr_rev); 675 676#ifdef MULTIPROCESSOR 677 sc->sc_pic.pic_cpus = kcpuset_running; 678#endif 679 pic_add(&sc->sc_pic, 0); 680 681 /* 682 * Force the GICD to IPL_HIGH and then enable interrupts. 683 */ 684 struct cpu_info * const ci = curcpu(); 685 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 686 armgic_set_priority(&sc->sc_pic, ci->ci_cpl); // set PMR 687 gicd_write(sc, GICD_CTRL, GICD_CTRL_Enable); // enable Distributer 688 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable CPU interrupts 689 ENABLE_INTERRUPT(); // allow interrupt exceptions 690 691 /* 692 * For each line that isn't valid, we set the intrsource for it to 693 * point at a dummy source so that pic_intr_establish will fail for it. 694 */ 695 for (size_t i = 0, group = 0; 696 i < sc->sc_pic.pic_maxsources; 697 i += 32, group++) { 698 uint32_t invalid = ~sc->sc_gic_valid_lines[group]; 699 for (size_t j = 0; invalid && j < 32; j++, invalid >>= 1) { 700 if (invalid & 1) { 701 sc->sc_pic.pic_sources[i + j] = 702 &armgic_dummy_source; 703 } 704 } 705 } 706#ifdef __HAVE_PIC_FAST_SOFTINTS 707 intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, 708 pic_handle_softint, (void *)SOFTINT_BIO, "softint bio"); 709 intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, 710 pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock"); 711 intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, 712 pic_handle_softint, (void *)SOFTINT_NET, "softint net"); 713 intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, 714 pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial"); 715#endif 716#ifdef MULTIPROCESSOR 717 armgic_cpu_init(&sc->sc_pic, curcpu()); 718 719 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_AST, IPL_VM, 720 IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 721 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_XCALL, IPL_HIGH, 722 IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 723 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_GENERIC, IPL_HIGH, 724 IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 725 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_NOP, IPL_VM, 726 IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 727 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_SHOOTDOWN, IPL_SCHED, 728 IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 729#ifdef DDB 730 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_DDB, IPL_HIGH, 731 IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 732#endif 733#ifdef __HAVE_PREEMPTION 734 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_KPREEMPT, IPL_VM, 735 IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 736#endif 737#endif 738 739 const u_int ppis = popcount32(sc->sc_gic_valid_lines[0] >> 16); 740 const u_int sgis = popcount32(sc->sc_gic_valid_lines[0] & 0xffff); 741 aprint_normal_dev(sc->sc_dev, "%u Priorities, %zu SPIs, %u PPIs, " 742 "%u SGIs\n", priorities, sc->sc_gic_lines - ppis - sgis, ppis, 743 sgis); 744} 745 746CFATTACH_DECL_NEW(armgic, 0, 747 armgic_match, armgic_attach, NULL, NULL); 748