gic.c revision 1.48
1/* $NetBSD: gic.c,v 1.48 2021/08/10 15:33:09 jmcneill Exp $ */ 2/*- 3 * Copyright (c) 2012 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas of 3am Software Foundry. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31#include "opt_ddb.h" 32#include "opt_multiprocessor.h" 33 34#define _INTR_PRIVATE 35 36#include <sys/cdefs.h> 37__KERNEL_RCSID(0, "$NetBSD: gic.c,v 1.48 2021/08/10 15:33:09 jmcneill Exp $"); 38 39#include <sys/param.h> 40#include <sys/bus.h> 41#include <sys/cpu.h> 42#include <sys/device.h> 43#include <sys/evcnt.h> 44#include <sys/intr.h> 45#include <sys/proc.h> 46#include <sys/atomic.h> 47 48#include <arm/armreg.h> 49#include <arm/cpufunc.h> 50#include <arm/locore.h> 51 52#include <arm/cortex/gic_reg.h> 53#include <arm/cortex/gic_splfuncs.h> 54#include <arm/cortex/mpcore_var.h> 55 56void armgic_irq_handler(void *); 57 58#define ARMGIC_SGI_IPIBASE 0 59 60/* 61 * SGIs 8-16 are reserved for use by ARM Trusted Firmware. 62 */ 63__CTASSERT(ARMGIC_SGI_IPIBASE + NIPI <= 8); 64 65static int armgic_match(device_t, cfdata_t, void *); 66static void armgic_attach(device_t, device_t, void *); 67 68static void armgic_set_priority(struct pic_softc *, int); 69static void armgic_unblock_irqs(struct pic_softc *, size_t, uint32_t); 70static void armgic_block_irqs(struct pic_softc *, size_t, uint32_t); 71static void armgic_establish_irq(struct pic_softc *, struct intrsource *); 72#if 0 73static void armgic_source_name(struct pic_softc *, int, char *, size_t); 74#endif 75 76#ifdef MULTIPROCESSOR 77static void armgic_cpu_init(struct pic_softc *, struct cpu_info *); 78static void armgic_ipi_send(struct pic_softc *, const kcpuset_t *, u_long); 79static void armgic_get_affinity(struct pic_softc *, size_t, kcpuset_t *); 80static int armgic_set_affinity(struct pic_softc *, size_t, const kcpuset_t *); 81#endif 82 83static const struct pic_ops armgic_picops = { 84 .pic_unblock_irqs = armgic_unblock_irqs, 85 .pic_block_irqs = armgic_block_irqs, 86 .pic_establish_irq = armgic_establish_irq, 87#if 0 88 .pic_source_name = armgic_source_name, 89#endif 90 .pic_set_priority = armgic_set_priority, 91#ifdef MULTIPROCESSOR 92 .pic_cpu_init = armgic_cpu_init, 93 .pic_ipi_send = armgic_ipi_send, 94 .pic_get_affinity = armgic_get_affinity, 95 .pic_set_affinity = armgic_set_affinity, 96#endif 97}; 98 99#define PICTOSOFTC(pic) ((struct armgic_softc *)(pic)) 100 101static struct armgic_softc { 102 struct pic_softc sc_pic; 103 device_t sc_dev; 104 bus_space_tag_t sc_memt; 105 bus_space_handle_t sc_gicch; 106 bus_space_handle_t sc_gicdh; 107 size_t sc_gic_lines; 108 uint32_t sc_gic_type; 109 uint32_t sc_gic_valid_lines[1024/32]; 110 uint32_t sc_enabled_local; 111#ifdef MULTIPROCESSOR 112 uint32_t sc_target[MAXCPUS]; 113 uint32_t sc_mptargets; 114#endif 115 uint32_t sc_bptargets; 116} armgic_softc = { 117 .sc_pic = { 118 .pic_ops = &armgic_picops, 119 .pic_name = "armgic", 120 }, 121}; 122 123static struct intrsource armgic_dummy_source; 124 125__CTASSERT(NIPL == 8); 126 127/* 128 * GIC register are always in little-endian. It is assumed the bus_space 129 * will do any endian conversion required. 130 */ 131static inline uint32_t 132gicc_read(struct armgic_softc *sc, bus_size_t o) 133{ 134 return bus_space_read_4(sc->sc_memt, sc->sc_gicch, o); 135} 136 137static inline void 138gicc_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 139{ 140 bus_space_write_4(sc->sc_memt, sc->sc_gicch, o, v); 141} 142 143static inline uint32_t 144gicd_read(struct armgic_softc *sc, bus_size_t o) 145{ 146 return bus_space_read_4(sc->sc_memt, sc->sc_gicdh, o); 147} 148 149static inline void 150gicd_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 151{ 152 bus_space_write_4(sc->sc_memt, sc->sc_gicdh, o, v); 153} 154 155static uint32_t 156gicd_find_targets(struct armgic_softc *sc) 157{ 158 uint32_t targets = 0; 159 160 /* 161 * GICD_ITARGETSR0 through 7 are read-only, and each field returns 162 * a value that corresponds only to the processor reading the 163 * register. Use this to determine the current processor's 164 * CPU interface number. 165 */ 166 for (int i = 0; i < 8; i++) { 167 targets = gicd_read(sc, GICD_ITARGETSRn(i)); 168 if (targets != 0) 169 break; 170 } 171 targets |= (targets >> 16); 172 targets |= (targets >> 8); 173 targets &= 0xff; 174 175 return targets ? targets : 1; 176} 177 178/* 179 * In the GIC prioritization scheme, lower numbers have higher priority. 180 * Only write priorities that could be non-secure. 181 */ 182static inline uint32_t 183armgic_ipl_to_priority(int ipl) 184{ 185 return GICC_PMR_NONSECURE 186 | ((IPL_HIGH - ipl) * GICC_PMR_NS_PRIORITIES / NIPL); 187} 188 189#if 0 190static inline int 191armgic_priority_to_ipl(uint32_t priority) 192{ 193 return IPL_HIGH 194 - (priority & ~GICC_PMR_NONSECURE) * NIPL / GICC_PMR_NS_PRIORITIES; 195} 196#endif 197 198static void 199armgic_unblock_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 200{ 201 struct armgic_softc * const sc = PICTOSOFTC(pic); 202 const size_t group = irq_base / 32; 203 204 if (group == 0) 205 sc->sc_enabled_local |= irq_mask; 206 207 gicd_write(sc, GICD_ISENABLERn(group), irq_mask); 208} 209 210static void 211armgic_block_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 212{ 213 struct armgic_softc * const sc = PICTOSOFTC(pic); 214 const size_t group = irq_base / 32; 215 216 if (group == 0) 217 sc->sc_enabled_local &= ~irq_mask; 218 219 gicd_write(sc, GICD_ICENABLERn(group), irq_mask); 220} 221 222static void 223armgic_set_priority(struct pic_softc *pic, int ipl) 224{ 225 struct armgic_softc * const sc = PICTOSOFTC(pic); 226 struct cpu_info * const ci = curcpu(); 227 228 if (ipl < ci->ci_hwpl) { 229 /* Lowering priority mask */ 230 ci->ci_hwpl = ipl; 231 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 232 } 233} 234 235#ifdef MULTIPROCESSOR 236static void 237armgic_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 238{ 239 struct armgic_softc * const sc = PICTOSOFTC(pic); 240 const size_t group = irq / 32; 241 int n; 242 243 kcpuset_zero(affinity); 244 if (group == 0) { 245 /* All CPUs are targets for group 0 (SGI/PPI) */ 246 for (n = 0; n < MAXCPUS; n++) { 247 if (sc->sc_target[n] != 0) 248 kcpuset_set(affinity, n); 249 } 250 } else { 251 /* Find distributor targets (SPI) */ 252 const u_int byte_shift = 8 * (irq & 3); 253 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 254 const uint32_t targets = gicd_read(sc, targets_reg); 255 const uint32_t targets_val = (targets >> byte_shift) & 0xff; 256 257 for (n = 0; n < MAXCPUS; n++) { 258 if (sc->sc_target[n] & targets_val) 259 kcpuset_set(affinity, n); 260 } 261 } 262} 263 264static int 265armgic_set_affinity(struct pic_softc *pic, size_t irq, 266 const kcpuset_t *affinity) 267{ 268 struct armgic_softc * const sc = PICTOSOFTC(pic); 269 const size_t group = irq / 32; 270 if (group == 0) 271 return EINVAL; 272 273 const u_int byte_shift = 8 * (irq & 3); 274 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 275 uint32_t targets_val = 0; 276 int n; 277 278 for (n = 0; n < MAXCPUS; n++) { 279 if (kcpuset_isset(affinity, n)) 280 targets_val |= sc->sc_target[n]; 281 } 282 283 uint32_t targets = gicd_read(sc, targets_reg); 284 targets &= ~(0xff << byte_shift); 285 targets |= (targets_val << byte_shift); 286 gicd_write(sc, targets_reg, targets); 287 288 return 0; 289} 290#endif 291 292#ifdef __HAVE_PIC_FAST_SOFTINTS 293void 294softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep_p) 295{ 296 lwp_t **lp = &l->l_cpu->ci_softlwps[level]; 297 KASSERT(*lp == NULL || *lp == l); 298 *lp = l; 299 /* 300 * Really easy. Just tell it to trigger the local CPU. 301 */ 302 *machdep_p = GICD_SGIR_TargetListFilter_Me 303 | __SHIFTIN(level, GICD_SGIR_SGIINTID); 304} 305 306void 307softint_trigger(uintptr_t machdep) 308{ 309 310 gicd_write(&armgic_softc, GICD_SGIR, machdep); 311} 312#endif 313 314void 315armgic_irq_handler(void *tf) 316{ 317 struct cpu_info * const ci = curcpu(); 318 struct armgic_softc * const sc = &armgic_softc; 319 const int old_ipl = ci->ci_cpl; 320#ifdef DIAGNOSTIC 321 const int old_mtx_count = ci->ci_mtx_count; 322 const int old_l_biglocks = ci->ci_curlwp->l_biglocks; 323#endif 324#ifdef DEBUG 325 size_t n = 0; 326#endif 327 328 ci->ci_data.cpu_nintr++; 329 330 if (ci->ci_hwpl != old_ipl) { 331 ci->ci_hwpl = old_ipl; 332 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(old_ipl)); 333 if (old_ipl == IPL_HIGH) { 334 return; 335 } 336 } 337 338 for (;;) { 339 uint32_t iar = gicc_read(sc, GICC_IAR); 340 uint32_t irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 341 342 if (irq == GICC_IAR_IRQ_SPURIOUS || 343 irq == GICC_IAR_IRQ_SSPURIOUS) { 344 iar = gicc_read(sc, GICC_IAR); 345 irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 346 if (irq == GICC_IAR_IRQ_SPURIOUS) 347 break; 348 if (irq == GICC_IAR_IRQ_SSPURIOUS) { 349 break; 350 } 351 } 352 353 KASSERTMSG(old_ipl != IPL_HIGH, "old_ipl %d pmr %#x hppir %#x", 354 old_ipl, gicc_read(sc, GICC_PMR), gicc_read(sc, GICC_HPPIR)); 355 356 //const uint32_t cpuid = __SHIFTOUT(iar, GICC_IAR_CPUID_MASK); 357 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 358 KASSERT(is != &armgic_dummy_source); 359 360 /* 361 * GIC has asserted IPL for us so we can just update ci_cpl. 362 * 363 * But it's not that simple. We may have already bumped ci_cpl 364 * due to a high priority interrupt and now we are about to 365 * dispatch one lower than the previous. It's possible for 366 * that previous interrupt to have deferred some interrupts 367 * so we need deal with those when lowering to the current 368 * interrupt's ipl. 369 * 370 * However, if are just raising ipl, we can just update ci_cpl. 371 */ 372 const int ipl = is->is_ipl; 373 if (__predict_false(ipl < ci->ci_cpl)) { 374 pic_do_pending_ints(I32_bit, ipl, tf); 375 KASSERT(ci->ci_cpl == ipl); 376 } else { 377 KASSERTMSG(ipl > ci->ci_cpl, "ipl %d cpl %d hw-ipl %#x", 378 ipl, ci->ci_cpl, 379 gicc_read(sc, GICC_PMR)); 380 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 381 ci->ci_hwpl = ci->ci_cpl = ipl; 382 } 383 ENABLE_INTERRUPT(); 384 pic_dispatch(is, tf); 385 DISABLE_INTERRUPT(); 386 gicc_write(sc, GICC_EOIR, iar); 387#ifdef DEBUG 388 n++; 389 KDASSERTMSG(n < 5, "%s: processed too many (%zu)", 390 ci->ci_data.cpu_name, n); 391#endif 392 } 393 394 /* 395 * Now handle any pending ints. 396 */ 397 pic_do_pending_ints(I32_bit, old_ipl, tf); 398 KASSERTMSG(ci->ci_cpl == old_ipl, "ci_cpl %d old_ipl %d", ci->ci_cpl, old_ipl); 399 KASSERT(old_mtx_count == ci->ci_mtx_count); 400 KASSERT(old_l_biglocks == ci->ci_curlwp->l_biglocks); 401} 402 403void 404armgic_establish_irq(struct pic_softc *pic, struct intrsource *is) 405{ 406 struct armgic_softc * const sc = PICTOSOFTC(pic); 407 const size_t group = is->is_irq / 32; 408 const u_int irq = is->is_irq & 31; 409 const u_int byte_shift = 8 * (irq & 3); 410 const u_int twopair_shift = 2 * (irq & 15); 411 412 KASSERTMSG(sc->sc_gic_valid_lines[group] & __BIT(irq), 413 "irq %u: not valid (group[%zu]=0x%08x [0x%08x])", 414 is->is_irq, group, sc->sc_gic_valid_lines[group], 415 (uint32_t)__BIT(irq)); 416 417 KASSERTMSG(is->is_type == IST_LEVEL || is->is_type == IST_EDGE, 418 "irq %u: type %u unsupported", is->is_irq, is->is_type); 419 420 const bus_size_t targets_reg = GICD_ITARGETSRn(is->is_irq / 4); 421 const bus_size_t cfg_reg = GICD_ICFGRn(is->is_irq / 16); 422 uint32_t targets = gicd_read(sc, targets_reg); 423 uint32_t cfg = gicd_read(sc, cfg_reg); 424 425 if (group > 0) { 426 /* 427 * There are 4 irqs per TARGETS register. For now bind 428 * to the primary cpu. 429 */ 430 targets &= ~(0xffU << byte_shift); 431#if 0 432#ifdef MULTIPROCESSOR 433 if (is->is_mpsafe) { 434 targets |= sc->sc_mptargets << byte_shift; 435 } else 436#endif 437#endif 438 targets |= sc->sc_bptargets << byte_shift; 439 gicd_write(sc, targets_reg, targets); 440 441 /* 442 * There are 16 irqs per CFG register. 10=EDGE 00=LEVEL 443 */ 444 uint32_t new_cfg = cfg; 445 uint32_t old_cfg = (cfg >> twopair_shift) & __BITS(1, 0); 446 if (is->is_type == IST_LEVEL && (old_cfg & __BIT(1)) != 0) { 447 new_cfg &= ~(__BITS(1, 0) << twopair_shift); 448 } else if (is->is_type == IST_EDGE && (old_cfg & 2) == 0) { 449 new_cfg |= __BIT(1) << twopair_shift; 450 } 451 if (new_cfg != cfg) { 452 gicd_write(sc, cfg_reg, new_cfg); 453 } 454#ifdef MULTIPROCESSOR 455 } else { 456 /* 457 * All group 0 interrupts are per processor and MPSAFE by 458 * default. 459 */ 460 is->is_mpsafe = true; 461#endif 462 } 463 464 /* 465 * There are 4 irqs per PRIORITY register. Map the IPL 466 * to GIC priority. 467 */ 468 const bus_size_t priority_reg = GICD_IPRIORITYRn(is->is_irq / 4); 469 uint32_t priority = gicd_read(sc, priority_reg); 470 priority &= ~(0xffU << byte_shift); 471 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 472 gicd_write(sc, priority_reg, priority); 473} 474 475#ifdef MULTIPROCESSOR 476static void 477armgic_cpu_init_priorities(struct armgic_softc *sc) 478{ 479 /* Set lowest priority, i.e. disable interrupts */ 480 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4) { 481 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 482 gicd_write(sc, priority_reg, ~0); 483 } 484} 485 486static void 487armgic_cpu_update_priorities(struct armgic_softc *sc) 488{ 489 uint32_t enabled = sc->sc_enabled_local; 490 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4, enabled >>= 4) { 491 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 492 uint32_t priority = gicd_read(sc, priority_reg); 493 uint32_t byte_mask = 0xff; 494 size_t byte_shift = 0; 495 for (size_t j = 0; j < 4; j++, byte_mask <<= 8, byte_shift += 8) { 496 struct intrsource * const is = sc->sc_pic.pic_sources[i+j]; 497 priority |= byte_mask; 498 if (is == NULL || is == &armgic_dummy_source) 499 continue; 500 priority &= ~byte_mask; 501 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 502 } 503 gicd_write(sc, priority_reg, priority); 504 } 505} 506 507static void 508armgic_cpu_init_targets(struct armgic_softc *sc) 509{ 510 /* 511 * Update the mpsafe targets 512 */ 513 for (size_t irq = 32; irq < sc->sc_pic.pic_maxsources; irq++) { 514 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 515 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 516 if (is != NULL && is->is_mpsafe) { 517 const u_int byte_shift = 8 * (irq & 3); 518 uint32_t targets = gicd_read(sc, targets_reg); 519#if 0 520 targets |= sc->sc_mptargets << byte_shift; 521#else 522 targets |= sc->sc_bptargets << byte_shift; 523#endif 524 gicd_write(sc, targets_reg, targets); 525 } 526 } 527} 528 529void 530armgic_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 531{ 532 struct armgic_softc * const sc = PICTOSOFTC(pic); 533 sc->sc_target[cpu_index(ci)] = gicd_find_targets(sc); 534 atomic_or_32(&sc->sc_mptargets, sc->sc_target[cpu_index(ci)]); 535 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 536 armgic_cpu_init_priorities(sc); 537 if (!CPU_IS_PRIMARY(ci)) { 538 if (popcount(sc->sc_mptargets) != 1) { 539 armgic_cpu_init_targets(sc); 540 } 541 if (sc->sc_enabled_local) { 542 armgic_cpu_update_priorities(sc); 543 gicd_write(sc, GICD_ISENABLERn(0), 544 sc->sc_enabled_local); 545 } 546 } 547 ci->ci_hwpl = ci->ci_cpl; 548 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ci->ci_cpl)); // set PMR 549 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable interrupt 550 ENABLE_INTERRUPT(); // allow IRQ exceptions 551} 552 553void 554armgic_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 555{ 556 struct armgic_softc * const sc = PICTOSOFTC(pic); 557 558#if 0 559 if (ipi == IPI_NOP) { 560 sev(); 561 return; 562 } 563#endif 564 565 uint32_t sgir = __SHIFTIN(ARMGIC_SGI_IPIBASE + ipi, GICD_SGIR_SGIINTID); 566 if (kcp != NULL) { 567 uint32_t targets_val = 0; 568 for (int n = 0; n < MAXCPUS; n++) { 569 if (kcpuset_isset(kcp, n)) 570 targets_val |= sc->sc_target[n]; 571 } 572 sgir |= __SHIFTIN(targets_val, GICD_SGIR_TargetList); 573 sgir |= GICD_SGIR_TargetListFilter_List; 574 } else { 575 if (ncpu == 1) 576 return; 577 sgir |= GICD_SGIR_TargetListFilter_NotMe; 578 } 579 580 gicd_write(sc, GICD_SGIR, sgir); 581} 582#endif 583 584int 585armgic_match(device_t parent, cfdata_t cf, void *aux) 586{ 587 struct mpcore_attach_args * const mpcaa = aux; 588 589 if (strcmp(cf->cf_name, mpcaa->mpcaa_name) != 0) 590 return 0; 591 592 return 1; 593} 594 595void 596armgic_attach(device_t parent, device_t self, void *aux) 597{ 598 struct armgic_softc * const sc = &armgic_softc; 599 struct mpcore_attach_args * const mpcaa = aux; 600 601 sc->sc_dev = self; 602 self->dv_private = sc; 603 604 sc->sc_memt = mpcaa->mpcaa_memt; /* provided for us */ 605 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off1, 606 4096, &sc->sc_gicdh); 607 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off2, 608 4096, &sc->sc_gicch); 609 610 sc->sc_gic_type = gicd_read(sc, GICD_TYPER); 611 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gic_type); 612 613 gicc_write(sc, GICC_CTRL, 0); /* disable all interrupts */ 614 gicd_write(sc, GICD_CTRL, 0); /* disable all interrupts */ 615 616 gicc_write(sc, GICC_PMR, 0xff); 617 uint32_t pmr = gicc_read(sc, GICC_PMR); 618 u_int priorities = 1 << popcount32(pmr); 619 620 const uint32_t iidr = gicc_read(sc, GICC_IIDR); 621 const int iidr_prod = __SHIFTOUT(iidr, GICC_IIDR_ProductID); 622 const int iidr_arch = __SHIFTOUT(iidr, GICC_IIDR_ArchVersion); 623 const int iidr_rev = __SHIFTOUT(iidr, GICC_IIDR_Revision); 624 const int iidr_imp = __SHIFTOUT(iidr, GICC_IIDR_Implementer); 625 626 /* 627 * Find the boot processor's CPU interface number. 628 */ 629 sc->sc_bptargets = gicd_find_targets(sc); 630 631 /* 632 * Let's find out how many real sources we have. 633 */ 634 for (size_t i = 0, group = 0; 635 i < sc->sc_pic.pic_maxsources; 636 i += 32, group++) { 637 /* 638 * To figure what sources are real, one enables all interrupts 639 * and then reads back the enable mask so which ones really 640 * got enabled. 641 */ 642 gicd_write(sc, GICD_ISENABLERn(group), 0xffffffff); 643 uint32_t valid = gicd_read(sc, GICD_ISENABLERn(group)); 644 645 /* 646 * Now disable (clear enable) them again. 647 */ 648 gicd_write(sc, GICD_ICENABLERn(group), valid); 649 650 /* 651 * Count how many are valid. 652 */ 653 sc->sc_gic_lines += popcount32(valid); 654 sc->sc_gic_valid_lines[group] = valid; 655 } 656 657 aprint_normal(": Generic Interrupt Controller, " 658 "%zu sources (%zu valid)\n", 659 sc->sc_pic.pic_maxsources, sc->sc_gic_lines); 660 aprint_debug_dev(sc->sc_dev, "Architecture version %d" 661 " (0x%x:%d rev %d)\n", iidr_arch, iidr_imp, iidr_prod, 662 iidr_rev); 663 664#ifdef MULTIPROCESSOR 665 sc->sc_pic.pic_cpus = kcpuset_running; 666#endif 667 pic_add(&sc->sc_pic, 0); 668 669 /* 670 * Force the GICD to IPL_HIGH and then enable interrupts. 671 */ 672 struct cpu_info * const ci = curcpu(); 673 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 674 armgic_set_priority(&sc->sc_pic, ci->ci_cpl); // set PMR 675 gicd_write(sc, GICD_CTRL, GICD_CTRL_Enable); // enable Distributer 676 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable CPU interrupts 677 ENABLE_INTERRUPT(); // allow interrupt exceptions 678 679 /* 680 * For each line that isn't valid, we set the intrsource for it to 681 * point at a dummy source so that pic_intr_establish will fail for it. 682 */ 683 for (size_t i = 0, group = 0; 684 i < sc->sc_pic.pic_maxsources; 685 i += 32, group++) { 686 uint32_t invalid = ~sc->sc_gic_valid_lines[group]; 687 for (size_t j = 0; invalid && j < 32; j++, invalid >>= 1) { 688 if (invalid & 1) { 689 sc->sc_pic.pic_sources[i + j] = 690 &armgic_dummy_source; 691 } 692 } 693 } 694#ifdef __HAVE_PIC_FAST_SOFTINTS 695 intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, 696 pic_handle_softint, (void *)SOFTINT_BIO, "softint bio"); 697 intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, 698 pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock"); 699 intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, 700 pic_handle_softint, (void *)SOFTINT_NET, "softint net"); 701 intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, 702 pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial"); 703#endif 704#ifdef MULTIPROCESSOR 705 armgic_cpu_init(&sc->sc_pic, curcpu()); 706 707 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_AST, IPL_VM, 708 IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 709 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_XCALL, IPL_HIGH, 710 IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 711 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_GENERIC, IPL_HIGH, 712 IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 713 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_NOP, IPL_VM, 714 IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 715 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_SHOOTDOWN, IPL_SCHED, 716 IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 717#ifdef DDB 718 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_DDB, IPL_HIGH, 719 IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 720#endif 721#ifdef __HAVE_PREEMPTION 722 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_KPREEMPT, IPL_VM, 723 IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 724#endif 725#endif 726 727 const u_int ppis = popcount32(sc->sc_gic_valid_lines[0] >> 16); 728 const u_int sgis = popcount32(sc->sc_gic_valid_lines[0] & 0xffff); 729 aprint_normal_dev(sc->sc_dev, "%u Priorities, %zu SPIs, %u PPIs, " 730 "%u SGIs\n", priorities, sc->sc_gic_lines - ppis - sgis, ppis, 731 sgis); 732 733 gic_spl_init(); 734} 735 736CFATTACH_DECL_NEW(armgic, 0, 737 armgic_match, armgic_attach, NULL, NULL); 738