e500_intr.c revision 1.32
1/* $NetBSD: e500_intr.c,v 1.32 2015/01/23 09:02:42 nonaka Exp $ */ 2/*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37#include "opt_mpc85xx.h" 38#include "opt_multiprocessor.h" 39#include "opt_ddb.h" 40 41#define __INTR_PRIVATE 42 43#include <sys/cdefs.h> 44__KERNEL_RCSID(0, "$NetBSD: e500_intr.c,v 1.32 2015/01/23 09:02:42 nonaka Exp $"); 45 46#include <sys/param.h> 47#include <sys/proc.h> 48#include <sys/intr.h> 49#include <sys/cpu.h> 50#include <sys/kmem.h> 51#include <sys/atomic.h> 52#include <sys/bus.h> 53#include <sys/xcall.h> 54#include <sys/ipi.h> 55#include <sys/bitops.h> 56 57#include <uvm/uvm_extern.h> 58 59#ifdef __HAVE_FAST_SOFTINTS 60#include <powerpc/softint.h> 61#endif 62 63#include <powerpc/spr.h> 64#include <powerpc/booke/spr.h> 65 66#include <powerpc/booke/cpuvar.h> 67#include <powerpc/booke/e500reg.h> 68#include <powerpc/booke/e500var.h> 69#include <powerpc/booke/openpicreg.h> 70 71#define IPL2CTPR(ipl) ((ipl) + 15 - IPL_HIGH) 72#define CTPR2IPL(ctpr) ((ctpr) - (15 - IPL_HIGH)) 73 74#define IST_PERCPU_P(ist) ((ist) >= IST_TIMER) 75 76struct e500_intr_irq_info { 77 bus_addr_t irq_vpr; 78 bus_addr_t irq_dr; 79 u_int irq_vector; 80}; 81 82struct intr_source { 83 int (*is_func)(void *); 84 void *is_arg; 85 int8_t is_ipl; 86 uint8_t is_ist; 87 uint8_t is_irq; 88 uint8_t is_refcnt; 89 bus_size_t is_vpr; 90 bus_size_t is_dr; 91}; 92 93#define INTR_SOURCE_INITIALIZER \ 94 { .is_func = e500_intr_spurious, .is_arg = NULL, \ 95 .is_irq = -1, .is_ipl = IPL_NONE, .is_ist = IST_NONE, } 96 97struct e500_intr_name { 98 uint8_t in_irq; 99 const char in_name[15]; 100}; 101 102static const struct e500_intr_name e500_onchip_intr_names[] = { 103 { ISOURCE_L2, "l2" }, 104 { ISOURCE_ECM, "ecm" }, 105 { ISOURCE_DDR, "ddr" }, 106 { ISOURCE_LBC, "lbc" }, 107 { ISOURCE_DMA_CHAN1, "dma-chan1" }, 108 { ISOURCE_DMA_CHAN2, "dma-chan2" }, 109 { ISOURCE_DMA_CHAN3, "dma-chan3" }, 110 { ISOURCE_DMA_CHAN4, "dma-chan4" }, 111 { ISOURCE_PCI1, "pci1" }, 112 { ISOURCE_PCIEX2, "pcie2" }, 113 { ISOURCE_PCIEX , "pcie1" }, 114 { ISOURCE_PCIEX3, "pcie3" }, 115 { ISOURCE_USB1, "usb1" }, 116 { ISOURCE_ETSEC1_TX, "etsec1-tx" }, 117 { ISOURCE_ETSEC1_RX, "etsec1-rx" }, 118 { ISOURCE_ETSEC3_TX, "etsec3-tx" }, 119 { ISOURCE_ETSEC3_RX, "etsec3-rx" }, 120 { ISOURCE_ETSEC3_ERR, "etsec3-err" }, 121 { ISOURCE_ETSEC1_ERR, "etsec1-err" }, 122 { ISOURCE_ETSEC2_TX, "etsec2-tx" }, 123 { ISOURCE_ETSEC2_RX, "etsec2-rx" }, 124 { ISOURCE_ETSEC4_TX, "etsec4-tx" }, 125 { ISOURCE_ETSEC4_RX, "etsec4-rx" }, 126 { ISOURCE_ETSEC4_ERR, "etsec4-err" }, 127 { ISOURCE_ETSEC2_ERR, "etsec2-err" }, 128 { ISOURCE_DUART, "duart" }, 129 { ISOURCE_I2C, "i2c" }, 130 { ISOURCE_PERFMON, "perfmon" }, 131 { ISOURCE_SECURITY1, "sec1" }, 132 { ISOURCE_GPIO, "gpio" }, 133 { ISOURCE_SRIO_EWPU, "srio-ewpu" }, 134 { ISOURCE_SRIO_ODBELL, "srio-odbell" }, 135 { ISOURCE_SRIO_IDBELL, "srio-idbell" }, 136 { ISOURCE_SRIO_OMU1, "srio-omu1" }, 137 { ISOURCE_SRIO_IMU1, "srio-imu1" }, 138 { ISOURCE_SRIO_OMU2, "srio-omu2" }, 139 { ISOURCE_SRIO_IMU2, "srio-imu2" }, 140 { ISOURCE_SECURITY2, "sec2" }, 141 { ISOURCE_SPI, "spi" }, 142 { ISOURCE_ETSEC1_PTP, "etsec1-ptp" }, 143 { ISOURCE_ETSEC2_PTP, "etsec2-ptp" }, 144 { ISOURCE_ETSEC3_PTP, "etsec3-ptp" }, 145 { ISOURCE_ETSEC4_PTP, "etsec4-ptp" }, 146 { ISOURCE_ESDHC, "esdhc" }, 147 { 0, "" }, 148}; 149 150const struct e500_intr_name default_external_intr_names[] = { 151 { 0, "" }, 152}; 153 154static const struct e500_intr_name e500_msigroup_intr_names[] = { 155 { 0, "msigroup0" }, 156 { 1, "msigroup1" }, 157 { 2, "msigroup2" }, 158 { 3, "msigroup3" }, 159 { 4, "msigroup4" }, 160 { 5, "msigroup5" }, 161 { 6, "msigroup6" }, 162 { 7, "msigroup7" }, 163 { 0, "" }, 164}; 165 166static const struct e500_intr_name e500_timer_intr_names[] = { 167 { 0, "timer0" }, 168 { 1, "timer1" }, 169 { 2, "timer2" }, 170 { 3, "timer3" }, 171 { 0, "" }, 172}; 173 174static const struct e500_intr_name e500_ipi_intr_names[] = { 175 { 0, "ipi0" }, 176 { 1, "ipi1" }, 177 { 2, "ipi2" }, 178 { 3, "ipi3" }, 179 { 0, "" }, 180}; 181 182static const struct e500_intr_name e500_mi_intr_names[] = { 183 { 0, "mi0" }, 184 { 1, "mi1" }, 185 { 2, "mi2" }, 186 { 3, "mi3" }, 187 { 0, "" }, 188}; 189 190struct e500_intr_info { 191 u_int ii_external_sources; 192 uint32_t ii_onchip_bitmap[2]; 193 u_int ii_onchip_sources; 194 u_int ii_msigroup_sources; 195 u_int ii_ipi_sources; /* per-cpu */ 196 u_int ii_timer_sources; /* per-cpu */ 197 u_int ii_mi_sources; /* per-cpu */ 198 u_int ii_percpu_sources; 199 const struct e500_intr_name *ii_external_intr_names; 200 const struct e500_intr_name *ii_onchip_intr_names; 201 u_int8_t ii_ist_vectors[IST_MAX+1]; 202}; 203 204static kmutex_t e500_intr_lock __cacheline_aligned; 205static struct e500_intr_info e500_intr_info; 206 207#define INTR_INFO_DECL(lc_chip, UC_CHIP) \ 208static const struct e500_intr_info lc_chip##_intr_info = { \ 209 .ii_external_sources = UC_CHIP ## _EXTERNALSOURCES, \ 210 .ii_onchip_bitmap = UC_CHIP ## _ONCHIPBITMAP, \ 211 .ii_onchip_sources = UC_CHIP ## _ONCHIPSOURCES, \ 212 .ii_msigroup_sources = UC_CHIP ## _MSIGROUPSOURCES, \ 213 .ii_timer_sources = UC_CHIP ## _TIMERSOURCES, \ 214 .ii_ipi_sources = UC_CHIP ## _IPISOURCES, \ 215 .ii_mi_sources = UC_CHIP ## _MISOURCES, \ 216 .ii_percpu_sources = UC_CHIP ## _TIMERSOURCES \ 217 + UC_CHIP ## _IPISOURCES + UC_CHIP ## _MISOURCES, \ 218 .ii_external_intr_names = lc_chip ## _external_intr_names, \ 219 .ii_onchip_intr_names = lc_chip ## _onchip_intr_names, \ 220 .ii_ist_vectors = { \ 221 [IST_NONE] = ~0, \ 222 [IST_EDGE] = 0, \ 223 [IST_LEVEL_LOW] = 0, \ 224 [IST_LEVEL_HIGH] = 0, \ 225 [IST_PULSE] = 0, \ 226 [IST_ONCHIP] = UC_CHIP ## _EXTERNALSOURCES, \ 227 [IST_MSIGROUP] = UC_CHIP ## _EXTERNALSOURCES \ 228 + UC_CHIP ## _ONCHIPSOURCES, \ 229 [IST_TIMER] = UC_CHIP ## _EXTERNALSOURCES \ 230 + UC_CHIP ## _ONCHIPSOURCES \ 231 + UC_CHIP ## _MSIGROUPSOURCES, \ 232 [IST_IPI] = UC_CHIP ## _EXTERNALSOURCES \ 233 + UC_CHIP ## _ONCHIPSOURCES \ 234 + UC_CHIP ## _MSIGROUPSOURCES \ 235 + UC_CHIP ## _TIMERSOURCES, \ 236 [IST_MI] = UC_CHIP ## _EXTERNALSOURCES \ 237 + UC_CHIP ## _ONCHIPSOURCES \ 238 + UC_CHIP ## _MSIGROUPSOURCES \ 239 + UC_CHIP ## _TIMERSOURCES \ 240 + UC_CHIP ## _IPISOURCES, \ 241 [IST_MAX] = UC_CHIP ## _EXTERNALSOURCES \ 242 + UC_CHIP ## _ONCHIPSOURCES \ 243 + UC_CHIP ## _MSIGROUPSOURCES \ 244 + UC_CHIP ## _TIMERSOURCES \ 245 + UC_CHIP ## _IPISOURCES \ 246 + UC_CHIP ## _MISOURCES, \ 247 }, \ 248} 249 250#ifdef MPC8536 251#define mpc8536_external_intr_names default_external_intr_names 252const struct e500_intr_name mpc8536_onchip_intr_names[] = { 253 { ISOURCE_SATA2, "sata2" }, 254 { ISOURCE_USB2, "usb2" }, 255 { ISOURCE_USB3, "usb3" }, 256 { ISOURCE_SATA1, "sata1" }, 257 { 0, "" }, 258}; 259 260INTR_INFO_DECL(mpc8536, MPC8536); 261#endif 262 263#ifdef MPC8544 264#define mpc8544_external_intr_names default_external_intr_names 265const struct e500_intr_name mpc8544_onchip_intr_names[] = { 266 { 0, "" }, 267}; 268 269INTR_INFO_DECL(mpc8544, MPC8544); 270#endif 271#ifdef MPC8548 272#define mpc8548_external_intr_names default_external_intr_names 273const struct e500_intr_name mpc8548_onchip_intr_names[] = { 274 { ISOURCE_PCI1, "pci1" }, 275 { ISOURCE_PCI2, "pci2" }, 276 { 0, "" }, 277}; 278 279INTR_INFO_DECL(mpc8548, MPC8548); 280#endif 281#ifdef MPC8555 282#define mpc8555_external_intr_names default_external_intr_names 283const struct e500_intr_name mpc8555_onchip_intr_names[] = { 284 { ISOURCE_PCI2, "pci2" }, 285 { ISOURCE_CPM, "CPM" }, 286 { 0, "" }, 287}; 288 289INTR_INFO_DECL(mpc8555, MPC8555); 290#endif 291#ifdef MPC8568 292#define mpc8568_external_intr_names default_external_intr_names 293const struct e500_intr_name mpc8568_onchip_intr_names[] = { 294 { ISOURCE_QEB_LOW, "QEB low" }, 295 { ISOURCE_QEB_PORT, "QEB port" }, 296 { ISOURCE_QEB_IECC, "QEB iram ecc" }, 297 { ISOURCE_QEB_MUECC, "QEB ram ecc" }, 298 { ISOURCE_TLU1, "tlu1" }, 299 { ISOURCE_QEB_HIGH, "QEB high" }, 300 { 0, "" }, 301}; 302 303INTR_INFO_DECL(mpc8568, MPC8568); 304#endif 305#ifdef MPC8572 306#define mpc8572_external_intr_names default_external_intr_names 307const struct e500_intr_name mpc8572_onchip_intr_names[] = { 308 { ISOURCE_PCIEX3_MPC8572, "pcie3" }, 309 { ISOURCE_FEC, "fec" }, 310 { ISOURCE_PME_GENERAL, "pme" }, 311 { ISOURCE_TLU1, "tlu1" }, 312 { ISOURCE_TLU2, "tlu2" }, 313 { ISOURCE_PME_CHAN1, "pme-chan1" }, 314 { ISOURCE_PME_CHAN2, "pme-chan2" }, 315 { ISOURCE_PME_CHAN3, "pme-chan3" }, 316 { ISOURCE_PME_CHAN4, "pme-chan4" }, 317 { ISOURCE_DMA2_CHAN1, "dma2-chan1" }, 318 { ISOURCE_DMA2_CHAN2, "dma2-chan2" }, 319 { ISOURCE_DMA2_CHAN3, "dma2-chan3" }, 320 { ISOURCE_DMA2_CHAN4, "dma2-chan4" }, 321 { 0, "" }, 322}; 323 324INTR_INFO_DECL(mpc8572, MPC8572); 325#endif 326 327#ifdef P1025 328#define p1025_external_intr_names default_external_intr_names 329const struct e500_intr_name p1025_onchip_intr_names[] = { 330 { ISOURCE_PCIEX3_MPC8572, "pcie3" }, 331 { ISOURCE_ETSEC1_G1_TX, "etsec1-g1-tx" }, 332 { ISOURCE_ETSEC1_G1_RX, "etsec1-g1-rx" }, 333 { ISOURCE_ETSEC1_G1_ERR, "etsec1-g1-error" }, 334 { ISOURCE_ETSEC2_G1_TX, "etsec2-g1-tx" }, 335 { ISOURCE_ETSEC2_G1_RX, "etsec2-g1-rx" }, 336 { ISOURCE_ETSEC2_G1_ERR, "etsec2-g1-error" }, 337 { ISOURCE_ETSEC3_G1_TX, "etsec3-g1-tx" }, 338 { ISOURCE_ETSEC3_G1_RX, "etsec3-g1-rx" }, 339 { ISOURCE_ETSEC3_G1_ERR, "etsec3-g1-error" }, 340 { ISOURCE_QEB_MUECC, "qeb-low" }, 341 { ISOURCE_QEB_HIGH, "qeb-crit" }, 342 { ISOURCE_DMA2_CHAN1, "dma2-chan1" }, 343 { ISOURCE_DMA2_CHAN2, "dma2-chan2" }, 344 { ISOURCE_DMA2_CHAN3, "dma2-chan3" }, 345 { ISOURCE_DMA2_CHAN4, "dma2-chan4" }, 346 { 0, "" }, 347}; 348 349INTR_INFO_DECL(p1025, P1025); 350#endif 351 352#ifdef P2020 353#define p20x0_external_intr_names default_external_intr_names 354const struct e500_intr_name p20x0_onchip_intr_names[] = { 355 { ISOURCE_PCIEX3_MPC8572, "pcie3" }, 356 { ISOURCE_DMA2_CHAN1, "dma2-chan1" }, 357 { ISOURCE_DMA2_CHAN2, "dma2-chan2" }, 358 { ISOURCE_DMA2_CHAN3, "dma2-chan3" }, 359 { ISOURCE_DMA2_CHAN4, "dma2-chan4" }, 360 { 0, "" }, 361}; 362 363INTR_INFO_DECL(p20x0, P20x0); 364#endif 365 366#ifdef P1023 367#define p1023_external_intr_names default_external_intr_names 368const struct e500_intr_name p1023_onchip_intr_names[] = { 369 { ISOURCE_FMAN, "fman" }, 370 { ISOURCE_MDIO, "mdio" }, 371 { ISOURCE_QMAN0, "qman0" }, 372 { ISOURCE_BMAN0, "bman0" }, 373 { ISOURCE_QMAN1, "qman1" }, 374 { ISOURCE_BMAN1, "bman1" }, 375 { ISOURCE_QMAN2, "qman2" }, 376 { ISOURCE_BMAN2, "bman2" }, 377 { ISOURCE_SECURITY2_P1023, "sec2" }, 378 { ISOURCE_SEC_GENERAL, "sec-general" }, 379 { ISOURCE_DMA2_CHAN1, "dma2-chan1" }, 380 { ISOURCE_DMA2_CHAN2, "dma2-chan2" }, 381 { ISOURCE_DMA2_CHAN3, "dma2-chan3" }, 382 { ISOURCE_DMA2_CHAN4, "dma2-chan4" }, 383 { 0, "" }, 384}; 385 386INTR_INFO_DECL(p1023, P1023); 387#endif 388 389static const char ist_names[][12] = { 390 [IST_NONE] = "none", 391 [IST_EDGE] = "edge", 392 [IST_LEVEL_LOW] = "level-", 393 [IST_LEVEL_HIGH] = "level+", 394 [IST_PULSE] = "pulse", 395 [IST_MSI] = "msi", 396 [IST_ONCHIP] = "onchip", 397 [IST_MSIGROUP] = "msigroup", 398 [IST_TIMER] = "timer", 399 [IST_IPI] = "ipi", 400 [IST_MI] = "msgint", 401}; 402 403static struct intr_source *e500_intr_sources; 404static const struct intr_source *e500_intr_last_source; 405 406static void *e500_intr_establish(int, int, int, int (*)(void *), void *); 407static void e500_intr_disestablish(void *); 408static void e500_intr_cpu_attach(struct cpu_info *ci); 409static void e500_intr_cpu_hatch(struct cpu_info *ci); 410static void e500_intr_cpu_send_ipi(cpuid_t, uintptr_t); 411static void e500_intr_init(void); 412static void e500_intr_init_precpu(void); 413static const char *e500_intr_string(int, int, char *, size_t); 414static const char *e500_intr_typename(int); 415static void e500_critintr(struct trapframe *tf); 416static void e500_decrintr(struct trapframe *tf); 417static void e500_extintr(struct trapframe *tf); 418static void e500_fitintr(struct trapframe *tf); 419static void e500_wdogintr(struct trapframe *tf); 420static void e500_spl0(void); 421static int e500_splraise(int); 422static void e500_splx(int); 423 424const struct intrsw e500_intrsw = { 425 .intrsw_establish = e500_intr_establish, 426 .intrsw_disestablish = e500_intr_disestablish, 427 .intrsw_init = e500_intr_init, 428 .intrsw_cpu_attach = e500_intr_cpu_attach, 429 .intrsw_cpu_hatch = e500_intr_cpu_hatch, 430 .intrsw_cpu_send_ipi = e500_intr_cpu_send_ipi, 431 .intrsw_string = e500_intr_string, 432 .intrsw_typename = e500_intr_typename, 433 434 .intrsw_critintr = e500_critintr, 435 .intrsw_decrintr = e500_decrintr, 436 .intrsw_extintr = e500_extintr, 437 .intrsw_fitintr = e500_fitintr, 438 .intrsw_wdogintr = e500_wdogintr, 439 440 .intrsw_splraise = e500_splraise, 441 .intrsw_splx = e500_splx, 442 .intrsw_spl0 = e500_spl0, 443 444#ifdef __HAVE_FAST_SOFTINTS 445 .intrsw_softint_init_md = powerpc_softint_init_md, 446 .intrsw_softint_trigger = powerpc_softint_trigger, 447#endif 448}; 449 450static bool wdog_barked; 451 452static inline uint32_t 453openpic_read(struct cpu_softc *cpu, bus_size_t offset) 454{ 455 456 return bus_space_read_4(cpu->cpu_bst, cpu->cpu_bsh, 457 OPENPIC_BASE + offset); 458} 459 460static inline void 461openpic_write(struct cpu_softc *cpu, bus_size_t offset, uint32_t val) 462{ 463 464 return bus_space_write_4(cpu->cpu_bst, cpu->cpu_bsh, 465 OPENPIC_BASE + offset, val); 466} 467 468static const char * 469e500_intr_external_name_lookup(int irq) 470{ 471 prop_array_t extirqs = board_info_get_object("external-irqs"); 472 prop_string_t irqname = prop_array_get(extirqs, irq); 473 KASSERT(irqname != NULL); 474 KASSERT(prop_object_type(irqname) == PROP_TYPE_STRING); 475 476 return prop_string_cstring_nocopy(irqname); 477} 478 479static const char * 480e500_intr_name_lookup(const struct e500_intr_name *names, int irq) 481{ 482 for (; names->in_name[0] != '\0'; names++) { 483 if (names->in_irq == irq) 484 return names->in_name; 485 } 486 487 return NULL; 488} 489 490static const char * 491e500_intr_onchip_name_lookup(int irq) 492{ 493 const char *name; 494 495 name = e500_intr_name_lookup(e500_intr_info.ii_onchip_intr_names, irq); 496 if (name == NULL) 497 name = e500_intr_name_lookup(e500_onchip_intr_names, irq); 498 499 return name; 500} 501 502static inline void 503e500_splset(struct cpu_info *ci, int ipl) 504{ 505 struct cpu_softc * const cpu = ci->ci_softc; 506 507 KASSERT((curlwp->l_pflag & LP_INTR) == 0 || ipl != IPL_NONE); 508 const u_int ctpr = IPL2CTPR(ipl); 509 KASSERT(openpic_read(cpu, OPENPIC_CTPR) == IPL2CTPR(ci->ci_cpl)); 510 openpic_write(cpu, OPENPIC_CTPR, ctpr); 511 KASSERT(openpic_read(cpu, OPENPIC_CTPR) == ctpr); 512#ifdef DIAGNOSTIC 513 cpu->cpu_spl_tb[ipl][ci->ci_cpl] = mftb(); 514#endif 515 ci->ci_cpl = ipl; 516} 517 518static void 519e500_spl0(void) 520{ 521 wrtee(0); 522 523 struct cpu_info * const ci = curcpu(); 524 525#ifdef __HAVE_FAST_SOFTINTS 526 if (__predict_false(ci->ci_data.cpu_softints != 0)) { 527 e500_splset(ci, IPL_HIGH); 528 wrtee(PSL_EE); 529 powerpc_softint(ci, IPL_NONE, 530 (vaddr_t)__builtin_return_address(0)); 531 wrtee(0); 532 } 533#endif /* __HAVE_FAST_SOFTINTS */ 534 e500_splset(ci, IPL_NONE); 535 536 wrtee(PSL_EE); 537} 538 539static void 540e500_splx(int ipl) 541{ 542 struct cpu_info * const ci = curcpu(); 543 const int old_ipl = ci->ci_cpl; 544 545 /* if we paniced because of watchdog, PSL_CE will be clear. */ 546 KASSERT(wdog_barked || (mfmsr() & PSL_CE)); 547 548 if (ipl == old_ipl) 549 return; 550 551 if (__predict_false(ipl > old_ipl)) { 552 printf("%s: %p: cpl=%u: ignoring splx(%u) to raise ipl\n", 553 __func__, __builtin_return_address(0), old_ipl, ipl); 554 if (old_ipl == IPL_NONE) 555 Debugger(); 556 } 557 558 // const 559 register_t msr = wrtee(0); 560#ifdef __HAVE_FAST_SOFTINTS 561 const u_int softints = ci->ci_data.cpu_softints & (IPL_SOFTMASK << ipl); 562 if (__predict_false(softints != 0)) { 563 e500_splset(ci, IPL_HIGH); 564 wrtee(msr); 565 powerpc_softint(ci, ipl, 566 (vaddr_t)__builtin_return_address(0)); 567 wrtee(0); 568 } 569#endif /* __HAVE_FAST_SOFTINTS */ 570 e500_splset(ci, ipl); 571#if 1 572 if (ipl < IPL_VM && old_ipl >= IPL_VM) 573 msr = PSL_EE; 574#endif 575 wrtee(msr); 576} 577 578static int 579e500_splraise(int ipl) 580{ 581 struct cpu_info * const ci = curcpu(); 582 const int old_ipl = ci->ci_cpl; 583 584 /* if we paniced because of watchdog, PSL_CE will be clear. */ 585 KASSERT(wdog_barked || (mfmsr() & PSL_CE)); 586 587 if (old_ipl < ipl) { 588 //const 589 register_t msr = wrtee(0); 590 e500_splset(ci, ipl); 591#if 0 592 if (old_ipl < IPL_VM && ipl >= IPL_VM) 593 msr = 0; 594#endif 595 wrtee(msr); 596 } else if (ipl == IPL_NONE) { 597 panic("%s: %p: cpl=%u: attempt to splraise(IPL_NONE)", 598 __func__, __builtin_return_address(0), old_ipl); 599#if 0 600 } else if (old_ipl > ipl) { 601 printf("%s: %p: cpl=%u: ignoring splraise(%u) to lower ipl\n", 602 __func__, __builtin_return_address(0), old_ipl, ipl); 603#endif 604 } 605 606 return old_ipl; 607} 608 609static int 610e500_intr_spurious(void *arg) 611{ 612 return 0; 613} 614 615static bool 616e500_intr_irq_info_get(struct cpu_info *ci, u_int irq, int ipl, int ist, 617 struct e500_intr_irq_info *ii) 618{ 619 const struct e500_intr_info * const info = &e500_intr_info; 620 bool ok; 621 622#if DEBUG > 2 623 printf("%s(%p,irq=%u,ipl=%u,ist=%u,%p)\n", __func__, ci, irq, ipl, ist, ii); 624#endif 625 626 if (ipl < IPL_VM || ipl > IPL_HIGH) { 627#if DEBUG > 2 628 printf("%s:%d ipl=%u\n", __func__, __LINE__, ipl); 629#endif 630 return false; 631 } 632 633 if (ist <= IST_NONE || ist >= IST_MAX) { 634#if DEBUG > 2 635 printf("%s:%d ist=%u\n", __func__, __LINE__, ist); 636#endif 637 return false; 638 } 639 640 ii->irq_vector = irq + info->ii_ist_vectors[ist]; 641 if (IST_PERCPU_P(ist) && ist != IST_IPI) 642 ii->irq_vector += ci->ci_cpuid * info->ii_percpu_sources; 643 644 switch (ist) { 645 default: 646 ii->irq_vpr = OPENPIC_EIVPR(irq); 647 ii->irq_dr = OPENPIC_EIDR(irq); 648 ok = irq < info->ii_external_sources 649 && (ist == IST_EDGE 650 || ist == IST_LEVEL_LOW 651 || ist == IST_LEVEL_HIGH); 652 break; 653 case IST_PULSE: 654 ok = false; 655 break; 656 case IST_ONCHIP: 657 ii->irq_vpr = OPENPIC_IIVPR(irq); 658 ii->irq_dr = OPENPIC_IIDR(irq); 659 ok = irq < 32 * __arraycount(info->ii_onchip_bitmap); 660#if DEBUG > 2 661 printf("%s: irq=%u: ok=%u\n", __func__, irq, ok); 662#endif 663 ok = ok && (info->ii_onchip_bitmap[irq/32] & (1 << (irq & 31))); 664#if DEBUG > 2 665 printf("%s: %08x%08x -> %08x%08x: ok=%u\n", __func__, 666 irq < 32 ? 0 : (1 << irq), irq < 32 ? (1 << irq) : 0, 667 info->ii_onchip_bitmap[1], info->ii_onchip_bitmap[0], 668 ok); 669#endif 670 break; 671 case IST_MSIGROUP: 672 ii->irq_vpr = OPENPIC_MSIVPR(irq); 673 ii->irq_dr = OPENPIC_MSIDR(irq); 674 ok = irq < info->ii_msigroup_sources 675 && ipl == IPL_VM; 676 break; 677 case IST_TIMER: 678 ii->irq_vpr = OPENPIC_GTVPR(ci->ci_cpuid, irq); 679 ii->irq_dr = OPENPIC_GTDR(ci->ci_cpuid, irq); 680 ok = irq < info->ii_timer_sources; 681#if DEBUG > 2 682 printf("%s: IST_TIMER irq=%u: ok=%u\n", __func__, irq, ok); 683#endif 684 break; 685 case IST_IPI: 686 ii->irq_vpr = OPENPIC_IPIVPR(irq); 687 ii->irq_dr = OPENPIC_IPIDR(irq); 688 ok = irq < info->ii_ipi_sources; 689 break; 690 case IST_MI: 691 ii->irq_vpr = OPENPIC_MIVPR(irq); 692 ii->irq_dr = OPENPIC_MIDR(irq); 693 ok = irq < info->ii_mi_sources; 694 break; 695 } 696 697 return ok; 698} 699 700static const char * 701e500_intr_string(int irq, int ist, char *buf, size_t len) 702{ 703 struct cpu_info * const ci = curcpu(); 704 struct cpu_softc * const cpu = ci->ci_softc; 705 struct e500_intr_irq_info ii; 706 707 if (!e500_intr_irq_info_get(ci, irq, IPL_VM, ist, &ii)) 708 return NULL; 709 710 strlcpy(buf, cpu->cpu_evcnt_intrs[ii.irq_vector].ev_name, len); 711 return buf; 712} 713 714__CTASSERT(__arraycount(ist_names) == IST_MAX); 715 716static const char * 717e500_intr_typename(int ist) 718{ 719 if (IST_NONE <= ist && ist < IST_MAX) 720 return ist_names[ist]; 721 722 return NULL; 723} 724 725static void * 726e500_intr_cpu_establish(struct cpu_info *ci, int irq, int ipl, int ist, 727 int (*handler)(void *), void *arg) 728{ 729 struct cpu_softc * const cpu = ci->ci_softc; 730 struct e500_intr_irq_info ii; 731 732 KASSERT(ipl >= IPL_VM && ipl <= IPL_HIGH); 733 KASSERT(ist > IST_NONE && ist < IST_MAX && ist != IST_MSI); 734 735 if (!e500_intr_irq_info_get(ci, irq, ipl, ist, &ii)) { 736 printf("%s: e500_intr_irq_info_get(%p,%u,%u,%u,%p) failed\n", 737 __func__, ci, irq, ipl, ist, &ii); 738 return NULL; 739 } 740 741 struct intr_source * const is = &e500_intr_sources[ii.irq_vector]; 742 mutex_enter(&e500_intr_lock); 743 if (is->is_ipl != IPL_NONE) { 744 /* XXX IPI0 is shared by all CPU. */ 745 if (is->is_ist != IST_IPI || 746 is->is_irq != irq || 747 is->is_ipl != ipl || 748 is->is_ist != ist || 749 is->is_func != handler || 750 is->is_arg != arg) { 751 mutex_exit(&e500_intr_lock); 752 return NULL; 753 } 754 } 755 756 is->is_func = handler; 757 is->is_arg = arg; 758 is->is_ipl = ipl; 759 is->is_ist = ist; 760 is->is_irq = irq; 761 is->is_refcnt++; 762 is->is_vpr = ii.irq_vpr; 763 is->is_dr = ii.irq_dr; 764 765 uint32_t vpr = VPR_PRIORITY_MAKE(IPL2CTPR(ipl)) 766 | VPR_VECTOR_MAKE(((ii.irq_vector + 1) << 4) | ipl) 767 | (ist == IST_LEVEL_LOW 768 ? VPR_LEVEL_LOW 769 : (ist == IST_LEVEL_HIGH 770 ? VPR_LEVEL_HIGH 771 : (ist == IST_ONCHIP 772 ? VPR_P_HIGH 773 : 0))); 774 775 /* 776 * All interrupts go to the primary except per-cpu interrupts which get 777 * routed to the appropriate cpu. 778 */ 779 uint32_t dr = openpic_read(cpu, ii.irq_dr); 780 781 dr |= 1 << (IST_PERCPU_P(ist) ? ci->ci_cpuid : 0); 782 783 /* 784 * Update the vector/priority and destination registers keeping the 785 * interrupt masked. 786 */ 787 const register_t msr = wrtee(0); /* disable interrupts */ 788 openpic_write(cpu, ii.irq_vpr, vpr | VPR_MSK); 789 openpic_write(cpu, ii.irq_dr, dr); 790 791 /* 792 * Now unmask the interrupt. 793 */ 794 openpic_write(cpu, ii.irq_vpr, vpr); 795 796 wrtee(msr); /* re-enable interrupts */ 797 798 mutex_exit(&e500_intr_lock); 799 800 return is; 801} 802 803static void * 804e500_intr_establish(int irq, int ipl, int ist, 805 int (*handler)(void *), void *arg) 806{ 807 return e500_intr_cpu_establish(curcpu(), irq, ipl, ist, handler, arg); 808} 809 810static void 811e500_intr_disestablish(void *vis) 812{ 813 struct cpu_softc * const cpu = curcpu()->ci_softc; 814 struct intr_source * const is = vis; 815 struct e500_intr_irq_info ii; 816 817 KASSERT(e500_intr_sources <= is); 818 KASSERT(is < e500_intr_last_source); 819 KASSERT(!cpu_intr_p()); 820 821 bool ok = e500_intr_irq_info_get(curcpu(), is->is_irq, is->is_ipl, 822 is->is_ist, &ii); 823 (void)ok; /* appease gcc */ 824 KASSERT(ok); 825 KASSERT(is - e500_intr_sources == ii.irq_vector); 826 827 mutex_enter(&e500_intr_lock); 828 829 if (is->is_refcnt-- > 1) { 830 mutex_exit(&e500_intr_lock); 831 return; 832 } 833 834 /* 835 * Mask the source using the mask (MSK) bit in the vector/priority reg. 836 */ 837 uint32_t vpr = openpic_read(cpu, ii.irq_vpr); 838 openpic_write(cpu, ii.irq_vpr, VPR_MSK | vpr); 839 840 /* 841 * Wait for the Activity (A) bit for the source to be cleared. 842 */ 843 while (openpic_read(cpu, ii.irq_vpr) & VPR_A) 844 ; 845 846 /* 847 * Now the source can be modified. 848 */ 849 openpic_write(cpu, ii.irq_dr, 0); /* stop delivery */ 850 openpic_write(cpu, ii.irq_vpr, VPR_MSK); /* mask/reset it */ 851 852 *is = (struct intr_source)INTR_SOURCE_INITIALIZER; 853 854 mutex_exit(&e500_intr_lock); 855} 856 857static void 858e500_critintr(struct trapframe *tf) 859{ 860 panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1); 861} 862 863static void 864e500_decrintr(struct trapframe *tf) 865{ 866 panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1); 867} 868 869static void 870e500_fitintr(struct trapframe *tf) 871{ 872 panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1); 873} 874 875static void 876e500_wdogintr(struct trapframe *tf) 877{ 878 struct cpu_info * const ci = curcpu(); 879 mtspr(SPR_TSR, TSR_ENW|TSR_WIS); 880 wdog_barked = true; 881 dump_splhist(ci, NULL); 882 dump_trapframe(tf, NULL); 883 panic("%s: tf=%p tb=%"PRId64" srr0/srr1=%#lx/%#lx" 884 " cpl=%d idepth=%d, mtxcount=%d", 885 __func__, tf, mftb(), tf->tf_srr0, tf->tf_srr1, 886 ci->ci_cpl, ci->ci_idepth, ci->ci_mtx_count); 887} 888 889static void 890e500_extintr(struct trapframe *tf) 891{ 892 struct cpu_info * const ci = curcpu(); 893 struct cpu_softc * const cpu = ci->ci_softc; 894 const int old_ipl = ci->ci_cpl; 895 896 /* if we paniced because of watchdog, PSL_CE will be clear. */ 897 KASSERT(wdog_barked || (mfmsr() & PSL_CE)); 898 899#if 0 900// printf("%s(%p): idepth=%d enter\n", __func__, tf, ci->ci_idepth); 901 if ((register_t)tf >= (register_t)curlwp->l_addr + USPACE 902 || (register_t)tf < (register_t)curlwp->l_addr + NBPG) { 903 printf("%s(entry): pid %d.%d (%s): srr0/srr1=%#lx/%#lx: invalid tf addr %p\n", 904 __func__, curlwp->l_proc->p_pid, curlwp->l_lid, 905 curlwp->l_proc->p_comm, tf->tf_srr0, tf->tf_srr1, tf); 906 } 907#endif 908 909 910 ci->ci_data.cpu_nintr++; 911 tf->tf_cf.cf_idepth = ci->ci_idepth++; 912 cpu->cpu_pcpls[ci->ci_idepth] = old_ipl; 913#if 1 914 if (mfmsr() & PSL_EE) 915 panic("%s(%p): MSR[EE] is on (%#lx)!", __func__, tf, mfmsr()); 916 if (old_ipl == IPL_HIGH 917 || IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 918 panic("%s(%p): old_ipl(%u) == IPL_HIGH(%u) " 919 "|| old_ipl + %u != OPENPIC_CTPR (%u)", 920 __func__, tf, old_ipl, IPL_HIGH, 921 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 922#else 923 if (old_ipl >= IPL_VM) 924 panic("%s(%p): old_ipl(%u) >= IPL_VM(%u) CTPR=%u", 925 __func__, tf, old_ipl, IPL_VM, openpic_read(cpu, OPENPIC_CTPR)); 926#endif 927 928 for (;;) { 929 /* 930 * Find out the pending interrupt. 931 */ 932 KASSERTMSG((mfmsr() & PSL_EE) == 0, 933 "%s(%p): MSR[EE] left on (%#lx)!", __func__, tf, mfmsr()); 934 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 935 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 936 __func__, tf, __LINE__, old_ipl, 937 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 938 const uint32_t iack = openpic_read(cpu, OPENPIC_IACK); 939#ifdef DIAGNOSTIC 940 const int ipl = iack & 0xf; 941#endif 942 const int irq = (iack >> 4) - 1; 943#if 0 944 printf("%s: iack=%d ipl=%d irq=%d <%s>\n", 945 __func__, iack, ipl, irq, 946 (iack != IRQ_SPURIOUS ? 947 cpu->cpu_evcnt_intrs[irq].ev_name : "spurious")); 948#endif 949 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 950 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 951 __func__, tf, __LINE__, old_ipl, 952 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 953 if (iack == IRQ_SPURIOUS) 954 break; 955 956 struct intr_source * const is = &e500_intr_sources[irq]; 957 if (__predict_true(is < e500_intr_last_source)) { 958 /* 959 * Timer interrupts get their argument overriden with 960 * the pointer to the trapframe. 961 */ 962 KASSERTMSG(is->is_ipl == ipl, 963 "iack %#x: is %p: irq %d ipl %d != iack ipl %d", 964 iack, is, irq, is->is_ipl, ipl); 965 void *arg = (is->is_ist == IST_TIMER ? tf : is->is_arg); 966 if (is->is_ipl <= old_ipl) 967 panic("%s(%p): %s (%u): is->is_ipl (%u) <= old_ipl (%u)\n", 968 __func__, tf, 969 cpu->cpu_evcnt_intrs[irq].ev_name, irq, 970 is->is_ipl, old_ipl); 971 KASSERT(is->is_ipl > old_ipl); 972 e500_splset(ci, is->is_ipl); /* change IPL */ 973 if (__predict_false(is->is_func == NULL)) { 974 aprint_error_dev(ci->ci_dev, 975 "interrupt from unestablished irq %d\n", 976 irq); 977 } else { 978 int (*func)(void *) = is->is_func; 979 wrtee(PSL_EE); 980 int rv = (*func)(arg); 981 wrtee(0); 982#if DEBUG > 2 983 printf("%s: %s handler %p(%p) returned %d\n", 984 __func__, 985 cpu->cpu_evcnt_intrs[irq].ev_name, 986 func, arg, rv); 987#endif 988 if (rv == 0) 989 cpu->cpu_evcnt_spurious_intr.ev_count++; 990 } 991 e500_splset(ci, old_ipl); /* restore IPL */ 992 cpu->cpu_evcnt_intrs[irq].ev_count++; 993 } else { 994 aprint_error_dev(ci->ci_dev, 995 "interrupt from illegal irq %d\n", irq); 996 cpu->cpu_evcnt_spurious_intr.ev_count++; 997 } 998 /* 999 * If this is a nested interrupt, simply ack it and exit 1000 * because the loop we interrupted will complete looking 1001 * for interrupts. 1002 */ 1003 KASSERTMSG((mfmsr() & PSL_EE) == 0, 1004 "%s(%p): MSR[EE] left on (%#lx)!", __func__, tf, mfmsr()); 1005 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 1006 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 1007 __func__, tf, __LINE__, old_ipl, 1008 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 1009 1010 openpic_write(cpu, OPENPIC_EOI, 0); 1011 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 1012 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 1013 __func__, tf, __LINE__, old_ipl, 1014 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 1015 if (ci->ci_idepth > 0) 1016 break; 1017 } 1018 1019 ci->ci_idepth--; 1020 1021#ifdef __HAVE_FAST_SOFTINTS 1022 /* 1023 * Before exiting, deal with any softints that need to be dealt with. 1024 */ 1025 const u_int softints = ci->ci_data.cpu_softints & (IPL_SOFTMASK << old_ipl); 1026 if (__predict_false(softints != 0)) { 1027 KASSERT(old_ipl < IPL_VM); 1028 e500_splset(ci, IPL_HIGH); /* pop to high */ 1029 wrtee(PSL_EE); /* reenable interrupts */ 1030 powerpc_softint(ci, old_ipl, /* deal with them */ 1031 tf->tf_srr0); 1032 wrtee(0); /* disable interrupts */ 1033 e500_splset(ci, old_ipl); /* and drop back */ 1034 } 1035#endif /* __HAVE_FAST_SOFTINTS */ 1036 KASSERT(ci->ci_cpl == old_ipl); 1037 1038 /* 1039 * If we interrupted while power-saving and we need to exit idle, 1040 * we need to clear PSL_POW so we won't go back into power-saving. 1041 */ 1042 if (__predict_false(tf->tf_srr1 & PSL_POW) && ci->ci_want_resched) 1043 tf->tf_srr1 &= ~PSL_POW; 1044 1045// printf("%s(%p): idepth=%d exit\n", __func__, tf, ci->ci_idepth); 1046} 1047 1048static void 1049e500_intr_init(void) 1050{ 1051 struct cpu_info * const ci = curcpu(); 1052 struct cpu_softc * const cpu = ci->ci_softc; 1053 const uint32_t frr = openpic_read(cpu, OPENPIC_FRR); 1054 const u_int nirq = FRR_NIRQ_GET(frr) + 1; 1055// const u_int ncpu = FRR_NCPU_GET(frr) + 1; 1056 struct intr_source *is; 1057 struct e500_intr_info * const ii = &e500_intr_info; 1058 1059 const uint16_t svr = (mfspr(SPR_SVR) & ~0x80000) >> 16; 1060 switch (svr) { 1061#ifdef MPC8536 1062 case SVR_MPC8536v1 >> 16: 1063 *ii = mpc8536_intr_info; 1064 break; 1065#endif 1066#ifdef MPC8544 1067 case SVR_MPC8544v1 >> 16: 1068 *ii = mpc8544_intr_info; 1069 break; 1070#endif 1071#ifdef MPC8548 1072 case SVR_MPC8543v1 >> 16: 1073 case SVR_MPC8548v1 >> 16: 1074 *ii = mpc8548_intr_info; 1075 break; 1076#endif 1077#ifdef MPC8555 1078 case SVR_MPC8541v1 >> 16: 1079 case SVR_MPC8555v1 >> 16: 1080 *ii = mpc8555_intr_info; 1081 break; 1082#endif 1083#ifdef MPC8568 1084 case SVR_MPC8568v1 >> 16: 1085 *ii = mpc8568_intr_info; 1086 break; 1087#endif 1088#ifdef MPC8572 1089 case SVR_MPC8572v1 >> 16: 1090 *ii = mpc8572_intr_info; 1091 break; 1092#endif 1093#ifdef P1023 1094 case SVR_P1017v1 >> 16: 1095 case SVR_P1023v1 >> 16: 1096 *ii = p1023_intr_info; 1097 break; 1098#endif 1099#ifdef P1025 1100 case SVR_P1016v1 >> 16: 1101 case SVR_P1025v1 >> 16: 1102 *ii = p1025_intr_info; 1103 break; 1104#endif 1105#ifdef P2020 1106 case SVR_P2010v2 >> 16: 1107 case SVR_P2020v2 >> 16: 1108 *ii = p20x0_intr_info; 1109 break; 1110#endif 1111 default: 1112 panic("%s: don't know how to deal with SVR %#lx", 1113 __func__, mfspr(SPR_SVR)); 1114 } 1115 1116 /* 1117 * Initialize interrupt handler lock 1118 */ 1119 mutex_init(&e500_intr_lock, MUTEX_DEFAULT, IPL_HIGH); 1120 1121 /* 1122 * We need to be in mixed mode. 1123 */ 1124 openpic_write(cpu, OPENPIC_GCR, GCR_M); 1125 1126 /* 1127 * Make we and the openpic both agree about the current SPL level. 1128 */ 1129 e500_splset(ci, ci->ci_cpl); 1130 1131 /* 1132 * Allow the required number of interrupt sources. 1133 */ 1134 is = kmem_zalloc(nirq * sizeof(*is), KM_SLEEP); 1135 KASSERT(is); 1136 e500_intr_sources = is; 1137 e500_intr_last_source = is + nirq; 1138 1139 /* 1140 * Initialize all the external interrupts as active low. 1141 */ 1142 for (u_int irq = 0; irq < e500_intr_info.ii_external_sources; irq++) { 1143 openpic_write(cpu, OPENPIC_EIVPR(irq), 1144 VPR_VECTOR_MAKE(irq) | VPR_LEVEL_LOW); 1145 } 1146} 1147 1148static void 1149e500_intr_init_precpu(void) 1150{ 1151 struct cpu_info const *ci = curcpu(); 1152 struct cpu_softc * const cpu = ci->ci_softc; 1153 bus_addr_t dr; 1154 1155 /* 1156 * timer's DR is set to be delivered to cpu0 as initial value. 1157 */ 1158 for (u_int irq = 0; irq < e500_intr_info.ii_timer_sources; irq++) { 1159 dr = OPENPIC_GTDR(ci->ci_cpuid, irq); 1160 openpic_write(cpu, dr, 0); /* stop delivery */ 1161 } 1162} 1163 1164static void 1165e500_idlespin(void) 1166{ 1167 KASSERTMSG(curcpu()->ci_cpl == IPL_NONE, 1168 "%s: cpu%u: ci_cpl (%d) != 0", __func__, cpu_number(), 1169 curcpu()->ci_cpl); 1170 KASSERTMSG(CTPR2IPL(openpic_read(curcpu()->ci_softc, OPENPIC_CTPR)) == IPL_NONE, 1171 "%s: cpu%u: CTPR (%d) != IPL_NONE", __func__, cpu_number(), 1172 CTPR2IPL(openpic_read(curcpu()->ci_softc, OPENPIC_CTPR))); 1173 KASSERT(mfmsr() & PSL_EE); 1174 1175 if (powersave > 0) 1176 mtmsr(mfmsr() | PSL_POW); 1177} 1178 1179static void 1180e500_intr_cpu_attach(struct cpu_info *ci) 1181{ 1182 struct cpu_softc * const cpu = ci->ci_softc; 1183 const char * const xname = device_xname(ci->ci_dev); 1184 1185 const u_int32_t frr = openpic_read(cpu, OPENPIC_FRR); 1186 const u_int nirq = FRR_NIRQ_GET(frr) + 1; 1187// const u_int ncpu = FRR_NCPU_GET(frr) + 1; 1188 1189 const struct e500_intr_info * const info = &e500_intr_info; 1190 1191 cpu->cpu_clock_gtbcr = OPENPIC_GTBCR(ci->ci_cpuid, E500_CLOCK_TIMER); 1192 1193 cpu->cpu_evcnt_intrs = 1194 kmem_zalloc(nirq * sizeof(cpu->cpu_evcnt_intrs[0]), KM_SLEEP); 1195 KASSERT(cpu->cpu_evcnt_intrs); 1196 1197 struct evcnt *evcnt = cpu->cpu_evcnt_intrs; 1198 for (size_t j = 0; j < info->ii_external_sources; j++, evcnt++) { 1199 const char *name = e500_intr_external_name_lookup(j); 1200 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, NULL, xname, name); 1201 } 1202 KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_ONCHIP]); 1203 for (size_t j = 0; j < info->ii_onchip_sources; j++, evcnt++) { 1204 if (info->ii_onchip_bitmap[j / 32] & __BIT(j & 31)) { 1205 const char *name = e500_intr_onchip_name_lookup(j); 1206 if (name != NULL) { 1207 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1208 NULL, xname, name); 1209#ifdef DIAGNOSTIC 1210 } else { 1211 printf("%s: missing evcnt for onchip irq %zu\n", 1212 __func__, j); 1213#endif 1214 } 1215 } 1216 } 1217 1218 KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_MSIGROUP]); 1219 for (size_t j = 0; j < info->ii_msigroup_sources; j++, evcnt++) { 1220 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1221 NULL, xname, e500_msigroup_intr_names[j].in_name); 1222 } 1223 1224 KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_TIMER]); 1225 evcnt += ci->ci_cpuid * info->ii_percpu_sources; 1226 for (size_t j = 0; j < info->ii_timer_sources; j++, evcnt++) { 1227 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1228 NULL, xname, e500_timer_intr_names[j].in_name); 1229 } 1230 1231 for (size_t j = 0; j < info->ii_ipi_sources; j++, evcnt++) { 1232 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1233 NULL, xname, e500_ipi_intr_names[j].in_name); 1234 } 1235 1236 for (size_t j = 0; j < info->ii_mi_sources; j++, evcnt++) { 1237 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1238 NULL, xname, e500_mi_intr_names[j].in_name); 1239 } 1240 1241 ci->ci_idlespin = e500_idlespin; 1242} 1243 1244static void 1245e500_intr_cpu_send_ipi(cpuid_t target, uint32_t ipimsg) 1246{ 1247 struct cpu_info * const ci = curcpu(); 1248 struct cpu_softc * const cpu = ci->ci_softc; 1249 uint32_t dstmask; 1250 1251 if (target >= CPU_MAXNUM) { 1252 CPU_INFO_ITERATOR cii; 1253 struct cpu_info *dst_ci; 1254 1255 KASSERT(target == IPI_DST_NOTME || target == IPI_DST_ALL); 1256 1257 dstmask = 0; 1258 for (CPU_INFO_FOREACH(cii, dst_ci)) { 1259 if (target == IPI_DST_ALL || ci != dst_ci) { 1260 dstmask |= 1 << cpu_index(ci); 1261 if (ipimsg) 1262 atomic_or_32(&dst_ci->ci_pending_ipis, 1263 ipimsg); 1264 } 1265 } 1266 } else { 1267 struct cpu_info * const dst_ci = cpu_lookup(target); 1268 KASSERT(dst_ci != NULL); 1269 KASSERTMSG(target == cpu_index(dst_ci), 1270 "%s: target (%lu) != cpu_index(cpu%u)", 1271 __func__, target, cpu_index(dst_ci)); 1272 dstmask = (1 << target); 1273 if (ipimsg) 1274 atomic_or_32(&dst_ci->ci_pending_ipis, ipimsg); 1275 } 1276 1277 openpic_write(cpu, OPENPIC_IPIDR(0), dstmask); 1278} 1279 1280typedef void (*ipifunc_t)(void); 1281 1282#ifdef __HAVE_PREEEMPTION 1283static void 1284e500_ipi_kpreempt(void) 1285{ 1286 poowerpc_softint_trigger(1 << IPL_NONE); 1287} 1288#endif 1289 1290static void 1291e500_ipi_suspend(void) 1292{ 1293 1294#ifdef MULTIPROCESSOR 1295 cpu_pause(NULL); 1296#endif /* MULTIPROCESSOR */ 1297} 1298 1299static const ipifunc_t e500_ipifuncs[] = { 1300 [ilog2(IPI_XCALL)] = xc_ipi_handler, 1301 [ilog2(IPI_GENERIC)] = ipi_cpu_handler, 1302 [ilog2(IPI_HALT)] = e500_ipi_halt, 1303#ifdef __HAVE_PREEMPTION 1304 [ilog2(IPI_KPREEMPT)] = e500_ipi_kpreempt, 1305#endif 1306 [ilog2(IPI_TLB1SYNC)] = e500_tlb1_sync, 1307 [ilog2(IPI_SUSPEND)] = e500_ipi_suspend, 1308}; 1309 1310static int 1311e500_ipi_intr(void *v) 1312{ 1313 struct cpu_info * const ci = curcpu(); 1314 1315 ci->ci_ev_ipi.ev_count++; 1316 1317 uint32_t pending_ipis = atomic_swap_32(&ci->ci_pending_ipis, 0); 1318 for (u_int ipi = 31; pending_ipis != 0; ipi--, pending_ipis <<= 1) { 1319 const u_int bits = __builtin_clz(pending_ipis); 1320 ipi -= bits; 1321 pending_ipis <<= bits; 1322 KASSERT(e500_ipifuncs[ipi] != NULL); 1323 (*e500_ipifuncs[ipi])(); 1324 } 1325 1326 return 1; 1327} 1328 1329static void 1330e500_intr_cpu_hatch(struct cpu_info *ci) 1331{ 1332 1333 /* Initialize percpu interupts. */ 1334 e500_intr_init_precpu(); 1335 1336 /* 1337 * Establish clock interrupt for this CPU. 1338 */ 1339 if (e500_intr_cpu_establish(ci, E500_CLOCK_TIMER, IPL_CLOCK, IST_TIMER, 1340 e500_clock_intr, NULL) == NULL) 1341 panic("%s: failed to establish clock interrupt!", __func__); 1342 1343 /* 1344 * Establish the IPI interrupts for this CPU. 1345 */ 1346 if (e500_intr_cpu_establish(ci, 0, IPL_VM, IST_IPI, e500_ipi_intr, 1347 NULL) == NULL) 1348 panic("%s: failed to establish ipi interrupt!", __func__); 1349 1350 /* 1351 * Enable watchdog interrupts. 1352 */ 1353 uint32_t tcr = mfspr(SPR_TCR); 1354 tcr |= TCR_WIE; 1355 mtspr(SPR_TCR, tcr); 1356} 1357