1/*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * Copyright (c) 2023 Arm Ltd 4 * 5 * This software was developed by Andrew Turner under 6 * the sponsorship of the FreeBSD Foundation. 7 * 8 * This software was developed by Semihalf under 9 * the sponsorship of the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33#include "opt_acpi.h" 34#include "opt_platform.h" 35#include "opt_iommu.h" 36 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/bus.h> 40#include <sys/cpuset.h> 41#include <sys/domainset.h> 42#include <sys/endian.h> 43#include <sys/kernel.h> 44#include <sys/lock.h> 45#include <sys/malloc.h> 46#include <sys/module.h> 47#include <sys/mutex.h> 48#include <sys/physmem.h> 49#include <sys/proc.h> 50#include <sys/taskqueue.h> 51#include <sys/tree.h> 52#include <sys/queue.h> 53#include <sys/rman.h> 54#include <sys/sbuf.h> 55#include <sys/smp.h> 56#include <sys/sysctl.h> 57#include <sys/vmem.h> 58 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#include <vm/vm_page.h> 62 63#include <machine/bus.h> 64#include <machine/intr.h> 65 66#include <arm/arm/gic_common.h> 67#include <arm64/arm64/gic_v3_reg.h> 68#include <arm64/arm64/gic_v3_var.h> 69 70#ifdef FDT 71#include <dev/ofw/openfirm.h> 72#include <dev/ofw/ofw_bus.h> 73#include <dev/ofw/ofw_bus_subr.h> 74#endif 75#include <dev/pci/pcireg.h> 76#include <dev/pci/pcivar.h> 77 78#ifdef IOMMU 79#include <dev/iommu/iommu.h> 80#include <dev/iommu/iommu_gas.h> 81#endif 82 83#include "pcib_if.h" 84#include "pic_if.h" 85#include "msi_if.h" 86 87MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 88 "ARM GICv3 Interrupt Translation Service"); 89 90#define LPI_NIRQS (64 * 1024) 91 92/* The size and alignment of the command circular buffer */ 93#define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 94#define ITS_CMDQ_ALIGN (64 * 1024) 95 96#define LPI_CONFTAB_SIZE LPI_NIRQS 97#define LPI_CONFTAB_ALIGN (64 * 1024) 98#define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 99 100/* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 101#define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 102#define LPI_PENDTAB_ALIGN (64 * 1024) 103#define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 104 105#define LPI_INT_TRANS_TAB_ALIGN 256 106#define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 107 108/* ITS commands encoding */ 109#define ITS_CMD_MOVI (0x01) 110#define ITS_CMD_SYNC (0x05) 111#define ITS_CMD_MAPD (0x08) 112#define ITS_CMD_MAPC (0x09) 113#define ITS_CMD_MAPTI (0x0a) 114#define ITS_CMD_MAPI (0x0b) 115#define ITS_CMD_INV (0x0c) 116#define ITS_CMD_INVALL (0x0d) 117/* Command */ 118#define CMD_COMMAND_MASK (0xFFUL) 119/* PCI device ID */ 120#define CMD_DEVID_SHIFT (32) 121#define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 122/* Size of IRQ ID bitfield */ 123#define CMD_SIZE_MASK (0xFFUL) 124/* Virtual LPI ID */ 125#define CMD_ID_MASK (0xFFFFFFFFUL) 126/* Physical LPI ID */ 127#define CMD_PID_SHIFT (32) 128#define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 129/* Collection */ 130#define CMD_COL_MASK (0xFFFFUL) 131/* Target (CPU or Re-Distributor) */ 132#define CMD_TARGET_SHIFT (16) 133#define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 134/* Interrupt Translation Table address */ 135#define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 136/* Valid command bit */ 137#define CMD_VALID_SHIFT (63) 138#define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 139 140#define ITS_TARGET_NONE 0xFBADBEEF 141 142/* LPI chunk owned by ITS device */ 143struct lpi_chunk { 144 u_int lpi_base; 145 u_int lpi_free; /* First free LPI in set */ 146 u_int lpi_num; /* Total number of LPIs in chunk */ 147 u_int lpi_busy; /* Number of busy LPIs in chink */ 148}; 149 150/* ITS device */ 151struct its_dev { 152 TAILQ_ENTRY(its_dev) entry; 153 /* PCI device */ 154 device_t pci_dev; 155 /* Device ID (i.e. PCI device ID) */ 156 uint32_t devid; 157 /* List of assigned LPIs */ 158 struct lpi_chunk lpis; 159 /* Virtual address of ITT */ 160 void *itt; 161 size_t itt_size; 162}; 163 164/* 165 * ITS command descriptor. 166 * Idea for command description passing taken from Linux. 167 */ 168struct its_cmd_desc { 169 uint8_t cmd_type; 170 171 union { 172 struct { 173 struct its_dev *its_dev; 174 struct its_col *col; 175 uint32_t id; 176 } cmd_desc_movi; 177 178 struct { 179 struct its_col *col; 180 } cmd_desc_sync; 181 182 struct { 183 struct its_col *col; 184 uint8_t valid; 185 } cmd_desc_mapc; 186 187 struct { 188 struct its_dev *its_dev; 189 struct its_col *col; 190 uint32_t pid; 191 uint32_t id; 192 } cmd_desc_mapvi; 193 194 struct { 195 struct its_dev *its_dev; 196 struct its_col *col; 197 uint32_t pid; 198 } cmd_desc_mapi; 199 200 struct { 201 struct its_dev *its_dev; 202 uint8_t valid; 203 } cmd_desc_mapd; 204 205 struct { 206 struct its_dev *its_dev; 207 struct its_col *col; 208 uint32_t pid; 209 } cmd_desc_inv; 210 211 struct { 212 struct its_col *col; 213 } cmd_desc_invall; 214 }; 215}; 216 217/* ITS command. Each command is 32 bytes long */ 218struct its_cmd { 219 uint64_t cmd_dword[4]; /* ITS command double word */ 220}; 221 222/* An ITS private table */ 223struct its_ptable { 224 void *ptab_vaddr; 225 /* Size of the L1 and L2 tables */ 226 size_t ptab_l1_size; 227 size_t ptab_l2_size; 228 /* Number of L1 and L2 entries */ 229 int ptab_l1_nidents; 230 int ptab_l2_nidents; 231 232 int ptab_page_size; 233 int ptab_share; 234 bool ptab_indirect; 235}; 236 237/* ITS collection description. */ 238struct its_col { 239 uint64_t col_target; /* Target Re-Distributor */ 240 uint64_t col_id; /* Collection ID */ 241}; 242 243struct gicv3_its_irqsrc { 244 struct intr_irqsrc gi_isrc; 245 u_int gi_id; 246 u_int gi_lpi; 247 struct its_dev *gi_its_dev; 248 TAILQ_ENTRY(gicv3_its_irqsrc) gi_link; 249}; 250 251struct gicv3_its_softc { 252 device_t dev; 253 struct intr_pic *sc_pic; 254 struct resource *sc_its_res; 255 256 cpuset_t sc_cpus; 257 struct domainset *sc_ds; 258 u_int gic_irq_cpu; 259 int sc_devbits; 260 int sc_dev_table_idx; 261 262 struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 263 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 264 265 /* 266 * TODO: We should get these from the parent as we only want a 267 * single copy of each across the interrupt controller. 268 */ 269 uint8_t *sc_conf_base; 270 void *sc_pend_base[MAXCPU]; 271 272 /* Command handling */ 273 struct mtx sc_its_cmd_lock; 274 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 275 size_t sc_its_cmd_next_idx; 276 277 vmem_t *sc_irq_alloc; 278 struct gicv3_its_irqsrc **sc_irqs; 279 u_int sc_irq_base; 280 u_int sc_irq_length; 281 u_int sc_irq_count; 282 283 struct mtx sc_its_dev_lock; 284 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 285 TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs; 286 287#define ITS_FLAGS_CMDQ_FLUSH 0x00000001 288#define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 289#define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 290#define ITS_FLAGS_LPI_PREALLOC 0x00000008 291 u_int sc_its_flags; 292 bool trace_enable; 293 vm_page_t ma; /* fake msi page */ 294}; 295 296typedef void (its_quirk_func_t)(device_t); 297static its_quirk_func_t its_quirk_cavium_22375; 298 299static const struct { 300 const char *desc; 301 uint32_t iidr; 302 uint32_t iidr_mask; 303 its_quirk_func_t *func; 304} its_quirks[] = { 305 { 306 /* Cavium ThunderX Pass 1.x */ 307 .desc = "Cavium ThunderX errata: 22375, 24313", 308 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 309 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 310 .iidr_mask = ~GITS_IIDR_REVISION_MASK, 311 .func = its_quirk_cavium_22375, 312 }, 313}; 314 315#define gic_its_read_4(sc, reg) \ 316 bus_read_4((sc)->sc_its_res, (reg)) 317#define gic_its_read_8(sc, reg) \ 318 bus_read_8((sc)->sc_its_res, (reg)) 319 320#define gic_its_write_4(sc, reg, val) \ 321 bus_write_4((sc)->sc_its_res, (reg), (val)) 322#define gic_its_write_8(sc, reg, val) \ 323 bus_write_8((sc)->sc_its_res, (reg), (val)) 324 325static device_attach_t gicv3_its_attach; 326static device_detach_t gicv3_its_detach; 327 328static pic_disable_intr_t gicv3_its_disable_intr; 329static pic_enable_intr_t gicv3_its_enable_intr; 330static pic_map_intr_t gicv3_its_map_intr; 331static pic_setup_intr_t gicv3_its_setup_intr; 332static pic_post_filter_t gicv3_its_post_filter; 333static pic_post_ithread_t gicv3_its_post_ithread; 334static pic_pre_ithread_t gicv3_its_pre_ithread; 335static pic_bind_intr_t gicv3_its_bind_intr; 336#ifdef SMP 337static pic_init_secondary_t gicv3_its_init_secondary; 338#endif 339static msi_alloc_msi_t gicv3_its_alloc_msi; 340static msi_release_msi_t gicv3_its_release_msi; 341static msi_alloc_msix_t gicv3_its_alloc_msix; 342static msi_release_msix_t gicv3_its_release_msix; 343static msi_map_msi_t gicv3_its_map_msi; 344#ifdef IOMMU 345static msi_iommu_init_t gicv3_iommu_init; 346static msi_iommu_deinit_t gicv3_iommu_deinit; 347#endif 348 349static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 350static void its_cmd_mapc(device_t, struct its_col *, uint8_t); 351static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 352static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 353static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 354static void its_cmd_invall(device_t, struct its_col *); 355 356static device_method_t gicv3_its_methods[] = { 357 /* Device interface */ 358 DEVMETHOD(device_detach, gicv3_its_detach), 359 360 /* Interrupt controller interface */ 361 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 362 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 363 DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 364 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 365 DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 366 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 367 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 368#ifdef SMP 369 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 370 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 371#endif 372 373 /* MSI/MSI-X */ 374 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 375 DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 376 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 377 DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 378 DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 379#ifdef IOMMU 380 DEVMETHOD(msi_iommu_init, gicv3_iommu_init), 381 DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit), 382#endif 383 384 /* End */ 385 DEVMETHOD_END 386}; 387 388static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 389 sizeof(struct gicv3_its_softc)); 390 391static void 392gicv3_its_cmdq_init(struct gicv3_its_softc *sc) 393{ 394 vm_paddr_t cmd_paddr; 395 uint64_t reg, tmp; 396 397 /* Set up the command circular buffer */ 398 sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS, 399 sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 400 0); 401 sc->sc_its_cmd_next_idx = 0; 402 403 cmd_paddr = vtophys(sc->sc_its_cmd_base); 404 405 /* Set the base of the command buffer */ 406 reg = GITS_CBASER_VALID | 407 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 408 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 409 (ITS_CMDQ_SIZE / 4096 - 1); 410 gic_its_write_8(sc, GITS_CBASER, reg); 411 412 /* Read back to check for fixed value fields */ 413 tmp = gic_its_read_8(sc, GITS_CBASER); 414 415 if ((tmp & GITS_CBASER_SHARE_MASK) != 416 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 417 /* Check if the hardware reported non-shareable */ 418 if ((tmp & GITS_CBASER_SHARE_MASK) == 419 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 420 /* If so remove the cache attribute */ 421 reg &= ~GITS_CBASER_CACHE_MASK; 422 reg &= ~GITS_CBASER_SHARE_MASK; 423 /* Set to Non-cacheable, Non-shareable */ 424 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 425 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 426 427 gic_its_write_8(sc, GITS_CBASER, reg); 428 } 429 430 /* The command queue has to be flushed after each command */ 431 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 432 } 433 434 /* Get the next command from the start of the buffer */ 435 gic_its_write_8(sc, GITS_CWRITER, 0x0); 436} 437 438static int 439gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table) 440{ 441 uint64_t reg, tmp; 442 int page_size; 443 444 page_size = PAGE_SIZE_64K; 445 reg = gic_its_read_8(sc, GITS_BASER(table)); 446 447 while (1) { 448 reg &= GITS_BASER_PSZ_MASK; 449 switch (page_size) { 450 case PAGE_SIZE_4K: /* 4KB */ 451 reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 452 break; 453 case PAGE_SIZE_16K: /* 16KB */ 454 reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 455 break; 456 case PAGE_SIZE_64K: /* 64KB */ 457 reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 458 break; 459 } 460 461 /* Write the new page size */ 462 gic_its_write_8(sc, GITS_BASER(table), reg); 463 464 /* Read back to check */ 465 tmp = gic_its_read_8(sc, GITS_BASER(table)); 466 467 /* The page size is correct */ 468 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK)) 469 return (page_size); 470 471 switch (page_size) { 472 default: 473 return (-1); 474 case PAGE_SIZE_16K: 475 page_size = PAGE_SIZE_4K; 476 break; 477 case PAGE_SIZE_64K: 478 page_size = PAGE_SIZE_16K; 479 break; 480 } 481 } 482} 483 484static bool 485gicv3_its_table_supports_indirect(struct gicv3_its_softc *sc, int table) 486{ 487 uint64_t reg; 488 489 reg = gic_its_read_8(sc, GITS_BASER(table)); 490 491 /* Try setting the indirect flag */ 492 reg |= GITS_BASER_INDIRECT; 493 gic_its_write_8(sc, GITS_BASER(table), reg); 494 495 /* Read back to check */ 496 reg = gic_its_read_8(sc, GITS_BASER(table)); 497 return ((reg & GITS_BASER_INDIRECT) != 0); 498} 499 500 501static int 502gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 503{ 504 void *table; 505 vm_paddr_t paddr; 506 uint64_t cache, reg, share, tmp, type; 507 size_t its_tbl_size, nitspages, npages; 508 size_t l1_esize, l2_esize, l1_nidents, l2_nidents; 509 int i, page_size; 510 int devbits; 511 bool indirect; 512 513 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 514 /* 515 * GITS_TYPER[17:13] of ThunderX reports that device IDs 516 * are to be 21 bits in length. The entry size of the ITS 517 * table can be read from GITS_BASERn[52:48] and on ThunderX 518 * is supposed to be 8 bytes in length (for device table). 519 * Finally the page size that is to be used by ITS to access 520 * this table will be set to 64KB. 521 * 522 * This gives 0x200000 entries of size 0x8 bytes covered by 523 * 256 pages each of which 64KB in size. The number of pages 524 * (minus 1) should then be written to GITS_BASERn[7:0]. In 525 * that case this value would be 0xFF but on ThunderX the 526 * maximum value that HW accepts is 0xFD. 527 * 528 * Set an arbitrary number of device ID bits to 20 in order 529 * to limit the number of entries in ITS device table to 530 * 0x100000 and the table size to 8MB. 531 */ 532 devbits = 20; 533 cache = 0; 534 } else { 535 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 536 cache = GITS_BASER_CACHE_WAWB; 537 } 538 sc->sc_devbits = devbits; 539 share = GITS_BASER_SHARE_IS; 540 541 for (i = 0; i < GITS_BASER_NUM; i++) { 542 reg = gic_its_read_8(sc, GITS_BASER(i)); 543 /* The type of table */ 544 type = GITS_BASER_TYPE(reg); 545 if (type == GITS_BASER_TYPE_UNIMPL) 546 continue; 547 548 /* The table entry size */ 549 l1_esize = GITS_BASER_ESIZE(reg); 550 551 /* Find the tables page size */ 552 page_size = gicv3_its_table_page_size(sc, i); 553 if (page_size == -1) { 554 device_printf(dev, "No valid page size for table %d\n", 555 i); 556 return (EINVAL); 557 } 558 559 indirect = false; 560 l2_nidents = 0; 561 l2_esize = 0; 562 switch(type) { 563 case GITS_BASER_TYPE_DEV: 564 if (sc->sc_dev_table_idx != -1) 565 device_printf(dev, 566 "Warning: Multiple device tables found\n"); 567 568 sc->sc_dev_table_idx = i; 569 l1_nidents = (1 << devbits); 570 if ((l1_esize * l1_nidents) > (page_size * 2)) { 571 indirect = 572 gicv3_its_table_supports_indirect(sc, i); 573 if (indirect) { 574 /* 575 * Each l1 entry is 8 bytes and points 576 * to an l2 table of size page_size. 577 * Calculate how many entries this is 578 * and use this to find how many 579 * 8 byte l1 idents we need. 580 */ 581 l2_esize = l1_esize; 582 l2_nidents = page_size / l2_esize; 583 l1_nidents = l1_nidents / l2_nidents; 584 l1_esize = GITS_INDIRECT_L1_ESIZE; 585 } 586 } 587 its_tbl_size = l1_esize * l1_nidents; 588 its_tbl_size = roundup2(its_tbl_size, page_size); 589 break; 590 case GITS_BASER_TYPE_VP: 591 case GITS_BASER_TYPE_PP: /* Undocumented? */ 592 case GITS_BASER_TYPE_IC: 593 its_tbl_size = page_size; 594 break; 595 default: 596 if (bootverbose) 597 device_printf(dev, "Unhandled table type %lx\n", 598 type); 599 continue; 600 } 601 npages = howmany(its_tbl_size, PAGE_SIZE); 602 603 /* Allocate the table */ 604 table = contigmalloc_domainset(npages * PAGE_SIZE, 605 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, 606 (1ul << 48) - 1, PAGE_SIZE_64K, 0); 607 608 sc->sc_its_ptab[i].ptab_vaddr = table; 609 sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size; 610 sc->sc_its_ptab[i].ptab_l1_nidents = l1_nidents; 611 sc->sc_its_ptab[i].ptab_l2_size = page_size; 612 sc->sc_its_ptab[i].ptab_l2_nidents = l2_nidents; 613 614 sc->sc_its_ptab[i].ptab_indirect = indirect; 615 sc->sc_its_ptab[i].ptab_page_size = page_size; 616 617 paddr = vtophys(table); 618 619 while (1) { 620 nitspages = howmany(its_tbl_size, page_size); 621 622 /* Clear the fields we will be setting */ 623 reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT | 624 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 625 GITS_BASER_PA_MASK | 626 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 627 GITS_BASER_SIZE_MASK); 628 /* Set the new values */ 629 reg |= GITS_BASER_VALID | 630 (indirect ? GITS_BASER_INDIRECT : 0) | 631 (cache << GITS_BASER_CACHE_SHIFT) | 632 (type << GITS_BASER_TYPE_SHIFT) | 633 paddr | (share << GITS_BASER_SHARE_SHIFT) | 634 (nitspages - 1); 635 636 switch (page_size) { 637 case PAGE_SIZE_4K: /* 4KB */ 638 reg |= 639 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 640 break; 641 case PAGE_SIZE_16K: /* 16KB */ 642 reg |= 643 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 644 break; 645 case PAGE_SIZE_64K: /* 64KB */ 646 reg |= 647 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 648 break; 649 } 650 651 gic_its_write_8(sc, GITS_BASER(i), reg); 652 653 /* Read back to check */ 654 tmp = gic_its_read_8(sc, GITS_BASER(i)); 655 656 /* Do the shareability masks line up? */ 657 if ((tmp & GITS_BASER_SHARE_MASK) != 658 (reg & GITS_BASER_SHARE_MASK)) { 659 share = (tmp & GITS_BASER_SHARE_MASK) >> 660 GITS_BASER_SHARE_SHIFT; 661 continue; 662 } 663 664 if (tmp != reg) { 665 device_printf(dev, "GITS_BASER%d: " 666 "unable to be updated: %lx != %lx\n", 667 i, reg, tmp); 668 return (ENXIO); 669 } 670 671 sc->sc_its_ptab[i].ptab_share = share; 672 /* We should have made all needed changes */ 673 break; 674 } 675 } 676 677 return (0); 678} 679 680static void 681gicv3_its_conftable_init(struct gicv3_its_softc *sc) 682{ 683 /* note: we assume the ITS children are serialized by the parent */ 684 static void *conf_table; 685 int extra_flags = 0; 686 device_t gicv3; 687 uint32_t ctlr; 688 vm_paddr_t conf_pa; 689 vm_offset_t conf_va; 690 691 /* 692 * The PROPBASER is a singleton in our parent. We only set it up the 693 * first time through. conf_table is effectively global to all the units 694 * and we rely on subr_bus to serialize probe/attach. 695 */ 696 if (conf_table != NULL) { 697 sc->sc_conf_base = conf_table; 698 return; 699 } 700 701 gicv3 = device_get_parent(sc->dev); 702 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 703 if ((ctlr & GICR_CTLR_LPI_ENABLE) != 0) { 704 conf_pa = gic_r_read_8(gicv3, GICR_PROPBASER); 705 conf_pa &= GICR_PROPBASER_PA_MASK; 706 /* 707 * If there was a pre-existing PROPBASER, then we need to honor 708 * it because implementation defined behavior in gicv3 makes it 709 * impossible to quiesce to change it out. We will only see a 710 * pre-existing one when we've been kexec'd from a Linux kernel, 711 * or from a LinuxBoot environment. 712 * 713 * Linux provides us with a MEMRESERVE table that we put into 714 * the excluded physmem area. If PROPBASER isn't in this tabke, 715 * the system cannot run due to random memory corruption, 716 * so we panic for this case. 717 */ 718 if (!physmem_excluded(conf_pa, LPI_CONFTAB_SIZE)) 719 panic("gicv3 PROPBASER needs to reuse %#lx, but not reserved", 720 conf_pa); 721 conf_va = PHYS_TO_DMAP(conf_pa); 722 if (!pmap_klookup(conf_va, NULL)) 723 panic("Cannot map prior LPI mapping into KVA"); 724 conf_table = (void *)conf_va; 725 extra_flags = ITS_FLAGS_LPI_PREALLOC | ITS_FLAGS_LPI_CONF_FLUSH; 726 if (bootverbose) 727 device_printf(sc->dev, 728 "LPI enabled, conf table using pa %#lx va %lx\n", 729 conf_pa, conf_va); 730 } else { 731 /* 732 * Otherwise just allocate contiguous pages. We'll configure the 733 * PROPBASER register later in its_init_cpu_lpi(). 734 */ 735 conf_table = contigmalloc(LPI_CONFTAB_SIZE, 736 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, 737 LPI_CONFTAB_ALIGN, 0); 738 } 739 sc->sc_conf_base = conf_table; 740 sc->sc_its_flags |= extra_flags; 741 742 /* Set the default configuration */ 743 memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 744 LPI_CONFTAB_SIZE); 745 746 /* Flush the table to memory */ 747 cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE); 748} 749 750static void 751gicv3_its_pendtables_init(struct gicv3_its_softc *sc) 752{ 753 754 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) { 755 for (int i = 0; i <= mp_maxid; i++) { 756 if (CPU_ISSET(i, &sc->sc_cpus) == 0) 757 continue; 758 759 sc->sc_pend_base[i] = contigmalloc( 760 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 761 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 762 763 /* Flush so the ITS can see the memory */ 764 cpu_dcache_wb_range(sc->sc_pend_base[i], 765 LPI_PENDTAB_SIZE); 766 } 767 } 768} 769 770static void 771its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) 772{ 773 device_t gicv3; 774 uint64_t xbaser, tmp, size; 775 uint32_t ctlr; 776 u_int cpuid; 777 778 gicv3 = device_get_parent(dev); 779 cpuid = PCPU_GET(cpuid); 780 781 /* 782 * Set the redistributor base. If we're reusing what we found on boot 783 * since the gic was already running, then don't touch it here. We also 784 * don't need to disable / enable LPI if we're not changing PROPBASER, 785 * so only do that if we're not prealloced. 786 */ 787 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) { 788 /* Disable LPIs */ 789 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 790 ctlr &= ~GICR_CTLR_LPI_ENABLE; 791 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 792 793 /* Make sure changes are observable my the GIC */ 794 dsb(sy); 795 796 size = (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 797 798 xbaser = vtophys(sc->sc_conf_base) | 799 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 800 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 801 size; 802 803 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 804 805 /* Check the cache attributes we set */ 806 tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 807 808 if ((tmp & GICR_PROPBASER_SHARE_MASK) != 809 (xbaser & GICR_PROPBASER_SHARE_MASK)) { 810 if ((tmp & GICR_PROPBASER_SHARE_MASK) == 811 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 812 /* We need to mark as non-cacheable */ 813 xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 814 GICR_PROPBASER_CACHE_MASK); 815 /* Non-cacheable */ 816 xbaser |= GICR_PROPBASER_CACHE_NIN << 817 GICR_PROPBASER_CACHE_SHIFT; 818 /* Non-shareable */ 819 xbaser |= GICR_PROPBASER_SHARE_NS << 820 GICR_PROPBASER_SHARE_SHIFT; 821 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 822 } 823 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 824 } 825 826 /* 827 * Set the LPI pending table base 828 */ 829 xbaser = vtophys(sc->sc_pend_base[cpuid]) | 830 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 831 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 832 833 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 834 835 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 836 837 if ((tmp & GICR_PENDBASER_SHARE_MASK) == 838 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 839 /* Clear the cahce and shareability bits */ 840 xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 841 GICR_PENDBASER_SHARE_MASK); 842 /* Mark as non-shareable */ 843 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 844 /* And non-cacheable */ 845 xbaser |= GICR_PENDBASER_CACHE_NIN << 846 GICR_PENDBASER_CACHE_SHIFT; 847 } 848 849 /* Enable LPIs */ 850 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 851 ctlr |= GICR_CTLR_LPI_ENABLE; 852 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 853 854 /* Make sure the GIC has seen everything */ 855 dsb(sy); 856 } else { 857 KASSERT(sc->sc_pend_base[cpuid] == NULL, 858 ("PREALLOC too soon cpuid %d", cpuid)); 859 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 860 tmp &= GICR_PENDBASER_PA_MASK; 861 if (!physmem_excluded(tmp, LPI_PENDTAB_SIZE)) 862 panic("gicv3 PENDBASER on cpu %d needs to reuse 0x%#lx, but not reserved\n", 863 cpuid, tmp); 864 sc->sc_pend_base[cpuid] = (void *)PHYS_TO_DMAP(tmp); 865 } 866 867 868 if (bootverbose) 869 device_printf(gicv3, "using %sPENDBASE of %#lx on cpu %d\n", 870 (sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) ? "pre-existing " : "", 871 vtophys(sc->sc_pend_base[cpuid]), cpuid); 872} 873 874static int 875its_init_cpu(device_t dev, struct gicv3_its_softc *sc) 876{ 877 device_t gicv3; 878 vm_paddr_t target; 879 u_int cpuid; 880 struct redist_pcpu *rpcpu; 881 882 gicv3 = device_get_parent(dev); 883 cpuid = PCPU_GET(cpuid); 884 if (!CPU_ISSET(cpuid, &sc->sc_cpus)) 885 return (0); 886 887 /* Check if the ITS is enabled on this CPU */ 888 if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) 889 return (ENXIO); 890 891 rpcpu = gicv3_get_redist(dev); 892 893 /* Do per-cpu LPI init once */ 894 if (!rpcpu->lpi_enabled) { 895 its_init_cpu_lpi(dev, sc); 896 rpcpu->lpi_enabled = true; 897 } 898 899 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 900 /* This ITS wants the redistributor physical address */ 901 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) + 902 rpcpu->offset); 903 } else { 904 /* This ITS wants the unique processor number */ 905 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) << 906 CMD_TARGET_SHIFT; 907 } 908 909 sc->sc_its_cols[cpuid]->col_target = target; 910 sc->sc_its_cols[cpuid]->col_id = cpuid; 911 912 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 913 its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 914 915 return (0); 916} 917 918static int 919gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS) 920{ 921 struct gicv3_its_softc *sc; 922 int rv; 923 924 sc = arg1; 925 926 rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req); 927 if (rv != 0 || req->newptr == NULL) 928 return (rv); 929 if (sc->trace_enable) 930 gic_its_write_8(sc, GITS_TRKCTLR, 3); 931 else 932 gic_its_write_8(sc, GITS_TRKCTLR, 0); 933 934 return (0); 935} 936 937static int 938gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS) 939{ 940 struct gicv3_its_softc *sc; 941 struct sbuf *sb; 942 int err; 943 944 sc = arg1; 945 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 946 if (sb == NULL) { 947 device_printf(sc->dev, "Could not allocate sbuf for output.\n"); 948 return (ENOMEM); 949 } 950 sbuf_cat(sb, "\n"); 951 sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n", 952 gic_its_read_4(sc, GITS_TRKCTLR)); 953 sbuf_printf(sb, "GITS_TRKR: 0x%08X\n", 954 gic_its_read_4(sc, GITS_TRKR)); 955 sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n", 956 gic_its_read_4(sc, GITS_TRKDIDR)); 957 sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n", 958 gic_its_read_4(sc, GITS_TRKPIDR)); 959 sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n", 960 gic_its_read_4(sc, GITS_TRKVIDR)); 961 sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n", 962 gic_its_read_4(sc, GITS_TRKTGTR)); 963 964 err = sbuf_finish(sb); 965 if (err) 966 device_printf(sc->dev, "Error finishing sbuf: %d\n", err); 967 sbuf_delete(sb); 968 return(err); 969} 970 971static int 972gicv3_its_init_sysctl(struct gicv3_its_softc *sc) 973{ 974 struct sysctl_oid *oid, *child; 975 struct sysctl_ctx_list *ctx_list; 976 977 ctx_list = device_get_sysctl_ctx(sc->dev); 978 child = device_get_sysctl_tree(sc->dev); 979 oid = SYSCTL_ADD_NODE(ctx_list, 980 SYSCTL_CHILDREN(child), OID_AUTO, "tracing", 981 CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing"); 982 if (oid == NULL) 983 return (ENXIO); 984 985 /* Add registers */ 986 SYSCTL_ADD_PROC(ctx_list, 987 SYSCTL_CHILDREN(oid), OID_AUTO, "enable", 988 CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 989 gicv3_its_sysctl_trace_enable, "CU", "Enable tracing"); 990 SYSCTL_ADD_PROC(ctx_list, 991 SYSCTL_CHILDREN(oid), OID_AUTO, "capture", 992 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 993 gicv3_its_sysctl_trace_regs, "", "Captured tracing registers."); 994 995 return (0); 996} 997 998static int 999gicv3_its_attach(device_t dev) 1000{ 1001 struct gicv3_its_softc *sc; 1002 int domain, err, i, rid; 1003 uint64_t phys; 1004 uint32_t ctlr, iidr; 1005 1006 sc = device_get_softc(dev); 1007 1008 sc->sc_dev_table_idx = -1; 1009 sc->sc_irq_length = gicv3_get_nirqs(dev); 1010 sc->sc_irq_base = GIC_FIRST_LPI; 1011 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; 1012 1013 rid = 0; 1014 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 1015 RF_ACTIVE); 1016 if (sc->sc_its_res == NULL) { 1017 device_printf(dev, "Could not allocate memory\n"); 1018 return (ENXIO); 1019 } 1020 1021 phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) + 1022 GITS_TRANSLATER, PAGE_SIZE); 1023 sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO); 1024 vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT); 1025 1026 CPU_COPY(&all_cpus, &sc->sc_cpus); 1027 iidr = gic_its_read_4(sc, GITS_IIDR); 1028 for (i = 0; i < nitems(its_quirks); i++) { 1029 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 1030 if (bootverbose) { 1031 device_printf(dev, "Applying %s\n", 1032 its_quirks[i].desc); 1033 } 1034 its_quirks[i].func(dev); 1035 break; 1036 } 1037 } 1038 1039 if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) { 1040 sc->sc_ds = DOMAINSET_PREF(domain); 1041 } else { 1042 sc->sc_ds = DOMAINSET_RR(); 1043 } 1044 1045 /* 1046 * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be 1047 * coming in via, for instance, a kexec/kboot style setup where a 1048 * previous kernel has configured then relinquished control. Clear it 1049 * so that we can reconfigure GITS_BASER*. 1050 */ 1051 ctlr = gic_its_read_4(sc, GITS_CTLR); 1052 if ((ctlr & GITS_CTLR_EN) != 0) { 1053 ctlr &= ~GITS_CTLR_EN; 1054 gic_its_write_4(sc, GITS_CTLR, ctlr); 1055 } 1056 1057 /* Allocate the private tables */ 1058 err = gicv3_its_table_init(dev, sc); 1059 if (err != 0) 1060 return (err); 1061 1062 /* Protects access to the device list */ 1063 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 1064 1065 /* Protects access to the ITS command circular buffer. */ 1066 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 1067 1068 /* Allocate the command circular buffer */ 1069 gicv3_its_cmdq_init(sc); 1070 1071 /* Allocate the per-CPU collections */ 1072 for (int cpu = 0; cpu <= mp_maxid; cpu++) 1073 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0) 1074 sc->sc_its_cols[cpu] = malloc_domainset( 1075 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 1076 DOMAINSET_PREF(pcpu_find(cpu)->pc_domain), 1077 M_WAITOK | M_ZERO); 1078 else 1079 sc->sc_its_cols[cpu] = NULL; 1080 1081 /* Enable the ITS */ 1082 gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN); 1083 1084 /* Create the LPI configuration table */ 1085 gicv3_its_conftable_init(sc); 1086 1087 /* And the pending tebles */ 1088 gicv3_its_pendtables_init(sc); 1089 1090 /* Enable LPIs on this CPU */ 1091 its_init_cpu(dev, sc); 1092 1093 TAILQ_INIT(&sc->sc_its_dev_list); 1094 TAILQ_INIT(&sc->sc_free_irqs); 1095 1096 /* 1097 * Create the vmem object to allocate INTRNG IRQs from. We try to 1098 * use all IRQs not already used by the GICv3. 1099 * XXX: This assumes there are no other interrupt controllers in the 1100 * system. 1101 */ 1102 sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0, 1103 gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK); 1104 1105 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length, 1106 M_GICV3_ITS, M_WAITOK | M_ZERO); 1107 1108 /* For GIC-500 install tracking sysctls. */ 1109 if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) == 1110 GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0)) 1111 gicv3_its_init_sysctl(sc); 1112 1113 return (0); 1114} 1115 1116static int 1117gicv3_its_detach(device_t dev) 1118{ 1119 1120 return (ENXIO); 1121} 1122 1123static void 1124its_quirk_cavium_22375(device_t dev) 1125{ 1126 struct gicv3_its_softc *sc; 1127 int domain; 1128 1129 sc = device_get_softc(dev); 1130 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 1131 1132 /* 1133 * We need to limit which CPUs we send these interrupts to on 1134 * the original dual socket ThunderX as it is unable to 1135 * forward them between the two sockets. 1136 */ 1137 if (bus_get_domain(dev, &domain) == 0) { 1138 if (domain < MAXMEMDOM) { 1139 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus); 1140 } else { 1141 CPU_ZERO(&sc->sc_cpus); 1142 } 1143 } 1144} 1145 1146static void 1147gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 1148{ 1149 struct gicv3_its_softc *sc; 1150 struct gicv3_its_irqsrc *girq; 1151 uint8_t *conf; 1152 1153 sc = device_get_softc(dev); 1154 girq = (struct gicv3_its_irqsrc *)isrc; 1155 conf = sc->sc_conf_base; 1156 1157 conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE; 1158 1159 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 1160 /* Clean D-cache under command. */ 1161 cpu_dcache_wb_range(&conf[girq->gi_lpi], 1); 1162 } else { 1163 /* DSB inner shareable, store */ 1164 dsb(ishst); 1165 } 1166 1167 its_cmd_inv(dev, girq->gi_its_dev, girq); 1168} 1169 1170static void 1171gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 1172{ 1173 struct gicv3_its_softc *sc; 1174 struct gicv3_its_irqsrc *girq; 1175 uint8_t *conf; 1176 1177 sc = device_get_softc(dev); 1178 girq = (struct gicv3_its_irqsrc *)isrc; 1179 conf = sc->sc_conf_base; 1180 1181 conf[girq->gi_lpi] |= LPI_CONF_ENABLE; 1182 1183 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 1184 /* Clean D-cache under command. */ 1185 cpu_dcache_wb_range(&conf[girq->gi_lpi], 1); 1186 } else { 1187 /* DSB inner shareable, store */ 1188 dsb(ishst); 1189 } 1190 1191 its_cmd_inv(dev, girq->gi_its_dev, girq); 1192} 1193 1194static int 1195gicv3_its_intr(void *arg, uintptr_t irq) 1196{ 1197 struct gicv3_its_softc *sc = arg; 1198 struct gicv3_its_irqsrc *girq; 1199 struct trapframe *tf; 1200 1201 irq -= sc->sc_irq_base; 1202 girq = sc->sc_irqs[irq]; 1203 if (girq == NULL) 1204 panic("gicv3_its_intr: Invalid interrupt %ld", 1205 irq + sc->sc_irq_base); 1206 1207 tf = curthread->td_intr_frame; 1208 intr_isrc_dispatch(&girq->gi_isrc, tf); 1209 return (FILTER_HANDLED); 1210} 1211 1212static void 1213gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 1214{ 1215 struct gicv3_its_irqsrc *girq; 1216 1217 girq = (struct gicv3_its_irqsrc *)isrc; 1218 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 1219} 1220 1221static void 1222gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 1223{ 1224 1225} 1226 1227static void 1228gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 1229{ 1230 struct gicv3_its_irqsrc *girq; 1231 1232 girq = (struct gicv3_its_irqsrc *)isrc; 1233 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 1234} 1235 1236static int 1237gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc) 1238{ 1239 struct gicv3_its_softc *sc; 1240 1241 sc = device_get_softc(dev); 1242 if (CPU_EMPTY(&isrc->isrc_cpu)) { 1243 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu, 1244 &sc->sc_cpus); 1245 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu); 1246 } 1247 1248 return (0); 1249} 1250 1251static int 1252gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 1253{ 1254 struct gicv3_its_irqsrc *girq; 1255 1256 gicv3_its_select_cpu(dev, isrc); 1257 1258 girq = (struct gicv3_its_irqsrc *)isrc; 1259 its_cmd_movi(dev, girq); 1260 return (0); 1261} 1262 1263static int 1264gicv3_its_map_intr(device_t dev, struct intr_map_data *data, 1265 struct intr_irqsrc **isrcp) 1266{ 1267 1268 /* 1269 * This should never happen, we only call this function to map 1270 * interrupts found before the controller driver is ready. 1271 */ 1272 panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 1273} 1274 1275static int 1276gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 1277 struct resource *res, struct intr_map_data *data) 1278{ 1279 1280 /* Bind the interrupt to a CPU */ 1281 gicv3_its_bind_intr(dev, isrc); 1282 1283 return (0); 1284} 1285 1286#ifdef SMP 1287static void 1288gicv3_its_init_secondary(device_t dev) 1289{ 1290 struct gicv3_its_softc *sc; 1291 1292 sc = device_get_softc(dev); 1293 1294 /* 1295 * This is fatal as otherwise we may bind interrupts to this CPU. 1296 * We need a way to tell the interrupt framework to only bind to a 1297 * subset of given CPUs when it performs the shuffle. 1298 */ 1299 if (its_init_cpu(dev, sc) != 0) 1300 panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 1301 PCPU_GET(cpuid)); 1302} 1303#endif 1304 1305static uint32_t 1306its_get_devid(device_t pci_dev) 1307{ 1308 uintptr_t id; 1309 1310 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 1311 panic("%s: %s: Unable to get the MSI DeviceID", __func__, 1312 device_get_nameunit(pci_dev)); 1313 1314 return (id); 1315} 1316 1317static struct its_dev * 1318its_device_find(device_t dev, device_t child) 1319{ 1320 struct gicv3_its_softc *sc; 1321 struct its_dev *its_dev = NULL; 1322 1323 sc = device_get_softc(dev); 1324 1325 mtx_lock_spin(&sc->sc_its_dev_lock); 1326 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 1327 if (its_dev->pci_dev == child) 1328 break; 1329 } 1330 mtx_unlock_spin(&sc->sc_its_dev_lock); 1331 1332 return (its_dev); 1333} 1334 1335static bool 1336its_device_alloc(struct gicv3_its_softc *sc, int devid) 1337{ 1338 struct its_ptable *ptable; 1339 void *l2_table; 1340 uint64_t *table; 1341 uint32_t index; 1342 bool shareable; 1343 1344 /* No device table */ 1345 if (sc->sc_dev_table_idx < 0) { 1346 if (devid >= (1 << sc->sc_devbits)) { 1347 if (bootverbose) { 1348 device_printf(sc->dev, 1349 "%s: Device out of range for hardware " 1350 "(%x >= %x)\n", __func__, devid, 1351 1 << sc->sc_devbits); 1352 } 1353 return (false); 1354 } 1355 return (true); 1356 } 1357 1358 ptable = &sc->sc_its_ptab[sc->sc_dev_table_idx]; 1359 /* Check the devid is within the table limit */ 1360 if (!ptable->ptab_indirect) { 1361 if (devid >= ptable->ptab_l1_nidents) { 1362 if (bootverbose) { 1363 device_printf(sc->dev, 1364 "%s: Device out of range for table " 1365 "(%x >= %x)\n", __func__, devid, 1366 ptable->ptab_l1_nidents); 1367 } 1368 return (false); 1369 } 1370 1371 return (true); 1372 } 1373 1374 /* Check the devid is within the allocated range */ 1375 index = devid / ptable->ptab_l2_nidents; 1376 if (index >= ptable->ptab_l1_nidents) { 1377 if (bootverbose) { 1378 device_printf(sc->dev, 1379 "%s: Index out of range for table (%x >= %x)\n", 1380 __func__, index, ptable->ptab_l1_nidents); 1381 } 1382 return (false); 1383 } 1384 1385 table = (uint64_t *)ptable->ptab_vaddr; 1386 /* We have an second level table */ 1387 if ((table[index] & GITS_BASER_VALID) != 0) 1388 return (true); 1389 1390 shareable = true; 1391 if ((ptable->ptab_share & GITS_BASER_SHARE_MASK) == GITS_BASER_SHARE_NS) 1392 shareable = false; 1393 1394 l2_table = contigmalloc_domainset(ptable->ptab_l2_size, 1395 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, 1396 ptable->ptab_page_size, 0); 1397 1398 if (!shareable) 1399 cpu_dcache_wb_range(l2_table, ptable->ptab_l2_size); 1400 1401 table[index] = vtophys(l2_table) | GITS_BASER_VALID; 1402 if (!shareable) 1403 cpu_dcache_wb_range(&table[index], sizeof(table[index])); 1404 1405 dsb(sy); 1406 return (true); 1407} 1408 1409static struct its_dev * 1410its_device_get(device_t dev, device_t child, u_int nvecs) 1411{ 1412 struct gicv3_its_softc *sc; 1413 struct its_dev *its_dev; 1414 vmem_addr_t irq_base; 1415 size_t esize; 1416 1417 sc = device_get_softc(dev); 1418 1419 its_dev = its_device_find(dev, child); 1420 if (its_dev != NULL) 1421 return (its_dev); 1422 1423 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 1424 if (its_dev == NULL) 1425 return (NULL); 1426 1427 its_dev->pci_dev = child; 1428 its_dev->devid = its_get_devid(child); 1429 1430 its_dev->lpis.lpi_busy = 0; 1431 its_dev->lpis.lpi_num = nvecs; 1432 its_dev->lpis.lpi_free = nvecs; 1433 1434 if (!its_device_alloc(sc, its_dev->devid)) { 1435 free(its_dev, M_GICV3_ITS); 1436 return (NULL); 1437 } 1438 1439 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 1440 &irq_base) != 0) { 1441 free(its_dev, M_GICV3_ITS); 1442 return (NULL); 1443 } 1444 its_dev->lpis.lpi_base = irq_base; 1445 1446 /* Get ITT entry size */ 1447 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 1448 1449 /* 1450 * Allocate ITT for this device. 1451 * PA has to be 256 B aligned. At least two entries for device. 1452 */ 1453 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 1454 its_dev->itt = contigmalloc_domainset(its_dev->itt_size, 1455 M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0, 1456 LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0); 1457 if (its_dev->itt == NULL) { 1458 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 1459 free(its_dev, M_GICV3_ITS); 1460 return (NULL); 1461 } 1462 1463 /* Make sure device sees zeroed ITT. */ 1464 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) 1465 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size); 1466 1467 mtx_lock_spin(&sc->sc_its_dev_lock); 1468 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 1469 mtx_unlock_spin(&sc->sc_its_dev_lock); 1470 1471 /* Map device to its ITT */ 1472 its_cmd_mapd(dev, its_dev, 1); 1473 1474 return (its_dev); 1475} 1476 1477static void 1478its_device_release(device_t dev, struct its_dev *its_dev) 1479{ 1480 struct gicv3_its_softc *sc; 1481 1482 KASSERT(its_dev->lpis.lpi_busy == 0, 1483 ("its_device_release: Trying to release an inuse ITS device")); 1484 1485 /* Unmap device in ITS */ 1486 its_cmd_mapd(dev, its_dev, 0); 1487 1488 sc = device_get_softc(dev); 1489 1490 /* Remove the device from the list of devices */ 1491 mtx_lock_spin(&sc->sc_its_dev_lock); 1492 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 1493 mtx_unlock_spin(&sc->sc_its_dev_lock); 1494 1495 /* Free ITT */ 1496 KASSERT(its_dev->itt != NULL, ("Invalid ITT in valid ITS device")); 1497 contigfree(its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 1498 1499 /* Free the IRQ allocation */ 1500 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 1501 its_dev->lpis.lpi_num); 1502 1503 free(its_dev, M_GICV3_ITS); 1504} 1505 1506static struct gicv3_its_irqsrc * 1507gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq) 1508{ 1509 struct gicv3_its_irqsrc *girq = NULL; 1510 1511 KASSERT(sc->sc_irqs[irq] == NULL, 1512 ("%s: Interrupt %u already allocated", __func__, irq)); 1513 mtx_lock_spin(&sc->sc_its_dev_lock); 1514 if (!TAILQ_EMPTY(&sc->sc_free_irqs)) { 1515 girq = TAILQ_FIRST(&sc->sc_free_irqs); 1516 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link); 1517 } 1518 mtx_unlock_spin(&sc->sc_its_dev_lock); 1519 if (girq == NULL) { 1520 girq = malloc(sizeof(*girq), M_GICV3_ITS, 1521 M_NOWAIT | M_ZERO); 1522 if (girq == NULL) 1523 return (NULL); 1524 girq->gi_id = -1; 1525 if (intr_isrc_register(&girq->gi_isrc, dev, 0, 1526 "%s,%u", device_get_nameunit(dev), irq) != 0) { 1527 free(girq, M_GICV3_ITS); 1528 return (NULL); 1529 } 1530 } 1531 girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI; 1532 sc->sc_irqs[irq] = girq; 1533 1534 return (girq); 1535} 1536 1537static void 1538gicv3_its_release_irqsrc(struct gicv3_its_softc *sc, 1539 struct gicv3_its_irqsrc *girq) 1540{ 1541 u_int irq; 1542 1543 mtx_assert(&sc->sc_its_dev_lock, MA_OWNED); 1544 1545 irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base; 1546 sc->sc_irqs[irq] = NULL; 1547 1548 girq->gi_id = -1; 1549 girq->gi_its_dev = NULL; 1550 TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link); 1551} 1552 1553static int 1554gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 1555 device_t *pic, struct intr_irqsrc **srcs) 1556{ 1557 struct gicv3_its_softc *sc; 1558 struct gicv3_its_irqsrc *girq; 1559 struct its_dev *its_dev; 1560 u_int irq; 1561 int i; 1562 1563 its_dev = its_device_get(dev, child, count); 1564 if (its_dev == NULL) 1565 return (ENXIO); 1566 1567 KASSERT(its_dev->lpis.lpi_free >= count, 1568 ("gicv3_its_alloc_msi: No free LPIs")); 1569 sc = device_get_softc(dev); 1570 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1571 its_dev->lpis.lpi_free; 1572 1573 /* Allocate the irqsrc for each MSI */ 1574 for (i = 0; i < count; i++, irq++) { 1575 its_dev->lpis.lpi_free--; 1576 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev, 1577 sc, irq); 1578 if (srcs[i] == NULL) 1579 break; 1580 } 1581 1582 /* The allocation failed, release them */ 1583 if (i != count) { 1584 mtx_lock_spin(&sc->sc_its_dev_lock); 1585 for (i = 0; i < count; i++) { 1586 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1587 if (girq == NULL) 1588 break; 1589 gicv3_its_release_irqsrc(sc, girq); 1590 srcs[i] = NULL; 1591 } 1592 mtx_unlock_spin(&sc->sc_its_dev_lock); 1593 return (ENXIO); 1594 } 1595 1596 /* Finish the allocation now we have all MSI irqsrcs */ 1597 for (i = 0; i < count; i++) { 1598 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1599 girq->gi_id = i; 1600 girq->gi_its_dev = its_dev; 1601 1602 /* Map the message to the given IRQ */ 1603 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1604 its_cmd_mapti(dev, girq); 1605 } 1606 its_dev->lpis.lpi_busy += count; 1607 *pic = dev; 1608 1609 return (0); 1610} 1611 1612static int 1613gicv3_its_release_msi(device_t dev, device_t child, int count, 1614 struct intr_irqsrc **isrc) 1615{ 1616 struct gicv3_its_softc *sc; 1617 struct gicv3_its_irqsrc *girq; 1618 struct its_dev *its_dev; 1619 int i; 1620 1621 its_dev = its_device_find(dev, child); 1622 1623 KASSERT(its_dev != NULL, 1624 ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1625 "no ITS device")); 1626 KASSERT(its_dev->lpis.lpi_busy >= count, 1627 ("gicv3_its_release_msi: Releasing more interrupts than " 1628 "were allocated: releasing %d, allocated %d", count, 1629 its_dev->lpis.lpi_busy)); 1630 1631 sc = device_get_softc(dev); 1632 mtx_lock_spin(&sc->sc_its_dev_lock); 1633 for (i = 0; i < count; i++) { 1634 girq = (struct gicv3_its_irqsrc *)isrc[i]; 1635 gicv3_its_release_irqsrc(sc, girq); 1636 } 1637 mtx_unlock_spin(&sc->sc_its_dev_lock); 1638 its_dev->lpis.lpi_busy -= count; 1639 1640 if (its_dev->lpis.lpi_busy == 0) 1641 its_device_release(dev, its_dev); 1642 1643 return (0); 1644} 1645 1646static int 1647gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1648 struct intr_irqsrc **isrcp) 1649{ 1650 struct gicv3_its_softc *sc; 1651 struct gicv3_its_irqsrc *girq; 1652 struct its_dev *its_dev; 1653 u_int nvecs, irq; 1654 1655 nvecs = pci_msix_count(child); 1656 its_dev = its_device_get(dev, child, nvecs); 1657 if (its_dev == NULL) 1658 return (ENXIO); 1659 1660 KASSERT(its_dev->lpis.lpi_free > 0, 1661 ("gicv3_its_alloc_msix: No free LPIs")); 1662 sc = device_get_softc(dev); 1663 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1664 its_dev->lpis.lpi_free; 1665 1666 girq = gicv3_its_alloc_irqsrc(dev, sc, irq); 1667 if (girq == NULL) 1668 return (ENXIO); 1669 girq->gi_id = its_dev->lpis.lpi_busy; 1670 girq->gi_its_dev = its_dev; 1671 1672 its_dev->lpis.lpi_free--; 1673 its_dev->lpis.lpi_busy++; 1674 1675 /* Map the message to the given IRQ */ 1676 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1677 its_cmd_mapti(dev, girq); 1678 1679 *pic = dev; 1680 *isrcp = (struct intr_irqsrc *)girq; 1681 1682 return (0); 1683} 1684 1685static int 1686gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1687{ 1688 struct gicv3_its_softc *sc; 1689 struct gicv3_its_irqsrc *girq; 1690 struct its_dev *its_dev; 1691 1692 its_dev = its_device_find(dev, child); 1693 1694 KASSERT(its_dev != NULL, 1695 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1696 "no ITS device")); 1697 KASSERT(its_dev->lpis.lpi_busy > 0, 1698 ("gicv3_its_release_msix: Releasing more interrupts than " 1699 "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1700 1701 sc = device_get_softc(dev); 1702 girq = (struct gicv3_its_irqsrc *)isrc; 1703 mtx_lock_spin(&sc->sc_its_dev_lock); 1704 gicv3_its_release_irqsrc(sc, girq); 1705 mtx_unlock_spin(&sc->sc_its_dev_lock); 1706 its_dev->lpis.lpi_busy--; 1707 1708 if (its_dev->lpis.lpi_busy == 0) 1709 its_device_release(dev, its_dev); 1710 1711 return (0); 1712} 1713 1714static int 1715gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1716 uint64_t *addr, uint32_t *data) 1717{ 1718 struct gicv3_its_softc *sc; 1719 struct gicv3_its_irqsrc *girq; 1720 1721 sc = device_get_softc(dev); 1722 girq = (struct gicv3_its_irqsrc *)isrc; 1723 1724 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1725 *data = girq->gi_id; 1726 1727 return (0); 1728} 1729 1730#ifdef IOMMU 1731static int 1732gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain) 1733{ 1734 struct gicv3_its_softc *sc; 1735 struct iommu_ctx *ctx; 1736 int error; 1737 1738 sc = device_get_softc(dev); 1739 ctx = iommu_get_dev_ctx(child); 1740 if (ctx == NULL) 1741 return (ENXIO); 1742 /* Map the page containing the GITS_TRANSLATER register. */ 1743 error = iommu_map_msi(ctx, PAGE_SIZE, 0, 1744 IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma); 1745 *domain = iommu_get_ctx_domain(ctx); 1746 1747 return (error); 1748} 1749 1750static void 1751gicv3_iommu_deinit(device_t dev, device_t child) 1752{ 1753 struct iommu_ctx *ctx; 1754 1755 ctx = iommu_get_dev_ctx(child); 1756 if (ctx == NULL) 1757 return; 1758 1759 iommu_unmap_msi(ctx); 1760} 1761#endif 1762 1763/* 1764 * Commands handling. 1765 */ 1766 1767static __inline void 1768cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1769{ 1770 /* Command field: DW0 [7:0] */ 1771 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1772 cmd->cmd_dword[0] |= htole64(cmd_type); 1773} 1774 1775static __inline void 1776cmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1777{ 1778 /* Device ID field: DW0 [63:32] */ 1779 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1780 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1781} 1782 1783static __inline void 1784cmd_format_size(struct its_cmd *cmd, uint16_t size) 1785{ 1786 /* Size field: DW1 [4:0] */ 1787 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1788 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1789} 1790 1791static __inline void 1792cmd_format_id(struct its_cmd *cmd, uint32_t id) 1793{ 1794 /* ID field: DW1 [31:0] */ 1795 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1796 cmd->cmd_dword[1] |= htole64(id); 1797} 1798 1799static __inline void 1800cmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1801{ 1802 /* Physical ID field: DW1 [63:32] */ 1803 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1804 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1805} 1806 1807static __inline void 1808cmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1809{ 1810 /* Collection field: DW2 [16:0] */ 1811 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1812 cmd->cmd_dword[2] |= htole64(col_id); 1813} 1814 1815static __inline void 1816cmd_format_target(struct its_cmd *cmd, uint64_t target) 1817{ 1818 /* Target Address field: DW2 [47:16] */ 1819 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1820 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1821} 1822 1823static __inline void 1824cmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1825{ 1826 /* ITT Address field: DW2 [47:8] */ 1827 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1828 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1829} 1830 1831static __inline void 1832cmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1833{ 1834 /* Valid field: DW2 [63] */ 1835 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1836 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1837} 1838 1839static inline bool 1840its_cmd_queue_full(struct gicv3_its_softc *sc) 1841{ 1842 size_t read_idx, next_write_idx; 1843 1844 /* Get the index of the next command */ 1845 next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1846 (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1847 /* And the index of the current command being read */ 1848 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1849 1850 /* 1851 * The queue is full when the write offset points 1852 * at the command before the current read offset. 1853 */ 1854 return (next_write_idx == read_idx); 1855} 1856 1857static inline void 1858its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1859{ 1860 1861 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1862 /* Clean D-cache under command. */ 1863 cpu_dcache_wb_range(cmd, sizeof(*cmd)); 1864 } else { 1865 /* DSB inner shareable, store */ 1866 dsb(ishst); 1867 } 1868 1869} 1870 1871static inline uint64_t 1872its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1873{ 1874 uint64_t off; 1875 1876 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1877 1878 return (off); 1879} 1880 1881static void 1882its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1883 struct its_cmd *cmd_last) 1884{ 1885 struct gicv3_its_softc *sc; 1886 uint64_t first, last, read; 1887 size_t us_left; 1888 1889 sc = device_get_softc(dev); 1890 1891 /* 1892 * XXX ARM64TODO: This is obviously a significant delay. 1893 * The reason for that is that currently the time frames for 1894 * the command to complete are not known. 1895 */ 1896 us_left = 1000000; 1897 1898 first = its_cmd_cwriter_offset(sc, cmd_first); 1899 last = its_cmd_cwriter_offset(sc, cmd_last); 1900 1901 for (;;) { 1902 read = gic_its_read_8(sc, GITS_CREADR); 1903 if (first < last) { 1904 if (read < first || read >= last) 1905 break; 1906 } else if (read < first && read >= last) 1907 break; 1908 1909 if (us_left-- == 0) { 1910 /* This means timeout */ 1911 device_printf(dev, 1912 "Timeout while waiting for CMD completion.\n"); 1913 return; 1914 } 1915 DELAY(1); 1916 } 1917} 1918 1919static struct its_cmd * 1920its_cmd_alloc_locked(device_t dev) 1921{ 1922 struct gicv3_its_softc *sc; 1923 struct its_cmd *cmd; 1924 size_t us_left; 1925 1926 sc = device_get_softc(dev); 1927 1928 /* 1929 * XXX ARM64TODO: This is obviously a significant delay. 1930 * The reason for that is that currently the time frames for 1931 * the command to complete (and therefore free the descriptor) 1932 * are not known. 1933 */ 1934 us_left = 1000000; 1935 1936 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1937 while (its_cmd_queue_full(sc)) { 1938 if (us_left-- == 0) { 1939 /* Timeout while waiting for free command */ 1940 device_printf(dev, 1941 "Timeout while waiting for free command\n"); 1942 return (NULL); 1943 } 1944 DELAY(1); 1945 } 1946 1947 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1948 sc->sc_its_cmd_next_idx++; 1949 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1950 1951 return (cmd); 1952} 1953 1954static uint64_t 1955its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1956{ 1957 uint64_t target; 1958 uint8_t cmd_type; 1959 u_int size; 1960 1961 cmd_type = desc->cmd_type; 1962 target = ITS_TARGET_NONE; 1963 1964 switch (cmd_type) { 1965 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1966 target = desc->cmd_desc_movi.col->col_target; 1967 cmd_format_command(cmd, ITS_CMD_MOVI); 1968 cmd_format_id(cmd, desc->cmd_desc_movi.id); 1969 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1970 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1971 break; 1972 case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1973 target = desc->cmd_desc_sync.col->col_target; 1974 cmd_format_command(cmd, ITS_CMD_SYNC); 1975 cmd_format_target(cmd, target); 1976 break; 1977 case ITS_CMD_MAPD: /* Assign ITT to device */ 1978 cmd_format_command(cmd, ITS_CMD_MAPD); 1979 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1980 /* 1981 * Size describes number of bits to encode interrupt IDs 1982 * supported by the device minus one. 1983 * When V (valid) bit is zero, this field should be written 1984 * as zero. 1985 */ 1986 if (desc->cmd_desc_mapd.valid != 0) { 1987 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1988 size = MAX(1, size) - 1; 1989 } else 1990 size = 0; 1991 1992 cmd_format_size(cmd, size); 1993 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1994 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1995 break; 1996 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1997 target = desc->cmd_desc_mapc.col->col_target; 1998 cmd_format_command(cmd, ITS_CMD_MAPC); 1999 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 2000 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 2001 cmd_format_target(cmd, target); 2002 break; 2003 case ITS_CMD_MAPTI: 2004 target = desc->cmd_desc_mapvi.col->col_target; 2005 cmd_format_command(cmd, ITS_CMD_MAPTI); 2006 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 2007 cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 2008 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 2009 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 2010 break; 2011 case ITS_CMD_MAPI: 2012 target = desc->cmd_desc_mapi.col->col_target; 2013 cmd_format_command(cmd, ITS_CMD_MAPI); 2014 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 2015 cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 2016 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 2017 break; 2018 case ITS_CMD_INV: 2019 target = desc->cmd_desc_inv.col->col_target; 2020 cmd_format_command(cmd, ITS_CMD_INV); 2021 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 2022 cmd_format_id(cmd, desc->cmd_desc_inv.pid); 2023 break; 2024 case ITS_CMD_INVALL: 2025 cmd_format_command(cmd, ITS_CMD_INVALL); 2026 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 2027 break; 2028 default: 2029 panic("its_cmd_prepare: Invalid command: %x", cmd_type); 2030 } 2031 2032 return (target); 2033} 2034 2035static int 2036its_cmd_send(device_t dev, struct its_cmd_desc *desc) 2037{ 2038 struct gicv3_its_softc *sc; 2039 struct its_cmd *cmd, *cmd_sync, *cmd_write; 2040 struct its_col col_sync; 2041 struct its_cmd_desc desc_sync; 2042 uint64_t target, cwriter; 2043 2044 sc = device_get_softc(dev); 2045 mtx_lock_spin(&sc->sc_its_cmd_lock); 2046 cmd = its_cmd_alloc_locked(dev); 2047 if (cmd == NULL) { 2048 device_printf(dev, "could not allocate ITS command\n"); 2049 mtx_unlock_spin(&sc->sc_its_cmd_lock); 2050 return (EBUSY); 2051 } 2052 2053 target = its_cmd_prepare(cmd, desc); 2054 its_cmd_sync(sc, cmd); 2055 2056 if (target != ITS_TARGET_NONE) { 2057 cmd_sync = its_cmd_alloc_locked(dev); 2058 if (cmd_sync != NULL) { 2059 desc_sync.cmd_type = ITS_CMD_SYNC; 2060 col_sync.col_target = target; 2061 desc_sync.cmd_desc_sync.col = &col_sync; 2062 its_cmd_prepare(cmd_sync, &desc_sync); 2063 its_cmd_sync(sc, cmd_sync); 2064 } 2065 } 2066 2067 /* Update GITS_CWRITER */ 2068 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 2069 gic_its_write_8(sc, GITS_CWRITER, cwriter); 2070 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 2071 mtx_unlock_spin(&sc->sc_its_cmd_lock); 2072 2073 its_cmd_wait_completion(dev, cmd, cmd_write); 2074 2075 return (0); 2076} 2077 2078/* Handlers to send commands */ 2079static void 2080its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 2081{ 2082 struct gicv3_its_softc *sc; 2083 struct its_cmd_desc desc; 2084 struct its_col *col; 2085 2086 sc = device_get_softc(dev); 2087 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 2088 2089 desc.cmd_type = ITS_CMD_MOVI; 2090 desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 2091 desc.cmd_desc_movi.col = col; 2092 desc.cmd_desc_movi.id = girq->gi_id; 2093 2094 its_cmd_send(dev, &desc); 2095} 2096 2097static void 2098its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 2099{ 2100 struct its_cmd_desc desc; 2101 2102 desc.cmd_type = ITS_CMD_MAPC; 2103 desc.cmd_desc_mapc.col = col; 2104 /* 2105 * Valid bit set - map the collection. 2106 * Valid bit cleared - unmap the collection. 2107 */ 2108 desc.cmd_desc_mapc.valid = valid; 2109 2110 its_cmd_send(dev, &desc); 2111} 2112 2113static void 2114its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 2115{ 2116 struct gicv3_its_softc *sc; 2117 struct its_cmd_desc desc; 2118 struct its_col *col; 2119 u_int col_id; 2120 2121 sc = device_get_softc(dev); 2122 2123 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 2124 col = sc->sc_its_cols[col_id]; 2125 2126 desc.cmd_type = ITS_CMD_MAPTI; 2127 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 2128 desc.cmd_desc_mapvi.col = col; 2129 /* The EventID sent to the device */ 2130 desc.cmd_desc_mapvi.id = girq->gi_id; 2131 /* The physical interrupt presented to softeware */ 2132 desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI; 2133 2134 its_cmd_send(dev, &desc); 2135} 2136 2137static void 2138its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 2139{ 2140 struct its_cmd_desc desc; 2141 2142 desc.cmd_type = ITS_CMD_MAPD; 2143 desc.cmd_desc_mapd.its_dev = its_dev; 2144 desc.cmd_desc_mapd.valid = valid; 2145 2146 its_cmd_send(dev, &desc); 2147} 2148 2149static void 2150its_cmd_inv(device_t dev, struct its_dev *its_dev, 2151 struct gicv3_its_irqsrc *girq) 2152{ 2153 struct gicv3_its_softc *sc; 2154 struct its_cmd_desc desc; 2155 struct its_col *col; 2156 2157 sc = device_get_softc(dev); 2158 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 2159 2160 desc.cmd_type = ITS_CMD_INV; 2161 /* The EventID sent to the device */ 2162 desc.cmd_desc_inv.pid = girq->gi_id; 2163 desc.cmd_desc_inv.its_dev = its_dev; 2164 desc.cmd_desc_inv.col = col; 2165 2166 its_cmd_send(dev, &desc); 2167} 2168 2169static void 2170its_cmd_invall(device_t dev, struct its_col *col) 2171{ 2172 struct its_cmd_desc desc; 2173 2174 desc.cmd_type = ITS_CMD_INVALL; 2175 desc.cmd_desc_invall.col = col; 2176 2177 its_cmd_send(dev, &desc); 2178} 2179 2180#ifdef FDT 2181static device_probe_t gicv3_its_fdt_probe; 2182static device_attach_t gicv3_its_fdt_attach; 2183 2184static device_method_t gicv3_its_fdt_methods[] = { 2185 /* Device interface */ 2186 DEVMETHOD(device_probe, gicv3_its_fdt_probe), 2187 DEVMETHOD(device_attach, gicv3_its_fdt_attach), 2188 2189 /* End */ 2190 DEVMETHOD_END 2191}; 2192 2193#define its_baseclasses its_fdt_baseclasses 2194DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 2195 sizeof(struct gicv3_its_softc), gicv3_its_driver); 2196#undef its_baseclasses 2197 2198EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0, 2199 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 2200 2201static int 2202gicv3_its_fdt_probe(device_t dev) 2203{ 2204 2205 if (!ofw_bus_status_okay(dev)) 2206 return (ENXIO); 2207 2208 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 2209 return (ENXIO); 2210 2211 if (!gicv3_get_support_lpis(dev)) 2212 return (ENXIO); 2213 2214 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 2215 return (BUS_PROBE_DEFAULT); 2216} 2217 2218static int 2219gicv3_its_fdt_attach(device_t dev) 2220{ 2221 struct gicv3_its_softc *sc; 2222 phandle_t xref; 2223 int err; 2224 2225 sc = device_get_softc(dev); 2226 sc->dev = dev; 2227 err = gicv3_its_attach(dev); 2228 if (err != 0) 2229 return (err); 2230 2231 /* Register this device as a interrupt controller */ 2232 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 2233 sc->sc_pic = intr_pic_register(dev, xref); 2234 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 2235 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 2236 if (err != 0) { 2237 device_printf(dev, "Failed to add PIC handler: %d\n", err); 2238 return (err); 2239 } 2240 2241 /* Register this device to handle MSI interrupts */ 2242 err = intr_msi_register(dev, xref); 2243 if (err != 0) { 2244 device_printf(dev, "Failed to register for MSIs: %d\n", err); 2245 return (err); 2246 } 2247 2248 return (0); 2249} 2250#endif 2251 2252#ifdef DEV_ACPI 2253static device_probe_t gicv3_its_acpi_probe; 2254static device_attach_t gicv3_its_acpi_attach; 2255 2256static device_method_t gicv3_its_acpi_methods[] = { 2257 /* Device interface */ 2258 DEVMETHOD(device_probe, gicv3_its_acpi_probe), 2259 DEVMETHOD(device_attach, gicv3_its_acpi_attach), 2260 2261 /* End */ 2262 DEVMETHOD_END 2263}; 2264 2265#define its_baseclasses its_acpi_baseclasses 2266DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods, 2267 sizeof(struct gicv3_its_softc), gicv3_its_driver); 2268#undef its_baseclasses 2269 2270EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0, 2271 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 2272 2273static int 2274gicv3_its_acpi_probe(device_t dev) 2275{ 2276 2277 if (gic_get_bus(dev) != GIC_BUS_ACPI) 2278 return (EINVAL); 2279 2280 if (gic_get_hw_rev(dev) < 3) 2281 return (EINVAL); 2282 2283 if (!gicv3_get_support_lpis(dev)) 2284 return (ENXIO); 2285 2286 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 2287 return (BUS_PROBE_DEFAULT); 2288} 2289 2290static int 2291gicv3_its_acpi_attach(device_t dev) 2292{ 2293 struct gicv3_its_softc *sc; 2294 struct gic_v3_devinfo *di; 2295 int err; 2296 2297 sc = device_get_softc(dev); 2298 sc->dev = dev; 2299 err = gicv3_its_attach(dev); 2300 if (err != 0) 2301 return (err); 2302 2303 di = device_get_ivars(dev); 2304 sc->sc_pic = intr_pic_register(dev, di->msi_xref); 2305 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 2306 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 2307 if (err != 0) { 2308 device_printf(dev, "Failed to add PIC handler: %d\n", err); 2309 return (err); 2310 } 2311 2312 /* Register this device to handle MSI interrupts */ 2313 err = intr_msi_register(dev, di->msi_xref); 2314 if (err != 0) { 2315 device_printf(dev, "Failed to register for MSIs: %d\n", err); 2316 return (err); 2317 } 2318 2319 return (0); 2320} 2321#endif 2322