gicv3_its.c revision 301265
1/*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Andrew Turner under 6 * the sponsorship of the FreeBSD Foundation. 7 * 8 * This software was developed by Semihalf under 9 * the sponsorship of the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33#include "opt_platform.h" 34 35#include <sys/cdefs.h> 36__FBSDID("$FreeBSD: head/sys/arm64/arm64/gicv3_its.c 301265 2016-06-03 10:28:06Z andrew $"); 37 38#include <sys/param.h> 39#include <sys/systm.h> 40#include <sys/bus.h> 41#include <sys/cpuset.h> 42#include <sys/endian.h> 43#include <sys/kernel.h> 44#include <sys/malloc.h> 45#include <sys/module.h> 46#include <sys/proc.h> 47#include <sys/queue.h> 48#include <sys/rman.h> 49#include <sys/smp.h> 50#include <sys/vmem.h> 51 52#include <vm/vm.h> 53#include <vm/pmap.h> 54 55#include <machine/bus.h> 56#include <machine/intr.h> 57 58#include <arm64/arm64/gic_v3_reg.h> 59#include <arm64/arm64/gic_v3_var.h> 60 61#ifdef FDT 62#include <dev/ofw/openfirm.h> 63#include <dev/ofw/ofw_bus.h> 64#include <dev/ofw/ofw_bus_subr.h> 65#endif 66#include <dev/pci/pcireg.h> 67#include <dev/pci/pcivar.h> 68 69#include "pcib_if.h" 70#include "pic_if.h" 71#include "msi_if.h" 72 73MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 74 "ARM GICv3 Interrupt Translation Service"); 75 76#define LPI_NIRQS (64 * 1024) 77 78/* The size and alignment of the command circular buffer */ 79#define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 80#define ITS_CMDQ_ALIGN (64 * 1024) 81 82#define LPI_CONFTAB_SIZE LPI_NIRQS 83#define LPI_CONFTAB_ALIGN (64 * 1024) 84#define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 85 86/* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 87#define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 88#define LPI_PENDTAB_ALIGN (64 * 1024) 89#define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 90 91#define LPI_INT_TRANS_TAB_ALIGN 256 92#define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 93 94/* ITS commands encoding */ 95#define ITS_CMD_MOVI (0x01) 96#define ITS_CMD_SYNC (0x05) 97#define ITS_CMD_MAPD (0x08) 98#define ITS_CMD_MAPC (0x09) 99#define ITS_CMD_MAPTI (0x0a) 100#define ITS_CMD_MAPI (0x0b) 101#define ITS_CMD_INV (0x0c) 102#define ITS_CMD_INVALL (0x0d) 103/* Command */ 104#define CMD_COMMAND_MASK (0xFFUL) 105/* PCI device ID */ 106#define CMD_DEVID_SHIFT (32) 107#define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 108/* Size of IRQ ID bitfield */ 109#define CMD_SIZE_MASK (0xFFUL) 110/* Virtual LPI ID */ 111#define CMD_ID_MASK (0xFFFFFFFFUL) 112/* Physical LPI ID */ 113#define CMD_PID_SHIFT (32) 114#define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 115/* Collection */ 116#define CMD_COL_MASK (0xFFFFUL) 117/* Target (CPU or Re-Distributor) */ 118#define CMD_TARGET_SHIFT (16) 119#define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 120/* Interrupt Translation Table address */ 121#define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 122/* Valid command bit */ 123#define CMD_VALID_SHIFT (63) 124#define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 125 126/* ITS command. Each command is 32 bytes long */ 127struct its_cmd { 128 uint64_t cmd_dword[4]; /* ITS command double word */ 129}; 130 131/* An ITS private table */ 132struct its_ptable { 133 vm_offset_t ptab_vaddr; 134 unsigned long ptab_size; 135}; 136 137/* ITS collection description. */ 138struct its_col { 139 uint64_t col_target; /* Target Re-Distributor */ 140 uint64_t col_id; /* Collection ID */ 141}; 142 143struct gicv3_its_irqsrc { 144 struct intr_irqsrc gi_isrc; 145 u_int gi_irq; 146 struct its_dev *gi_its_dev; 147}; 148 149struct gicv3_its_softc { 150 struct intr_pic *sc_pic; 151 struct resource *sc_its_res; 152 153 struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 154 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 155 156 /* 157 * TODO: We should get these from the parent as we only want a 158 * single copy of each across the interrupt controller. 159 */ 160 vm_offset_t sc_conf_base; 161 vm_offset_t sc_pend_base[MAXCPU]; 162 163 /* Command handling */ 164 struct mtx sc_its_cmd_lock; 165 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 166 size_t sc_its_cmd_next_idx; 167 168 vmem_t *sc_irq_alloc; 169 struct gicv3_its_irqsrc *sc_irqs; 170 171 struct mtx sc_its_dev_lock; 172 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 173 174#define ITS_FLAGS_CMDQ_FLUSH 0x00000001 175#define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 176#define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 177 u_int sc_its_flags; 178}; 179 180typedef void (its_quirk_func_t)(device_t); 181static its_quirk_func_t its_quirk_cavium_22375; 182 183static const struct { 184 const char *desc; 185 uint32_t iidr; 186 uint32_t iidr_mask; 187 its_quirk_func_t *func; 188} its_quirks[] = { 189 { 190 /* Cavium ThunderX Pass 1.x */ 191 .desc = "Cavoum ThunderX errata: 22375, 24313", 192 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 193 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 194 .iidr_mask = ~GITS_IIDR_REVISION_MASK, 195 .func = its_quirk_cavium_22375, 196 }, 197}; 198 199static u_int gic_irq_cpu; 200 201#define gic_its_read_4(sc, reg) \ 202 bus_read_4((sc)->sc_its_res, (reg)) 203#define gic_its_read_8(sc, reg) \ 204 bus_read_8((sc)->sc_its_res, (reg)) 205 206#define gic_its_write_4(sc, reg, val) \ 207 bus_write_4((sc)->sc_its_res, (reg), (val)) 208#define gic_its_write_8(sc, reg, val) \ 209 bus_write_8((sc)->sc_its_res, (reg), (val)) 210 211static device_attach_t gicv3_its_attach; 212static device_detach_t gicv3_its_detach; 213 214static pic_disable_intr_t gicv3_its_disable_intr; 215static pic_enable_intr_t gicv3_its_enable_intr; 216static pic_map_intr_t gicv3_its_map_intr; 217static pic_setup_intr_t gicv3_its_setup_intr; 218static pic_post_filter_t gicv3_its_post_filter; 219static pic_post_ithread_t gicv3_its_post_ithread; 220static pic_pre_ithread_t gicv3_its_pre_ithread; 221static pic_bind_intr_t gicv3_its_bind_intr; 222#ifdef SMP 223static pic_init_secondary_t gicv3_its_init_secondary; 224#endif 225static msi_alloc_msi_t gicv3_its_alloc_msi; 226static msi_release_msi_t gicv3_its_release_msi; 227static msi_alloc_msix_t gicv3_its_alloc_msix; 228static msi_release_msix_t gicv3_its_release_msix; 229static msi_map_msi_t gicv3_its_map_msi; 230 231static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 232static void its_cmd_mapc(device_t, struct its_col *, uint8_t); 233static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 234static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 235static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 236static void its_cmd_invall(device_t, struct its_col *); 237 238static device_method_t gicv3_its_methods[] = { 239 /* Device interface */ 240 DEVMETHOD(device_detach, gicv3_its_detach), 241 242 /* Interrupt controller interface */ 243 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 244 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 245 DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 246 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 247 DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 248 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 249 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 250#ifdef SMP 251 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 252 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 253#endif 254 255 /* MSI/MSI-X */ 256 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 257 DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 258 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 259 DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 260 DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 261 262 /* End */ 263 DEVMETHOD_END 264}; 265 266static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 267 sizeof(struct gicv3_its_softc)); 268 269static void 270gicv3_its_cmdq_init(struct gicv3_its_softc *sc) 271{ 272 vm_paddr_t cmd_paddr; 273 uint64_t reg, tmp; 274 275 /* Set up the command circular buffer */ 276 sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS, 277 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); 278 sc->sc_its_cmd_next_idx = 0; 279 280 cmd_paddr = vtophys(sc->sc_its_cmd_base); 281 282 /* Set the base of the command buffer */ 283 reg = GITS_CBASER_VALID | 284 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 285 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 286 (ITS_CMDQ_SIZE / 4096 - 1); 287 gic_its_write_8(sc, GITS_CBASER, reg); 288 289 /* Read back to check for fixed value fields */ 290 tmp = gic_its_read_8(sc, GITS_CBASER); 291 292 if ((tmp & GITS_CBASER_SHARE_MASK) != 293 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 294 /* Check if the hardware reported non-shareable */ 295 if ((tmp & GITS_CBASER_SHARE_MASK) == 296 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 297 /* If so remove the cache attribute */ 298 reg &= ~GITS_CBASER_CACHE_MASK; 299 reg &= ~GITS_CBASER_SHARE_MASK; 300 /* Set to Non-cacheable, Non-shareable */ 301 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 302 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 303 304 gic_its_write_8(sc, GITS_CBASER, reg); 305 } 306 307 /* The command queue has to be flushed after each command */ 308 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 309 } 310 311 /* Get the next command from the start of the buffer */ 312 gic_its_write_8(sc, GITS_CWRITER, 0x0); 313} 314 315static int 316gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 317{ 318 vm_offset_t table; 319 vm_paddr_t paddr; 320 uint64_t cache, reg, share, tmp, type; 321 size_t esize, its_tbl_size, nidents, nitspages, npages; 322 int i, page_size; 323 int devbits; 324 325 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 326 /* 327 * GITS_TYPER[17:13] of ThunderX reports that device IDs 328 * are to be 21 bits in length. The entry size of the ITS 329 * table can be read from GITS_BASERn[52:48] and on ThunderX 330 * is supposed to be 8 bytes in length (for device table). 331 * Finally the page size that is to be used by ITS to access 332 * this table will be set to 64KB. 333 * 334 * This gives 0x200000 entries of size 0x8 bytes covered by 335 * 256 pages each of which 64KB in size. The number of pages 336 * (minus 1) should then be written to GITS_BASERn[7:0]. In 337 * that case this value would be 0xFF but on ThunderX the 338 * maximum value that HW accepts is 0xFD. 339 * 340 * Set an arbitrary number of device ID bits to 20 in order 341 * to limit the number of entries in ITS device table to 342 * 0x100000 and the table size to 8MB. 343 */ 344 devbits = 20; 345 cache = 0; 346 } else { 347 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 348 cache = GITS_BASER_CACHE_WAWB; 349 } 350 share = GITS_BASER_SHARE_IS; 351 page_size = PAGE_SIZE_64K; 352 353 for (i = 0; i < GITS_BASER_NUM; i++) { 354 reg = gic_its_read_8(sc, GITS_BASER(i)); 355 /* The type of table */ 356 type = GITS_BASER_TYPE(reg); 357 /* The table entry size */ 358 esize = GITS_BASER_ESIZE(reg); 359 360 switch(type) { 361 case GITS_BASER_TYPE_DEV: 362 nidents = (1 << devbits); 363 its_tbl_size = esize * nidents; 364 its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); 365 break; 366 case GITS_BASER_TYPE_VP: 367 case GITS_BASER_TYPE_PP: /* Undocumented? */ 368 case GITS_BASER_TYPE_IC: 369 its_tbl_size = page_size; 370 break; 371 default: 372 continue; 373 } 374 npages = howmany(its_tbl_size, PAGE_SIZE); 375 376 /* Allocate the table */ 377 table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, 378 M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, 379 PAGE_SIZE, 0); 380 381 sc->sc_its_ptab[i].ptab_vaddr = table; 382 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; 383 384 paddr = vtophys(table); 385 386 while (1) { 387 nitspages = howmany(its_tbl_size, page_size); 388 389 /* Clear the fields we will be setting */ 390 reg &= ~(GITS_BASER_VALID | 391 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 392 GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | 393 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 394 GITS_BASER_SIZE_MASK); 395 /* Set the new values */ 396 reg |= GITS_BASER_VALID | 397 (cache << GITS_BASER_CACHE_SHIFT) | 398 (type << GITS_BASER_TYPE_SHIFT) | 399 ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | 400 paddr | (share << GITS_BASER_SHARE_SHIFT) | 401 (nitspages - 1); 402 403 switch (page_size) { 404 case PAGE_SIZE: /* 4KB */ 405 reg |= 406 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 407 break; 408 case PAGE_SIZE_16K: /* 16KB */ 409 reg |= 410 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 411 break; 412 case PAGE_SIZE_64K: /* 64KB */ 413 reg |= 414 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 415 break; 416 } 417 418 gic_its_write_8(sc, GITS_BASER(i), reg); 419 420 /* Read back to check */ 421 tmp = gic_its_read_8(sc, GITS_BASER(i)); 422 423 /* Do the snareability masks line up? */ 424 if ((tmp & GITS_BASER_SHARE_MASK) != 425 (reg & GITS_BASER_SHARE_MASK)) { 426 share = (tmp & GITS_BASER_SHARE_MASK) >> 427 GITS_BASER_SHARE_SHIFT; 428 continue; 429 } 430 431 if ((tmp & GITS_BASER_PSZ_MASK) != 432 (reg & GITS_BASER_PSZ_MASK)) { 433 switch (page_size) { 434 case PAGE_SIZE_16K: 435 page_size = PAGE_SIZE; 436 continue; 437 case PAGE_SIZE_64K: 438 page_size = PAGE_SIZE_16K; 439 continue; 440 } 441 } 442 443 if (tmp != reg) { 444 device_printf(dev, "GITS_BASER%d: " 445 "unable to be updated: %lx != %lx\n", 446 i, reg, tmp); 447 return (ENXIO); 448 } 449 450 /* We should have made all needed changes */ 451 break; 452 } 453 } 454 455 return (0); 456} 457 458static void 459gicv3_its_conftable_init(struct gicv3_its_softc *sc) 460{ 461 462 sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, 463 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN, 464 0); 465 466 /* Set the default configuration */ 467 memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 468 LPI_CONFTAB_SIZE); 469 470 /* Flush the table to memory */ 471 cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE); 472} 473 474static void 475gicv3_its_pendtables_init(struct gicv3_its_softc *sc) 476{ 477 int i; 478 479 for (i = 0; i < mp_ncpus; i++) { 480 if (CPU_ISSET(i, &all_cpus) == 0) 481 continue; 482 483 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( 484 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 485 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 486 487 /* Flush so the ITS can see the memory */ 488 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base, 489 LPI_PENDTAB_SIZE); 490 } 491} 492 493static int 494its_init_cpu(device_t dev, struct gicv3_its_softc *sc) 495{ 496 device_t gicv3; 497 vm_paddr_t target; 498 uint64_t xbaser, tmp; 499 uint32_t ctlr; 500 u_int cpuid; 501 502 gicv3 = device_get_parent(dev); 503 cpuid = PCPU_GET(cpuid); 504 505 /* Check if the ITS is enabled on this CPU */ 506 if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { 507 return (ENXIO); 508 } 509 510 /* Disable LPIs */ 511 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 512 ctlr &= ~GICR_CTLR_LPI_ENABLE; 513 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 514 515 /* Make sure changes are observable my the GIC */ 516 dsb(sy); 517 518 /* 519 * Set the redistributor base 520 */ 521 xbaser = vtophys(sc->sc_conf_base) | 522 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 523 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 524 (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 525 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 526 527 /* Check the cache attributes we set */ 528 tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 529 530 if ((tmp & GICR_PROPBASER_SHARE_MASK) != 531 (xbaser & GICR_PROPBASER_SHARE_MASK)) { 532 if ((tmp & GICR_PROPBASER_SHARE_MASK) == 533 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 534 /* We need to mark as non-cacheable */ 535 xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 536 GICR_PROPBASER_CACHE_MASK); 537 /* Non-cacheable */ 538 xbaser |= GICR_PROPBASER_CACHE_NIN << 539 GICR_PROPBASER_CACHE_SHIFT; 540 /* Non-sareable */ 541 xbaser |= GICR_PROPBASER_SHARE_NS << 542 GICR_PROPBASER_SHARE_SHIFT; 543 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 544 } 545 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 546 } 547 548 /* 549 * Set the LPI pending table base 550 */ 551 xbaser = vtophys(sc->sc_pend_base[cpuid]) | 552 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 553 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 554 555 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 556 557 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 558 559 if ((tmp & GICR_PENDBASER_SHARE_MASK) == 560 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 561 /* Clear the cahce and shareability bits */ 562 xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 563 GICR_PENDBASER_SHARE_MASK); 564 /* Mark as non-shareable */ 565 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 566 /* And non-cacheable */ 567 xbaser |= GICR_PENDBASER_CACHE_NIN << 568 GICR_PENDBASER_CACHE_SHIFT; 569 } 570 571 /* Enable LPIs */ 572 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 573 ctlr |= GICR_CTLR_LPI_ENABLE; 574 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 575 576 /* Make sure the GIC has seen everything */ 577 dsb(sy); 578 579 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 580 /* This ITS wants the redistributor physical address */ 581 target = vtophys(gicv3_get_redist_vaddr(dev)); 582 } else { 583 /* This ITS wants the unique processor number */ 584 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)); 585 } 586 587 sc->sc_its_cols[cpuid]->col_target = target; 588 sc->sc_its_cols[cpuid]->col_id = cpuid; 589 590 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 591 its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 592 593 return (0); 594} 595 596static int 597gicv3_its_attach(device_t dev) 598{ 599 struct gicv3_its_softc *sc; 600 const char *name; 601 uint32_t iidr; 602 int err, i, rid; 603 604 sc = device_get_softc(dev); 605 606 rid = 0; 607 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 608 RF_ACTIVE); 609 if (sc->sc_its_res == NULL) { 610 device_printf(dev, "Could not allocate memory\n"); 611 return (ENXIO); 612 } 613 614 iidr = gic_its_read_4(sc, GITS_IIDR); 615 for (i = 0; i < nitems(its_quirks); i++) { 616 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 617 if (bootverbose) { 618 device_printf(dev, "Applying %s\n", 619 its_quirks[i].desc); 620 } 621 its_quirks[i].func(dev); 622 break; 623 } 624 } 625 626 /* Allocate the private tables */ 627 err = gicv3_its_table_init(dev, sc); 628 if (err != 0) 629 return (err); 630 631 /* Protects access to the device list */ 632 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 633 634 /* Protects access to the ITS command circular buffer. */ 635 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 636 637 /* Allocate the command circular buffer */ 638 gicv3_its_cmdq_init(sc); 639 640 /* Allocate the per-CPU collections */ 641 for (int cpu = 0; cpu < mp_ncpus; cpu++) 642 if (CPU_ISSET(cpu, &all_cpus) != 0) 643 sc->sc_its_cols[cpu] = malloc( 644 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 645 M_WAITOK | M_ZERO); 646 else 647 sc->sc_its_cols[cpu] = NULL; 648 649 /* Enable the ITS */ 650 gic_its_write_4(sc, GITS_CTLR, 651 gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); 652 653 /* Create the LPI configuration table */ 654 gicv3_its_conftable_init(sc); 655 656 /* And the pending tebles */ 657 gicv3_its_pendtables_init(sc); 658 659 /* Enable LPIs on this CPU */ 660 its_init_cpu(dev, sc); 661 662 TAILQ_INIT(&sc->sc_its_dev_list); 663 664 /* 665 * Create the vmem object to allocate IRQs from. We try to use all 666 * IRQs not already used by the GICv3. 667 * XXX: This assumes there are no other interrupt controllers in the 668 * system. 669 */ 670 sc->sc_irq_alloc = vmem_create("GICv3 ITS IRQs", 0, 671 NIRQ - gicv3_get_nirqs(dev), 1, 1, M_FIRSTFIT | M_WAITOK); 672 673 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * LPI_NIRQS, M_GICV3_ITS, 674 M_WAITOK | M_ZERO); 675 name = device_get_nameunit(dev); 676 for (i = 0; i < LPI_NIRQS; i++) { 677 sc->sc_irqs[i].gi_irq = i; 678 err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0, 679 "%s,%u", name, i); 680 } 681 682 return (0); 683} 684 685static int 686gicv3_its_detach(device_t dev) 687{ 688 689 return (ENXIO); 690} 691 692static void 693its_quirk_cavium_22375(device_t dev) 694{ 695 struct gicv3_its_softc *sc; 696 697 sc = device_get_softc(dev); 698 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 699} 700 701static void 702gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 703{ 704 struct gicv3_its_softc *sc; 705 struct gicv3_its_irqsrc *girq; 706 uint8_t *conf; 707 708 sc = device_get_softc(dev); 709 girq = (struct gicv3_its_irqsrc *)isrc; 710 conf = (uint8_t *)sc->sc_conf_base; 711 712 conf[girq->gi_irq] &= ~LPI_CONF_ENABLE; 713 714 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 715 /* Clean D-cache under command. */ 716 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 717 } else { 718 /* DSB inner shareable, store */ 719 dsb(ishst); 720 } 721 722 its_cmd_inv(dev, girq->gi_its_dev, girq); 723} 724 725static void 726gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 727{ 728 struct gicv3_its_softc *sc; 729 struct gicv3_its_irqsrc *girq; 730 uint8_t *conf; 731 732 sc = device_get_softc(dev); 733 girq = (struct gicv3_its_irqsrc *)isrc; 734 conf = (uint8_t *)sc->sc_conf_base; 735 736 conf[girq->gi_irq] |= LPI_CONF_ENABLE; 737 738 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 739 /* Clean D-cache under command. */ 740 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 741 } else { 742 /* DSB inner shareable, store */ 743 dsb(ishst); 744 } 745 746 its_cmd_inv(dev, girq->gi_its_dev, girq); 747} 748 749static int 750gicv3_its_intr(void *arg, uintptr_t irq) 751{ 752 struct gicv3_its_softc *sc = arg; 753 struct gicv3_its_irqsrc *girq; 754 struct trapframe *tf; 755 756 irq -= GIC_FIRST_LPI; 757 girq = &sc->sc_irqs[irq]; 758 if (girq == NULL) 759 panic("gicv3_its_intr: Invalid interrupt %ld", 760 irq + GIC_FIRST_LPI); 761 762 tf = curthread->td_intr_frame; 763 intr_isrc_dispatch(&girq->gi_isrc, tf); 764 return (FILTER_HANDLED); 765} 766 767static void 768gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 769{ 770 struct gicv3_its_irqsrc *girq; 771 772 girq = (struct gicv3_its_irqsrc *)isrc; 773 gicv3_its_disable_intr(dev, isrc); 774 gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); 775} 776 777static void 778gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 779{ 780 781 gicv3_its_enable_intr(dev, isrc); 782} 783 784static void 785gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 786{ 787 struct gicv3_its_irqsrc *girq; 788 789 girq = (struct gicv3_its_irqsrc *)isrc; 790 gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); 791} 792 793static int 794gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 795{ 796 struct gicv3_its_irqsrc *girq; 797 798 girq = (struct gicv3_its_irqsrc *)isrc; 799 if (CPU_EMPTY(&isrc->isrc_cpu)) { 800 gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); 801 CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); 802 } 803 804 its_cmd_movi(dev, girq); 805 806 return (0); 807} 808 809static int 810gicv3_its_map_intr(device_t dev, struct intr_map_data *data, 811 struct intr_irqsrc **isrcp) 812{ 813 814 /* 815 * This should never happen, we only call this function to map 816 * interrupts found before the controller driver is ready. 817 */ 818 panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 819} 820 821static int 822gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 823 struct resource *res, struct intr_map_data *data) 824{ 825 826 /* Bind the interrupt to a CPU */ 827 gicv3_its_bind_intr(dev, isrc); 828 829 return (0); 830} 831 832#ifdef SMP 833static void 834gicv3_its_init_secondary(device_t dev) 835{ 836 struct gicv3_its_softc *sc; 837 838 sc = device_get_softc(dev); 839 840 /* 841 * This is fatal as otherwise we may bind interrupts to this CPU. 842 * We need a way to tell the interrupt framework to only bind to a 843 * subset of given CPUs when it performs the shuffle. 844 */ 845 if (its_init_cpu(dev, sc) != 0) 846 panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 847 PCPU_GET(cpuid)); 848} 849#endif 850 851static uint32_t 852its_get_devid(device_t pci_dev) 853{ 854 uintptr_t id; 855 856 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 857 panic("its_get_devid: Unable to get the MSI DeviceID"); 858 859 return (id); 860} 861 862static struct its_dev * 863its_device_find(device_t dev, device_t child) 864{ 865 struct gicv3_its_softc *sc; 866 struct its_dev *its_dev = NULL; 867 868 sc = device_get_softc(dev); 869 870 mtx_lock_spin(&sc->sc_its_dev_lock); 871 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 872 if (its_dev->pci_dev == child) 873 break; 874 } 875 mtx_unlock_spin(&sc->sc_its_dev_lock); 876 877 return (its_dev); 878} 879 880static struct its_dev * 881its_device_get(device_t dev, device_t child, u_int nvecs) 882{ 883 struct gicv3_its_softc *sc; 884 struct its_dev *its_dev; 885 vmem_addr_t irq_base; 886 size_t esize; 887 888 sc = device_get_softc(dev); 889 890 its_dev = its_device_find(dev, child); 891 if (its_dev != NULL) 892 return (its_dev); 893 894 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 895 if (its_dev == NULL) 896 return (NULL); 897 898 its_dev->pci_dev = child; 899 its_dev->devid = its_get_devid(child); 900 901 its_dev->lpis.lpi_busy = 0; 902 its_dev->lpis.lpi_num = nvecs; 903 its_dev->lpis.lpi_free = nvecs; 904 905 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 906 &irq_base) != 0) { 907 free(its_dev, M_GICV3_ITS); 908 return (NULL); 909 } 910 its_dev->lpis.lpi_base = irq_base; 911 912 /* Get ITT entry size */ 913 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 914 915 /* 916 * Allocate ITT for this device. 917 * PA has to be 256 B aligned. At least two entries for device. 918 */ 919 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 920 its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size, 921 M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, 922 LPI_INT_TRANS_TAB_ALIGN, 0); 923 if (its_dev->itt == 0) { 924 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 925 free(its_dev, M_GICV3_ITS); 926 return (NULL); 927 } 928 929 mtx_lock_spin(&sc->sc_its_dev_lock); 930 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 931 mtx_unlock_spin(&sc->sc_its_dev_lock); 932 933 /* Map device to its ITT */ 934 its_cmd_mapd(dev, its_dev, 1); 935 936 return (its_dev); 937} 938 939static void 940its_device_release(device_t dev, struct its_dev *its_dev) 941{ 942 struct gicv3_its_softc *sc; 943 944 KASSERT(its_dev->lpis.lpi_busy == 0, 945 ("its_device_release: Trying to release an inuse ITS device")); 946 947 /* Unmap device in ITS */ 948 its_cmd_mapd(dev, its_dev, 0); 949 950 sc = device_get_softc(dev); 951 952 /* Remove the device from the list of devices */ 953 mtx_lock_spin(&sc->sc_its_dev_lock); 954 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 955 mtx_unlock_spin(&sc->sc_its_dev_lock); 956 957 /* Free ITT */ 958 KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); 959 contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 960 961 /* Free the IRQ allocation */ 962 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 963 its_dev->lpis.lpi_num); 964 965 free(its_dev, M_GICV3_ITS); 966} 967 968static int 969gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 970 device_t *pic, struct intr_irqsrc **srcs) 971{ 972 struct gicv3_its_softc *sc; 973 struct gicv3_its_irqsrc *girq; 974 struct its_dev *its_dev; 975 u_int irq; 976 int i; 977 978 its_dev = its_device_get(dev, child, count); 979 if (its_dev == NULL) 980 return (ENXIO); 981 982 KASSERT(its_dev->lpis.lpi_free >= count, 983 ("gicv3_its_alloc_msi: No free LPIs")); 984 sc = device_get_softc(dev); 985 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 986 its_dev->lpis.lpi_free; 987 for (i = 0; i < count; i++, irq++) { 988 its_dev->lpis.lpi_free--; 989 girq = &sc->sc_irqs[irq]; 990 girq->gi_its_dev = its_dev; 991 srcs[i] = (struct intr_irqsrc *)girq; 992 } 993 its_dev->lpis.lpi_busy += count; 994 *pic = dev; 995 996 return (0); 997} 998 999static int 1000gicv3_its_release_msi(device_t dev, device_t child, int count, 1001 struct intr_irqsrc **isrc) 1002{ 1003 struct gicv3_its_softc *sc; 1004 struct gicv3_its_irqsrc *girq; 1005 struct its_dev *its_dev; 1006 int i; 1007 1008 sc = device_get_softc(dev); 1009 its_dev = its_device_find(dev, child); 1010 1011 KASSERT(its_dev != NULL, 1012 ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1013 "no ITS device")); 1014 KASSERT(its_dev->lpis.lpi_busy >= count, 1015 ("gicv3_its_release_msi: Releasing more interrupts than " 1016 "were allocated: releasing %d, allocated %d", count, 1017 its_dev->lpis.lpi_busy)); 1018 for (i = 0; i < count; i++) { 1019 girq = (struct gicv3_its_irqsrc *)isrc[i]; 1020 girq->gi_its_dev = NULL; 1021 } 1022 its_dev->lpis.lpi_busy -= count; 1023 1024 if (its_dev->lpis.lpi_busy == 0) 1025 its_device_release(dev, its_dev); 1026 1027 return (0); 1028} 1029 1030static int 1031gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1032 struct intr_irqsrc **isrcp) 1033{ 1034 struct gicv3_its_softc *sc; 1035 struct gicv3_its_irqsrc *girq; 1036 struct its_dev *its_dev; 1037 u_int nvecs, irq; 1038 1039 nvecs = pci_msix_count(child); 1040 its_dev = its_device_get(dev, child, nvecs); 1041 if (its_dev == NULL) 1042 return (ENXIO); 1043 1044 KASSERT(its_dev->lpis.lpi_free > 0, 1045 ("gicv3_its_alloc_msix: No free LPIs")); 1046 sc = device_get_softc(dev); 1047 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1048 its_dev->lpis.lpi_free; 1049 its_dev->lpis.lpi_free--; 1050 its_dev->lpis.lpi_busy++; 1051 girq = &sc->sc_irqs[irq]; 1052 girq->gi_its_dev = its_dev; 1053 1054 *pic = dev; 1055 *isrcp = (struct intr_irqsrc *)girq; 1056 1057 return (0); 1058} 1059 1060static int 1061gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1062{ 1063 struct gicv3_its_softc *sc; 1064 struct gicv3_its_irqsrc *girq; 1065 struct its_dev *its_dev; 1066 1067 sc = device_get_softc(dev); 1068 its_dev = its_device_find(dev, child); 1069 1070 KASSERT(its_dev != NULL, 1071 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1072 "no ITS device")); 1073 KASSERT(its_dev->lpis.lpi_busy > 0, 1074 ("gicv3_its_release_msix: Releasing more interrupts than " 1075 "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1076 girq = (struct gicv3_its_irqsrc *)isrc; 1077 girq->gi_its_dev = NULL; 1078 its_dev->lpis.lpi_busy--; 1079 1080 if (its_dev->lpis.lpi_busy == 0) 1081 its_device_release(dev, its_dev); 1082 1083 return (0); 1084} 1085 1086static int 1087gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1088 uint64_t *addr, uint32_t *data) 1089{ 1090 struct gicv3_its_softc *sc; 1091 struct gicv3_its_irqsrc *girq; 1092 1093 sc = device_get_softc(dev); 1094 girq = (struct gicv3_its_irqsrc *)isrc; 1095 1096 /* Map the message to the given IRQ */ 1097 its_cmd_mapti(dev, girq); 1098 1099 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1100 *data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1101 1102 return (0); 1103} 1104 1105/* 1106 * Commands handling. 1107 */ 1108 1109static __inline void 1110cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1111{ 1112 /* Command field: DW0 [7:0] */ 1113 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1114 cmd->cmd_dword[0] |= htole64(cmd_type); 1115} 1116 1117static __inline void 1118cmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1119{ 1120 /* Device ID field: DW0 [63:32] */ 1121 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1122 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1123} 1124 1125static __inline void 1126cmd_format_size(struct its_cmd *cmd, uint16_t size) 1127{ 1128 /* Size field: DW1 [4:0] */ 1129 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1130 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1131} 1132 1133static __inline void 1134cmd_format_id(struct its_cmd *cmd, uint32_t id) 1135{ 1136 /* ID field: DW1 [31:0] */ 1137 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1138 cmd->cmd_dword[1] |= htole64(id); 1139} 1140 1141static __inline void 1142cmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1143{ 1144 /* Physical ID field: DW1 [63:32] */ 1145 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1146 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1147} 1148 1149static __inline void 1150cmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1151{ 1152 /* Collection field: DW2 [16:0] */ 1153 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1154 cmd->cmd_dword[2] |= htole64(col_id); 1155} 1156 1157static __inline void 1158cmd_format_target(struct its_cmd *cmd, uint64_t target) 1159{ 1160 /* Target Address field: DW2 [47:16] */ 1161 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1162 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1163} 1164 1165static __inline void 1166cmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1167{ 1168 /* ITT Address field: DW2 [47:8] */ 1169 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1170 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1171} 1172 1173static __inline void 1174cmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1175{ 1176 /* Valid field: DW2 [63] */ 1177 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1178 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1179} 1180 1181static inline bool 1182its_cmd_queue_full(struct gicv3_its_softc *sc) 1183{ 1184 size_t read_idx, next_write_idx; 1185 1186 /* Get the index of the next command */ 1187 next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1188 (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1189 /* And the index of the current command being read */ 1190 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1191 1192 /* 1193 * The queue is full when the write offset points 1194 * at the command before the current read offset. 1195 */ 1196 return (next_write_idx == read_idx); 1197} 1198 1199static inline void 1200its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1201{ 1202 1203 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1204 /* Clean D-cache under command. */ 1205 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); 1206 } else { 1207 /* DSB inner shareable, store */ 1208 dsb(ishst); 1209 } 1210 1211} 1212 1213static inline uint64_t 1214its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1215{ 1216 uint64_t off; 1217 1218 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1219 1220 return (off); 1221} 1222 1223static void 1224its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1225 struct its_cmd *cmd_last) 1226{ 1227 struct gicv3_its_softc *sc; 1228 uint64_t first, last, read; 1229 size_t us_left; 1230 1231 sc = device_get_softc(dev); 1232 1233 /* 1234 * XXX ARM64TODO: This is obviously a significant delay. 1235 * The reason for that is that currently the time frames for 1236 * the command to complete are not known. 1237 */ 1238 us_left = 1000000; 1239 1240 first = its_cmd_cwriter_offset(sc, cmd_first); 1241 last = its_cmd_cwriter_offset(sc, cmd_last); 1242 1243 for (;;) { 1244 read = gic_its_read_8(sc, GITS_CREADR); 1245 if (first < last) { 1246 if (read < first || read >= last) 1247 break; 1248 } else if (read < first && read >= last) 1249 break; 1250 1251 if (us_left-- == 0) { 1252 /* This means timeout */ 1253 device_printf(dev, 1254 "Timeout while waiting for CMD completion.\n"); 1255 return; 1256 } 1257 DELAY(1); 1258 } 1259} 1260 1261 1262static struct its_cmd * 1263its_cmd_alloc_locked(device_t dev) 1264{ 1265 struct gicv3_its_softc *sc; 1266 struct its_cmd *cmd; 1267 size_t us_left; 1268 1269 sc = device_get_softc(dev); 1270 1271 /* 1272 * XXX ARM64TODO: This is obviously a significant delay. 1273 * The reason for that is that currently the time frames for 1274 * the command to complete (and therefore free the descriptor) 1275 * are not known. 1276 */ 1277 us_left = 1000000; 1278 1279 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1280 while (its_cmd_queue_full(sc)) { 1281 if (us_left-- == 0) { 1282 /* Timeout while waiting for free command */ 1283 device_printf(dev, 1284 "Timeout while waiting for free command\n"); 1285 return (NULL); 1286 } 1287 DELAY(1); 1288 } 1289 1290 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1291 sc->sc_its_cmd_next_idx++; 1292 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1293 1294 return (cmd); 1295} 1296 1297static uint64_t 1298its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1299{ 1300 uint64_t target; 1301 uint8_t cmd_type; 1302 u_int size; 1303 boolean_t error; 1304 1305 error = FALSE; 1306 cmd_type = desc->cmd_type; 1307 target = ITS_TARGET_NONE; 1308 1309 switch (cmd_type) { 1310 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1311 target = desc->cmd_desc_movi.col->col_target; 1312 cmd_format_command(cmd, ITS_CMD_MOVI); 1313 cmd_format_id(cmd, desc->cmd_desc_movi.id); 1314 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1315 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1316 break; 1317 case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1318 target = desc->cmd_desc_sync.col->col_target; 1319 cmd_format_command(cmd, ITS_CMD_SYNC); 1320 cmd_format_target(cmd, target); 1321 break; 1322 case ITS_CMD_MAPD: /* Assign ITT to device */ 1323 cmd_format_command(cmd, ITS_CMD_MAPD); 1324 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1325 /* 1326 * Size describes number of bits to encode interrupt IDs 1327 * supported by the device minus one. 1328 * When V (valid) bit is zero, this field should be written 1329 * as zero. 1330 */ 1331 if (desc->cmd_desc_mapd.valid != 0) { 1332 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1333 size = MAX(1, size) - 1; 1334 } else 1335 size = 0; 1336 1337 cmd_format_size(cmd, size); 1338 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1339 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1340 break; 1341 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1342 target = desc->cmd_desc_mapc.col->col_target; 1343 cmd_format_command(cmd, ITS_CMD_MAPC); 1344 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 1345 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 1346 cmd_format_target(cmd, target); 1347 break; 1348 case ITS_CMD_MAPTI: 1349 target = desc->cmd_desc_mapvi.col->col_target; 1350 cmd_format_command(cmd, ITS_CMD_MAPTI); 1351 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 1352 cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 1353 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 1354 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 1355 break; 1356 case ITS_CMD_MAPI: 1357 target = desc->cmd_desc_mapi.col->col_target; 1358 cmd_format_command(cmd, ITS_CMD_MAPI); 1359 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 1360 cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 1361 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 1362 break; 1363 case ITS_CMD_INV: 1364 target = desc->cmd_desc_inv.col->col_target; 1365 cmd_format_command(cmd, ITS_CMD_INV); 1366 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 1367 cmd_format_id(cmd, desc->cmd_desc_inv.pid); 1368 break; 1369 case ITS_CMD_INVALL: 1370 cmd_format_command(cmd, ITS_CMD_INVALL); 1371 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 1372 break; 1373 default: 1374 panic("its_cmd_prepare: Invalid command: %x", cmd_type); 1375 } 1376 1377 return (target); 1378} 1379 1380static int 1381its_cmd_send(device_t dev, struct its_cmd_desc *desc) 1382{ 1383 struct gicv3_its_softc *sc; 1384 struct its_cmd *cmd, *cmd_sync, *cmd_write; 1385 struct its_col col_sync; 1386 struct its_cmd_desc desc_sync; 1387 uint64_t target, cwriter; 1388 1389 sc = device_get_softc(dev); 1390 mtx_lock_spin(&sc->sc_its_cmd_lock); 1391 cmd = its_cmd_alloc_locked(dev); 1392 if (cmd == NULL) { 1393 device_printf(dev, "could not allocate ITS command\n"); 1394 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1395 return (EBUSY); 1396 } 1397 1398 target = its_cmd_prepare(cmd, desc); 1399 its_cmd_sync(sc, cmd); 1400 1401 if (target != ITS_TARGET_NONE) { 1402 cmd_sync = its_cmd_alloc_locked(dev); 1403 if (cmd_sync != NULL) { 1404 desc_sync.cmd_type = ITS_CMD_SYNC; 1405 col_sync.col_target = target; 1406 desc_sync.cmd_desc_sync.col = &col_sync; 1407 its_cmd_prepare(cmd_sync, &desc_sync); 1408 its_cmd_sync(sc, cmd_sync); 1409 } 1410 } 1411 1412 /* Update GITS_CWRITER */ 1413 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 1414 gic_its_write_8(sc, GITS_CWRITER, cwriter); 1415 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1416 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1417 1418 its_cmd_wait_completion(dev, cmd, cmd_write); 1419 1420 return (0); 1421} 1422 1423/* Handlers to send commands */ 1424static void 1425its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 1426{ 1427 struct gicv3_its_softc *sc; 1428 struct its_cmd_desc desc; 1429 struct its_col *col; 1430 1431 sc = device_get_softc(dev); 1432 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1433 1434 desc.cmd_type = ITS_CMD_MOVI; 1435 desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 1436 desc.cmd_desc_movi.col = col; 1437 desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1438 1439 its_cmd_send(dev, &desc); 1440} 1441 1442static void 1443its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 1444{ 1445 struct its_cmd_desc desc; 1446 1447 desc.cmd_type = ITS_CMD_MAPC; 1448 desc.cmd_desc_mapc.col = col; 1449 /* 1450 * Valid bit set - map the collection. 1451 * Valid bit cleared - unmap the collection. 1452 */ 1453 desc.cmd_desc_mapc.valid = valid; 1454 1455 its_cmd_send(dev, &desc); 1456} 1457 1458static void 1459its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 1460{ 1461 struct gicv3_its_softc *sc; 1462 struct its_cmd_desc desc; 1463 struct its_col *col; 1464 u_int col_id; 1465 1466 sc = device_get_softc(dev); 1467 1468 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 1469 col = sc->sc_its_cols[col_id]; 1470 1471 desc.cmd_type = ITS_CMD_MAPTI; 1472 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 1473 desc.cmd_desc_mapvi.col = col; 1474 /* The EventID sent to the device */ 1475 desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1476 /* The physical interrupt presented to softeware */ 1477 desc.cmd_desc_mapvi.pid = girq->gi_irq + GIC_FIRST_LPI; 1478 1479 its_cmd_send(dev, &desc); 1480} 1481 1482static void 1483its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 1484{ 1485 struct its_cmd_desc desc; 1486 1487 desc.cmd_type = ITS_CMD_MAPD; 1488 desc.cmd_desc_mapd.its_dev = its_dev; 1489 desc.cmd_desc_mapd.valid = valid; 1490 1491 its_cmd_send(dev, &desc); 1492} 1493 1494static void 1495its_cmd_inv(device_t dev, struct its_dev *its_dev, 1496 struct gicv3_its_irqsrc *girq) 1497{ 1498 struct gicv3_its_softc *sc; 1499 struct its_cmd_desc desc; 1500 struct its_col *col; 1501 1502 sc = device_get_softc(dev); 1503 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1504 1505 desc.cmd_type = ITS_CMD_INV; 1506 /* The EventID sent to the device */ 1507 desc.cmd_desc_inv.pid = girq->gi_irq - its_dev->lpis.lpi_base; 1508 desc.cmd_desc_inv.its_dev = its_dev; 1509 desc.cmd_desc_inv.col = col; 1510 1511 its_cmd_send(dev, &desc); 1512} 1513 1514static void 1515its_cmd_invall(device_t dev, struct its_col *col) 1516{ 1517 struct its_cmd_desc desc; 1518 1519 desc.cmd_type = ITS_CMD_INVALL; 1520 desc.cmd_desc_invall.col = col; 1521 1522 its_cmd_send(dev, &desc); 1523} 1524 1525#ifdef FDT 1526static device_probe_t gicv3_its_fdt_probe; 1527static device_attach_t gicv3_its_fdt_attach; 1528 1529static device_method_t gicv3_its_fdt_methods[] = { 1530 /* Device interface */ 1531 DEVMETHOD(device_probe, gicv3_its_fdt_probe), 1532 DEVMETHOD(device_attach, gicv3_its_fdt_attach), 1533 1534 /* End */ 1535 DEVMETHOD_END 1536}; 1537 1538#define its_baseclasses its_fdt_baseclasses 1539DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 1540 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1541#undef its_baseclasses 1542static devclass_t gicv3_its_fdt_devclass; 1543 1544EARLY_DRIVER_MODULE(its, gic, gicv3_its_fdt_driver, 1545 gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1546 1547static int 1548gicv3_its_fdt_probe(device_t dev) 1549{ 1550 1551 if (!ofw_bus_status_okay(dev)) 1552 return (ENXIO); 1553 1554 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 1555 return (ENXIO); 1556 1557 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1558 return (BUS_PROBE_DEFAULT); 1559} 1560 1561static int 1562gicv3_its_fdt_attach(device_t dev) 1563{ 1564 struct gicv3_its_softc *sc; 1565 phandle_t xref; 1566 int err; 1567 1568 err = gicv3_its_attach(dev); 1569 if (err != 0) 1570 return (err); 1571 1572 sc = device_get_softc(dev); 1573 1574 /* Register this device as a interrupt controller */ 1575 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 1576 sc->sc_pic = intr_pic_register(dev, xref); 1577 intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1578 gicv3_its_intr, sc, GIC_FIRST_LPI, LPI_NIRQS); 1579 1580 /* Register this device to handle MSI interrupts */ 1581 intr_msi_register(dev, xref); 1582 1583 return (0); 1584} 1585#endif 1586