gicv3_its.c revision 301265
1301265Sandrew/*- 2301265Sandrew * Copyright (c) 2015-2016 The FreeBSD Foundation 3301265Sandrew * All rights reserved. 4301265Sandrew * 5301265Sandrew * This software was developed by Andrew Turner under 6301265Sandrew * the sponsorship of the FreeBSD Foundation. 7301265Sandrew * 8301265Sandrew * This software was developed by Semihalf under 9301265Sandrew * the sponsorship of the FreeBSD Foundation. 10301265Sandrew * 11301265Sandrew * Redistribution and use in source and binary forms, with or without 12301265Sandrew * modification, are permitted provided that the following conditions 13301265Sandrew * are met: 14301265Sandrew * 1. Redistributions of source code must retain the above copyright 15301265Sandrew * notice, this list of conditions and the following disclaimer. 16301265Sandrew * 2. Redistributions in binary form must reproduce the above copyright 17301265Sandrew * notice, this list of conditions and the following disclaimer in the 18301265Sandrew * documentation and/or other materials provided with the distribution. 19301265Sandrew * 20301265Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21301265Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22301265Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23301265Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24301265Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25301265Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26301265Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27301265Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28301265Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29301265Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30301265Sandrew * SUCH DAMAGE. 31301265Sandrew */ 32301265Sandrew 33301265Sandrew#include "opt_platform.h" 34301265Sandrew 35301265Sandrew#include <sys/cdefs.h> 36301265Sandrew__FBSDID("$FreeBSD: head/sys/arm64/arm64/gicv3_its.c 301265 2016-06-03 10:28:06Z andrew $"); 37301265Sandrew 38301265Sandrew#include <sys/param.h> 39301265Sandrew#include <sys/systm.h> 40301265Sandrew#include <sys/bus.h> 41301265Sandrew#include <sys/cpuset.h> 42301265Sandrew#include <sys/endian.h> 43301265Sandrew#include <sys/kernel.h> 44301265Sandrew#include <sys/malloc.h> 45301265Sandrew#include <sys/module.h> 46301265Sandrew#include <sys/proc.h> 47301265Sandrew#include <sys/queue.h> 48301265Sandrew#include <sys/rman.h> 49301265Sandrew#include <sys/smp.h> 50301265Sandrew#include <sys/vmem.h> 51301265Sandrew 52301265Sandrew#include <vm/vm.h> 53301265Sandrew#include <vm/pmap.h> 54301265Sandrew 55301265Sandrew#include <machine/bus.h> 56301265Sandrew#include <machine/intr.h> 57301265Sandrew 58301265Sandrew#include <arm64/arm64/gic_v3_reg.h> 59301265Sandrew#include <arm64/arm64/gic_v3_var.h> 60301265Sandrew 61301265Sandrew#ifdef FDT 62301265Sandrew#include <dev/ofw/openfirm.h> 63301265Sandrew#include <dev/ofw/ofw_bus.h> 64301265Sandrew#include <dev/ofw/ofw_bus_subr.h> 65301265Sandrew#endif 66301265Sandrew#include <dev/pci/pcireg.h> 67301265Sandrew#include <dev/pci/pcivar.h> 68301265Sandrew 69301265Sandrew#include "pcib_if.h" 70301265Sandrew#include "pic_if.h" 71301265Sandrew#include "msi_if.h" 72301265Sandrew 73301265SandrewMALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 74301265Sandrew "ARM GICv3 Interrupt Translation Service"); 75301265Sandrew 76301265Sandrew#define LPI_NIRQS (64 * 1024) 77301265Sandrew 78301265Sandrew/* The size and alignment of the command circular buffer */ 79301265Sandrew#define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 80301265Sandrew#define ITS_CMDQ_ALIGN (64 * 1024) 81301265Sandrew 82301265Sandrew#define LPI_CONFTAB_SIZE LPI_NIRQS 83301265Sandrew#define LPI_CONFTAB_ALIGN (64 * 1024) 84301265Sandrew#define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 85301265Sandrew 86301265Sandrew/* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 87301265Sandrew#define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 88301265Sandrew#define LPI_PENDTAB_ALIGN (64 * 1024) 89301265Sandrew#define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 90301265Sandrew 91301265Sandrew#define LPI_INT_TRANS_TAB_ALIGN 256 92301265Sandrew#define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 93301265Sandrew 94301265Sandrew/* ITS commands encoding */ 95301265Sandrew#define ITS_CMD_MOVI (0x01) 96301265Sandrew#define ITS_CMD_SYNC (0x05) 97301265Sandrew#define ITS_CMD_MAPD (0x08) 98301265Sandrew#define ITS_CMD_MAPC (0x09) 99301265Sandrew#define ITS_CMD_MAPTI (0x0a) 100301265Sandrew#define ITS_CMD_MAPI (0x0b) 101301265Sandrew#define ITS_CMD_INV (0x0c) 102301265Sandrew#define ITS_CMD_INVALL (0x0d) 103301265Sandrew/* Command */ 104301265Sandrew#define CMD_COMMAND_MASK (0xFFUL) 105301265Sandrew/* PCI device ID */ 106301265Sandrew#define CMD_DEVID_SHIFT (32) 107301265Sandrew#define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 108301265Sandrew/* Size of IRQ ID bitfield */ 109301265Sandrew#define CMD_SIZE_MASK (0xFFUL) 110301265Sandrew/* Virtual LPI ID */ 111301265Sandrew#define CMD_ID_MASK (0xFFFFFFFFUL) 112301265Sandrew/* Physical LPI ID */ 113301265Sandrew#define CMD_PID_SHIFT (32) 114301265Sandrew#define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 115301265Sandrew/* Collection */ 116301265Sandrew#define CMD_COL_MASK (0xFFFFUL) 117301265Sandrew/* Target (CPU or Re-Distributor) */ 118301265Sandrew#define CMD_TARGET_SHIFT (16) 119301265Sandrew#define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 120301265Sandrew/* Interrupt Translation Table address */ 121301265Sandrew#define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 122301265Sandrew/* Valid command bit */ 123301265Sandrew#define CMD_VALID_SHIFT (63) 124301265Sandrew#define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 125301265Sandrew 126301265Sandrew/* ITS command. Each command is 32 bytes long */ 127301265Sandrewstruct its_cmd { 128301265Sandrew uint64_t cmd_dword[4]; /* ITS command double word */ 129301265Sandrew}; 130301265Sandrew 131301265Sandrew/* An ITS private table */ 132301265Sandrewstruct its_ptable { 133301265Sandrew vm_offset_t ptab_vaddr; 134301265Sandrew unsigned long ptab_size; 135301265Sandrew}; 136301265Sandrew 137301265Sandrew/* ITS collection description. */ 138301265Sandrewstruct its_col { 139301265Sandrew uint64_t col_target; /* Target Re-Distributor */ 140301265Sandrew uint64_t col_id; /* Collection ID */ 141301265Sandrew}; 142301265Sandrew 143301265Sandrewstruct gicv3_its_irqsrc { 144301265Sandrew struct intr_irqsrc gi_isrc; 145301265Sandrew u_int gi_irq; 146301265Sandrew struct its_dev *gi_its_dev; 147301265Sandrew}; 148301265Sandrew 149301265Sandrewstruct gicv3_its_softc { 150301265Sandrew struct intr_pic *sc_pic; 151301265Sandrew struct resource *sc_its_res; 152301265Sandrew 153301265Sandrew struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 154301265Sandrew struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 155301265Sandrew 156301265Sandrew /* 157301265Sandrew * TODO: We should get these from the parent as we only want a 158301265Sandrew * single copy of each across the interrupt controller. 159301265Sandrew */ 160301265Sandrew vm_offset_t sc_conf_base; 161301265Sandrew vm_offset_t sc_pend_base[MAXCPU]; 162301265Sandrew 163301265Sandrew /* Command handling */ 164301265Sandrew struct mtx sc_its_cmd_lock; 165301265Sandrew struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 166301265Sandrew size_t sc_its_cmd_next_idx; 167301265Sandrew 168301265Sandrew vmem_t *sc_irq_alloc; 169301265Sandrew struct gicv3_its_irqsrc *sc_irqs; 170301265Sandrew 171301265Sandrew struct mtx sc_its_dev_lock; 172301265Sandrew TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 173301265Sandrew 174301265Sandrew#define ITS_FLAGS_CMDQ_FLUSH 0x00000001 175301265Sandrew#define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 176301265Sandrew#define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 177301265Sandrew u_int sc_its_flags; 178301265Sandrew}; 179301265Sandrew 180301265Sandrewtypedef void (its_quirk_func_t)(device_t); 181301265Sandrewstatic its_quirk_func_t its_quirk_cavium_22375; 182301265Sandrew 183301265Sandrewstatic const struct { 184301265Sandrew const char *desc; 185301265Sandrew uint32_t iidr; 186301265Sandrew uint32_t iidr_mask; 187301265Sandrew its_quirk_func_t *func; 188301265Sandrew} its_quirks[] = { 189301265Sandrew { 190301265Sandrew /* Cavium ThunderX Pass 1.x */ 191301265Sandrew .desc = "Cavoum ThunderX errata: 22375, 24313", 192301265Sandrew .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 193301265Sandrew GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 194301265Sandrew .iidr_mask = ~GITS_IIDR_REVISION_MASK, 195301265Sandrew .func = its_quirk_cavium_22375, 196301265Sandrew }, 197301265Sandrew}; 198301265Sandrew 199301265Sandrewstatic u_int gic_irq_cpu; 200301265Sandrew 201301265Sandrew#define gic_its_read_4(sc, reg) \ 202301265Sandrew bus_read_4((sc)->sc_its_res, (reg)) 203301265Sandrew#define gic_its_read_8(sc, reg) \ 204301265Sandrew bus_read_8((sc)->sc_its_res, (reg)) 205301265Sandrew 206301265Sandrew#define gic_its_write_4(sc, reg, val) \ 207301265Sandrew bus_write_4((sc)->sc_its_res, (reg), (val)) 208301265Sandrew#define gic_its_write_8(sc, reg, val) \ 209301265Sandrew bus_write_8((sc)->sc_its_res, (reg), (val)) 210301265Sandrew 211301265Sandrewstatic device_attach_t gicv3_its_attach; 212301265Sandrewstatic device_detach_t gicv3_its_detach; 213301265Sandrew 214301265Sandrewstatic pic_disable_intr_t gicv3_its_disable_intr; 215301265Sandrewstatic pic_enable_intr_t gicv3_its_enable_intr; 216301265Sandrewstatic pic_map_intr_t gicv3_its_map_intr; 217301265Sandrewstatic pic_setup_intr_t gicv3_its_setup_intr; 218301265Sandrewstatic pic_post_filter_t gicv3_its_post_filter; 219301265Sandrewstatic pic_post_ithread_t gicv3_its_post_ithread; 220301265Sandrewstatic pic_pre_ithread_t gicv3_its_pre_ithread; 221301265Sandrewstatic pic_bind_intr_t gicv3_its_bind_intr; 222301265Sandrew#ifdef SMP 223301265Sandrewstatic pic_init_secondary_t gicv3_its_init_secondary; 224301265Sandrew#endif 225301265Sandrewstatic msi_alloc_msi_t gicv3_its_alloc_msi; 226301265Sandrewstatic msi_release_msi_t gicv3_its_release_msi; 227301265Sandrewstatic msi_alloc_msix_t gicv3_its_alloc_msix; 228301265Sandrewstatic msi_release_msix_t gicv3_its_release_msix; 229301265Sandrewstatic msi_map_msi_t gicv3_its_map_msi; 230301265Sandrew 231301265Sandrewstatic void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 232301265Sandrewstatic void its_cmd_mapc(device_t, struct its_col *, uint8_t); 233301265Sandrewstatic void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 234301265Sandrewstatic void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 235301265Sandrewstatic void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 236301265Sandrewstatic void its_cmd_invall(device_t, struct its_col *); 237301265Sandrew 238301265Sandrewstatic device_method_t gicv3_its_methods[] = { 239301265Sandrew /* Device interface */ 240301265Sandrew DEVMETHOD(device_detach, gicv3_its_detach), 241301265Sandrew 242301265Sandrew /* Interrupt controller interface */ 243301265Sandrew DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 244301265Sandrew DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 245301265Sandrew DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 246301265Sandrew DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 247301265Sandrew DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 248301265Sandrew DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 249301265Sandrew DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 250301265Sandrew#ifdef SMP 251301265Sandrew DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 252301265Sandrew DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 253301265Sandrew#endif 254301265Sandrew 255301265Sandrew /* MSI/MSI-X */ 256301265Sandrew DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 257301265Sandrew DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 258301265Sandrew DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 259301265Sandrew DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 260301265Sandrew DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 261301265Sandrew 262301265Sandrew /* End */ 263301265Sandrew DEVMETHOD_END 264301265Sandrew}; 265301265Sandrew 266301265Sandrewstatic DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 267301265Sandrew sizeof(struct gicv3_its_softc)); 268301265Sandrew 269301265Sandrewstatic void 270301265Sandrewgicv3_its_cmdq_init(struct gicv3_its_softc *sc) 271301265Sandrew{ 272301265Sandrew vm_paddr_t cmd_paddr; 273301265Sandrew uint64_t reg, tmp; 274301265Sandrew 275301265Sandrew /* Set up the command circular buffer */ 276301265Sandrew sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS, 277301265Sandrew M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); 278301265Sandrew sc->sc_its_cmd_next_idx = 0; 279301265Sandrew 280301265Sandrew cmd_paddr = vtophys(sc->sc_its_cmd_base); 281301265Sandrew 282301265Sandrew /* Set the base of the command buffer */ 283301265Sandrew reg = GITS_CBASER_VALID | 284301265Sandrew (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 285301265Sandrew cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 286301265Sandrew (ITS_CMDQ_SIZE / 4096 - 1); 287301265Sandrew gic_its_write_8(sc, GITS_CBASER, reg); 288301265Sandrew 289301265Sandrew /* Read back to check for fixed value fields */ 290301265Sandrew tmp = gic_its_read_8(sc, GITS_CBASER); 291301265Sandrew 292301265Sandrew if ((tmp & GITS_CBASER_SHARE_MASK) != 293301265Sandrew (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 294301265Sandrew /* Check if the hardware reported non-shareable */ 295301265Sandrew if ((tmp & GITS_CBASER_SHARE_MASK) == 296301265Sandrew (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 297301265Sandrew /* If so remove the cache attribute */ 298301265Sandrew reg &= ~GITS_CBASER_CACHE_MASK; 299301265Sandrew reg &= ~GITS_CBASER_SHARE_MASK; 300301265Sandrew /* Set to Non-cacheable, Non-shareable */ 301301265Sandrew reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 302301265Sandrew reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 303301265Sandrew 304301265Sandrew gic_its_write_8(sc, GITS_CBASER, reg); 305301265Sandrew } 306301265Sandrew 307301265Sandrew /* The command queue has to be flushed after each command */ 308301265Sandrew sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 309301265Sandrew } 310301265Sandrew 311301265Sandrew /* Get the next command from the start of the buffer */ 312301265Sandrew gic_its_write_8(sc, GITS_CWRITER, 0x0); 313301265Sandrew} 314301265Sandrew 315301265Sandrewstatic int 316301265Sandrewgicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 317301265Sandrew{ 318301265Sandrew vm_offset_t table; 319301265Sandrew vm_paddr_t paddr; 320301265Sandrew uint64_t cache, reg, share, tmp, type; 321301265Sandrew size_t esize, its_tbl_size, nidents, nitspages, npages; 322301265Sandrew int i, page_size; 323301265Sandrew int devbits; 324301265Sandrew 325301265Sandrew if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 326301265Sandrew /* 327301265Sandrew * GITS_TYPER[17:13] of ThunderX reports that device IDs 328301265Sandrew * are to be 21 bits in length. The entry size of the ITS 329301265Sandrew * table can be read from GITS_BASERn[52:48] and on ThunderX 330301265Sandrew * is supposed to be 8 bytes in length (for device table). 331301265Sandrew * Finally the page size that is to be used by ITS to access 332301265Sandrew * this table will be set to 64KB. 333301265Sandrew * 334301265Sandrew * This gives 0x200000 entries of size 0x8 bytes covered by 335301265Sandrew * 256 pages each of which 64KB in size. The number of pages 336301265Sandrew * (minus 1) should then be written to GITS_BASERn[7:0]. In 337301265Sandrew * that case this value would be 0xFF but on ThunderX the 338301265Sandrew * maximum value that HW accepts is 0xFD. 339301265Sandrew * 340301265Sandrew * Set an arbitrary number of device ID bits to 20 in order 341301265Sandrew * to limit the number of entries in ITS device table to 342301265Sandrew * 0x100000 and the table size to 8MB. 343301265Sandrew */ 344301265Sandrew devbits = 20; 345301265Sandrew cache = 0; 346301265Sandrew } else { 347301265Sandrew devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 348301265Sandrew cache = GITS_BASER_CACHE_WAWB; 349301265Sandrew } 350301265Sandrew share = GITS_BASER_SHARE_IS; 351301265Sandrew page_size = PAGE_SIZE_64K; 352301265Sandrew 353301265Sandrew for (i = 0; i < GITS_BASER_NUM; i++) { 354301265Sandrew reg = gic_its_read_8(sc, GITS_BASER(i)); 355301265Sandrew /* The type of table */ 356301265Sandrew type = GITS_BASER_TYPE(reg); 357301265Sandrew /* The table entry size */ 358301265Sandrew esize = GITS_BASER_ESIZE(reg); 359301265Sandrew 360301265Sandrew switch(type) { 361301265Sandrew case GITS_BASER_TYPE_DEV: 362301265Sandrew nidents = (1 << devbits); 363301265Sandrew its_tbl_size = esize * nidents; 364301265Sandrew its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); 365301265Sandrew break; 366301265Sandrew case GITS_BASER_TYPE_VP: 367301265Sandrew case GITS_BASER_TYPE_PP: /* Undocumented? */ 368301265Sandrew case GITS_BASER_TYPE_IC: 369301265Sandrew its_tbl_size = page_size; 370301265Sandrew break; 371301265Sandrew default: 372301265Sandrew continue; 373301265Sandrew } 374301265Sandrew npages = howmany(its_tbl_size, PAGE_SIZE); 375301265Sandrew 376301265Sandrew /* Allocate the table */ 377301265Sandrew table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, 378301265Sandrew M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, 379301265Sandrew PAGE_SIZE, 0); 380301265Sandrew 381301265Sandrew sc->sc_its_ptab[i].ptab_vaddr = table; 382301265Sandrew sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; 383301265Sandrew 384301265Sandrew paddr = vtophys(table); 385301265Sandrew 386301265Sandrew while (1) { 387301265Sandrew nitspages = howmany(its_tbl_size, page_size); 388301265Sandrew 389301265Sandrew /* Clear the fields we will be setting */ 390301265Sandrew reg &= ~(GITS_BASER_VALID | 391301265Sandrew GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 392301265Sandrew GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | 393301265Sandrew GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 394301265Sandrew GITS_BASER_SIZE_MASK); 395301265Sandrew /* Set the new values */ 396301265Sandrew reg |= GITS_BASER_VALID | 397301265Sandrew (cache << GITS_BASER_CACHE_SHIFT) | 398301265Sandrew (type << GITS_BASER_TYPE_SHIFT) | 399301265Sandrew ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | 400301265Sandrew paddr | (share << GITS_BASER_SHARE_SHIFT) | 401301265Sandrew (nitspages - 1); 402301265Sandrew 403301265Sandrew switch (page_size) { 404301265Sandrew case PAGE_SIZE: /* 4KB */ 405301265Sandrew reg |= 406301265Sandrew GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 407301265Sandrew break; 408301265Sandrew case PAGE_SIZE_16K: /* 16KB */ 409301265Sandrew reg |= 410301265Sandrew GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 411301265Sandrew break; 412301265Sandrew case PAGE_SIZE_64K: /* 64KB */ 413301265Sandrew reg |= 414301265Sandrew GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 415301265Sandrew break; 416301265Sandrew } 417301265Sandrew 418301265Sandrew gic_its_write_8(sc, GITS_BASER(i), reg); 419301265Sandrew 420301265Sandrew /* Read back to check */ 421301265Sandrew tmp = gic_its_read_8(sc, GITS_BASER(i)); 422301265Sandrew 423301265Sandrew /* Do the snareability masks line up? */ 424301265Sandrew if ((tmp & GITS_BASER_SHARE_MASK) != 425301265Sandrew (reg & GITS_BASER_SHARE_MASK)) { 426301265Sandrew share = (tmp & GITS_BASER_SHARE_MASK) >> 427301265Sandrew GITS_BASER_SHARE_SHIFT; 428301265Sandrew continue; 429301265Sandrew } 430301265Sandrew 431301265Sandrew if ((tmp & GITS_BASER_PSZ_MASK) != 432301265Sandrew (reg & GITS_BASER_PSZ_MASK)) { 433301265Sandrew switch (page_size) { 434301265Sandrew case PAGE_SIZE_16K: 435301265Sandrew page_size = PAGE_SIZE; 436301265Sandrew continue; 437301265Sandrew case PAGE_SIZE_64K: 438301265Sandrew page_size = PAGE_SIZE_16K; 439301265Sandrew continue; 440301265Sandrew } 441301265Sandrew } 442301265Sandrew 443301265Sandrew if (tmp != reg) { 444301265Sandrew device_printf(dev, "GITS_BASER%d: " 445301265Sandrew "unable to be updated: %lx != %lx\n", 446301265Sandrew i, reg, tmp); 447301265Sandrew return (ENXIO); 448301265Sandrew } 449301265Sandrew 450301265Sandrew /* We should have made all needed changes */ 451301265Sandrew break; 452301265Sandrew } 453301265Sandrew } 454301265Sandrew 455301265Sandrew return (0); 456301265Sandrew} 457301265Sandrew 458301265Sandrewstatic void 459301265Sandrewgicv3_its_conftable_init(struct gicv3_its_softc *sc) 460301265Sandrew{ 461301265Sandrew 462301265Sandrew sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, 463301265Sandrew M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN, 464301265Sandrew 0); 465301265Sandrew 466301265Sandrew /* Set the default configuration */ 467301265Sandrew memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 468301265Sandrew LPI_CONFTAB_SIZE); 469301265Sandrew 470301265Sandrew /* Flush the table to memory */ 471301265Sandrew cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE); 472301265Sandrew} 473301265Sandrew 474301265Sandrewstatic void 475301265Sandrewgicv3_its_pendtables_init(struct gicv3_its_softc *sc) 476301265Sandrew{ 477301265Sandrew int i; 478301265Sandrew 479301265Sandrew for (i = 0; i < mp_ncpus; i++) { 480301265Sandrew if (CPU_ISSET(i, &all_cpus) == 0) 481301265Sandrew continue; 482301265Sandrew 483301265Sandrew sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( 484301265Sandrew LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 485301265Sandrew 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 486301265Sandrew 487301265Sandrew /* Flush so the ITS can see the memory */ 488301265Sandrew cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base, 489301265Sandrew LPI_PENDTAB_SIZE); 490301265Sandrew } 491301265Sandrew} 492301265Sandrew 493301265Sandrewstatic int 494301265Sandrewits_init_cpu(device_t dev, struct gicv3_its_softc *sc) 495301265Sandrew{ 496301265Sandrew device_t gicv3; 497301265Sandrew vm_paddr_t target; 498301265Sandrew uint64_t xbaser, tmp; 499301265Sandrew uint32_t ctlr; 500301265Sandrew u_int cpuid; 501301265Sandrew 502301265Sandrew gicv3 = device_get_parent(dev); 503301265Sandrew cpuid = PCPU_GET(cpuid); 504301265Sandrew 505301265Sandrew /* Check if the ITS is enabled on this CPU */ 506301265Sandrew if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { 507301265Sandrew return (ENXIO); 508301265Sandrew } 509301265Sandrew 510301265Sandrew /* Disable LPIs */ 511301265Sandrew ctlr = gic_r_read_4(gicv3, GICR_CTLR); 512301265Sandrew ctlr &= ~GICR_CTLR_LPI_ENABLE; 513301265Sandrew gic_r_write_4(gicv3, GICR_CTLR, ctlr); 514301265Sandrew 515301265Sandrew /* Make sure changes are observable my the GIC */ 516301265Sandrew dsb(sy); 517301265Sandrew 518301265Sandrew /* 519301265Sandrew * Set the redistributor base 520301265Sandrew */ 521301265Sandrew xbaser = vtophys(sc->sc_conf_base) | 522301265Sandrew (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 523301265Sandrew (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 524301265Sandrew (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 525301265Sandrew gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 526301265Sandrew 527301265Sandrew /* Check the cache attributes we set */ 528301265Sandrew tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 529301265Sandrew 530301265Sandrew if ((tmp & GICR_PROPBASER_SHARE_MASK) != 531301265Sandrew (xbaser & GICR_PROPBASER_SHARE_MASK)) { 532301265Sandrew if ((tmp & GICR_PROPBASER_SHARE_MASK) == 533301265Sandrew (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 534301265Sandrew /* We need to mark as non-cacheable */ 535301265Sandrew xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 536301265Sandrew GICR_PROPBASER_CACHE_MASK); 537301265Sandrew /* Non-cacheable */ 538301265Sandrew xbaser |= GICR_PROPBASER_CACHE_NIN << 539301265Sandrew GICR_PROPBASER_CACHE_SHIFT; 540301265Sandrew /* Non-sareable */ 541301265Sandrew xbaser |= GICR_PROPBASER_SHARE_NS << 542301265Sandrew GICR_PROPBASER_SHARE_SHIFT; 543301265Sandrew gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 544301265Sandrew } 545301265Sandrew sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 546301265Sandrew } 547301265Sandrew 548301265Sandrew /* 549301265Sandrew * Set the LPI pending table base 550301265Sandrew */ 551301265Sandrew xbaser = vtophys(sc->sc_pend_base[cpuid]) | 552301265Sandrew (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 553301265Sandrew (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 554301265Sandrew 555301265Sandrew gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 556301265Sandrew 557301265Sandrew tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 558301265Sandrew 559301265Sandrew if ((tmp & GICR_PENDBASER_SHARE_MASK) == 560301265Sandrew (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 561301265Sandrew /* Clear the cahce and shareability bits */ 562301265Sandrew xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 563301265Sandrew GICR_PENDBASER_SHARE_MASK); 564301265Sandrew /* Mark as non-shareable */ 565301265Sandrew xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 566301265Sandrew /* And non-cacheable */ 567301265Sandrew xbaser |= GICR_PENDBASER_CACHE_NIN << 568301265Sandrew GICR_PENDBASER_CACHE_SHIFT; 569301265Sandrew } 570301265Sandrew 571301265Sandrew /* Enable LPIs */ 572301265Sandrew ctlr = gic_r_read_4(gicv3, GICR_CTLR); 573301265Sandrew ctlr |= GICR_CTLR_LPI_ENABLE; 574301265Sandrew gic_r_write_4(gicv3, GICR_CTLR, ctlr); 575301265Sandrew 576301265Sandrew /* Make sure the GIC has seen everything */ 577301265Sandrew dsb(sy); 578301265Sandrew 579301265Sandrew if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 580301265Sandrew /* This ITS wants the redistributor physical address */ 581301265Sandrew target = vtophys(gicv3_get_redist_vaddr(dev)); 582301265Sandrew } else { 583301265Sandrew /* This ITS wants the unique processor number */ 584301265Sandrew target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)); 585301265Sandrew } 586301265Sandrew 587301265Sandrew sc->sc_its_cols[cpuid]->col_target = target; 588301265Sandrew sc->sc_its_cols[cpuid]->col_id = cpuid; 589301265Sandrew 590301265Sandrew its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 591301265Sandrew its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 592301265Sandrew 593301265Sandrew return (0); 594301265Sandrew} 595301265Sandrew 596301265Sandrewstatic int 597301265Sandrewgicv3_its_attach(device_t dev) 598301265Sandrew{ 599301265Sandrew struct gicv3_its_softc *sc; 600301265Sandrew const char *name; 601301265Sandrew uint32_t iidr; 602301265Sandrew int err, i, rid; 603301265Sandrew 604301265Sandrew sc = device_get_softc(dev); 605301265Sandrew 606301265Sandrew rid = 0; 607301265Sandrew sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 608301265Sandrew RF_ACTIVE); 609301265Sandrew if (sc->sc_its_res == NULL) { 610301265Sandrew device_printf(dev, "Could not allocate memory\n"); 611301265Sandrew return (ENXIO); 612301265Sandrew } 613301265Sandrew 614301265Sandrew iidr = gic_its_read_4(sc, GITS_IIDR); 615301265Sandrew for (i = 0; i < nitems(its_quirks); i++) { 616301265Sandrew if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 617301265Sandrew if (bootverbose) { 618301265Sandrew device_printf(dev, "Applying %s\n", 619301265Sandrew its_quirks[i].desc); 620301265Sandrew } 621301265Sandrew its_quirks[i].func(dev); 622301265Sandrew break; 623301265Sandrew } 624301265Sandrew } 625301265Sandrew 626301265Sandrew /* Allocate the private tables */ 627301265Sandrew err = gicv3_its_table_init(dev, sc); 628301265Sandrew if (err != 0) 629301265Sandrew return (err); 630301265Sandrew 631301265Sandrew /* Protects access to the device list */ 632301265Sandrew mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 633301265Sandrew 634301265Sandrew /* Protects access to the ITS command circular buffer. */ 635301265Sandrew mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 636301265Sandrew 637301265Sandrew /* Allocate the command circular buffer */ 638301265Sandrew gicv3_its_cmdq_init(sc); 639301265Sandrew 640301265Sandrew /* Allocate the per-CPU collections */ 641301265Sandrew for (int cpu = 0; cpu < mp_ncpus; cpu++) 642301265Sandrew if (CPU_ISSET(cpu, &all_cpus) != 0) 643301265Sandrew sc->sc_its_cols[cpu] = malloc( 644301265Sandrew sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 645301265Sandrew M_WAITOK | M_ZERO); 646301265Sandrew else 647301265Sandrew sc->sc_its_cols[cpu] = NULL; 648301265Sandrew 649301265Sandrew /* Enable the ITS */ 650301265Sandrew gic_its_write_4(sc, GITS_CTLR, 651301265Sandrew gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); 652301265Sandrew 653301265Sandrew /* Create the LPI configuration table */ 654301265Sandrew gicv3_its_conftable_init(sc); 655301265Sandrew 656301265Sandrew /* And the pending tebles */ 657301265Sandrew gicv3_its_pendtables_init(sc); 658301265Sandrew 659301265Sandrew /* Enable LPIs on this CPU */ 660301265Sandrew its_init_cpu(dev, sc); 661301265Sandrew 662301265Sandrew TAILQ_INIT(&sc->sc_its_dev_list); 663301265Sandrew 664301265Sandrew /* 665301265Sandrew * Create the vmem object to allocate IRQs from. We try to use all 666301265Sandrew * IRQs not already used by the GICv3. 667301265Sandrew * XXX: This assumes there are no other interrupt controllers in the 668301265Sandrew * system. 669301265Sandrew */ 670301265Sandrew sc->sc_irq_alloc = vmem_create("GICv3 ITS IRQs", 0, 671301265Sandrew NIRQ - gicv3_get_nirqs(dev), 1, 1, M_FIRSTFIT | M_WAITOK); 672301265Sandrew 673301265Sandrew sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * LPI_NIRQS, M_GICV3_ITS, 674301265Sandrew M_WAITOK | M_ZERO); 675301265Sandrew name = device_get_nameunit(dev); 676301265Sandrew for (i = 0; i < LPI_NIRQS; i++) { 677301265Sandrew sc->sc_irqs[i].gi_irq = i; 678301265Sandrew err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0, 679301265Sandrew "%s,%u", name, i); 680301265Sandrew } 681301265Sandrew 682301265Sandrew return (0); 683301265Sandrew} 684301265Sandrew 685301265Sandrewstatic int 686301265Sandrewgicv3_its_detach(device_t dev) 687301265Sandrew{ 688301265Sandrew 689301265Sandrew return (ENXIO); 690301265Sandrew} 691301265Sandrew 692301265Sandrewstatic void 693301265Sandrewits_quirk_cavium_22375(device_t dev) 694301265Sandrew{ 695301265Sandrew struct gicv3_its_softc *sc; 696301265Sandrew 697301265Sandrew sc = device_get_softc(dev); 698301265Sandrew sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 699301265Sandrew} 700301265Sandrew 701301265Sandrewstatic void 702301265Sandrewgicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 703301265Sandrew{ 704301265Sandrew struct gicv3_its_softc *sc; 705301265Sandrew struct gicv3_its_irqsrc *girq; 706301265Sandrew uint8_t *conf; 707301265Sandrew 708301265Sandrew sc = device_get_softc(dev); 709301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc; 710301265Sandrew conf = (uint8_t *)sc->sc_conf_base; 711301265Sandrew 712301265Sandrew conf[girq->gi_irq] &= ~LPI_CONF_ENABLE; 713301265Sandrew 714301265Sandrew if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 715301265Sandrew /* Clean D-cache under command. */ 716301265Sandrew cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 717301265Sandrew } else { 718301265Sandrew /* DSB inner shareable, store */ 719301265Sandrew dsb(ishst); 720301265Sandrew } 721301265Sandrew 722301265Sandrew its_cmd_inv(dev, girq->gi_its_dev, girq); 723301265Sandrew} 724301265Sandrew 725301265Sandrewstatic void 726301265Sandrewgicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 727301265Sandrew{ 728301265Sandrew struct gicv3_its_softc *sc; 729301265Sandrew struct gicv3_its_irqsrc *girq; 730301265Sandrew uint8_t *conf; 731301265Sandrew 732301265Sandrew sc = device_get_softc(dev); 733301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc; 734301265Sandrew conf = (uint8_t *)sc->sc_conf_base; 735301265Sandrew 736301265Sandrew conf[girq->gi_irq] |= LPI_CONF_ENABLE; 737301265Sandrew 738301265Sandrew if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 739301265Sandrew /* Clean D-cache under command. */ 740301265Sandrew cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 741301265Sandrew } else { 742301265Sandrew /* DSB inner shareable, store */ 743301265Sandrew dsb(ishst); 744301265Sandrew } 745301265Sandrew 746301265Sandrew its_cmd_inv(dev, girq->gi_its_dev, girq); 747301265Sandrew} 748301265Sandrew 749301265Sandrewstatic int 750301265Sandrewgicv3_its_intr(void *arg, uintptr_t irq) 751301265Sandrew{ 752301265Sandrew struct gicv3_its_softc *sc = arg; 753301265Sandrew struct gicv3_its_irqsrc *girq; 754301265Sandrew struct trapframe *tf; 755301265Sandrew 756301265Sandrew irq -= GIC_FIRST_LPI; 757301265Sandrew girq = &sc->sc_irqs[irq]; 758301265Sandrew if (girq == NULL) 759301265Sandrew panic("gicv3_its_intr: Invalid interrupt %ld", 760301265Sandrew irq + GIC_FIRST_LPI); 761301265Sandrew 762301265Sandrew tf = curthread->td_intr_frame; 763301265Sandrew intr_isrc_dispatch(&girq->gi_isrc, tf); 764301265Sandrew return (FILTER_HANDLED); 765301265Sandrew} 766301265Sandrew 767301265Sandrewstatic void 768301265Sandrewgicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 769301265Sandrew{ 770301265Sandrew struct gicv3_its_irqsrc *girq; 771301265Sandrew 772301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc; 773301265Sandrew gicv3_its_disable_intr(dev, isrc); 774301265Sandrew gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); 775301265Sandrew} 776301265Sandrew 777301265Sandrewstatic void 778301265Sandrewgicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 779301265Sandrew{ 780301265Sandrew 781301265Sandrew gicv3_its_enable_intr(dev, isrc); 782301265Sandrew} 783301265Sandrew 784301265Sandrewstatic void 785301265Sandrewgicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 786301265Sandrew{ 787301265Sandrew struct gicv3_its_irqsrc *girq; 788301265Sandrew 789301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc; 790301265Sandrew gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); 791301265Sandrew} 792301265Sandrew 793301265Sandrewstatic int 794301265Sandrewgicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 795301265Sandrew{ 796301265Sandrew struct gicv3_its_irqsrc *girq; 797301265Sandrew 798301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc; 799301265Sandrew if (CPU_EMPTY(&isrc->isrc_cpu)) { 800301265Sandrew gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); 801301265Sandrew CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); 802301265Sandrew } 803301265Sandrew 804301265Sandrew its_cmd_movi(dev, girq); 805301265Sandrew 806301265Sandrew return (0); 807301265Sandrew} 808301265Sandrew 809301265Sandrewstatic int 810301265Sandrewgicv3_its_map_intr(device_t dev, struct intr_map_data *data, 811301265Sandrew struct intr_irqsrc **isrcp) 812301265Sandrew{ 813301265Sandrew 814301265Sandrew /* 815301265Sandrew * This should never happen, we only call this function to map 816301265Sandrew * interrupts found before the controller driver is ready. 817301265Sandrew */ 818301265Sandrew panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 819301265Sandrew} 820301265Sandrew 821301265Sandrewstatic int 822301265Sandrewgicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 823301265Sandrew struct resource *res, struct intr_map_data *data) 824301265Sandrew{ 825301265Sandrew 826301265Sandrew /* Bind the interrupt to a CPU */ 827301265Sandrew gicv3_its_bind_intr(dev, isrc); 828301265Sandrew 829301265Sandrew return (0); 830301265Sandrew} 831301265Sandrew 832301265Sandrew#ifdef SMP 833301265Sandrewstatic void 834301265Sandrewgicv3_its_init_secondary(device_t dev) 835301265Sandrew{ 836301265Sandrew struct gicv3_its_softc *sc; 837301265Sandrew 838301265Sandrew sc = device_get_softc(dev); 839301265Sandrew 840301265Sandrew /* 841301265Sandrew * This is fatal as otherwise we may bind interrupts to this CPU. 842301265Sandrew * We need a way to tell the interrupt framework to only bind to a 843301265Sandrew * subset of given CPUs when it performs the shuffle. 844301265Sandrew */ 845301265Sandrew if (its_init_cpu(dev, sc) != 0) 846301265Sandrew panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 847301265Sandrew PCPU_GET(cpuid)); 848301265Sandrew} 849301265Sandrew#endif 850301265Sandrew 851301265Sandrewstatic uint32_t 852301265Sandrewits_get_devid(device_t pci_dev) 853301265Sandrew{ 854301265Sandrew uintptr_t id; 855301265Sandrew 856301265Sandrew if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 857301265Sandrew panic("its_get_devid: Unable to get the MSI DeviceID"); 858301265Sandrew 859301265Sandrew return (id); 860301265Sandrew} 861301265Sandrew 862301265Sandrewstatic struct its_dev * 863301265Sandrewits_device_find(device_t dev, device_t child) 864301265Sandrew{ 865301265Sandrew struct gicv3_its_softc *sc; 866301265Sandrew struct its_dev *its_dev = NULL; 867301265Sandrew 868301265Sandrew sc = device_get_softc(dev); 869301265Sandrew 870301265Sandrew mtx_lock_spin(&sc->sc_its_dev_lock); 871301265Sandrew TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 872301265Sandrew if (its_dev->pci_dev == child) 873301265Sandrew break; 874301265Sandrew } 875301265Sandrew mtx_unlock_spin(&sc->sc_its_dev_lock); 876301265Sandrew 877301265Sandrew return (its_dev); 878301265Sandrew} 879301265Sandrew 880301265Sandrewstatic struct its_dev * 881301265Sandrewits_device_get(device_t dev, device_t child, u_int nvecs) 882301265Sandrew{ 883301265Sandrew struct gicv3_its_softc *sc; 884301265Sandrew struct its_dev *its_dev; 885301265Sandrew vmem_addr_t irq_base; 886301265Sandrew size_t esize; 887301265Sandrew 888301265Sandrew sc = device_get_softc(dev); 889301265Sandrew 890301265Sandrew its_dev = its_device_find(dev, child); 891301265Sandrew if (its_dev != NULL) 892301265Sandrew return (its_dev); 893301265Sandrew 894301265Sandrew its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 895301265Sandrew if (its_dev == NULL) 896301265Sandrew return (NULL); 897301265Sandrew 898301265Sandrew its_dev->pci_dev = child; 899301265Sandrew its_dev->devid = its_get_devid(child); 900301265Sandrew 901301265Sandrew its_dev->lpis.lpi_busy = 0; 902301265Sandrew its_dev->lpis.lpi_num = nvecs; 903301265Sandrew its_dev->lpis.lpi_free = nvecs; 904301265Sandrew 905301265Sandrew if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 906301265Sandrew &irq_base) != 0) { 907301265Sandrew free(its_dev, M_GICV3_ITS); 908301265Sandrew return (NULL); 909301265Sandrew } 910301265Sandrew its_dev->lpis.lpi_base = irq_base; 911301265Sandrew 912301265Sandrew /* Get ITT entry size */ 913301265Sandrew esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 914301265Sandrew 915301265Sandrew /* 916301265Sandrew * Allocate ITT for this device. 917301265Sandrew * PA has to be 256 B aligned. At least two entries for device. 918301265Sandrew */ 919301265Sandrew its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 920301265Sandrew its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size, 921301265Sandrew M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, 922301265Sandrew LPI_INT_TRANS_TAB_ALIGN, 0); 923301265Sandrew if (its_dev->itt == 0) { 924301265Sandrew vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 925301265Sandrew free(its_dev, M_GICV3_ITS); 926301265Sandrew return (NULL); 927301265Sandrew } 928301265Sandrew 929301265Sandrew mtx_lock_spin(&sc->sc_its_dev_lock); 930301265Sandrew TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 931301265Sandrew mtx_unlock_spin(&sc->sc_its_dev_lock); 932301265Sandrew 933301265Sandrew /* Map device to its ITT */ 934301265Sandrew its_cmd_mapd(dev, its_dev, 1); 935301265Sandrew 936301265Sandrew return (its_dev); 937301265Sandrew} 938301265Sandrew 939301265Sandrewstatic void 940301265Sandrewits_device_release(device_t dev, struct its_dev *its_dev) 941301265Sandrew{ 942301265Sandrew struct gicv3_its_softc *sc; 943301265Sandrew 944301265Sandrew KASSERT(its_dev->lpis.lpi_busy == 0, 945301265Sandrew ("its_device_release: Trying to release an inuse ITS device")); 946301265Sandrew 947301265Sandrew /* Unmap device in ITS */ 948301265Sandrew its_cmd_mapd(dev, its_dev, 0); 949301265Sandrew 950301265Sandrew sc = device_get_softc(dev); 951301265Sandrew 952301265Sandrew /* Remove the device from the list of devices */ 953301265Sandrew mtx_lock_spin(&sc->sc_its_dev_lock); 954301265Sandrew TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 955301265Sandrew mtx_unlock_spin(&sc->sc_its_dev_lock); 956301265Sandrew 957301265Sandrew /* Free ITT */ 958301265Sandrew KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); 959301265Sandrew contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 960301265Sandrew 961301265Sandrew /* Free the IRQ allocation */ 962301265Sandrew vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 963301265Sandrew its_dev->lpis.lpi_num); 964301265Sandrew 965301265Sandrew free(its_dev, M_GICV3_ITS); 966301265Sandrew} 967301265Sandrew 968301265Sandrewstatic int 969301265Sandrewgicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 970301265Sandrew device_t *pic, struct intr_irqsrc **srcs) 971301265Sandrew{ 972301265Sandrew struct gicv3_its_softc *sc; 973301265Sandrew struct gicv3_its_irqsrc *girq; 974301265Sandrew struct its_dev *its_dev; 975301265Sandrew u_int irq; 976301265Sandrew int i; 977301265Sandrew 978301265Sandrew its_dev = its_device_get(dev, child, count); 979301265Sandrew if (its_dev == NULL) 980301265Sandrew return (ENXIO); 981301265Sandrew 982301265Sandrew KASSERT(its_dev->lpis.lpi_free >= count, 983301265Sandrew ("gicv3_its_alloc_msi: No free LPIs")); 984301265Sandrew sc = device_get_softc(dev); 985301265Sandrew irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 986301265Sandrew its_dev->lpis.lpi_free; 987301265Sandrew for (i = 0; i < count; i++, irq++) { 988301265Sandrew its_dev->lpis.lpi_free--; 989301265Sandrew girq = &sc->sc_irqs[irq]; 990301265Sandrew girq->gi_its_dev = its_dev; 991301265Sandrew srcs[i] = (struct intr_irqsrc *)girq; 992301265Sandrew } 993301265Sandrew its_dev->lpis.lpi_busy += count; 994301265Sandrew *pic = dev; 995301265Sandrew 996301265Sandrew return (0); 997301265Sandrew} 998301265Sandrew 999301265Sandrewstatic int 1000301265Sandrewgicv3_its_release_msi(device_t dev, device_t child, int count, 1001301265Sandrew struct intr_irqsrc **isrc) 1002301265Sandrew{ 1003301265Sandrew struct gicv3_its_softc *sc; 1004301265Sandrew struct gicv3_its_irqsrc *girq; 1005301265Sandrew struct its_dev *its_dev; 1006301265Sandrew int i; 1007301265Sandrew 1008301265Sandrew sc = device_get_softc(dev); 1009301265Sandrew its_dev = its_device_find(dev, child); 1010301265Sandrew 1011301265Sandrew KASSERT(its_dev != NULL, 1012301265Sandrew ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1013301265Sandrew "no ITS device")); 1014301265Sandrew KASSERT(its_dev->lpis.lpi_busy >= count, 1015301265Sandrew ("gicv3_its_release_msi: Releasing more interrupts than " 1016301265Sandrew "were allocated: releasing %d, allocated %d", count, 1017301265Sandrew its_dev->lpis.lpi_busy)); 1018301265Sandrew for (i = 0; i < count; i++) { 1019301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc[i]; 1020301265Sandrew girq->gi_its_dev = NULL; 1021301265Sandrew } 1022301265Sandrew its_dev->lpis.lpi_busy -= count; 1023301265Sandrew 1024301265Sandrew if (its_dev->lpis.lpi_busy == 0) 1025301265Sandrew its_device_release(dev, its_dev); 1026301265Sandrew 1027301265Sandrew return (0); 1028301265Sandrew} 1029301265Sandrew 1030301265Sandrewstatic int 1031301265Sandrewgicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1032301265Sandrew struct intr_irqsrc **isrcp) 1033301265Sandrew{ 1034301265Sandrew struct gicv3_its_softc *sc; 1035301265Sandrew struct gicv3_its_irqsrc *girq; 1036301265Sandrew struct its_dev *its_dev; 1037301265Sandrew u_int nvecs, irq; 1038301265Sandrew 1039301265Sandrew nvecs = pci_msix_count(child); 1040301265Sandrew its_dev = its_device_get(dev, child, nvecs); 1041301265Sandrew if (its_dev == NULL) 1042301265Sandrew return (ENXIO); 1043301265Sandrew 1044301265Sandrew KASSERT(its_dev->lpis.lpi_free > 0, 1045301265Sandrew ("gicv3_its_alloc_msix: No free LPIs")); 1046301265Sandrew sc = device_get_softc(dev); 1047301265Sandrew irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1048301265Sandrew its_dev->lpis.lpi_free; 1049301265Sandrew its_dev->lpis.lpi_free--; 1050301265Sandrew its_dev->lpis.lpi_busy++; 1051301265Sandrew girq = &sc->sc_irqs[irq]; 1052301265Sandrew girq->gi_its_dev = its_dev; 1053301265Sandrew 1054301265Sandrew *pic = dev; 1055301265Sandrew *isrcp = (struct intr_irqsrc *)girq; 1056301265Sandrew 1057301265Sandrew return (0); 1058301265Sandrew} 1059301265Sandrew 1060301265Sandrewstatic int 1061301265Sandrewgicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1062301265Sandrew{ 1063301265Sandrew struct gicv3_its_softc *sc; 1064301265Sandrew struct gicv3_its_irqsrc *girq; 1065301265Sandrew struct its_dev *its_dev; 1066301265Sandrew 1067301265Sandrew sc = device_get_softc(dev); 1068301265Sandrew its_dev = its_device_find(dev, child); 1069301265Sandrew 1070301265Sandrew KASSERT(its_dev != NULL, 1071301265Sandrew ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1072301265Sandrew "no ITS device")); 1073301265Sandrew KASSERT(its_dev->lpis.lpi_busy > 0, 1074301265Sandrew ("gicv3_its_release_msix: Releasing more interrupts than " 1075301265Sandrew "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1076301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc; 1077301265Sandrew girq->gi_its_dev = NULL; 1078301265Sandrew its_dev->lpis.lpi_busy--; 1079301265Sandrew 1080301265Sandrew if (its_dev->lpis.lpi_busy == 0) 1081301265Sandrew its_device_release(dev, its_dev); 1082301265Sandrew 1083301265Sandrew return (0); 1084301265Sandrew} 1085301265Sandrew 1086301265Sandrewstatic int 1087301265Sandrewgicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1088301265Sandrew uint64_t *addr, uint32_t *data) 1089301265Sandrew{ 1090301265Sandrew struct gicv3_its_softc *sc; 1091301265Sandrew struct gicv3_its_irqsrc *girq; 1092301265Sandrew 1093301265Sandrew sc = device_get_softc(dev); 1094301265Sandrew girq = (struct gicv3_its_irqsrc *)isrc; 1095301265Sandrew 1096301265Sandrew /* Map the message to the given IRQ */ 1097301265Sandrew its_cmd_mapti(dev, girq); 1098301265Sandrew 1099301265Sandrew *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1100301265Sandrew *data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1101301265Sandrew 1102301265Sandrew return (0); 1103301265Sandrew} 1104301265Sandrew 1105301265Sandrew/* 1106301265Sandrew * Commands handling. 1107301265Sandrew */ 1108301265Sandrew 1109301265Sandrewstatic __inline void 1110301265Sandrewcmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1111301265Sandrew{ 1112301265Sandrew /* Command field: DW0 [7:0] */ 1113301265Sandrew cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1114301265Sandrew cmd->cmd_dword[0] |= htole64(cmd_type); 1115301265Sandrew} 1116301265Sandrew 1117301265Sandrewstatic __inline void 1118301265Sandrewcmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1119301265Sandrew{ 1120301265Sandrew /* Device ID field: DW0 [63:32] */ 1121301265Sandrew cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1122301265Sandrew cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1123301265Sandrew} 1124301265Sandrew 1125301265Sandrewstatic __inline void 1126301265Sandrewcmd_format_size(struct its_cmd *cmd, uint16_t size) 1127301265Sandrew{ 1128301265Sandrew /* Size field: DW1 [4:0] */ 1129301265Sandrew cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1130301265Sandrew cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1131301265Sandrew} 1132301265Sandrew 1133301265Sandrewstatic __inline void 1134301265Sandrewcmd_format_id(struct its_cmd *cmd, uint32_t id) 1135301265Sandrew{ 1136301265Sandrew /* ID field: DW1 [31:0] */ 1137301265Sandrew cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1138301265Sandrew cmd->cmd_dword[1] |= htole64(id); 1139301265Sandrew} 1140301265Sandrew 1141301265Sandrewstatic __inline void 1142301265Sandrewcmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1143301265Sandrew{ 1144301265Sandrew /* Physical ID field: DW1 [63:32] */ 1145301265Sandrew cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1146301265Sandrew cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1147301265Sandrew} 1148301265Sandrew 1149301265Sandrewstatic __inline void 1150301265Sandrewcmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1151301265Sandrew{ 1152301265Sandrew /* Collection field: DW2 [16:0] */ 1153301265Sandrew cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1154301265Sandrew cmd->cmd_dword[2] |= htole64(col_id); 1155301265Sandrew} 1156301265Sandrew 1157301265Sandrewstatic __inline void 1158301265Sandrewcmd_format_target(struct its_cmd *cmd, uint64_t target) 1159301265Sandrew{ 1160301265Sandrew /* Target Address field: DW2 [47:16] */ 1161301265Sandrew cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1162301265Sandrew cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1163301265Sandrew} 1164301265Sandrew 1165301265Sandrewstatic __inline void 1166301265Sandrewcmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1167301265Sandrew{ 1168301265Sandrew /* ITT Address field: DW2 [47:8] */ 1169301265Sandrew cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1170301265Sandrew cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1171301265Sandrew} 1172301265Sandrew 1173301265Sandrewstatic __inline void 1174301265Sandrewcmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1175301265Sandrew{ 1176301265Sandrew /* Valid field: DW2 [63] */ 1177301265Sandrew cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1178301265Sandrew cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1179301265Sandrew} 1180301265Sandrew 1181301265Sandrewstatic inline bool 1182301265Sandrewits_cmd_queue_full(struct gicv3_its_softc *sc) 1183301265Sandrew{ 1184301265Sandrew size_t read_idx, next_write_idx; 1185301265Sandrew 1186301265Sandrew /* Get the index of the next command */ 1187301265Sandrew next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1188301265Sandrew (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1189301265Sandrew /* And the index of the current command being read */ 1190301265Sandrew read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1191301265Sandrew 1192301265Sandrew /* 1193301265Sandrew * The queue is full when the write offset points 1194301265Sandrew * at the command before the current read offset. 1195301265Sandrew */ 1196301265Sandrew return (next_write_idx == read_idx); 1197301265Sandrew} 1198301265Sandrew 1199301265Sandrewstatic inline void 1200301265Sandrewits_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1201301265Sandrew{ 1202301265Sandrew 1203301265Sandrew if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1204301265Sandrew /* Clean D-cache under command. */ 1205301265Sandrew cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); 1206301265Sandrew } else { 1207301265Sandrew /* DSB inner shareable, store */ 1208301265Sandrew dsb(ishst); 1209301265Sandrew } 1210301265Sandrew 1211301265Sandrew} 1212301265Sandrew 1213301265Sandrewstatic inline uint64_t 1214301265Sandrewits_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1215301265Sandrew{ 1216301265Sandrew uint64_t off; 1217301265Sandrew 1218301265Sandrew off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1219301265Sandrew 1220301265Sandrew return (off); 1221301265Sandrew} 1222301265Sandrew 1223301265Sandrewstatic void 1224301265Sandrewits_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1225301265Sandrew struct its_cmd *cmd_last) 1226301265Sandrew{ 1227301265Sandrew struct gicv3_its_softc *sc; 1228301265Sandrew uint64_t first, last, read; 1229301265Sandrew size_t us_left; 1230301265Sandrew 1231301265Sandrew sc = device_get_softc(dev); 1232301265Sandrew 1233301265Sandrew /* 1234301265Sandrew * XXX ARM64TODO: This is obviously a significant delay. 1235301265Sandrew * The reason for that is that currently the time frames for 1236301265Sandrew * the command to complete are not known. 1237301265Sandrew */ 1238301265Sandrew us_left = 1000000; 1239301265Sandrew 1240301265Sandrew first = its_cmd_cwriter_offset(sc, cmd_first); 1241301265Sandrew last = its_cmd_cwriter_offset(sc, cmd_last); 1242301265Sandrew 1243301265Sandrew for (;;) { 1244301265Sandrew read = gic_its_read_8(sc, GITS_CREADR); 1245301265Sandrew if (first < last) { 1246301265Sandrew if (read < first || read >= last) 1247301265Sandrew break; 1248301265Sandrew } else if (read < first && read >= last) 1249301265Sandrew break; 1250301265Sandrew 1251301265Sandrew if (us_left-- == 0) { 1252301265Sandrew /* This means timeout */ 1253301265Sandrew device_printf(dev, 1254301265Sandrew "Timeout while waiting for CMD completion.\n"); 1255301265Sandrew return; 1256301265Sandrew } 1257301265Sandrew DELAY(1); 1258301265Sandrew } 1259301265Sandrew} 1260301265Sandrew 1261301265Sandrew 1262301265Sandrewstatic struct its_cmd * 1263301265Sandrewits_cmd_alloc_locked(device_t dev) 1264301265Sandrew{ 1265301265Sandrew struct gicv3_its_softc *sc; 1266301265Sandrew struct its_cmd *cmd; 1267301265Sandrew size_t us_left; 1268301265Sandrew 1269301265Sandrew sc = device_get_softc(dev); 1270301265Sandrew 1271301265Sandrew /* 1272301265Sandrew * XXX ARM64TODO: This is obviously a significant delay. 1273301265Sandrew * The reason for that is that currently the time frames for 1274301265Sandrew * the command to complete (and therefore free the descriptor) 1275301265Sandrew * are not known. 1276301265Sandrew */ 1277301265Sandrew us_left = 1000000; 1278301265Sandrew 1279301265Sandrew mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1280301265Sandrew while (its_cmd_queue_full(sc)) { 1281301265Sandrew if (us_left-- == 0) { 1282301265Sandrew /* Timeout while waiting for free command */ 1283301265Sandrew device_printf(dev, 1284301265Sandrew "Timeout while waiting for free command\n"); 1285301265Sandrew return (NULL); 1286301265Sandrew } 1287301265Sandrew DELAY(1); 1288301265Sandrew } 1289301265Sandrew 1290301265Sandrew cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1291301265Sandrew sc->sc_its_cmd_next_idx++; 1292301265Sandrew sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1293301265Sandrew 1294301265Sandrew return (cmd); 1295301265Sandrew} 1296301265Sandrew 1297301265Sandrewstatic uint64_t 1298301265Sandrewits_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1299301265Sandrew{ 1300301265Sandrew uint64_t target; 1301301265Sandrew uint8_t cmd_type; 1302301265Sandrew u_int size; 1303301265Sandrew boolean_t error; 1304301265Sandrew 1305301265Sandrew error = FALSE; 1306301265Sandrew cmd_type = desc->cmd_type; 1307301265Sandrew target = ITS_TARGET_NONE; 1308301265Sandrew 1309301265Sandrew switch (cmd_type) { 1310301265Sandrew case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1311301265Sandrew target = desc->cmd_desc_movi.col->col_target; 1312301265Sandrew cmd_format_command(cmd, ITS_CMD_MOVI); 1313301265Sandrew cmd_format_id(cmd, desc->cmd_desc_movi.id); 1314301265Sandrew cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1315301265Sandrew cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1316301265Sandrew break; 1317301265Sandrew case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1318301265Sandrew target = desc->cmd_desc_sync.col->col_target; 1319301265Sandrew cmd_format_command(cmd, ITS_CMD_SYNC); 1320301265Sandrew cmd_format_target(cmd, target); 1321301265Sandrew break; 1322301265Sandrew case ITS_CMD_MAPD: /* Assign ITT to device */ 1323301265Sandrew cmd_format_command(cmd, ITS_CMD_MAPD); 1324301265Sandrew cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1325301265Sandrew /* 1326301265Sandrew * Size describes number of bits to encode interrupt IDs 1327301265Sandrew * supported by the device minus one. 1328301265Sandrew * When V (valid) bit is zero, this field should be written 1329301265Sandrew * as zero. 1330301265Sandrew */ 1331301265Sandrew if (desc->cmd_desc_mapd.valid != 0) { 1332301265Sandrew size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1333301265Sandrew size = MAX(1, size) - 1; 1334301265Sandrew } else 1335301265Sandrew size = 0; 1336301265Sandrew 1337301265Sandrew cmd_format_size(cmd, size); 1338301265Sandrew cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1339301265Sandrew cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1340301265Sandrew break; 1341301265Sandrew case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1342301265Sandrew target = desc->cmd_desc_mapc.col->col_target; 1343301265Sandrew cmd_format_command(cmd, ITS_CMD_MAPC); 1344301265Sandrew cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 1345301265Sandrew cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 1346301265Sandrew cmd_format_target(cmd, target); 1347301265Sandrew break; 1348301265Sandrew case ITS_CMD_MAPTI: 1349301265Sandrew target = desc->cmd_desc_mapvi.col->col_target; 1350301265Sandrew cmd_format_command(cmd, ITS_CMD_MAPTI); 1351301265Sandrew cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 1352301265Sandrew cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 1353301265Sandrew cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 1354301265Sandrew cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 1355301265Sandrew break; 1356301265Sandrew case ITS_CMD_MAPI: 1357301265Sandrew target = desc->cmd_desc_mapi.col->col_target; 1358301265Sandrew cmd_format_command(cmd, ITS_CMD_MAPI); 1359301265Sandrew cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 1360301265Sandrew cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 1361301265Sandrew cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 1362301265Sandrew break; 1363301265Sandrew case ITS_CMD_INV: 1364301265Sandrew target = desc->cmd_desc_inv.col->col_target; 1365301265Sandrew cmd_format_command(cmd, ITS_CMD_INV); 1366301265Sandrew cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 1367301265Sandrew cmd_format_id(cmd, desc->cmd_desc_inv.pid); 1368301265Sandrew break; 1369301265Sandrew case ITS_CMD_INVALL: 1370301265Sandrew cmd_format_command(cmd, ITS_CMD_INVALL); 1371301265Sandrew cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 1372301265Sandrew break; 1373301265Sandrew default: 1374301265Sandrew panic("its_cmd_prepare: Invalid command: %x", cmd_type); 1375301265Sandrew } 1376301265Sandrew 1377301265Sandrew return (target); 1378301265Sandrew} 1379301265Sandrew 1380301265Sandrewstatic int 1381301265Sandrewits_cmd_send(device_t dev, struct its_cmd_desc *desc) 1382301265Sandrew{ 1383301265Sandrew struct gicv3_its_softc *sc; 1384301265Sandrew struct its_cmd *cmd, *cmd_sync, *cmd_write; 1385301265Sandrew struct its_col col_sync; 1386301265Sandrew struct its_cmd_desc desc_sync; 1387301265Sandrew uint64_t target, cwriter; 1388301265Sandrew 1389301265Sandrew sc = device_get_softc(dev); 1390301265Sandrew mtx_lock_spin(&sc->sc_its_cmd_lock); 1391301265Sandrew cmd = its_cmd_alloc_locked(dev); 1392301265Sandrew if (cmd == NULL) { 1393301265Sandrew device_printf(dev, "could not allocate ITS command\n"); 1394301265Sandrew mtx_unlock_spin(&sc->sc_its_cmd_lock); 1395301265Sandrew return (EBUSY); 1396301265Sandrew } 1397301265Sandrew 1398301265Sandrew target = its_cmd_prepare(cmd, desc); 1399301265Sandrew its_cmd_sync(sc, cmd); 1400301265Sandrew 1401301265Sandrew if (target != ITS_TARGET_NONE) { 1402301265Sandrew cmd_sync = its_cmd_alloc_locked(dev); 1403301265Sandrew if (cmd_sync != NULL) { 1404301265Sandrew desc_sync.cmd_type = ITS_CMD_SYNC; 1405301265Sandrew col_sync.col_target = target; 1406301265Sandrew desc_sync.cmd_desc_sync.col = &col_sync; 1407301265Sandrew its_cmd_prepare(cmd_sync, &desc_sync); 1408301265Sandrew its_cmd_sync(sc, cmd_sync); 1409301265Sandrew } 1410301265Sandrew } 1411301265Sandrew 1412301265Sandrew /* Update GITS_CWRITER */ 1413301265Sandrew cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 1414301265Sandrew gic_its_write_8(sc, GITS_CWRITER, cwriter); 1415301265Sandrew cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1416301265Sandrew mtx_unlock_spin(&sc->sc_its_cmd_lock); 1417301265Sandrew 1418301265Sandrew its_cmd_wait_completion(dev, cmd, cmd_write); 1419301265Sandrew 1420301265Sandrew return (0); 1421301265Sandrew} 1422301265Sandrew 1423301265Sandrew/* Handlers to send commands */ 1424301265Sandrewstatic void 1425301265Sandrewits_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 1426301265Sandrew{ 1427301265Sandrew struct gicv3_its_softc *sc; 1428301265Sandrew struct its_cmd_desc desc; 1429301265Sandrew struct its_col *col; 1430301265Sandrew 1431301265Sandrew sc = device_get_softc(dev); 1432301265Sandrew col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1433301265Sandrew 1434301265Sandrew desc.cmd_type = ITS_CMD_MOVI; 1435301265Sandrew desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 1436301265Sandrew desc.cmd_desc_movi.col = col; 1437301265Sandrew desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1438301265Sandrew 1439301265Sandrew its_cmd_send(dev, &desc); 1440301265Sandrew} 1441301265Sandrew 1442301265Sandrewstatic void 1443301265Sandrewits_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 1444301265Sandrew{ 1445301265Sandrew struct its_cmd_desc desc; 1446301265Sandrew 1447301265Sandrew desc.cmd_type = ITS_CMD_MAPC; 1448301265Sandrew desc.cmd_desc_mapc.col = col; 1449301265Sandrew /* 1450301265Sandrew * Valid bit set - map the collection. 1451301265Sandrew * Valid bit cleared - unmap the collection. 1452301265Sandrew */ 1453301265Sandrew desc.cmd_desc_mapc.valid = valid; 1454301265Sandrew 1455301265Sandrew its_cmd_send(dev, &desc); 1456301265Sandrew} 1457301265Sandrew 1458301265Sandrewstatic void 1459301265Sandrewits_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 1460301265Sandrew{ 1461301265Sandrew struct gicv3_its_softc *sc; 1462301265Sandrew struct its_cmd_desc desc; 1463301265Sandrew struct its_col *col; 1464301265Sandrew u_int col_id; 1465301265Sandrew 1466301265Sandrew sc = device_get_softc(dev); 1467301265Sandrew 1468301265Sandrew col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 1469301265Sandrew col = sc->sc_its_cols[col_id]; 1470301265Sandrew 1471301265Sandrew desc.cmd_type = ITS_CMD_MAPTI; 1472301265Sandrew desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 1473301265Sandrew desc.cmd_desc_mapvi.col = col; 1474301265Sandrew /* The EventID sent to the device */ 1475301265Sandrew desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1476301265Sandrew /* The physical interrupt presented to softeware */ 1477301265Sandrew desc.cmd_desc_mapvi.pid = girq->gi_irq + GIC_FIRST_LPI; 1478301265Sandrew 1479301265Sandrew its_cmd_send(dev, &desc); 1480301265Sandrew} 1481301265Sandrew 1482301265Sandrewstatic void 1483301265Sandrewits_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 1484301265Sandrew{ 1485301265Sandrew struct its_cmd_desc desc; 1486301265Sandrew 1487301265Sandrew desc.cmd_type = ITS_CMD_MAPD; 1488301265Sandrew desc.cmd_desc_mapd.its_dev = its_dev; 1489301265Sandrew desc.cmd_desc_mapd.valid = valid; 1490301265Sandrew 1491301265Sandrew its_cmd_send(dev, &desc); 1492301265Sandrew} 1493301265Sandrew 1494301265Sandrewstatic void 1495301265Sandrewits_cmd_inv(device_t dev, struct its_dev *its_dev, 1496301265Sandrew struct gicv3_its_irqsrc *girq) 1497301265Sandrew{ 1498301265Sandrew struct gicv3_its_softc *sc; 1499301265Sandrew struct its_cmd_desc desc; 1500301265Sandrew struct its_col *col; 1501301265Sandrew 1502301265Sandrew sc = device_get_softc(dev); 1503301265Sandrew col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1504301265Sandrew 1505301265Sandrew desc.cmd_type = ITS_CMD_INV; 1506301265Sandrew /* The EventID sent to the device */ 1507301265Sandrew desc.cmd_desc_inv.pid = girq->gi_irq - its_dev->lpis.lpi_base; 1508301265Sandrew desc.cmd_desc_inv.its_dev = its_dev; 1509301265Sandrew desc.cmd_desc_inv.col = col; 1510301265Sandrew 1511301265Sandrew its_cmd_send(dev, &desc); 1512301265Sandrew} 1513301265Sandrew 1514301265Sandrewstatic void 1515301265Sandrewits_cmd_invall(device_t dev, struct its_col *col) 1516301265Sandrew{ 1517301265Sandrew struct its_cmd_desc desc; 1518301265Sandrew 1519301265Sandrew desc.cmd_type = ITS_CMD_INVALL; 1520301265Sandrew desc.cmd_desc_invall.col = col; 1521301265Sandrew 1522301265Sandrew its_cmd_send(dev, &desc); 1523301265Sandrew} 1524301265Sandrew 1525301265Sandrew#ifdef FDT 1526301265Sandrewstatic device_probe_t gicv3_its_fdt_probe; 1527301265Sandrewstatic device_attach_t gicv3_its_fdt_attach; 1528301265Sandrew 1529301265Sandrewstatic device_method_t gicv3_its_fdt_methods[] = { 1530301265Sandrew /* Device interface */ 1531301265Sandrew DEVMETHOD(device_probe, gicv3_its_fdt_probe), 1532301265Sandrew DEVMETHOD(device_attach, gicv3_its_fdt_attach), 1533301265Sandrew 1534301265Sandrew /* End */ 1535301265Sandrew DEVMETHOD_END 1536301265Sandrew}; 1537301265Sandrew 1538301265Sandrew#define its_baseclasses its_fdt_baseclasses 1539301265SandrewDEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 1540301265Sandrew sizeof(struct gicv3_its_softc), gicv3_its_driver); 1541301265Sandrew#undef its_baseclasses 1542301265Sandrewstatic devclass_t gicv3_its_fdt_devclass; 1543301265Sandrew 1544301265SandrewEARLY_DRIVER_MODULE(its, gic, gicv3_its_fdt_driver, 1545301265Sandrew gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1546301265Sandrew 1547301265Sandrewstatic int 1548301265Sandrewgicv3_its_fdt_probe(device_t dev) 1549301265Sandrew{ 1550301265Sandrew 1551301265Sandrew if (!ofw_bus_status_okay(dev)) 1552301265Sandrew return (ENXIO); 1553301265Sandrew 1554301265Sandrew if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 1555301265Sandrew return (ENXIO); 1556301265Sandrew 1557301265Sandrew device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1558301265Sandrew return (BUS_PROBE_DEFAULT); 1559301265Sandrew} 1560301265Sandrew 1561301265Sandrewstatic int 1562301265Sandrewgicv3_its_fdt_attach(device_t dev) 1563301265Sandrew{ 1564301265Sandrew struct gicv3_its_softc *sc; 1565301265Sandrew phandle_t xref; 1566301265Sandrew int err; 1567301265Sandrew 1568301265Sandrew err = gicv3_its_attach(dev); 1569301265Sandrew if (err != 0) 1570301265Sandrew return (err); 1571301265Sandrew 1572301265Sandrew sc = device_get_softc(dev); 1573301265Sandrew 1574301265Sandrew /* Register this device as a interrupt controller */ 1575301265Sandrew xref = OF_xref_from_node(ofw_bus_get_node(dev)); 1576301265Sandrew sc->sc_pic = intr_pic_register(dev, xref); 1577301265Sandrew intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1578301265Sandrew gicv3_its_intr, sc, GIC_FIRST_LPI, LPI_NIRQS); 1579301265Sandrew 1580301265Sandrew /* Register this device to handle MSI interrupts */ 1581301265Sandrew intr_msi_register(dev, xref); 1582301265Sandrew 1583301265Sandrew return (0); 1584301265Sandrew} 1585301265Sandrew#endif 1586