1/* 2 * Copyright 2019, DornerWorks 3 * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230) 4 * 5 * SPDX-License-Identifier: GPL-2.0-only 6 */ 7 8#include <config.h> 9#include <types.h> 10#include <arch/machine/gic_v3.h> 11 12#define IRQ_SET_ALL 0xffffffff 13 14#define RDIST_BANK_SZ 0x00010000 15/* One GICR region and one GICR_SGI region */ 16#define GICR_PER_CORE_SIZE (0x20000) 17/* Assume 8 cores */ 18#define GICR_SIZE (0x100000) 19 20#define GIC_DEADLINE_MS 2 21#define GIC_REG_WIDTH 32 22 23#ifdef CONFIG_ARCH_AARCH64 24#define ICC_SGI1R_EL1 "S3_0_C12_C11_5" 25#else 26#define ICC_SGI1R_EL1 "p15, 0, %Q0, %R0, c12" 27#endif 28 29#define ICC_SGI1R_INTID_SHIFT (24) 30#define ICC_SGI1R_AFF1_SHIFT (16) 31#define ICC_SGI1R_IRM_BIT (40) 32#define ICC_SGI1R_CPUTARGETLIST_MASK 0xffff 33 34volatile struct gic_dist_map *const gic_dist = (volatile struct gic_dist_map *)(GICD_PPTR); 35volatile void *const gicr_base = (volatile uint8_t *)(GICR_PPTR); 36 37word_t active_irq[CONFIG_MAX_NUM_NODES] = {IRQ_NONE}; 38volatile struct gic_rdist_map *gic_rdist_map[CONFIG_MAX_NUM_NODES] = { 0 }; 39volatile struct gic_rdist_sgi_ppi_map *gic_rdist_sgi_ppi_map[CONFIG_MAX_NUM_NODES] = { 0 }; 40 41#ifdef CONFIG_ARCH_AARCH64 42#define MPIDR_AFF0(x) (x & 0xff) 43#define MPIDR_AFF1(x) ((x >> 8) & 0xff) 44#define MPIDR_AFF2(x) ((x >> 16) & 0xff) 45#define MPIDR_AFF3(x) ((x >> 32) & 0xff) 46#else 47#define MPIDR_AFF0(x) (x & 0xff) 48#define MPIDR_AFF1(x) ((x >> 8) & 0xff) 49#define MPIDR_AFF2(x) ((x >> 16) & 0xff) 50#define MPIDR_AFF3(x) (0) 51#endif 52#define MPIDR_MT(x) (x & BIT(24)) 53#define MPIDR_AFF_MASK(x) (x & 0xff00ffffff) 54 55static word_t mpidr_map[CONFIG_MAX_NUM_NODES]; 56 57static inline word_t get_mpidr(word_t core_id) 58{ 59 return mpidr_map[core_id]; 60} 61 62static inline word_t get_current_mpidr(void) 63{ 64 word_t core_id = CURRENT_CPU_INDEX(); 65 return get_mpidr(core_id); 66} 67 68static inline uint64_t mpidr_to_gic_affinity(void) 69{ 70 word_t mpidr = get_current_mpidr(); 71 uint64_t affinity = 0; 72 affinity = (uint64_t)MPIDR_AFF3(mpidr) << 32 | MPIDR_AFF2(mpidr) << 16 | 73 MPIDR_AFF1(mpidr) << 8 | MPIDR_AFF0(mpidr); 74 return affinity; 75} 76 77/* Wait for completion of a distributor change */ 78static uint32_t gicv3_do_wait_for_rwp(volatile uint32_t *ctlr_addr) 79{ 80 uint32_t val; 81 bool_t waiting = true; 82 uint32_t ret = 0; 83 84 uint32_t gpt_cnt_tval = 0; 85 uint32_t deadline_ms = GIC_DEADLINE_MS; 86 uint32_t gpt_cnt_ciel; 87 88 /* Check the value before reading the generic timer */ 89 val = *ctlr_addr; 90 if (!(val & GICD_CTLR_RWP)) { 91 return 0; 92 } 93 SYSTEM_READ_WORD(CNTFRQ, gpt_cnt_tval); 94 gpt_cnt_ciel = gpt_cnt_tval + (deadline_ms * TICKS_PER_MS); 95 96 while (waiting) { 97 SYSTEM_READ_WORD(CNTFRQ, gpt_cnt_tval); 98 val = *ctlr_addr; 99 100 if (gpt_cnt_tval >= gpt_cnt_ciel) { 101 printf("GICV3 RWP Timeout after %u ms\n", deadline_ms); 102 ret = 1; 103 waiting = false; 104 105 } else if (!(val & GICD_CTLR_RWP)) { 106 ret = 0; 107 waiting = false; 108 } 109 } 110 return ret; 111} 112 113static void gicv3_dist_wait_for_rwp(void) 114{ 115 gicv3_do_wait_for_rwp(&gic_dist->ctlr); 116} 117 118static void gicv3_redist_wait_for_rwp(void) 119{ 120 gicv3_do_wait_for_rwp(&gic_rdist_map[CURRENT_CPU_INDEX()]->ctlr); 121} 122 123static void gicv3_enable_sre(void) 124{ 125 uint32_t val = 0; 126 127 /* ICC_SRE_EL1 */ 128 SYSTEM_READ_WORD(ICC_SRE_EL1, val); 129 val |= GICC_SRE_EL1_SRE; 130 131 SYSTEM_WRITE_WORD(ICC_SRE_EL1, val); 132 isb(); 133} 134 135 136BOOT_CODE static void dist_init(void) 137{ 138 word_t i; 139 uint32_t type; 140 unsigned int nr_lines; 141 uint64_t affinity; 142 uint32_t priority; 143 144 /* Disable GIC Distributor */ 145 gic_dist->ctlr = 0; 146 gicv3_dist_wait_for_rwp(); 147 148 type = gic_dist->typer; 149 150 nr_lines = GIC_REG_WIDTH * ((type & GICD_TYPE_LINESNR) + 1); 151 152 /* Assume level-triggered */ 153 for (i = SPI_START; i < nr_lines; i += 16) { 154 gic_dist->icfgrn[(i / 16)] = 0; 155 } 156 157 /* Default priority for global interrupts */ 158 priority = (GIC_PRI_IRQ << 24 | GIC_PRI_IRQ << 16 | GIC_PRI_IRQ << 8 | 159 GIC_PRI_IRQ); 160 for (i = SPI_START; i < nr_lines; i += 4) { 161 gic_dist->ipriorityrn[(i / 4)] = priority; 162 } 163 /* Disable and clear all global interrupts */ 164 for (i = SPI_START; i < nr_lines; i += 32) { 165 gic_dist->icenablern[(i / 32)] = IRQ_SET_ALL; 166 gic_dist->icpendrn[(i / 32)] = IRQ_SET_ALL; 167 } 168 169 /* Turn on the distributor */ 170 gic_dist->ctlr = GICD_CTL_ENABLE | GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1NS | GICD_CTLR_ENABLE_G0; 171 gicv3_dist_wait_for_rwp(); 172 173 /* Route all global IRQs to this CPU */ 174 affinity = mpidr_to_gic_affinity(); 175 for (i = SPI_START; i < nr_lines; i++) { 176 gic_dist->iroutern[i] = affinity; 177 } 178} 179 180BOOT_CODE static void gicr_locate_interface(void) 181{ 182 word_t offset; 183 int core_id = SMP_TERNARY(getCurrentCPUIndex(), 0); 184 word_t mpidr = get_current_mpidr(); 185 uint32_t val; 186 187 /* 188 * Iterate through all redistributor interfaces looking for one that matches 189 * our mpidr. 190 */ 191 for (offset = 0; offset < GICR_SIZE; offset += GICR_PER_CORE_SIZE) { 192 193 uint64_t typer = ((struct gic_rdist_map *)((word_t)gicr_base + offset))->typer; 194 if ((typer >> 32) == ((MPIDR_AFF3(mpidr) << 24) | 195 (MPIDR_AFF2(mpidr) << 16) | 196 (MPIDR_AFF1(mpidr) << 8) | 197 MPIDR_AFF0(mpidr))) { 198 199 word_t gicr = (word_t)gicr_base + offset; 200 if (gic_rdist_map[core_id] != NULL || gic_rdist_sgi_ppi_map[core_id] != NULL) { 201 printf("GICv3: %s[%d] %p is not null\n", 202 gic_rdist_map[core_id] == NULL ? "gic_rdist_map" : "gic_rdist_sgi_ppi_map", 203 core_id, 204 gic_rdist_map[core_id] == NULL ? (void *)gic_rdist_map[core_id] : (void *)gic_rdist_sgi_ppi_map[core_id]); 205 halt(); 206 } 207 gic_rdist_map[core_id] = (void *)gicr; 208 gic_rdist_sgi_ppi_map[core_id] = (void *)(gicr + RDIST_BANK_SZ); 209 210 /* 211 * GICR_WAKER should be Read-all-zeros in Non-secure world 212 * and we expect redistributors to be alread awoken by an earlier loader. 213 * However if we get a value back then something is probably wrong. 214 */ 215 val = gic_rdist_map[core_id]->waker; 216 if (val & GICR_WAKER_ChildrenAsleep) { 217 printf("GICv3: GICR_WAKER returned non-zero %x\n", val); 218 halt(); 219 } 220 221 break; 222 } 223 } 224 if (offset >= GICR_SIZE) { 225 printf("GICv3: GICR base for CPU %d %d %d %d (Logic ID %d) not found\n", 226 (int)MPIDR_AFF3(mpidr), (int)MPIDR_AFF2(mpidr), 227 (int)MPIDR_AFF1(mpidr), (int)MPIDR_AFF0(mpidr), core_id); 228 halt(); 229 } 230 231 232} 233 234BOOT_CODE static void gicr_init(void) 235{ 236 int i; 237 uint32_t priority; 238 239 /* Find redistributor for this core. */ 240 gicr_locate_interface(); 241 242 /* Deactivate SGIs/PPIs */ 243 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icactiver0 = ~0; 244 245 /* Set priority on PPI and SGI interrupts */ 246 priority = (GIC_PRI_IRQ << 24 | GIC_PRI_IRQ << 16 | GIC_PRI_IRQ << 8 | 247 GIC_PRI_IRQ); 248 for (i = 0; i < SPI_START; i += 4) { 249 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->ipriorityrn[i / 4] = priority; 250 } 251 252 /* 253 * Disable all PPI interrupts, ensure all SGI interrupts are 254 * enabled. 255 */ 256 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icenabler0 = 0xffff0000; 257 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->isenabler0 = 0x0000ffff; 258 259 /* Set ICFGR1 for PPIs as level-triggered */ 260 gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icfgr1 = 0x0; 261 262 gicv3_redist_wait_for_rwp(); 263} 264 265BOOT_CODE static void cpu_iface_init(void) 266{ 267 uint32_t icc_ctlr = 0; 268 269 /* Enable system registers */ 270 gicv3_enable_sre(); 271 272 /* No priority grouping: ICC_BPR1_EL1 */ 273 SYSTEM_WRITE_WORD(ICC_BPR1_EL1, 0); 274 275 /* Set priority mask register: ICC_PMR_EL1 */ 276 SYSTEM_WRITE_WORD(ICC_PMR_EL1, DEFAULT_PMR_VALUE); 277 278 /* EOI drops priority and deactivates the interrupt: ICC_CTLR_EL1 */ 279 SYSTEM_READ_WORD(ICC_CTLR_EL1, icc_ctlr); 280 icc_ctlr &= ~BIT(GICC_CTLR_EL1_EOImode_drop); 281 SYSTEM_WRITE_WORD(ICC_CTLR_EL1, icc_ctlr); 282 283 /* Enable Group1 interrupts: ICC_IGRPEN1_EL1 */ 284 SYSTEM_WRITE_WORD(ICC_IGRPEN1_EL1, 1); 285 286 /* Sync at once at the end of cpu interface configuration */ 287 isb(); 288} 289 290void setIRQTrigger(irq_t irq, bool_t trigger) 291{ 292 293 /* GICv3 has read-only GICR_ICFG0 for SGI with 294 * default value 0xaaaaaaaa, and read-write GICR_ICFG1 295 * for PPI with default 0x00000000.*/ 296 word_t hw_irq = IRQT_TO_IRQ(irq); 297 word_t core = IRQT_TO_CORE(irq); 298 if (HW_IRQ_IS_SGI(hw_irq)) { 299 return; 300 } 301 int word = hw_irq >> 4; 302 int bit = ((hw_irq & 0xf) * 2); 303 uint32_t icfgr = 0; 304 if (HW_IRQ_IS_PPI(hw_irq)) { 305 icfgr = gic_rdist_sgi_ppi_map[core]->icfgr1; 306 } else { 307 icfgr = gic_dist->icfgrn[word]; 308 } 309 310 if (trigger) { 311 icfgr |= (2 << bit); 312 } else { 313 icfgr &= ~(3 << bit); 314 } 315 316 if (HW_IRQ_IS_PPI(hw_irq)) { 317 gic_rdist_sgi_ppi_map[core]->icfgr1 = icfgr; 318 } else { 319 /* Update GICD_ICFGR<n>. Note that the interrupt should 320 * be disabled before changing the field, and this function 321 * assumes the caller has disabled the interrupt. */ 322 gic_dist->icfgrn[word] = icfgr; 323 } 324 325 return; 326} 327 328BOOT_CODE void initIRQController(void) 329{ 330 dist_init(); 331} 332 333BOOT_CODE void cpu_initLocalIRQController(void) 334{ 335 word_t mpidr = 0; 336 SYSTEM_READ_WORD(MPIDR, mpidr); 337 338 mpidr_map[CURRENT_CPU_INDEX()] = mpidr; 339 340 gicr_init(); 341 cpu_iface_init(); 342} 343 344#ifdef ENABLE_SMP_SUPPORT 345#define MPIDR_MT(x) (x & BIT(24)) 346 347void ipi_send_target(irq_t irq, word_t cpuTargetList) 348{ 349 uint64_t sgi1r_base = ((word_t) IRQT_TO_IRQ(irq)) << ICC_SGI1R_INTID_SHIFT; 350 word_t sgi1r[CONFIG_MAX_NUM_NODES]; 351 word_t last_aff1 = 0; 352 353 for (word_t i = 0; i < CONFIG_MAX_NUM_NODES; i++) { 354 sgi1r[i] = 0; 355 if (cpuTargetList & BIT(i)) { 356 word_t mpidr = mpidr_map[i]; 357 word_t aff1 = MPIDR_AFF1(mpidr); 358 word_t aff0 = MPIDR_AFF0(mpidr); 359 // AFF1 is assumed to be contiguous and less than CONFIG_MAX_NUM_NODES. 360 // The targets are grouped by AFF1. 361 assert(aff1 >= 0 && aff1 < CONFIG_MAX_NUM_NODES); 362 sgi1r[aff1] |= sgi1r_base | (aff1 << ICC_SGI1R_AFF1_SHIFT) | (1 << aff0); 363 if (aff1 > last_aff1) { 364 last_aff1 = aff1; 365 } 366 } 367 } 368 for (word_t i = 0; i <= last_aff1; i++) { 369 if (sgi1r[i] != 0) { 370 SYSTEM_WRITE_64(ICC_SGI1R_EL1, sgi1r[i]); 371 } 372 } 373 isb(); 374} 375 376void setIRQTarget(irq_t irq, seL4_Word target) 377{ 378 if (IRQ_IS_PPI(irq)) { 379 fail("PPI can't have designated target core\n"); 380 return; 381 } 382 383 word_t hw_irq = IRQT_TO_IRQ(irq); 384 gic_dist->iroutern[hw_irq] = MPIDR_AFF_MASK(mpidr_map[target]); 385} 386 387#endif /* ENABLE_SMP_SUPPORT */ 388