1/*
2 * Copyright 2019, DornerWorks
3 * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230)
4 *
5 * SPDX-License-Identifier: GPL-2.0-only
6 */
7
8#include <config.h>
9#include <types.h>
10#include <arch/machine/gic_v3.h>
11
12#define IRQ_SET_ALL 0xffffffff
13
14#define RDIST_BANK_SZ 0x00010000
15/* One GICR region and one GICR_SGI region */
16#define GICR_PER_CORE_SIZE  (0x20000)
17/* Assume 8 cores */
18#define GICR_SIZE           (0x100000)
19
20#define GIC_DEADLINE_MS 2
21#define GIC_REG_WIDTH   32
22
23#ifdef CONFIG_ARCH_AARCH64
24#define ICC_SGI1R_EL1 "S3_0_C12_C11_5"
25#else
26#define ICC_SGI1R_EL1 "p15, 0, %Q0, %R0, c12"
27#endif
28
29#define ICC_SGI1R_INTID_SHIFT          (24)
30#define ICC_SGI1R_AFF1_SHIFT           (16)
31#define ICC_SGI1R_IRM_BIT              (40)
32#define ICC_SGI1R_CPUTARGETLIST_MASK   0xffff
33
34volatile struct gic_dist_map *const gic_dist = (volatile struct gic_dist_map *)(GICD_PPTR);
35volatile void *const gicr_base = (volatile uint8_t *)(GICR_PPTR);
36
37word_t active_irq[CONFIG_MAX_NUM_NODES] = {IRQ_NONE};
38volatile struct gic_rdist_map *gic_rdist_map[CONFIG_MAX_NUM_NODES] = { 0 };
39volatile struct gic_rdist_sgi_ppi_map *gic_rdist_sgi_ppi_map[CONFIG_MAX_NUM_NODES] = { 0 };
40
41#ifdef CONFIG_ARCH_AARCH64
42#define MPIDR_AFF0(x) (x & 0xff)
43#define MPIDR_AFF1(x) ((x >> 8) & 0xff)
44#define MPIDR_AFF2(x) ((x >> 16) & 0xff)
45#define MPIDR_AFF3(x) ((x >> 32) & 0xff)
46#else
47#define MPIDR_AFF0(x) (x & 0xff)
48#define MPIDR_AFF1(x) ((x >> 8) & 0xff)
49#define MPIDR_AFF2(x) ((x >> 16) & 0xff)
50#define MPIDR_AFF3(x) (0)
51#endif
52#define MPIDR_MT(x)   (x & BIT(24))
53#define MPIDR_AFF_MASK(x) (x & 0xff00ffffff)
54
55static word_t mpidr_map[CONFIG_MAX_NUM_NODES];
56
57static inline word_t get_mpidr(word_t core_id)
58{
59    return mpidr_map[core_id];
60}
61
62static inline word_t get_current_mpidr(void)
63{
64    word_t core_id = CURRENT_CPU_INDEX();
65    return get_mpidr(core_id);
66}
67
68static inline uint64_t mpidr_to_gic_affinity(void)
69{
70    word_t mpidr = get_current_mpidr();
71    uint64_t affinity = 0;
72    affinity = (uint64_t)MPIDR_AFF3(mpidr) << 32 | MPIDR_AFF2(mpidr) << 16 |
73               MPIDR_AFF1(mpidr) << 8  | MPIDR_AFF0(mpidr);
74    return affinity;
75}
76
77/* Wait for completion of a distributor change */
78static uint32_t gicv3_do_wait_for_rwp(volatile uint32_t *ctlr_addr)
79{
80    uint32_t val;
81    bool_t waiting = true;
82    uint32_t ret = 0;
83
84    uint32_t gpt_cnt_tval = 0;
85    uint32_t deadline_ms =  GIC_DEADLINE_MS;
86    uint32_t gpt_cnt_ciel;
87
88    /* Check the value before reading the generic timer */
89    val = *ctlr_addr;
90    if (!(val & GICD_CTLR_RWP)) {
91        return 0;
92    }
93    SYSTEM_READ_WORD(CNTFRQ, gpt_cnt_tval);
94    gpt_cnt_ciel = gpt_cnt_tval + (deadline_ms * TICKS_PER_MS);
95
96    while (waiting) {
97        SYSTEM_READ_WORD(CNTFRQ, gpt_cnt_tval);
98        val = *ctlr_addr;
99
100        if (gpt_cnt_tval >= gpt_cnt_ciel) {
101            printf("GICV3 RWP Timeout after %u ms\n", deadline_ms);
102            ret = 1;
103            waiting = false;
104
105        } else if (!(val & GICD_CTLR_RWP)) {
106            ret = 0;
107            waiting = false;
108        }
109    }
110    return ret;
111}
112
113static void gicv3_dist_wait_for_rwp(void)
114{
115    gicv3_do_wait_for_rwp(&gic_dist->ctlr);
116}
117
118static void gicv3_redist_wait_for_rwp(void)
119{
120    gicv3_do_wait_for_rwp(&gic_rdist_map[CURRENT_CPU_INDEX()]->ctlr);
121}
122
123static void gicv3_enable_sre(void)
124{
125    uint32_t val = 0;
126
127    /* ICC_SRE_EL1 */
128    SYSTEM_READ_WORD(ICC_SRE_EL1, val);
129    val |= GICC_SRE_EL1_SRE;
130
131    SYSTEM_WRITE_WORD(ICC_SRE_EL1, val);
132    isb();
133}
134
135
136BOOT_CODE static void dist_init(void)
137{
138    word_t i;
139    uint32_t type;
140    unsigned int nr_lines;
141    uint64_t affinity;
142    uint32_t priority;
143
144    /* Disable GIC Distributor */
145    gic_dist->ctlr = 0;
146    gicv3_dist_wait_for_rwp();
147
148    type = gic_dist->typer;
149
150    nr_lines = GIC_REG_WIDTH * ((type & GICD_TYPE_LINESNR) + 1);
151
152    /* Assume level-triggered */
153    for (i = SPI_START; i < nr_lines; i += 16) {
154        gic_dist->icfgrn[(i / 16)] = 0;
155    }
156
157    /* Default priority for global interrupts */
158    priority = (GIC_PRI_IRQ << 24 | GIC_PRI_IRQ << 16 | GIC_PRI_IRQ << 8 |
159                GIC_PRI_IRQ);
160    for (i = SPI_START; i < nr_lines; i += 4) {
161        gic_dist->ipriorityrn[(i / 4)] = priority;
162    }
163    /* Disable and clear all global interrupts */
164    for (i = SPI_START; i < nr_lines; i += 32) {
165        gic_dist->icenablern[(i / 32)] = IRQ_SET_ALL;
166        gic_dist->icpendrn[(i / 32)] = IRQ_SET_ALL;
167    }
168
169    /* Turn on the distributor */
170    gic_dist->ctlr = GICD_CTL_ENABLE | GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1NS | GICD_CTLR_ENABLE_G0;
171    gicv3_dist_wait_for_rwp();
172
173    /* Route all global IRQs to this CPU */
174    affinity = mpidr_to_gic_affinity();
175    for (i = SPI_START; i < nr_lines; i++) {
176        gic_dist->iroutern[i] = affinity;
177    }
178}
179
180BOOT_CODE static void gicr_locate_interface(void)
181{
182    word_t offset;
183    int core_id = SMP_TERNARY(getCurrentCPUIndex(), 0);
184    word_t mpidr = get_current_mpidr();
185    uint32_t val;
186
187    /*
188     * Iterate through all redistributor interfaces looking for one that matches
189     * our mpidr.
190     */
191    for (offset = 0; offset < GICR_SIZE; offset += GICR_PER_CORE_SIZE) {
192
193        uint64_t typer = ((struct gic_rdist_map *)((word_t)gicr_base + offset))->typer;
194        if ((typer >> 32) == ((MPIDR_AFF3(mpidr) << 24) |
195                              (MPIDR_AFF2(mpidr) << 16) |
196                              (MPIDR_AFF1(mpidr) <<  8) |
197                              MPIDR_AFF0(mpidr))) {
198
199            word_t gicr = (word_t)gicr_base + offset;
200            if (gic_rdist_map[core_id] != NULL || gic_rdist_sgi_ppi_map[core_id] != NULL) {
201                printf("GICv3: %s[%d] %p is not null\n",
202                       gic_rdist_map[core_id] == NULL ? "gic_rdist_map" : "gic_rdist_sgi_ppi_map",
203                       core_id,
204                       gic_rdist_map[core_id] == NULL ? (void *)gic_rdist_map[core_id] : (void *)gic_rdist_sgi_ppi_map[core_id]);
205                halt();
206            }
207            gic_rdist_map[core_id] = (void *)gicr;
208            gic_rdist_sgi_ppi_map[core_id] = (void *)(gicr + RDIST_BANK_SZ);
209
210            /*
211             * GICR_WAKER should be Read-all-zeros in Non-secure world
212             * and we expect redistributors to be alread awoken by an earlier loader.
213             * However if we get a value back then something is probably wrong.
214             */
215            val = gic_rdist_map[core_id]->waker;
216            if (val & GICR_WAKER_ChildrenAsleep) {
217                printf("GICv3: GICR_WAKER returned non-zero %x\n", val);
218                halt();
219            }
220
221            break;
222        }
223    }
224    if (offset >= GICR_SIZE) {
225        printf("GICv3: GICR base for CPU %d %d %d %d (Logic ID %d) not found\n",
226               (int)MPIDR_AFF3(mpidr), (int)MPIDR_AFF2(mpidr),
227               (int)MPIDR_AFF1(mpidr), (int)MPIDR_AFF0(mpidr), core_id);
228        halt();
229    }
230
231
232}
233
234BOOT_CODE static void gicr_init(void)
235{
236    int i;
237    uint32_t priority;
238
239    /* Find redistributor for this core. */
240    gicr_locate_interface();
241
242    /* Deactivate SGIs/PPIs */
243    gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icactiver0 = ~0;
244
245    /* Set priority on PPI and SGI interrupts */
246    priority = (GIC_PRI_IRQ << 24 | GIC_PRI_IRQ << 16 | GIC_PRI_IRQ << 8 |
247                GIC_PRI_IRQ);
248    for (i = 0; i < SPI_START; i += 4) {
249        gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->ipriorityrn[i / 4] = priority;
250    }
251
252    /*
253     * Disable all PPI interrupts, ensure all SGI interrupts are
254     * enabled.
255     */
256    gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icenabler0 = 0xffff0000;
257    gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->isenabler0 = 0x0000ffff;
258
259    /* Set ICFGR1 for PPIs as level-triggered */
260    gic_rdist_sgi_ppi_map[CURRENT_CPU_INDEX()]->icfgr1 = 0x0;
261
262    gicv3_redist_wait_for_rwp();
263}
264
265BOOT_CODE static void cpu_iface_init(void)
266{
267    uint32_t icc_ctlr = 0;
268
269    /* Enable system registers */
270    gicv3_enable_sre();
271
272    /* No priority grouping: ICC_BPR1_EL1 */
273    SYSTEM_WRITE_WORD(ICC_BPR1_EL1, 0);
274
275    /* Set priority mask register: ICC_PMR_EL1 */
276    SYSTEM_WRITE_WORD(ICC_PMR_EL1, DEFAULT_PMR_VALUE);
277
278    /* EOI drops priority and deactivates the interrupt: ICC_CTLR_EL1 */
279    SYSTEM_READ_WORD(ICC_CTLR_EL1, icc_ctlr);
280    icc_ctlr &= ~BIT(GICC_CTLR_EL1_EOImode_drop);
281    SYSTEM_WRITE_WORD(ICC_CTLR_EL1, icc_ctlr);
282
283    /* Enable Group1 interrupts: ICC_IGRPEN1_EL1 */
284    SYSTEM_WRITE_WORD(ICC_IGRPEN1_EL1, 1);
285
286    /* Sync at once at the end of cpu interface configuration */
287    isb();
288}
289
290void setIRQTrigger(irq_t irq, bool_t trigger)
291{
292
293    /* GICv3 has read-only GICR_ICFG0 for SGI with
294     * default value 0xaaaaaaaa, and read-write GICR_ICFG1
295     * for PPI with default 0x00000000.*/
296    word_t hw_irq = IRQT_TO_IRQ(irq);
297    word_t core = IRQT_TO_CORE(irq);
298    if (HW_IRQ_IS_SGI(hw_irq)) {
299        return;
300    }
301    int word = hw_irq >> 4;
302    int bit = ((hw_irq & 0xf) * 2);
303    uint32_t icfgr = 0;
304    if (HW_IRQ_IS_PPI(hw_irq)) {
305        icfgr = gic_rdist_sgi_ppi_map[core]->icfgr1;
306    } else {
307        icfgr = gic_dist->icfgrn[word];
308    }
309
310    if (trigger) {
311        icfgr |= (2 << bit);
312    } else {
313        icfgr &= ~(3 << bit);
314    }
315
316    if (HW_IRQ_IS_PPI(hw_irq)) {
317        gic_rdist_sgi_ppi_map[core]->icfgr1 = icfgr;
318    } else {
319        /* Update GICD_ICFGR<n>. Note that the interrupt should
320         * be disabled before changing the field, and this function
321         * assumes the caller has disabled the interrupt. */
322        gic_dist->icfgrn[word] = icfgr;
323    }
324
325    return;
326}
327
328BOOT_CODE void initIRQController(void)
329{
330    dist_init();
331}
332
333BOOT_CODE void cpu_initLocalIRQController(void)
334{
335    word_t mpidr = 0;
336    SYSTEM_READ_WORD(MPIDR, mpidr);
337
338    mpidr_map[CURRENT_CPU_INDEX()] = mpidr;
339
340    gicr_init();
341    cpu_iface_init();
342}
343
344#ifdef ENABLE_SMP_SUPPORT
345#define MPIDR_MT(x)   (x & BIT(24))
346
347void ipi_send_target(irq_t irq, word_t cpuTargetList)
348{
349    uint64_t sgi1r_base = ((word_t) IRQT_TO_IRQ(irq)) << ICC_SGI1R_INTID_SHIFT;
350    word_t sgi1r[CONFIG_MAX_NUM_NODES];
351    word_t last_aff1 = 0;
352
353    for (word_t i = 0; i < CONFIG_MAX_NUM_NODES; i++) {
354        sgi1r[i] = 0;
355        if (cpuTargetList & BIT(i)) {
356            word_t mpidr = mpidr_map[i];
357            word_t aff1 = MPIDR_AFF1(mpidr);
358            word_t aff0 = MPIDR_AFF0(mpidr);
359            // AFF1 is assumed to be contiguous and less than CONFIG_MAX_NUM_NODES.
360            // The targets are grouped by AFF1.
361            assert(aff1 >= 0 && aff1 < CONFIG_MAX_NUM_NODES);
362            sgi1r[aff1] |= sgi1r_base | (aff1 << ICC_SGI1R_AFF1_SHIFT) | (1 << aff0);
363            if (aff1 > last_aff1) {
364                last_aff1 = aff1;
365            }
366        }
367    }
368    for (word_t i = 0; i <= last_aff1; i++) {
369        if (sgi1r[i] != 0) {
370            SYSTEM_WRITE_64(ICC_SGI1R_EL1, sgi1r[i]);
371        }
372    }
373    isb();
374}
375
376void setIRQTarget(irq_t irq, seL4_Word target)
377{
378    if (IRQ_IS_PPI(irq)) {
379        fail("PPI can't have designated target core\n");
380        return;
381    }
382
383    word_t hw_irq = IRQT_TO_IRQ(irq);
384    gic_dist->iroutern[hw_irq] = MPIDR_AFF_MASK(mpidr_map[target]);
385}
386
387#endif /* ENABLE_SMP_SUPPORT */
388