1/*
2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#pragma once
8
9#include <config.h>
10
11#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT
12
13#include <arch/object/vcpu.h>
14#include <drivers/timer/arm_generic.h>
15
16/* Note that the HCR_DC for ARMv8 disables S1 translation if enabled */
17#ifdef CONFIG_DISABLE_WFI_WFE_TRAPS
18/* Trap SMC and override CPSR.AIF */
19#define HCR_COMMON ( HCR_VM | HCR_RW | HCR_AMO | HCR_IMO | HCR_FMO )
20#else
21/* Trap WFI/WFE/SMC and override CPSR.AIF */
22#define HCR_COMMON ( HCR_TWI | HCR_TWE | HCR_VM | HCR_RW | HCR_AMO | HCR_IMO | HCR_FMO )
23#endif
24
25/* Allow native tasks to run at EL0, but restrict access */
26#define HCR_NATIVE ( HCR_COMMON | HCR_TGE | HCR_TVM | HCR_TTLB | HCR_DC \
27                   | HCR_TAC | HCR_SWIO |  HCR_TSC | HCR_IMO | HCR_FMO | HCR_AMO)
28#define HCR_VCPU   ( HCR_COMMON | HCR_TSC)
29
30#define SCTLR_EL1_UCI       BIT(26)     /* Enable EL0 access to DC CVAU, DC CIVAC, DC CVAC,
31                                           and IC IVAU in AArch64 state   */
32#define SCTLR_EL1_C         BIT(2)      /* Enable data and unified caches */
33#define SCTLR_EL1_I         BIT(12)     /* Enable instruction cache       */
34#define SCTLR_EL1_CP15BEN   BIT(5)      /* AArch32 CP15 barrier enable    */
35#define SCTLR_EL1_UTC       BIT(15)     /* Enable EL0 access to CTR_EL0   */
36#define SCTLR_EL1_NTWI      BIT(16)     /* WFI executed as normal         */
37#define SCTLR_EL1_NTWE      BIT(18)     /* WFE executed as normal         */
38
39/* Disable MMU, SP alignment check, and alignment check */
40/* A57 default value */
41#define SCTLR_EL1_RES      0x30d00800   /* Reserved value */
42#define SCTLR_EL1          ( SCTLR_EL1_RES | SCTLR_EL1_CP15BEN | SCTLR_EL1_UTC \
43                           | SCTLR_EL1_NTWI | SCTLR_EL1_NTWE )
44#define SCTLR_EL1_NATIVE   (SCTLR_EL1 | SCTLR_EL1_C | SCTLR_EL1_I | SCTLR_EL1_UCI)
45#define SCTLR_EL1_VM       (SCTLR_EL1 | SCTLR_EL1_UCI)
46#define SCTLR_DEFAULT      SCTLR_EL1_NATIVE
47
48#define UNKNOWN_FAULT       0x2000000
49#define ESR_EC_TFP          0x7         /* Trap instructions that access FPU registers */
50#define ESR_EC_CPACR        0x18        /* Trap access to CPACR                        */
51#define ESR_EC(x)           (((x) & 0xfc000000) >> 26)
52
53#define VTCR_EL2_T0SZ(x)    ((x) & 0x3f)
54#define VTCR_EL2_SL0(x)     (((x) & 0x3) << 6)
55#define VTCR_EL2_IRGN0(x)   (((x) & 0x3) << 8)
56#define VTCR_EL2_ORGN0(x)   (((x) & 0x3) << 10)
57#define VTCR_EL2_SH0(x)     (((x) & 0x3) << 12)
58#define VTCR_EL2_TG0(x)     (((x) & 0x3) << 14)
59#define VTCR_EL2_PS(x)      (((x) & 0x7) << 16)
60
61/* Physical address size */
62#define PS_4G               0
63#define PS_64G              1
64#define PS_1T               2
65#define PS_4T               3
66#define PS_16T              4
67#define PS_256T             5
68
69/* Translation granule size */
70#define TG0_4K              0
71#define TG0_64K             1
72#define TG0_16K             2
73
74#define ID_AA64MMFR0_EL1_PARANGE(x) ((x) & 0xf)
75#define ID_AA64MMFR0_TGRAN4(x)      (((x) >> 28u) & 0xf)
76
77/* Shareability attributes */
78#define SH0_NONE            0
79#define SH0_OUTER           2
80#define SH0_INNER           3
81
82/* Cacheability attributes */
83#define NORMAL_NON_CACHEABLE    0
84#define NORMAL_WB_WA_CACHEABLE  1 /* write-back, write-allocate      */
85#define NORMAL_WT_CACHEABLE     2 /* write-through                   */
86#define NORMAL_WB_NWA_CACHEABLE 3 /* write-back, no write-allocate   */
87
88/* Start level  */
89#define SL0_4K_L2       0         /* 4K, start at level 2 */
90#define SL0_4K_L1       1         /* 4K, start at level 1 */
91#define SL0_4K_L0       2         /* 4K, start at level 0 */
92
93#define REG_SCTLR_EL1       "sctlr_el1"
94#define REG_TTBR0_EL1       "ttbr0_el1"
95#define REG_TTBR1_EL1       "ttbr1_el1"
96#define REG_TCR_EL1         "tcr_el1"
97#define REG_MAIR_EL1        "mair_el1"
98#define REG_AMAIR_EL1       "amair_el1"
99#define REG_CONTEXTIDR_EL1  "contextidr_el1"
100#define REG_ACTLR_EL1       "actlr_el1"
101#define REG_AFSR0_EL1       "afsr0_el1"
102#define REG_AFSR1_EL1       "afsr1_el1"
103#define REG_ESR_EL1         "esr_el1"
104#define REG_FAR_EL1         "far_el1"
105#define REG_ISR_EL1         "isr_el1"
106#define REG_VBAR_EL1        "vbar_el1"
107#define REG_TPIDR_EL1       "tpidr_el1"
108#define REG_SP_EL1          "sp_el1"
109#define REG_ELR_EL1         "elr_el1"
110#define REG_SPSR_EL1        "spsr_el1"
111#define REG_CPACR_EL1       "cpacr_el1"
112#define REG_CNTV_TVAL_EL0   "cntv_tval_el0"
113#define REG_CNTV_CTL_EL0    "cntv_ctl_el0"
114#define REG_CNTV_CVAL_EL0   "cntv_cval_el0"
115#define REG_CNTVOFF_EL2     "cntvoff_el2"
116#define REG_CNTKCTL_EL1     "cntkctl_el1"
117#define REG_HCR_EL2         "hcr_el2"
118#define REG_VTCR_EL2        "vtcr_el2"
119#define REG_VMPIDR_EL2      "vmpidr_el2"
120#define REG_ID_AA64MMFR0_EL1 "id_aa64mmfr0_el1"
121
122/* for EL1 SCTLR */
123static inline word_t getSCTLR(void)
124{
125    return readSystemControlRegister();
126}
127
128static inline void setSCTLR(word_t sctlr)
129{
130    writeSystemControlRegister(sctlr);
131}
132
133static inline word_t readTTBR0(void)
134{
135    word_t reg;
136    MRS(REG_TTBR0_EL1, reg);
137    return reg;
138}
139
140static inline void writeTTBR0(word_t reg)
141{
142    MSR(REG_TTBR0_EL1, reg);
143}
144
145static inline word_t readTTBR1(void)
146{
147    word_t reg;
148    MRS(REG_TTBR1_EL1, reg);
149    return reg;
150}
151
152static inline void writeTTBR1(word_t reg)
153{
154    MSR(REG_TTBR1_EL1, reg);
155}
156
157static inline word_t readTCR(void)
158{
159    word_t reg;
160    MRS(REG_TCR_EL1, reg);
161    return reg;
162}
163
164static inline void writeTCR(word_t reg)
165{
166    MSR(REG_TCR_EL1, reg);
167}
168
169static inline word_t readMAIR(void)
170{
171    word_t reg;
172    MRS(REG_MAIR_EL1, reg);
173    return reg;
174}
175
176static inline void writeMAIR(word_t reg)
177{
178    MSR(REG_MAIR_EL1, reg);
179}
180
181static inline word_t readAMAIR(void)
182{
183    word_t reg;
184    MRS(REG_AMAIR_EL1, reg);
185    return reg;
186}
187
188static inline void writeAMAIR(word_t reg)
189{
190    MSR(REG_AMAIR_EL1, reg);
191}
192
193static inline word_t readCIDR(void)
194{
195    uint32_t reg;
196    MRS(REG_CONTEXTIDR_EL1, reg);
197    return (word_t)reg;
198}
199
200static inline void writeCIDR(word_t reg)
201{
202    MSR(REG_CONTEXTIDR_EL1, (uint32_t)reg);
203}
204
205static inline word_t readACTLR(void)
206{
207    word_t reg;
208    MRS(REG_ACTLR_EL1, reg);
209    return reg;
210}
211
212static inline void writeACTLR(word_t reg)
213{
214    MSR(REG_ACTLR_EL1, reg);
215}
216
217static inline word_t readAFSR0(void)
218{
219    uint32_t reg;
220    MRS(REG_AFSR0_EL1, reg);
221    return (word_t)reg;
222}
223
224static inline void writeAFSR0(word_t reg)
225{
226    MSR(REG_AFSR0_EL1, (uint32_t)reg);
227}
228
229static inline word_t readAFSR1(void)
230{
231    uint32_t reg;
232    MRS(REG_AFSR1_EL1, reg);
233    return (word_t)reg;
234}
235
236static inline void writeAFSR1(word_t reg)
237{
238    MSR(REG_AFSR1_EL1, (uint32_t)reg);
239}
240
241static inline word_t readESR(void)
242{
243    uint32_t reg;
244    MRS(REG_ESR_EL1, reg);
245    return (word_t)reg;
246}
247
248static inline void writeESR(word_t reg)
249{
250    MSR(REG_ESR_EL1, (uint32_t)reg);
251}
252
253static inline word_t readFAR(void)
254{
255    word_t reg;
256    MRS(REG_FAR_EL1, reg);
257    return reg;
258}
259
260static inline void writeFAR(word_t reg)
261{
262    MSR(REG_FAR_EL1, reg);
263}
264
265/* ISR is read-only */
266static inline word_t readISR(void)
267{
268    uint32_t reg;
269    MRS(REG_ISR_EL1, reg);
270    return (word_t)reg;
271}
272
273static inline word_t readVBAR(void)
274{
275    word_t reg;
276    MRS(REG_VBAR_EL1, reg);
277    return reg;
278}
279
280static inline void writeVBAR(word_t reg)
281{
282    MSR(REG_VBAR_EL1, reg);
283}
284
285static inline word_t readSP_EL1(void)
286{
287    word_t reg;
288    MRS(REG_SP_EL1, reg);
289    return reg;
290}
291
292static inline void writeSP_EL1(word_t reg)
293{
294    MSR(REG_SP_EL1, reg);
295}
296
297static inline word_t readELR_EL1(void)
298{
299    word_t reg;
300    MRS(REG_ELR_EL1, reg);
301    return reg;
302}
303
304static inline void writeELR_EL1(word_t reg)
305{
306    MSR(REG_ELR_EL1, reg);
307}
308
309static inline word_t readSPSR_EL1(void)
310{
311    word_t reg;
312    MRS(REG_SPSR_EL1, reg);
313    return reg;
314}
315
316static inline void writeSPSR_EL1(word_t reg)
317{
318    MSR(REG_SPSR_EL1, reg);
319}
320
321static inline word_t readCPACR_EL1(void)
322{
323    word_t reg;
324    MRS(REG_CPACR_EL1, reg);
325    return reg;
326}
327
328static inline void writeCPACR_EL1(word_t reg)
329{
330    MSR(REG_CPACR_EL1, reg);
331}
332
333static inline word_t readCNTV_TVAL_EL0(void)
334{
335    word_t reg;
336    MRS(REG_CNTV_TVAL_EL0, reg);
337    return reg;
338}
339
340static inline void writeCNTV_TVAL_EL0(word_t reg)
341{
342    MSR(REG_CNTV_TVAL_EL0, reg);
343}
344
345static inline word_t readCNTV_CTL_EL0(void)
346{
347    word_t reg;
348    MRS(REG_CNTV_CTL_EL0, reg);
349    return reg;
350}
351
352static inline void writeCNTV_CTL_EL0(word_t reg)
353{
354    MSR(REG_CNTV_CTL_EL0, reg);
355}
356
357static inline word_t readCNTV_CVAL_EL0(void)
358{
359    word_t reg;
360    MRS(REG_CNTV_CVAL_EL0, reg);
361    return reg;
362}
363
364static inline void writeCNTV_CVAL_EL0(word_t reg)
365{
366    MSR(REG_CNTV_CVAL_EL0, reg);
367}
368
369static inline word_t readCNTVOFF_EL2(void)
370{
371    word_t reg;
372    MRS(REG_CNTVOFF_EL2, reg);
373    return reg;
374}
375
376static inline void writeCNTVOFF_EL2(word_t reg)
377{
378    MSR(REG_CNTVOFF_EL2, reg);
379}
380
381static inline word_t readCNTKCTL_EL1(void)
382{
383    word_t reg;
384    MRS(REG_CNTKCTL_EL1, reg);
385    return reg;
386}
387
388static inline void writeCNTKCTL_EL1(word_t reg)
389{
390    MSR(REG_CNTKCTL_EL1, reg);
391}
392
393static inline word_t readVMPIDR_EL2(void)
394{
395    word_t reg;
396    MRS(REG_VMPIDR_EL2, reg);
397    return reg;
398}
399
400static inline void writeVMPIDR_EL2(word_t reg)
401{
402    MSR(REG_VMPIDR_EL2, reg);
403}
404
405static word_t vcpu_hw_read_reg(word_t reg_index)
406{
407    word_t reg = 0;
408    switch (reg_index) {
409    case seL4_VCPUReg_SCTLR:
410        return getSCTLR();
411    case seL4_VCPUReg_TTBR0:
412        return readTTBR0();
413    case seL4_VCPUReg_TTBR1:
414        return readTTBR1();
415    case seL4_VCPUReg_TCR:
416        return readTCR();
417    case seL4_VCPUReg_MAIR:
418        return readMAIR();
419    case seL4_VCPUReg_AMAIR:
420        return readAMAIR();
421    case seL4_VCPUReg_CIDR:
422        return readCIDR();
423    case seL4_VCPUReg_ACTLR:
424        return readACTLR();
425    case seL4_VCPUReg_CPACR:
426        return readCPACR_EL1();
427    case seL4_VCPUReg_AFSR0:
428        return readAFSR0();
429    case seL4_VCPUReg_AFSR1:
430        return readAFSR1();
431    case seL4_VCPUReg_ESR:
432        return readESR();
433    case seL4_VCPUReg_FAR:
434        return readFAR();
435    case seL4_VCPUReg_ISR:
436        return readISR();
437    case seL4_VCPUReg_VBAR:
438        return readVBAR();
439    case seL4_VCPUReg_TPIDR_EL1:
440        return readTPIDR_EL1();
441    case seL4_VCPUReg_SP_EL1:
442        return readSP_EL1();
443    case seL4_VCPUReg_ELR_EL1:
444        return readELR_EL1();
445    case seL4_VCPUReg_SPSR_EL1:
446        return readSPSR_EL1();
447    case seL4_VCPUReg_CNTV_CTL:
448        return readCNTV_CTL_EL0();
449    case seL4_VCPUReg_CNTV_CVAL:
450        return readCNTV_CVAL_EL0();
451    case seL4_VCPUReg_CNTVOFF:
452        return readCNTVOFF_EL2();
453    case seL4_VCPUReg_CNTKCTL_EL1:
454        return readCNTKCTL_EL1();
455#ifdef ENABLE_SMP_SUPPORT
456    case seL4_VCPUReg_VMPIDR_EL2:
457        return readVMPIDR_EL2();
458#endif /* ENABLE_SMP_SUPPORT */
459    default:
460        fail("ARM/HYP: Invalid register index");
461    }
462
463    return reg;
464}
465
466static void vcpu_hw_write_reg(word_t reg_index, word_t reg)
467{
468    switch (reg_index) {
469    case seL4_VCPUReg_SCTLR:
470        return setSCTLR(reg);
471    case seL4_VCPUReg_TTBR0:
472        return writeTTBR0(reg);
473    case seL4_VCPUReg_TTBR1:
474        return writeTTBR1(reg);
475    case seL4_VCPUReg_TCR:
476        return writeTCR(reg);
477    case seL4_VCPUReg_MAIR:
478        return writeMAIR(reg);
479    case seL4_VCPUReg_AMAIR:
480        return writeAMAIR(reg);
481    case seL4_VCPUReg_CIDR:
482        return writeCIDR(reg);
483    case seL4_VCPUReg_ACTLR:
484        return writeACTLR(reg);
485    case seL4_VCPUReg_CPACR:
486        return writeCPACR_EL1(reg);
487    case seL4_VCPUReg_AFSR0:
488        return writeAFSR0(reg);
489    case seL4_VCPUReg_AFSR1:
490        return writeAFSR1(reg);
491    case seL4_VCPUReg_ESR:
492        return writeESR(reg);
493    case seL4_VCPUReg_FAR:
494        return writeFAR(reg);
495    case seL4_VCPUReg_ISR:
496        /* ISR is read-only */
497        return;
498    case seL4_VCPUReg_VBAR:
499        return writeVBAR(reg);
500    case seL4_VCPUReg_TPIDR_EL1:
501        return writeTPIDR_EL1(reg);
502    case seL4_VCPUReg_SP_EL1:
503        return writeSP_EL1(reg);
504    case seL4_VCPUReg_ELR_EL1:
505        return writeELR_EL1(reg);
506    case seL4_VCPUReg_SPSR_EL1:
507        return writeSPSR_EL1(reg);
508    case seL4_VCPUReg_CNTV_CTL:
509        return writeCNTV_CTL_EL0(reg);
510    case seL4_VCPUReg_CNTV_CVAL:
511        return writeCNTV_CVAL_EL0(reg);
512    case seL4_VCPUReg_CNTVOFF:
513        return writeCNTVOFF_EL2(reg);
514    case seL4_VCPUReg_CNTKCTL_EL1:
515        return writeCNTKCTL_EL1(reg);
516#ifdef ENABLE_SMP_SUPPORT
517    case seL4_VCPUReg_VMPIDR_EL2:
518        return writeVMPIDR_EL2(reg);
519#endif /* ENABLE_SMP_SUPPORT */
520    default:
521        fail("ARM/HYP: Invalid register index");
522    }
523
524    return;
525}
526
527static inline void vcpu_init_vtcr(void)
528{
529
530    /* check that the processor supports the configuration */
531    uint32_t val;
532    MRS(REG_ID_AA64MMFR0_EL1, val);
533    uint32_t pa_range = ID_AA64MMFR0_EL1_PARANGE(val);
534    if (config_set(CONFIG_ARM_PA_SIZE_BITS_40) && pa_range < PS_1T) {
535        fail("Processor does not support a 40 bit PA");
536    }
537    if (config_set(CONFIG_ARM_PA_SIZE_BITS_44) && pa_range < PS_16T) {
538        fail("Processor does not support a 44 bit PA");
539    }
540    uint32_t granule = ID_AA64MMFR0_TGRAN4(val);
541    if (granule) {
542        fail("Processor does not support 4KB");
543    }
544
545    /* Set up the stage-2 translation control register for cores supporting 44-bit PA */
546    uint32_t vtcr_el2;
547#ifdef CONFIG_ARM_PA_SIZE_BITS_40
548    vtcr_el2 = VTCR_EL2_T0SZ(24);                            // 40-bit input IPA
549    vtcr_el2 |= VTCR_EL2_PS(PS_1T);                          // 40-bit PA size
550    vtcr_el2 |= VTCR_EL2_SL0(SL0_4K_L1);                     // 4KiB, start at level 1
551#else
552    vtcr_el2 = VTCR_EL2_T0SZ(20);                            // 44-bit input IPA
553    vtcr_el2 |= VTCR_EL2_PS(PS_16T);                         // 44-bit PA size
554    vtcr_el2 |= VTCR_EL2_SL0(SL0_4K_L0);                     // 4KiB, start at level 0
555#endif
556    vtcr_el2 |= VTCR_EL2_IRGN0(NORMAL_WB_WA_CACHEABLE);      // inner write-back, read/write allocate
557    vtcr_el2 |= VTCR_EL2_ORGN0(NORMAL_WB_WA_CACHEABLE);      // outer write-back, read/write allocate
558    vtcr_el2 |= VTCR_EL2_SH0(SH0_INNER);                     // inner shareable
559    vtcr_el2 |= VTCR_EL2_TG0(TG0_4K);                        // 4KiB page size
560    vtcr_el2 |= BIT(31);                                     // reserved as 1
561
562    MSR(REG_VTCR_EL2, vtcr_el2);
563    isb();
564}
565
566static inline void armv_vcpu_boot_init(void)
567{
568    word_t hcr_el2 = 0;
569
570    vcpu_init_vtcr();
571
572    hcr_el2 = HCR_NATIVE;
573    MSR(REG_HCR_EL2, hcr_el2);
574    isb();
575
576    /* set the SCTLR_EL1 for running native seL4 threads */
577    MSR(REG_SCTLR_EL1, SCTLR_EL1_NATIVE);
578    isb();
579}
580
581static inline void armv_vcpu_save(vcpu_t *vcpu, UNUSED bool_t active)
582{
583    vcpu_save_reg_range(vcpu, seL4_VCPUReg_TTBR0, seL4_VCPUReg_SPSR_EL1);
584}
585
586static inline void vcpu_enable(vcpu_t *vcpu)
587{
588    MSR(REG_HCR_EL2, HCR_VCPU);
589    isb();
590    vcpu_restore_reg(vcpu, seL4_VCPUReg_SCTLR);
591    isb();
592
593    set_gic_vcpu_ctrl_hcr(vcpu->vgic.hcr);
594#ifdef CONFIG_HAVE_FPU
595    vcpu_restore_reg(vcpu, seL4_VCPUReg_CPACR);
596#endif
597    /* Restore virtual timer state */
598    restore_virt_timer(vcpu);
599}
600
601static inline void vcpu_disable(vcpu_t *vcpu)
602{
603
604    uint32_t hcr;
605    dsb();
606    if (likely(vcpu)) {
607        hcr = get_gic_vcpu_ctrl_hcr();
608        vcpu->vgic.hcr = hcr;
609        vcpu_save_reg(vcpu, seL4_VCPUReg_SCTLR);
610#ifdef CONFIG_HAVE_FPU
611        vcpu_save_reg(vcpu, seL4_VCPUReg_CPACR);
612#endif
613        isb();
614    }
615    /* Turn off the VGIC */
616    set_gic_vcpu_ctrl_hcr(0);
617    isb();
618
619    /* Stage 1 MMU off */
620    setSCTLR(SCTLR_DEFAULT);
621    isb();
622    MSR(REG_HCR_EL2, HCR_NATIVE);
623    isb();
624
625#ifdef CONFIG_HAVE_FPU
626    /* Allow FPU instructions in EL0 and EL1 for native
627     * threads by setting the CPACR_EL1. The CPTR_EL2 is
628     * used to trap the FPU instructions to EL2.
629     */
630    enableFpuEL01();
631#endif
632    if (likely(vcpu)) {
633        /* Save virtual timer state */
634        save_virt_timer(vcpu);
635        /* Mask the virtual timer interrupt */
636        maskInterrupt(true, CORE_IRQ_TO_IRQT(CURRENT_CPU_INDEX(), INTERRUPT_VTIMER_EVENT));
637    }
638}
639
640static inline void armv_vcpu_init(vcpu_t *vcpu)
641{
642    vcpu_write_reg(vcpu, seL4_VCPUReg_SCTLR, SCTLR_EL1_VM);
643}
644
645static inline bool_t armv_handleVCPUFault(word_t hsr)
646{
647#ifdef CONFIG_HAVE_FPU
648    if ((ESR_EC(hsr) == ESR_EC_TFP || ESR_EC(hsr) == ESR_EC_CPACR) && !isFpuEnable()) {
649        handleFPUFault();
650        setNextPC(NODE_STATE(ksCurThread), getRestartPC(NODE_STATE(ksCurThread)));
651        return true;
652    }
653#endif
654
655    if (hsr == UNKNOWN_FAULT) {
656        handleUserLevelFault(getESR(), 0);
657        return true;
658    }
659
660    return false;
661}
662
663#endif /* End of CONFIG_ARM_HYPERVISOR_SUPPORT */
664
665
666