1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#pragma once 8 9#include <config.h> 10 11#ifdef CONFIG_ARM_HYPERVISOR_SUPPORT 12 13#include <arch/object/vcpu.h> 14#include <drivers/timer/arm_generic.h> 15 16/* Note that the HCR_DC for ARMv8 disables S1 translation if enabled */ 17#ifdef CONFIG_DISABLE_WFI_WFE_TRAPS 18/* Trap SMC and override CPSR.AIF */ 19#define HCR_COMMON ( HCR_VM | HCR_RW | HCR_AMO | HCR_IMO | HCR_FMO ) 20#else 21/* Trap WFI/WFE/SMC and override CPSR.AIF */ 22#define HCR_COMMON ( HCR_TWI | HCR_TWE | HCR_VM | HCR_RW | HCR_AMO | HCR_IMO | HCR_FMO ) 23#endif 24 25/* Allow native tasks to run at EL0, but restrict access */ 26#define HCR_NATIVE ( HCR_COMMON | HCR_TGE | HCR_TVM | HCR_TTLB | HCR_DC \ 27 | HCR_TAC | HCR_SWIO | HCR_TSC | HCR_IMO | HCR_FMO | HCR_AMO) 28#define HCR_VCPU ( HCR_COMMON | HCR_TSC) 29 30#define SCTLR_EL1_UCI BIT(26) /* Enable EL0 access to DC CVAU, DC CIVAC, DC CVAC, 31 and IC IVAU in AArch64 state */ 32#define SCTLR_EL1_C BIT(2) /* Enable data and unified caches */ 33#define SCTLR_EL1_I BIT(12) /* Enable instruction cache */ 34#define SCTLR_EL1_CP15BEN BIT(5) /* AArch32 CP15 barrier enable */ 35#define SCTLR_EL1_UTC BIT(15) /* Enable EL0 access to CTR_EL0 */ 36#define SCTLR_EL1_NTWI BIT(16) /* WFI executed as normal */ 37#define SCTLR_EL1_NTWE BIT(18) /* WFE executed as normal */ 38 39/* Disable MMU, SP alignment check, and alignment check */ 40/* A57 default value */ 41#define SCTLR_EL1_RES 0x30d00800 /* Reserved value */ 42#define SCTLR_EL1 ( SCTLR_EL1_RES | SCTLR_EL1_CP15BEN | SCTLR_EL1_UTC \ 43 | SCTLR_EL1_NTWI | SCTLR_EL1_NTWE ) 44#define SCTLR_EL1_NATIVE (SCTLR_EL1 | SCTLR_EL1_C | SCTLR_EL1_I | SCTLR_EL1_UCI) 45#define SCTLR_EL1_VM (SCTLR_EL1 | SCTLR_EL1_UCI) 46#define SCTLR_DEFAULT SCTLR_EL1_NATIVE 47 48#define UNKNOWN_FAULT 0x2000000 49#define ESR_EC_TFP 0x7 /* Trap instructions that access FPU registers */ 50#define ESR_EC_CPACR 0x18 /* Trap access to CPACR */ 51#define ESR_EC(x) (((x) & 0xfc000000) >> 26) 52 53#define VTCR_EL2_T0SZ(x) ((x) & 0x3f) 54#define VTCR_EL2_SL0(x) (((x) & 0x3) << 6) 55#define VTCR_EL2_IRGN0(x) (((x) & 0x3) << 8) 56#define VTCR_EL2_ORGN0(x) (((x) & 0x3) << 10) 57#define VTCR_EL2_SH0(x) (((x) & 0x3) << 12) 58#define VTCR_EL2_TG0(x) (((x) & 0x3) << 14) 59#define VTCR_EL2_PS(x) (((x) & 0x7) << 16) 60 61/* Physical address size */ 62#define PS_4G 0 63#define PS_64G 1 64#define PS_1T 2 65#define PS_4T 3 66#define PS_16T 4 67#define PS_256T 5 68 69/* Translation granule size */ 70#define TG0_4K 0 71#define TG0_64K 1 72#define TG0_16K 2 73 74#define ID_AA64MMFR0_EL1_PARANGE(x) ((x) & 0xf) 75#define ID_AA64MMFR0_TGRAN4(x) (((x) >> 28u) & 0xf) 76 77/* Shareability attributes */ 78#define SH0_NONE 0 79#define SH0_OUTER 2 80#define SH0_INNER 3 81 82/* Cacheability attributes */ 83#define NORMAL_NON_CACHEABLE 0 84#define NORMAL_WB_WA_CACHEABLE 1 /* write-back, write-allocate */ 85#define NORMAL_WT_CACHEABLE 2 /* write-through */ 86#define NORMAL_WB_NWA_CACHEABLE 3 /* write-back, no write-allocate */ 87 88/* Start level */ 89#define SL0_4K_L2 0 /* 4K, start at level 2 */ 90#define SL0_4K_L1 1 /* 4K, start at level 1 */ 91#define SL0_4K_L0 2 /* 4K, start at level 0 */ 92 93#define REG_SCTLR_EL1 "sctlr_el1" 94#define REG_TTBR0_EL1 "ttbr0_el1" 95#define REG_TTBR1_EL1 "ttbr1_el1" 96#define REG_TCR_EL1 "tcr_el1" 97#define REG_MAIR_EL1 "mair_el1" 98#define REG_AMAIR_EL1 "amair_el1" 99#define REG_CONTEXTIDR_EL1 "contextidr_el1" 100#define REG_ACTLR_EL1 "actlr_el1" 101#define REG_AFSR0_EL1 "afsr0_el1" 102#define REG_AFSR1_EL1 "afsr1_el1" 103#define REG_ESR_EL1 "esr_el1" 104#define REG_FAR_EL1 "far_el1" 105#define REG_ISR_EL1 "isr_el1" 106#define REG_VBAR_EL1 "vbar_el1" 107#define REG_TPIDR_EL1 "tpidr_el1" 108#define REG_SP_EL1 "sp_el1" 109#define REG_ELR_EL1 "elr_el1" 110#define REG_SPSR_EL1 "spsr_el1" 111#define REG_CPACR_EL1 "cpacr_el1" 112#define REG_CNTV_TVAL_EL0 "cntv_tval_el0" 113#define REG_CNTV_CTL_EL0 "cntv_ctl_el0" 114#define REG_CNTV_CVAL_EL0 "cntv_cval_el0" 115#define REG_CNTVOFF_EL2 "cntvoff_el2" 116#define REG_CNTKCTL_EL1 "cntkctl_el1" 117#define REG_HCR_EL2 "hcr_el2" 118#define REG_VTCR_EL2 "vtcr_el2" 119#define REG_VMPIDR_EL2 "vmpidr_el2" 120#define REG_ID_AA64MMFR0_EL1 "id_aa64mmfr0_el1" 121 122/* for EL1 SCTLR */ 123static inline word_t getSCTLR(void) 124{ 125 return readSystemControlRegister(); 126} 127 128static inline void setSCTLR(word_t sctlr) 129{ 130 writeSystemControlRegister(sctlr); 131} 132 133static inline word_t readTTBR0(void) 134{ 135 word_t reg; 136 MRS(REG_TTBR0_EL1, reg); 137 return reg; 138} 139 140static inline void writeTTBR0(word_t reg) 141{ 142 MSR(REG_TTBR0_EL1, reg); 143} 144 145static inline word_t readTTBR1(void) 146{ 147 word_t reg; 148 MRS(REG_TTBR1_EL1, reg); 149 return reg; 150} 151 152static inline void writeTTBR1(word_t reg) 153{ 154 MSR(REG_TTBR1_EL1, reg); 155} 156 157static inline word_t readTCR(void) 158{ 159 word_t reg; 160 MRS(REG_TCR_EL1, reg); 161 return reg; 162} 163 164static inline void writeTCR(word_t reg) 165{ 166 MSR(REG_TCR_EL1, reg); 167} 168 169static inline word_t readMAIR(void) 170{ 171 word_t reg; 172 MRS(REG_MAIR_EL1, reg); 173 return reg; 174} 175 176static inline void writeMAIR(word_t reg) 177{ 178 MSR(REG_MAIR_EL1, reg); 179} 180 181static inline word_t readAMAIR(void) 182{ 183 word_t reg; 184 MRS(REG_AMAIR_EL1, reg); 185 return reg; 186} 187 188static inline void writeAMAIR(word_t reg) 189{ 190 MSR(REG_AMAIR_EL1, reg); 191} 192 193static inline word_t readCIDR(void) 194{ 195 uint32_t reg; 196 MRS(REG_CONTEXTIDR_EL1, reg); 197 return (word_t)reg; 198} 199 200static inline void writeCIDR(word_t reg) 201{ 202 MSR(REG_CONTEXTIDR_EL1, (uint32_t)reg); 203} 204 205static inline word_t readACTLR(void) 206{ 207 word_t reg; 208 MRS(REG_ACTLR_EL1, reg); 209 return reg; 210} 211 212static inline void writeACTLR(word_t reg) 213{ 214 MSR(REG_ACTLR_EL1, reg); 215} 216 217static inline word_t readAFSR0(void) 218{ 219 uint32_t reg; 220 MRS(REG_AFSR0_EL1, reg); 221 return (word_t)reg; 222} 223 224static inline void writeAFSR0(word_t reg) 225{ 226 MSR(REG_AFSR0_EL1, (uint32_t)reg); 227} 228 229static inline word_t readAFSR1(void) 230{ 231 uint32_t reg; 232 MRS(REG_AFSR1_EL1, reg); 233 return (word_t)reg; 234} 235 236static inline void writeAFSR1(word_t reg) 237{ 238 MSR(REG_AFSR1_EL1, (uint32_t)reg); 239} 240 241static inline word_t readESR(void) 242{ 243 uint32_t reg; 244 MRS(REG_ESR_EL1, reg); 245 return (word_t)reg; 246} 247 248static inline void writeESR(word_t reg) 249{ 250 MSR(REG_ESR_EL1, (uint32_t)reg); 251} 252 253static inline word_t readFAR(void) 254{ 255 word_t reg; 256 MRS(REG_FAR_EL1, reg); 257 return reg; 258} 259 260static inline void writeFAR(word_t reg) 261{ 262 MSR(REG_FAR_EL1, reg); 263} 264 265/* ISR is read-only */ 266static inline word_t readISR(void) 267{ 268 uint32_t reg; 269 MRS(REG_ISR_EL1, reg); 270 return (word_t)reg; 271} 272 273static inline word_t readVBAR(void) 274{ 275 word_t reg; 276 MRS(REG_VBAR_EL1, reg); 277 return reg; 278} 279 280static inline void writeVBAR(word_t reg) 281{ 282 MSR(REG_VBAR_EL1, reg); 283} 284 285static inline word_t readSP_EL1(void) 286{ 287 word_t reg; 288 MRS(REG_SP_EL1, reg); 289 return reg; 290} 291 292static inline void writeSP_EL1(word_t reg) 293{ 294 MSR(REG_SP_EL1, reg); 295} 296 297static inline word_t readELR_EL1(void) 298{ 299 word_t reg; 300 MRS(REG_ELR_EL1, reg); 301 return reg; 302} 303 304static inline void writeELR_EL1(word_t reg) 305{ 306 MSR(REG_ELR_EL1, reg); 307} 308 309static inline word_t readSPSR_EL1(void) 310{ 311 word_t reg; 312 MRS(REG_SPSR_EL1, reg); 313 return reg; 314} 315 316static inline void writeSPSR_EL1(word_t reg) 317{ 318 MSR(REG_SPSR_EL1, reg); 319} 320 321static inline word_t readCPACR_EL1(void) 322{ 323 word_t reg; 324 MRS(REG_CPACR_EL1, reg); 325 return reg; 326} 327 328static inline void writeCPACR_EL1(word_t reg) 329{ 330 MSR(REG_CPACR_EL1, reg); 331} 332 333static inline word_t readCNTV_TVAL_EL0(void) 334{ 335 word_t reg; 336 MRS(REG_CNTV_TVAL_EL0, reg); 337 return reg; 338} 339 340static inline void writeCNTV_TVAL_EL0(word_t reg) 341{ 342 MSR(REG_CNTV_TVAL_EL0, reg); 343} 344 345static inline word_t readCNTV_CTL_EL0(void) 346{ 347 word_t reg; 348 MRS(REG_CNTV_CTL_EL0, reg); 349 return reg; 350} 351 352static inline void writeCNTV_CTL_EL0(word_t reg) 353{ 354 MSR(REG_CNTV_CTL_EL0, reg); 355} 356 357static inline word_t readCNTV_CVAL_EL0(void) 358{ 359 word_t reg; 360 MRS(REG_CNTV_CVAL_EL0, reg); 361 return reg; 362} 363 364static inline void writeCNTV_CVAL_EL0(word_t reg) 365{ 366 MSR(REG_CNTV_CVAL_EL0, reg); 367} 368 369static inline word_t readCNTVOFF_EL2(void) 370{ 371 word_t reg; 372 MRS(REG_CNTVOFF_EL2, reg); 373 return reg; 374} 375 376static inline void writeCNTVOFF_EL2(word_t reg) 377{ 378 MSR(REG_CNTVOFF_EL2, reg); 379} 380 381static inline word_t readCNTKCTL_EL1(void) 382{ 383 word_t reg; 384 MRS(REG_CNTKCTL_EL1, reg); 385 return reg; 386} 387 388static inline void writeCNTKCTL_EL1(word_t reg) 389{ 390 MSR(REG_CNTKCTL_EL1, reg); 391} 392 393static inline word_t readVMPIDR_EL2(void) 394{ 395 word_t reg; 396 MRS(REG_VMPIDR_EL2, reg); 397 return reg; 398} 399 400static inline void writeVMPIDR_EL2(word_t reg) 401{ 402 MSR(REG_VMPIDR_EL2, reg); 403} 404 405static word_t vcpu_hw_read_reg(word_t reg_index) 406{ 407 word_t reg = 0; 408 switch (reg_index) { 409 case seL4_VCPUReg_SCTLR: 410 return getSCTLR(); 411 case seL4_VCPUReg_TTBR0: 412 return readTTBR0(); 413 case seL4_VCPUReg_TTBR1: 414 return readTTBR1(); 415 case seL4_VCPUReg_TCR: 416 return readTCR(); 417 case seL4_VCPUReg_MAIR: 418 return readMAIR(); 419 case seL4_VCPUReg_AMAIR: 420 return readAMAIR(); 421 case seL4_VCPUReg_CIDR: 422 return readCIDR(); 423 case seL4_VCPUReg_ACTLR: 424 return readACTLR(); 425 case seL4_VCPUReg_CPACR: 426 return readCPACR_EL1(); 427 case seL4_VCPUReg_AFSR0: 428 return readAFSR0(); 429 case seL4_VCPUReg_AFSR1: 430 return readAFSR1(); 431 case seL4_VCPUReg_ESR: 432 return readESR(); 433 case seL4_VCPUReg_FAR: 434 return readFAR(); 435 case seL4_VCPUReg_ISR: 436 return readISR(); 437 case seL4_VCPUReg_VBAR: 438 return readVBAR(); 439 case seL4_VCPUReg_TPIDR_EL1: 440 return readTPIDR_EL1(); 441 case seL4_VCPUReg_SP_EL1: 442 return readSP_EL1(); 443 case seL4_VCPUReg_ELR_EL1: 444 return readELR_EL1(); 445 case seL4_VCPUReg_SPSR_EL1: 446 return readSPSR_EL1(); 447 case seL4_VCPUReg_CNTV_CTL: 448 return readCNTV_CTL_EL0(); 449 case seL4_VCPUReg_CNTV_CVAL: 450 return readCNTV_CVAL_EL0(); 451 case seL4_VCPUReg_CNTVOFF: 452 return readCNTVOFF_EL2(); 453 case seL4_VCPUReg_CNTKCTL_EL1: 454 return readCNTKCTL_EL1(); 455#ifdef ENABLE_SMP_SUPPORT 456 case seL4_VCPUReg_VMPIDR_EL2: 457 return readVMPIDR_EL2(); 458#endif /* ENABLE_SMP_SUPPORT */ 459 default: 460 fail("ARM/HYP: Invalid register index"); 461 } 462 463 return reg; 464} 465 466static void vcpu_hw_write_reg(word_t reg_index, word_t reg) 467{ 468 switch (reg_index) { 469 case seL4_VCPUReg_SCTLR: 470 return setSCTLR(reg); 471 case seL4_VCPUReg_TTBR0: 472 return writeTTBR0(reg); 473 case seL4_VCPUReg_TTBR1: 474 return writeTTBR1(reg); 475 case seL4_VCPUReg_TCR: 476 return writeTCR(reg); 477 case seL4_VCPUReg_MAIR: 478 return writeMAIR(reg); 479 case seL4_VCPUReg_AMAIR: 480 return writeAMAIR(reg); 481 case seL4_VCPUReg_CIDR: 482 return writeCIDR(reg); 483 case seL4_VCPUReg_ACTLR: 484 return writeACTLR(reg); 485 case seL4_VCPUReg_CPACR: 486 return writeCPACR_EL1(reg); 487 case seL4_VCPUReg_AFSR0: 488 return writeAFSR0(reg); 489 case seL4_VCPUReg_AFSR1: 490 return writeAFSR1(reg); 491 case seL4_VCPUReg_ESR: 492 return writeESR(reg); 493 case seL4_VCPUReg_FAR: 494 return writeFAR(reg); 495 case seL4_VCPUReg_ISR: 496 /* ISR is read-only */ 497 return; 498 case seL4_VCPUReg_VBAR: 499 return writeVBAR(reg); 500 case seL4_VCPUReg_TPIDR_EL1: 501 return writeTPIDR_EL1(reg); 502 case seL4_VCPUReg_SP_EL1: 503 return writeSP_EL1(reg); 504 case seL4_VCPUReg_ELR_EL1: 505 return writeELR_EL1(reg); 506 case seL4_VCPUReg_SPSR_EL1: 507 return writeSPSR_EL1(reg); 508 case seL4_VCPUReg_CNTV_CTL: 509 return writeCNTV_CTL_EL0(reg); 510 case seL4_VCPUReg_CNTV_CVAL: 511 return writeCNTV_CVAL_EL0(reg); 512 case seL4_VCPUReg_CNTVOFF: 513 return writeCNTVOFF_EL2(reg); 514 case seL4_VCPUReg_CNTKCTL_EL1: 515 return writeCNTKCTL_EL1(reg); 516#ifdef ENABLE_SMP_SUPPORT 517 case seL4_VCPUReg_VMPIDR_EL2: 518 return writeVMPIDR_EL2(reg); 519#endif /* ENABLE_SMP_SUPPORT */ 520 default: 521 fail("ARM/HYP: Invalid register index"); 522 } 523 524 return; 525} 526 527static inline void vcpu_init_vtcr(void) 528{ 529 530 /* check that the processor supports the configuration */ 531 uint32_t val; 532 MRS(REG_ID_AA64MMFR0_EL1, val); 533 uint32_t pa_range = ID_AA64MMFR0_EL1_PARANGE(val); 534 if (config_set(CONFIG_ARM_PA_SIZE_BITS_40) && pa_range < PS_1T) { 535 fail("Processor does not support a 40 bit PA"); 536 } 537 if (config_set(CONFIG_ARM_PA_SIZE_BITS_44) && pa_range < PS_16T) { 538 fail("Processor does not support a 44 bit PA"); 539 } 540 uint32_t granule = ID_AA64MMFR0_TGRAN4(val); 541 if (granule) { 542 fail("Processor does not support 4KB"); 543 } 544 545 /* Set up the stage-2 translation control register for cores supporting 44-bit PA */ 546 uint32_t vtcr_el2; 547#ifdef CONFIG_ARM_PA_SIZE_BITS_40 548 vtcr_el2 = VTCR_EL2_T0SZ(24); // 40-bit input IPA 549 vtcr_el2 |= VTCR_EL2_PS(PS_1T); // 40-bit PA size 550 vtcr_el2 |= VTCR_EL2_SL0(SL0_4K_L1); // 4KiB, start at level 1 551#else 552 vtcr_el2 = VTCR_EL2_T0SZ(20); // 44-bit input IPA 553 vtcr_el2 |= VTCR_EL2_PS(PS_16T); // 44-bit PA size 554 vtcr_el2 |= VTCR_EL2_SL0(SL0_4K_L0); // 4KiB, start at level 0 555#endif 556 vtcr_el2 |= VTCR_EL2_IRGN0(NORMAL_WB_WA_CACHEABLE); // inner write-back, read/write allocate 557 vtcr_el2 |= VTCR_EL2_ORGN0(NORMAL_WB_WA_CACHEABLE); // outer write-back, read/write allocate 558 vtcr_el2 |= VTCR_EL2_SH0(SH0_INNER); // inner shareable 559 vtcr_el2 |= VTCR_EL2_TG0(TG0_4K); // 4KiB page size 560 vtcr_el2 |= BIT(31); // reserved as 1 561 562 MSR(REG_VTCR_EL2, vtcr_el2); 563 isb(); 564} 565 566static inline void armv_vcpu_boot_init(void) 567{ 568 word_t hcr_el2 = 0; 569 570 vcpu_init_vtcr(); 571 572 hcr_el2 = HCR_NATIVE; 573 MSR(REG_HCR_EL2, hcr_el2); 574 isb(); 575 576 /* set the SCTLR_EL1 for running native seL4 threads */ 577 MSR(REG_SCTLR_EL1, SCTLR_EL1_NATIVE); 578 isb(); 579} 580 581static inline void armv_vcpu_save(vcpu_t *vcpu, UNUSED bool_t active) 582{ 583 vcpu_save_reg_range(vcpu, seL4_VCPUReg_TTBR0, seL4_VCPUReg_SPSR_EL1); 584} 585 586static inline void vcpu_enable(vcpu_t *vcpu) 587{ 588 MSR(REG_HCR_EL2, HCR_VCPU); 589 isb(); 590 vcpu_restore_reg(vcpu, seL4_VCPUReg_SCTLR); 591 isb(); 592 593 set_gic_vcpu_ctrl_hcr(vcpu->vgic.hcr); 594#ifdef CONFIG_HAVE_FPU 595 vcpu_restore_reg(vcpu, seL4_VCPUReg_CPACR); 596#endif 597 /* Restore virtual timer state */ 598 restore_virt_timer(vcpu); 599} 600 601static inline void vcpu_disable(vcpu_t *vcpu) 602{ 603 604 uint32_t hcr; 605 dsb(); 606 if (likely(vcpu)) { 607 hcr = get_gic_vcpu_ctrl_hcr(); 608 vcpu->vgic.hcr = hcr; 609 vcpu_save_reg(vcpu, seL4_VCPUReg_SCTLR); 610#ifdef CONFIG_HAVE_FPU 611 vcpu_save_reg(vcpu, seL4_VCPUReg_CPACR); 612#endif 613 isb(); 614 } 615 /* Turn off the VGIC */ 616 set_gic_vcpu_ctrl_hcr(0); 617 isb(); 618 619 /* Stage 1 MMU off */ 620 setSCTLR(SCTLR_DEFAULT); 621 isb(); 622 MSR(REG_HCR_EL2, HCR_NATIVE); 623 isb(); 624 625#ifdef CONFIG_HAVE_FPU 626 /* Allow FPU instructions in EL0 and EL1 for native 627 * threads by setting the CPACR_EL1. The CPTR_EL2 is 628 * used to trap the FPU instructions to EL2. 629 */ 630 enableFpuEL01(); 631#endif 632 if (likely(vcpu)) { 633 /* Save virtual timer state */ 634 save_virt_timer(vcpu); 635 /* Mask the virtual timer interrupt */ 636 maskInterrupt(true, CORE_IRQ_TO_IRQT(CURRENT_CPU_INDEX(), INTERRUPT_VTIMER_EVENT)); 637 } 638} 639 640static inline void armv_vcpu_init(vcpu_t *vcpu) 641{ 642 vcpu_write_reg(vcpu, seL4_VCPUReg_SCTLR, SCTLR_EL1_VM); 643} 644 645static inline bool_t armv_handleVCPUFault(word_t hsr) 646{ 647#ifdef CONFIG_HAVE_FPU 648 if ((ESR_EC(hsr) == ESR_EC_TFP || ESR_EC(hsr) == ESR_EC_CPACR) && !isFpuEnable()) { 649 handleFPUFault(); 650 setNextPC(NODE_STATE(ksCurThread), getRestartPC(NODE_STATE(ksCurThread))); 651 return true; 652 } 653#endif 654 655 if (hsr == UNKNOWN_FAULT) { 656 handleUserLevelFault(getESR(), 0); 657 return true; 658 } 659 660 return false; 661} 662 663#endif /* End of CONFIG_ARM_HYPERVISOR_SUPPORT */ 664 665 666