1281494Sandrew/*- 2281494Sandrew * Copyright (c) 2012-2014 Andrew Turner 3281494Sandrew * All rights reserved. 4281494Sandrew * 5281494Sandrew * Redistribution and use in source and binary forms, with or without 6281494Sandrew * modification, are permitted provided that the following conditions 7281494Sandrew * are met: 8281494Sandrew * 1. Redistributions of source code must retain the above copyright 9281494Sandrew * notice, this list of conditions and the following disclaimer. 10281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 11281494Sandrew * notice, this list of conditions and the following disclaimer in the 12281494Sandrew * documentation and/or other materials provided with the distribution. 13281494Sandrew * 14281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24281494Sandrew * SUCH DAMAGE. 25281494Sandrew * 26281494Sandrew * $FreeBSD: stable/11/sys/arm64/arm64/locore.S 331972 2018-04-04 02:22:56Z mmel $ 27281494Sandrew */ 28281494Sandrew 29281494Sandrew#include "assym.s" 30285627Szbb#include "opt_kstack_pages.h" 31281494Sandrew#include <sys/syscall.h> 32281494Sandrew#include <machine/asm.h> 33281494Sandrew#include <machine/armreg.h> 34281494Sandrew#include <machine/hypervisor.h> 35281494Sandrew#include <machine/param.h> 36281494Sandrew#include <machine/pte.h> 37297615Sandrew#include <machine/vmparam.h> 38281494Sandrew 39297446Sandrew#define VIRT_BITS 48 40297914Sandrew#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) 41281494Sandrew 42281494Sandrew .globl kernbase 43281494Sandrew .set kernbase, KERNBASE 44281494Sandrew 45281494Sandrew#define DEVICE_MEM 0 46281494Sandrew#define NORMAL_UNCACHED 1 47281494Sandrew#define NORMAL_MEM 2 48281494Sandrew 49281494Sandrew/* 50281494Sandrew * We assume: 51281494Sandrew * MMU on with an identity map, or off 52281494Sandrew * D-Cache: off 53281494Sandrew * I-Cache: on or off 54281494Sandrew * We are loaded at a 2MiB aligned address 55281494Sandrew */ 56281494Sandrew 57281494Sandrew .text 58281494Sandrew .globl _start 59281494Sandrew_start: 60281494Sandrew /* Drop to EL1 */ 61281494Sandrew bl drop_to_el1 62281494Sandrew 63281494Sandrew /* 64281494Sandrew * Disable the MMU. We may have entered the kernel with it on and 65281494Sandrew * will need to update the tables later. If this has been set up 66281494Sandrew * with anything other than a VA == PA map then this will fail, 67281494Sandrew * but in this case the code to find where we are running from 68281494Sandrew * would have also failed. 69281494Sandrew */ 70281494Sandrew dsb sy 71281494Sandrew mrs x2, sctlr_el1 72281494Sandrew bic x2, x2, SCTLR_M 73281494Sandrew msr sctlr_el1, x2 74281494Sandrew isb 75281494Sandrew 76285316Sandrew /* Set the context id */ 77285316Sandrew msr contextidr_el1, xzr 78281494Sandrew 79281494Sandrew /* Get the virt -> phys offset */ 80281494Sandrew bl get_virt_delta 81281494Sandrew 82281494Sandrew /* 83281494Sandrew * At this point: 84281494Sandrew * x29 = PA - VA 85281494Sandrew * x28 = Our physical load address 86281494Sandrew */ 87281494Sandrew 88281494Sandrew /* Create the page tables */ 89281494Sandrew bl create_pagetables 90281494Sandrew 91281494Sandrew /* 92281494Sandrew * At this point: 93281494Sandrew * x27 = TTBR0 table 94297446Sandrew * x26 = Kernel L1 table 95297446Sandrew * x24 = TTBR1 table 96281494Sandrew */ 97281494Sandrew 98281494Sandrew /* Enable the mmu */ 99281494Sandrew bl start_mmu 100281494Sandrew 101281494Sandrew /* Jump to the virtual address space */ 102281494Sandrew ldr x15, .Lvirtdone 103281494Sandrew br x15 104281494Sandrew 105281494Sandrewvirtdone: 106281494Sandrew /* Set up the stack */ 107281494Sandrew adr x25, initstack_end 108281494Sandrew mov sp, x25 109281494Sandrew sub sp, sp, #PCB_SIZE 110281494Sandrew 111281494Sandrew /* Zero the BSS */ 112281494Sandrew ldr x15, .Lbss 113281494Sandrew ldr x14, .Lend 114281494Sandrew1: 115281494Sandrew str xzr, [x15], #8 116281494Sandrew cmp x15, x14 117281494Sandrew b.lo 1b 118281494Sandrew 119281494Sandrew /* Backup the module pointer */ 120281494Sandrew mov x1, x0 121281494Sandrew 122281494Sandrew /* Make the page table base a virtual address */ 123281494Sandrew sub x26, x26, x29 124297446Sandrew sub x24, x24, x29 125281494Sandrew 126281494Sandrew sub sp, sp, #(64 * 4) 127281494Sandrew mov x0, sp 128281494Sandrew 129281494Sandrew /* Degate the delda so it is VA -> PA */ 130281494Sandrew neg x29, x29 131281494Sandrew 132281494Sandrew str x1, [x0] /* modulep */ 133281494Sandrew str x26, [x0, 8] /* kern_l1pt */ 134281494Sandrew str x29, [x0, 16] /* kern_delta */ 135281494Sandrew str x25, [x0, 24] /* kern_stack */ 136297446Sandrew str x24, [x0, 32] /* kern_l0pt */ 137281494Sandrew 138281494Sandrew /* trace back starts here */ 139281494Sandrew mov fp, #0 140281494Sandrew /* Branch to C code */ 141281494Sandrew bl initarm 142281494Sandrew bl mi_startup 143281494Sandrew 144281494Sandrew /* We should not get here */ 145281494Sandrew brk 0 146281494Sandrew 147281494Sandrew .align 3 148281494Sandrew.Lvirtdone: 149281494Sandrew .quad virtdone 150281494Sandrew.Lbss: 151281494Sandrew .quad __bss_start 152281494Sandrew.Lend: 153281494Sandrew .quad _end 154281494Sandrew 155285316Sandrew#ifdef SMP 156281494Sandrew/* 157285316Sandrew * mpentry(unsigned long) 158285316Sandrew * 159285316Sandrew * Called by a core when it is being brought online. 160285316Sandrew * The data in x0 is passed straight to init_secondary. 161285316Sandrew */ 162285316SandrewENTRY(mpentry) 163285316Sandrew /* Disable interrupts */ 164285316Sandrew msr daifset, #2 165285316Sandrew 166285316Sandrew /* Drop to EL1 */ 167285316Sandrew bl drop_to_el1 168285316Sandrew 169285316Sandrew /* Set the context id */ 170331972Smmel msr contextidr_el1, xzr 171285316Sandrew 172285316Sandrew /* Load the kernel page table */ 173297446Sandrew adr x24, pagetable_l0_ttbr1 174285316Sandrew /* Load the identity page table */ 175289581Sandrew adr x27, pagetable_l0_ttbr0 176285316Sandrew 177285316Sandrew /* Enable the mmu */ 178285316Sandrew bl start_mmu 179285316Sandrew 180285316Sandrew /* Jump to the virtual address space */ 181285316Sandrew ldr x15, =mp_virtdone 182285316Sandrew br x15 183285316Sandrew 184285316Sandrewmp_virtdone: 185285316Sandrew ldr x4, =secondary_stacks 186285316Sandrew mov x5, #(PAGE_SIZE * KSTACK_PAGES) 187285654Szbb mul x5, x0, x5 188285316Sandrew add sp, x4, x5 189285316Sandrew 190285316Sandrew b init_secondary 191285316SandrewEND(mpentry) 192285316Sandrew#endif 193285316Sandrew 194285316Sandrew/* 195281494Sandrew * If we are started in EL2, configure the required hypervisor 196281494Sandrew * registers and drop to EL1. 197281494Sandrew */ 198281494Sandrewdrop_to_el1: 199281494Sandrew mrs x1, CurrentEL 200281494Sandrew lsr x1, x1, #2 201281494Sandrew cmp x1, #0x2 202281494Sandrew b.eq 1f 203281494Sandrew ret 204281494Sandrew1: 205281494Sandrew /* Configure the Hypervisor */ 206281494Sandrew mov x2, #(HCR_RW) 207281494Sandrew msr hcr_el2, x2 208281494Sandrew 209281494Sandrew /* Load the Virtualization Process ID Register */ 210281494Sandrew mrs x2, midr_el1 211281494Sandrew msr vpidr_el2, x2 212281494Sandrew 213281494Sandrew /* Load the Virtualization Multiprocess ID Register */ 214281494Sandrew mrs x2, mpidr_el1 215281494Sandrew msr vmpidr_el2, x2 216281494Sandrew 217281494Sandrew /* Set the bits that need to be 1 in sctlr_el1 */ 218281494Sandrew ldr x2, .Lsctlr_res1 219281494Sandrew msr sctlr_el1, x2 220281494Sandrew 221281494Sandrew /* Don't trap to EL2 for exceptions */ 222281494Sandrew mov x2, #CPTR_RES1 223281494Sandrew msr cptr_el2, x2 224281494Sandrew 225281494Sandrew /* Don't trap to EL2 for CP15 traps */ 226281494Sandrew msr hstr_el2, xzr 227281494Sandrew 228286674Sandrew /* Enable access to the physical timers at EL1 */ 229286674Sandrew mrs x2, cnthctl_el2 230286674Sandrew orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 231286674Sandrew msr cnthctl_el2, x2 232286674Sandrew 233286674Sandrew /* Set the counter offset to a known value */ 234286674Sandrew msr cntvoff_el2, xzr 235286674Sandrew 236281494Sandrew /* Hypervisor trap functions */ 237281494Sandrew adr x2, hyp_vectors 238281494Sandrew msr vbar_el2, x2 239281494Sandrew 240281494Sandrew mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 241281494Sandrew msr spsr_el2, x2 242281494Sandrew 243282867Szbb /* Configure GICv3 CPU interface */ 244282867Szbb mrs x2, id_aa64pfr0_el1 245282867Szbb /* Extract GIC bits from the register */ 246282867Szbb ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 247282867Szbb /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 248282867Szbb cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 249282867Szbb b.ne 2f 250282867Szbb 251282867Szbb mrs x2, icc_sre_el2 252282867Szbb orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 253296256Swma orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 254282867Szbb msr icc_sre_el2, x2 255282867Szbb2: 256282867Szbb 257281494Sandrew /* Set the address to return to our return address */ 258281494Sandrew msr elr_el2, x30 259287532Sandrew isb 260281494Sandrew 261281494Sandrew eret 262281494Sandrew 263281494Sandrew .align 3 264281494Sandrew.Lsctlr_res1: 265281494Sandrew .quad SCTLR_RES1 266281494Sandrew 267281494Sandrew#define VECT_EMPTY \ 268281494Sandrew .align 7; \ 269281494Sandrew 1: b 1b 270281494Sandrew 271281494Sandrew .align 11 272281494Sandrewhyp_vectors: 273281494Sandrew VECT_EMPTY /* Synchronous EL2t */ 274281494Sandrew VECT_EMPTY /* IRQ EL2t */ 275281494Sandrew VECT_EMPTY /* FIQ EL2t */ 276281494Sandrew VECT_EMPTY /* Error EL2t */ 277281494Sandrew 278281494Sandrew VECT_EMPTY /* Synchronous EL2h */ 279281494Sandrew VECT_EMPTY /* IRQ EL2h */ 280281494Sandrew VECT_EMPTY /* FIQ EL2h */ 281281494Sandrew VECT_EMPTY /* Error EL2h */ 282281494Sandrew 283281494Sandrew VECT_EMPTY /* Synchronous 64-bit EL1 */ 284281494Sandrew VECT_EMPTY /* IRQ 64-bit EL1 */ 285281494Sandrew VECT_EMPTY /* FIQ 64-bit EL1 */ 286281494Sandrew VECT_EMPTY /* Error 64-bit EL1 */ 287281494Sandrew 288281494Sandrew VECT_EMPTY /* Synchronous 32-bit EL1 */ 289281494Sandrew VECT_EMPTY /* IRQ 32-bit EL1 */ 290281494Sandrew VECT_EMPTY /* FIQ 32-bit EL1 */ 291281494Sandrew VECT_EMPTY /* Error 32-bit EL1 */ 292281494Sandrew 293281494Sandrew/* 294281494Sandrew * Get the delta between the physical address we were loaded to and the 295281494Sandrew * virtual address we expect to run from. This is used when building the 296281494Sandrew * initial page table. 297281494Sandrew */ 298281494Sandrewget_virt_delta: 299281494Sandrew /* Load the physical address of virt_map */ 300281494Sandrew adr x29, virt_map 301281494Sandrew /* Load the virtual address of virt_map stored in virt_map */ 302281494Sandrew ldr x28, [x29] 303281494Sandrew /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 304281494Sandrew sub x29, x29, x28 305281494Sandrew /* Find the load address for the kernel */ 306281494Sandrew mov x28, #(KERNBASE) 307281494Sandrew add x28, x28, x29 308281494Sandrew ret 309281494Sandrew 310281494Sandrew .align 3 311281494Sandrewvirt_map: 312281494Sandrew .quad virt_map 313281494Sandrew 314281494Sandrew/* 315281494Sandrew * This builds the page tables containing the identity map, and the kernel 316281494Sandrew * virtual map. 317281494Sandrew * 318281494Sandrew * It relys on: 319281494Sandrew * We were loaded to an address that is on a 2MiB boundary 320281494Sandrew * All the memory must not cross a 1GiB boundaty 321281494Sandrew * x28 contains the physical address we were loaded from 322281494Sandrew * 323281494Sandrew * TODO: This is out of date. 324281494Sandrew * There are at least 5 pages before that address for the page tables 325281494Sandrew * The pages used are: 326297914Sandrew * - The Kernel L2 table 327297914Sandrew * - The Kernel L1 table 328297914Sandrew * - The Kernel L0 table (TTBR1) 329297914Sandrew * - The identity (PA = VA) L1 table 330297914Sandrew * - The identity (PA = VA) L0 table (TTBR0) 331297914Sandrew * - The DMAP L1 tables 332281494Sandrew */ 333281494Sandrewcreate_pagetables: 334281494Sandrew /* Save the Link register */ 335281494Sandrew mov x5, x30 336281494Sandrew 337281494Sandrew /* Clean the page table */ 338281494Sandrew adr x6, pagetable 339281494Sandrew mov x26, x6 340281494Sandrew adr x27, pagetable_end 341281494Sandrew1: 342281494Sandrew stp xzr, xzr, [x6], #16 343281494Sandrew stp xzr, xzr, [x6], #16 344281494Sandrew stp xzr, xzr, [x6], #16 345281494Sandrew stp xzr, xzr, [x6], #16 346281494Sandrew cmp x6, x27 347281494Sandrew b.lo 1b 348281494Sandrew 349281494Sandrew /* 350281494Sandrew * Build the TTBR1 maps. 351281494Sandrew */ 352281494Sandrew 353281494Sandrew /* Find the size of the kernel */ 354281494Sandrew mov x6, #(KERNBASE) 355281494Sandrew ldr x7, .Lend 356281494Sandrew /* Find the end - begin */ 357281494Sandrew sub x8, x7, x6 358281494Sandrew /* Get the number of l2 pages to allocate, rounded down */ 359281494Sandrew lsr x10, x8, #(L2_SHIFT) 360295154Sandrew /* Add 8 MiB for any rounding above and the module data */ 361295154Sandrew add x10, x10, #4 362281494Sandrew 363281494Sandrew /* Create the kernel space L2 table */ 364281494Sandrew mov x6, x26 365281494Sandrew mov x7, #NORMAL_MEM 366281494Sandrew mov x8, #(KERNBASE & L2_BLOCK_MASK) 367281494Sandrew mov x9, x28 368289455Sandrew bl build_l2_block_pagetable 369281494Sandrew 370281494Sandrew /* Move to the l1 table */ 371281494Sandrew add x26, x26, #PAGE_SIZE 372281494Sandrew 373281494Sandrew /* Link the l1 -> l2 table */ 374281494Sandrew mov x9, x6 375281494Sandrew mov x6, x26 376281494Sandrew bl link_l1_pagetable 377281494Sandrew 378297446Sandrew /* Move to the l0 table */ 379297446Sandrew add x24, x26, #PAGE_SIZE 380281494Sandrew 381297446Sandrew /* Link the l0 -> l1 table */ 382297446Sandrew mov x9, x6 383297446Sandrew mov x6, x24 384297616Sandrew mov x10, #1 385297446Sandrew bl link_l0_pagetable 386297446Sandrew 387297914Sandrew /* Link the DMAP tables */ 388297914Sandrew ldr x8, =DMAP_MIN_ADDRESS 389297914Sandrew adr x9, pagetable_dmap; 390297914Sandrew mov x10, #DMAP_TABLES 391297914Sandrew bl link_l0_pagetable 392297914Sandrew 393281494Sandrew /* 394281494Sandrew * Build the TTBR0 maps. 395281494Sandrew */ 396297446Sandrew add x27, x24, #PAGE_SIZE 397281494Sandrew 398289581Sandrew mov x6, x27 /* The initial page table */ 399281494Sandrew#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 400281494Sandrew /* Create a table for the UART */ 401281494Sandrew mov x7, #DEVICE_MEM 402281494Sandrew mov x8, #(SOCDEV_VA) /* VA start */ 403281494Sandrew mov x9, #(SOCDEV_PA) /* PA start */ 404289466Sandrew mov x10, #1 405289466Sandrew bl build_l1_block_pagetable 406281494Sandrew#endif 407281494Sandrew 408281494Sandrew /* Create the VA = PA map */ 409281494Sandrew mov x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */ 410281494Sandrew mov x9, x27 411281494Sandrew mov x8, x9 /* VA start (== PA start) */ 412289466Sandrew mov x10, #1 413289466Sandrew bl build_l1_block_pagetable 414281494Sandrew 415289581Sandrew /* Move to the l0 table */ 416289581Sandrew add x27, x27, #PAGE_SIZE 417289581Sandrew 418289581Sandrew /* Link the l0 -> l1 table */ 419289581Sandrew mov x9, x6 420289581Sandrew mov x6, x27 421297616Sandrew mov x10, #1 422289581Sandrew bl link_l0_pagetable 423289581Sandrew 424281494Sandrew /* Restore the Link register */ 425281494Sandrew mov x30, x5 426281494Sandrew ret 427281494Sandrew 428281494Sandrew/* 429289581Sandrew * Builds an L0 -> L1 table descriptor 430289581Sandrew * 431289581Sandrew * This is a link for a 512GiB block of memory with up to 1GiB regions mapped 432289581Sandrew * within it by build_l1_block_pagetable. 433289581Sandrew * 434289581Sandrew * x6 = L0 table 435289581Sandrew * x8 = Virtual Address 436289581Sandrew * x9 = L1 PA (trashed) 437297616Sandrew * x10 = Entry count 438289581Sandrew * x11, x12 and x13 are trashed 439289581Sandrew */ 440289581Sandrewlink_l0_pagetable: 441289581Sandrew /* 442289581Sandrew * Link an L0 -> L1 table entry. 443289581Sandrew */ 444289581Sandrew /* Find the table index */ 445289581Sandrew lsr x11, x8, #L0_SHIFT 446297446Sandrew and x11, x11, #L0_ADDR_MASK 447289581Sandrew 448289581Sandrew /* Build the L0 block entry */ 449289581Sandrew mov x12, #L0_TABLE 450289581Sandrew 451289581Sandrew /* Only use the output address bits */ 452297616Sandrew lsr x9, x9, #PAGE_SHIFT 453297616Sandrew1: orr x13, x12, x9, lsl #PAGE_SHIFT 454289581Sandrew 455289581Sandrew /* Store the entry */ 456297616Sandrew str x13, [x6, x11, lsl #3] 457289581Sandrew 458297616Sandrew sub x10, x10, #1 459297616Sandrew add x11, x11, #1 460297616Sandrew add x9, x9, #1 461297616Sandrew cbnz x10, 1b 462297616Sandrew 463289581Sandrew ret 464289581Sandrew 465289581Sandrew/* 466289466Sandrew * Builds an L1 -> L2 table descriptor 467289466Sandrew * 468289466Sandrew * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 469289466Sandrew * within it by build_l2_block_pagetable. 470289466Sandrew * 471289466Sandrew * x6 = L1 table 472289466Sandrew * x8 = Virtual Address 473289466Sandrew * x9 = L2 PA (trashed) 474281494Sandrew * x11, x12 and x13 are trashed 475281494Sandrew */ 476289466Sandrewlink_l1_pagetable: 477281494Sandrew /* 478289466Sandrew * Link an L1 -> L2 table entry. 479281494Sandrew */ 480281494Sandrew /* Find the table index */ 481281494Sandrew lsr x11, x8, #L1_SHIFT 482281494Sandrew and x11, x11, #Ln_ADDR_MASK 483281494Sandrew 484281494Sandrew /* Build the L1 block entry */ 485289466Sandrew mov x12, #L1_TABLE 486281494Sandrew 487281494Sandrew /* Only use the output address bits */ 488297616Sandrew lsr x9, x9, #PAGE_SHIFT 489297616Sandrew orr x13, x12, x9, lsl #PAGE_SHIFT 490281494Sandrew 491281494Sandrew /* Store the entry */ 492297616Sandrew str x13, [x6, x11, lsl #3] 493281494Sandrew 494281494Sandrew ret 495281494Sandrew 496281494Sandrew/* 497289466Sandrew * Builds count 1 GiB page table entry 498281494Sandrew * x6 = L1 table 499289466Sandrew * x7 = Type (0 = Device, 1 = Normal) 500289466Sandrew * x8 = VA start 501289466Sandrew * x9 = PA start (trashed) 502297616Sandrew * x10 = Entry count 503281494Sandrew * x11, x12 and x13 are trashed 504281494Sandrew */ 505289466Sandrewbuild_l1_block_pagetable: 506281494Sandrew /* 507289466Sandrew * Build the L1 table entry. 508281494Sandrew */ 509281494Sandrew /* Find the table index */ 510281494Sandrew lsr x11, x8, #L1_SHIFT 511281494Sandrew and x11, x11, #Ln_ADDR_MASK 512281494Sandrew 513281494Sandrew /* Build the L1 block entry */ 514289466Sandrew lsl x12, x7, #2 515289466Sandrew orr x12, x12, #L1_BLOCK 516289466Sandrew orr x12, x12, #(ATTR_AF) 517289466Sandrew#ifdef SMP 518289466Sandrew orr x12, x12, ATTR_SH(ATTR_SH_IS) 519289466Sandrew#endif 520281494Sandrew 521281494Sandrew /* Only use the output address bits */ 522289466Sandrew lsr x9, x9, #L1_SHIFT 523281494Sandrew 524289466Sandrew /* Set the physical address for this virtual address */ 525297616Sandrew1: orr x13, x12, x9, lsl #L1_SHIFT 526289466Sandrew 527281494Sandrew /* Store the entry */ 528297616Sandrew str x13, [x6, x11, lsl #3] 529281494Sandrew 530289466Sandrew sub x10, x10, #1 531289466Sandrew add x11, x11, #1 532289466Sandrew add x9, x9, #1 533289466Sandrew cbnz x10, 1b 534289466Sandrew 535297616Sandrew ret 536289466Sandrew 537281494Sandrew/* 538281494Sandrew * Builds count 2 MiB page table entry 539281494Sandrew * x6 = L2 table 540281494Sandrew * x7 = Type (0 = Device, 1 = Normal) 541281494Sandrew * x8 = VA start 542281494Sandrew * x9 = PA start (trashed) 543297616Sandrew * x10 = Entry count 544281494Sandrew * x11, x12 and x13 are trashed 545281494Sandrew */ 546289455Sandrewbuild_l2_block_pagetable: 547281494Sandrew /* 548281494Sandrew * Build the L2 table entry. 549281494Sandrew */ 550281494Sandrew /* Find the table index */ 551281494Sandrew lsr x11, x8, #L2_SHIFT 552281494Sandrew and x11, x11, #Ln_ADDR_MASK 553281494Sandrew 554281494Sandrew /* Build the L2 block entry */ 555281494Sandrew lsl x12, x7, #2 556281494Sandrew orr x12, x12, #L2_BLOCK 557281494Sandrew orr x12, x12, #(ATTR_AF) 558285537Sandrew#ifdef SMP 559285537Sandrew orr x12, x12, ATTR_SH(ATTR_SH_IS) 560285537Sandrew#endif 561281494Sandrew 562281494Sandrew /* Only use the output address bits */ 563281494Sandrew lsr x9, x9, #L2_SHIFT 564281494Sandrew 565281494Sandrew /* Set the physical address for this virtual address */ 566297616Sandrew1: orr x13, x12, x9, lsl #L2_SHIFT 567281494Sandrew 568281494Sandrew /* Store the entry */ 569297616Sandrew str x13, [x6, x11, lsl #3] 570281494Sandrew 571281494Sandrew sub x10, x10, #1 572281494Sandrew add x11, x11, #1 573281494Sandrew add x9, x9, #1 574281494Sandrew cbnz x10, 1b 575281494Sandrew 576297616Sandrew ret 577281494Sandrew 578281494Sandrewstart_mmu: 579281494Sandrew dsb sy 580281494Sandrew 581281494Sandrew /* Load the exception vectors */ 582281494Sandrew ldr x2, =exception_vectors 583281494Sandrew msr vbar_el1, x2 584281494Sandrew 585281494Sandrew /* Load ttbr0 and ttbr1 */ 586281494Sandrew msr ttbr0_el1, x27 587297446Sandrew msr ttbr1_el1, x24 588281494Sandrew isb 589281494Sandrew 590281494Sandrew /* Clear the Monitor Debug System control register */ 591281494Sandrew msr mdscr_el1, xzr 592281494Sandrew 593281494Sandrew /* Invalidate the TLB */ 594281494Sandrew tlbi vmalle1is 595281494Sandrew 596281494Sandrew ldr x2, mair 597281494Sandrew msr mair_el1, x2 598281494Sandrew 599289581Sandrew /* 600289581Sandrew * Setup TCR according to PARange bits from ID_AA64MMFR0_EL1. 601289581Sandrew */ 602297446Sandrew ldr x2, tcr 603281494Sandrew mrs x3, id_aa64mmfr0_el1 604281494Sandrew bfi x2, x3, #32, #3 605281494Sandrew msr tcr_el1, x2 606281494Sandrew 607281494Sandrew /* Setup SCTLR */ 608281494Sandrew ldr x2, sctlr_set 609281494Sandrew ldr x3, sctlr_clear 610281494Sandrew mrs x1, sctlr_el1 611281494Sandrew bic x1, x1, x3 /* Clear the required bits */ 612281494Sandrew orr x1, x1, x2 /* Set the required bits */ 613281494Sandrew msr sctlr_el1, x1 614281494Sandrew isb 615281494Sandrew 616281494Sandrew ret 617281494Sandrew 618281494Sandrew .align 3 619281494Sandrewmair: 620319213Sandrew .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, 0) | \ 621319213Sandrew MAIR_ATTR(MAIR_NORMAL_NC, 1) | \ 622319213Sandrew MAIR_ATTR(MAIR_NORMAL_WB, 2) | \ 623319213Sandrew MAIR_ATTR(MAIR_NORMAL_WT, 3) 624281494Sandrewtcr: 625285626Szbb .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K | \ 626285626Szbb TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 627281494Sandrewsctlr_set: 628281494Sandrew /* Bits to set */ 629319196Sandrew .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 630319196Sandrew SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 631295270Sandrew SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | SCTLR_M) 632281494Sandrewsctlr_clear: 633281494Sandrew /* Bits to clear */ 634319196Sandrew .quad (SCTLR_EE | SCTLR_EOE | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 635319196Sandrew SCTLR_ITD | SCTLR_THEE | SCTLR_CP15BEN | SCTLR_A) 636281494Sandrew 637281494Sandrew .globl abort 638281494Sandrewabort: 639281494Sandrew b abort 640281494Sandrew 641281494Sandrew //.section .init_pagetable 642281494Sandrew .align 12 /* 4KiB aligned */ 643281494Sandrew /* 644281494Sandrew * 3 initial tables (in the following order): 645281494Sandrew * L2 for kernel (High addresses) 646281494Sandrew * L1 for kernel 647281494Sandrew * L1 for user (Low addresses) 648281494Sandrew */ 649281494Sandrewpagetable: 650281494Sandrew .space PAGE_SIZE 651281494Sandrewpagetable_l1_ttbr1: 652281494Sandrew .space PAGE_SIZE 653297446Sandrewpagetable_l0_ttbr1: 654297446Sandrew .space PAGE_SIZE 655281494Sandrewpagetable_l1_ttbr0: 656281494Sandrew .space PAGE_SIZE 657289581Sandrewpagetable_l0_ttbr0: 658289581Sandrew .space PAGE_SIZE 659297914Sandrew 660297914Sandrew .globl pagetable_dmap 661297914Sandrewpagetable_dmap: 662297914Sandrew .space PAGE_SIZE * DMAP_TABLES 663281494Sandrewpagetable_end: 664281494Sandrew 665281494Sandrewel2_pagetable: 666281494Sandrew .space PAGE_SIZE 667281494Sandrew 668281494Sandrew .globl init_pt_va 669281494Sandrewinit_pt_va: 670281494Sandrew .quad pagetable /* XXX: Keep page tables VA */ 671281494Sandrew 672281494Sandrew .align 4 673281494Sandrewinitstack: 674281494Sandrew .space (PAGE_SIZE * KSTACK_PAGES) 675281494Sandrewinitstack_end: 676281494Sandrew 677281494Sandrew 678281494SandrewENTRY(sigcode) 679281494Sandrew mov x0, sp 680281494Sandrew add x0, x0, #SF_UC 681281494Sandrew 682281494Sandrew1: 683281494Sandrew mov x8, #SYS_sigreturn 684281494Sandrew svc 0 685281494Sandrew 686281494Sandrew /* sigreturn failed, exit */ 687281494Sandrew mov x8, #SYS_exit 688281494Sandrew svc 0 689281494Sandrew 690281494Sandrew b 1b 691281494SandrewEND(sigcode) 692281494Sandrew /* This may be copied to the stack, keep it 16-byte aligned */ 693281494Sandrew .align 3 694281494Sandrewesigcode: 695281494Sandrew 696281494Sandrew .data 697281494Sandrew .align 3 698281494Sandrew .global szsigcode 699281494Sandrewszsigcode: 700281494Sandrew .quad esigcode - sigcode 701