locore.S revision 297615
1281494Sandrew/*- 2281494Sandrew * Copyright (c) 2012-2014 Andrew Turner 3281494Sandrew * All rights reserved. 4281494Sandrew * 5281494Sandrew * Redistribution and use in source and binary forms, with or without 6281494Sandrew * modification, are permitted provided that the following conditions 7281494Sandrew * are met: 8281494Sandrew * 1. Redistributions of source code must retain the above copyright 9281494Sandrew * notice, this list of conditions and the following disclaimer. 10281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 11281494Sandrew * notice, this list of conditions and the following disclaimer in the 12281494Sandrew * documentation and/or other materials provided with the distribution. 13281494Sandrew * 14281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24281494Sandrew * SUCH DAMAGE. 25281494Sandrew * 26281494Sandrew * $FreeBSD: head/sys/arm64/arm64/locore.S 297615 2016-04-06 14:08:10Z andrew $ 27281494Sandrew */ 28281494Sandrew 29281494Sandrew#include "assym.s" 30285627Szbb#include "opt_kstack_pages.h" 31281494Sandrew#include <sys/syscall.h> 32281494Sandrew#include <machine/asm.h> 33281494Sandrew#include <machine/armreg.h> 34281494Sandrew#include <machine/hypervisor.h> 35281494Sandrew#include <machine/param.h> 36281494Sandrew#include <machine/pte.h> 37297615Sandrew#include <machine/vmparam.h> 38281494Sandrew 39297446Sandrew#define VIRT_BITS 48 40281494Sandrew 41281494Sandrew .globl kernbase 42281494Sandrew .set kernbase, KERNBASE 43281494Sandrew 44281494Sandrew#define DEVICE_MEM 0 45281494Sandrew#define NORMAL_UNCACHED 1 46281494Sandrew#define NORMAL_MEM 2 47281494Sandrew 48281494Sandrew/* 49281494Sandrew * We assume: 50281494Sandrew * MMU on with an identity map, or off 51281494Sandrew * D-Cache: off 52281494Sandrew * I-Cache: on or off 53281494Sandrew * We are loaded at a 2MiB aligned address 54281494Sandrew */ 55281494Sandrew 56281494Sandrew .text 57281494Sandrew .globl _start 58281494Sandrew_start: 59281494Sandrew /* Drop to EL1 */ 60281494Sandrew bl drop_to_el1 61281494Sandrew 62281494Sandrew /* 63281494Sandrew * Disable the MMU. We may have entered the kernel with it on and 64281494Sandrew * will need to update the tables later. If this has been set up 65281494Sandrew * with anything other than a VA == PA map then this will fail, 66281494Sandrew * but in this case the code to find where we are running from 67281494Sandrew * would have also failed. 68281494Sandrew */ 69281494Sandrew dsb sy 70281494Sandrew mrs x2, sctlr_el1 71281494Sandrew bic x2, x2, SCTLR_M 72281494Sandrew msr sctlr_el1, x2 73281494Sandrew isb 74281494Sandrew 75285316Sandrew /* Set the context id */ 76285316Sandrew msr contextidr_el1, xzr 77281494Sandrew 78281494Sandrew /* Get the virt -> phys offset */ 79281494Sandrew bl get_virt_delta 80281494Sandrew 81281494Sandrew /* 82281494Sandrew * At this point: 83281494Sandrew * x29 = PA - VA 84281494Sandrew * x28 = Our physical load address 85281494Sandrew */ 86281494Sandrew 87281494Sandrew /* Create the page tables */ 88281494Sandrew bl create_pagetables 89281494Sandrew 90281494Sandrew /* 91281494Sandrew * At this point: 92281494Sandrew * x27 = TTBR0 table 93297446Sandrew * x26 = Kernel L1 table 94297446Sandrew * x24 = TTBR1 table 95281494Sandrew */ 96281494Sandrew 97281494Sandrew /* Enable the mmu */ 98281494Sandrew bl start_mmu 99281494Sandrew 100281494Sandrew /* Jump to the virtual address space */ 101281494Sandrew ldr x15, .Lvirtdone 102281494Sandrew br x15 103281494Sandrew 104281494Sandrewvirtdone: 105281494Sandrew /* Set up the stack */ 106281494Sandrew adr x25, initstack_end 107281494Sandrew mov sp, x25 108281494Sandrew sub sp, sp, #PCB_SIZE 109281494Sandrew 110281494Sandrew /* Zero the BSS */ 111281494Sandrew ldr x15, .Lbss 112281494Sandrew ldr x14, .Lend 113281494Sandrew1: 114281494Sandrew str xzr, [x15], #8 115281494Sandrew cmp x15, x14 116281494Sandrew b.lo 1b 117281494Sandrew 118281494Sandrew /* Backup the module pointer */ 119281494Sandrew mov x1, x0 120281494Sandrew 121281494Sandrew /* Make the page table base a virtual address */ 122281494Sandrew sub x26, x26, x29 123297446Sandrew sub x24, x24, x29 124281494Sandrew 125281494Sandrew sub sp, sp, #(64 * 4) 126281494Sandrew mov x0, sp 127281494Sandrew 128281494Sandrew /* Degate the delda so it is VA -> PA */ 129281494Sandrew neg x29, x29 130281494Sandrew 131281494Sandrew str x1, [x0] /* modulep */ 132281494Sandrew str x26, [x0, 8] /* kern_l1pt */ 133281494Sandrew str x29, [x0, 16] /* kern_delta */ 134281494Sandrew str x25, [x0, 24] /* kern_stack */ 135297446Sandrew str x24, [x0, 32] /* kern_l0pt */ 136281494Sandrew 137281494Sandrew /* trace back starts here */ 138281494Sandrew mov fp, #0 139281494Sandrew /* Branch to C code */ 140281494Sandrew bl initarm 141281494Sandrew bl mi_startup 142281494Sandrew 143281494Sandrew /* We should not get here */ 144281494Sandrew brk 0 145281494Sandrew 146281494Sandrew .align 3 147281494Sandrew.Lvirtdone: 148281494Sandrew .quad virtdone 149281494Sandrew.Lbss: 150281494Sandrew .quad __bss_start 151281494Sandrew.Lend: 152281494Sandrew .quad _end 153281494Sandrew 154285316Sandrew#ifdef SMP 155281494Sandrew/* 156285316Sandrew * mpentry(unsigned long) 157285316Sandrew * 158285316Sandrew * Called by a core when it is being brought online. 159285316Sandrew * The data in x0 is passed straight to init_secondary. 160285316Sandrew */ 161285316SandrewENTRY(mpentry) 162285316Sandrew /* Disable interrupts */ 163285316Sandrew msr daifset, #2 164285316Sandrew 165285316Sandrew /* Drop to EL1 */ 166285316Sandrew bl drop_to_el1 167285316Sandrew 168285316Sandrew /* Set the context id */ 169285316Sandrew msr contextidr_el1, x1 170285316Sandrew 171285316Sandrew /* Load the kernel page table */ 172297446Sandrew adr x24, pagetable_l0_ttbr1 173285316Sandrew /* Load the identity page table */ 174289581Sandrew adr x27, pagetable_l0_ttbr0 175285316Sandrew 176285316Sandrew /* Enable the mmu */ 177285316Sandrew bl start_mmu 178285316Sandrew 179285316Sandrew /* Jump to the virtual address space */ 180285316Sandrew ldr x15, =mp_virtdone 181285316Sandrew br x15 182285316Sandrew 183285316Sandrewmp_virtdone: 184285316Sandrew ldr x4, =secondary_stacks 185285316Sandrew mov x5, #(PAGE_SIZE * KSTACK_PAGES) 186285654Szbb mul x5, x0, x5 187285316Sandrew add sp, x4, x5 188285316Sandrew 189285316Sandrew b init_secondary 190285316SandrewEND(mpentry) 191285316Sandrew#endif 192285316Sandrew 193285316Sandrew/* 194281494Sandrew * If we are started in EL2, configure the required hypervisor 195281494Sandrew * registers and drop to EL1. 196281494Sandrew */ 197281494Sandrewdrop_to_el1: 198281494Sandrew mrs x1, CurrentEL 199281494Sandrew lsr x1, x1, #2 200281494Sandrew cmp x1, #0x2 201281494Sandrew b.eq 1f 202281494Sandrew ret 203281494Sandrew1: 204281494Sandrew /* Configure the Hypervisor */ 205281494Sandrew mov x2, #(HCR_RW) 206281494Sandrew msr hcr_el2, x2 207281494Sandrew 208281494Sandrew /* Load the Virtualization Process ID Register */ 209281494Sandrew mrs x2, midr_el1 210281494Sandrew msr vpidr_el2, x2 211281494Sandrew 212281494Sandrew /* Load the Virtualization Multiprocess ID Register */ 213281494Sandrew mrs x2, mpidr_el1 214281494Sandrew msr vmpidr_el2, x2 215281494Sandrew 216281494Sandrew /* Set the bits that need to be 1 in sctlr_el1 */ 217281494Sandrew ldr x2, .Lsctlr_res1 218281494Sandrew msr sctlr_el1, x2 219281494Sandrew 220281494Sandrew /* Don't trap to EL2 for exceptions */ 221281494Sandrew mov x2, #CPTR_RES1 222281494Sandrew msr cptr_el2, x2 223281494Sandrew 224281494Sandrew /* Don't trap to EL2 for CP15 traps */ 225281494Sandrew msr hstr_el2, xzr 226281494Sandrew 227286674Sandrew /* Enable access to the physical timers at EL1 */ 228286674Sandrew mrs x2, cnthctl_el2 229286674Sandrew orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 230286674Sandrew msr cnthctl_el2, x2 231286674Sandrew 232286674Sandrew /* Set the counter offset to a known value */ 233286674Sandrew msr cntvoff_el2, xzr 234286674Sandrew 235281494Sandrew /* Hypervisor trap functions */ 236281494Sandrew adr x2, hyp_vectors 237281494Sandrew msr vbar_el2, x2 238281494Sandrew 239281494Sandrew mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 240281494Sandrew msr spsr_el2, x2 241281494Sandrew 242282867Szbb /* Configure GICv3 CPU interface */ 243282867Szbb mrs x2, id_aa64pfr0_el1 244282867Szbb /* Extract GIC bits from the register */ 245282867Szbb ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 246282867Szbb /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 247282867Szbb cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 248282867Szbb b.ne 2f 249282867Szbb 250282867Szbb mrs x2, icc_sre_el2 251282867Szbb orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 252296256Swma orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 253282867Szbb msr icc_sre_el2, x2 254282867Szbb2: 255282867Szbb 256281494Sandrew /* Set the address to return to our return address */ 257281494Sandrew msr elr_el2, x30 258287532Sandrew isb 259281494Sandrew 260281494Sandrew eret 261281494Sandrew 262281494Sandrew .align 3 263281494Sandrew.Lsctlr_res1: 264281494Sandrew .quad SCTLR_RES1 265281494Sandrew 266281494Sandrew#define VECT_EMPTY \ 267281494Sandrew .align 7; \ 268281494Sandrew 1: b 1b 269281494Sandrew 270281494Sandrew .align 11 271281494Sandrewhyp_vectors: 272281494Sandrew VECT_EMPTY /* Synchronous EL2t */ 273281494Sandrew VECT_EMPTY /* IRQ EL2t */ 274281494Sandrew VECT_EMPTY /* FIQ EL2t */ 275281494Sandrew VECT_EMPTY /* Error EL2t */ 276281494Sandrew 277281494Sandrew VECT_EMPTY /* Synchronous EL2h */ 278281494Sandrew VECT_EMPTY /* IRQ EL2h */ 279281494Sandrew VECT_EMPTY /* FIQ EL2h */ 280281494Sandrew VECT_EMPTY /* Error EL2h */ 281281494Sandrew 282281494Sandrew VECT_EMPTY /* Synchronous 64-bit EL1 */ 283281494Sandrew VECT_EMPTY /* IRQ 64-bit EL1 */ 284281494Sandrew VECT_EMPTY /* FIQ 64-bit EL1 */ 285281494Sandrew VECT_EMPTY /* Error 64-bit EL1 */ 286281494Sandrew 287281494Sandrew VECT_EMPTY /* Synchronous 32-bit EL1 */ 288281494Sandrew VECT_EMPTY /* IRQ 32-bit EL1 */ 289281494Sandrew VECT_EMPTY /* FIQ 32-bit EL1 */ 290281494Sandrew VECT_EMPTY /* Error 32-bit EL1 */ 291281494Sandrew 292281494Sandrew/* 293281494Sandrew * Get the delta between the physical address we were loaded to and the 294281494Sandrew * virtual address we expect to run from. This is used when building the 295281494Sandrew * initial page table. 296281494Sandrew */ 297281494Sandrewget_virt_delta: 298281494Sandrew /* Load the physical address of virt_map */ 299281494Sandrew adr x29, virt_map 300281494Sandrew /* Load the virtual address of virt_map stored in virt_map */ 301281494Sandrew ldr x28, [x29] 302281494Sandrew /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 303281494Sandrew sub x29, x29, x28 304281494Sandrew /* Find the load address for the kernel */ 305281494Sandrew mov x28, #(KERNBASE) 306281494Sandrew add x28, x28, x29 307281494Sandrew ret 308281494Sandrew 309281494Sandrew .align 3 310281494Sandrewvirt_map: 311281494Sandrew .quad virt_map 312281494Sandrew 313281494Sandrew/* 314281494Sandrew * This builds the page tables containing the identity map, and the kernel 315281494Sandrew * virtual map. 316281494Sandrew * 317281494Sandrew * It relys on: 318281494Sandrew * We were loaded to an address that is on a 2MiB boundary 319281494Sandrew * All the memory must not cross a 1GiB boundaty 320281494Sandrew * x28 contains the physical address we were loaded from 321281494Sandrew * 322281494Sandrew * TODO: This is out of date. 323281494Sandrew * There are at least 5 pages before that address for the page tables 324281494Sandrew * The pages used are: 325281494Sandrew * - The identity (PA = VA) table (TTBR0) 326281494Sandrew * - The Kernel L1 table (TTBR1)(not yet) 327281494Sandrew * - The PA != VA L2 table to jump into (not yet) 328281494Sandrew * - The FDT L2 table (not yet) 329281494Sandrew */ 330281494Sandrewcreate_pagetables: 331281494Sandrew /* Save the Link register */ 332281494Sandrew mov x5, x30 333281494Sandrew 334281494Sandrew /* Clean the page table */ 335281494Sandrew adr x6, pagetable 336281494Sandrew mov x26, x6 337281494Sandrew adr x27, pagetable_end 338281494Sandrew1: 339281494Sandrew stp xzr, xzr, [x6], #16 340281494Sandrew stp xzr, xzr, [x6], #16 341281494Sandrew stp xzr, xzr, [x6], #16 342281494Sandrew stp xzr, xzr, [x6], #16 343281494Sandrew cmp x6, x27 344281494Sandrew b.lo 1b 345281494Sandrew 346281494Sandrew /* 347281494Sandrew * Build the TTBR1 maps. 348281494Sandrew */ 349281494Sandrew 350281494Sandrew /* Find the size of the kernel */ 351281494Sandrew mov x6, #(KERNBASE) 352281494Sandrew ldr x7, .Lend 353281494Sandrew /* Find the end - begin */ 354281494Sandrew sub x8, x7, x6 355281494Sandrew /* Get the number of l2 pages to allocate, rounded down */ 356281494Sandrew lsr x10, x8, #(L2_SHIFT) 357295154Sandrew /* Add 8 MiB for any rounding above and the module data */ 358295154Sandrew add x10, x10, #4 359281494Sandrew 360281494Sandrew /* Create the kernel space L2 table */ 361281494Sandrew mov x6, x26 362281494Sandrew mov x7, #NORMAL_MEM 363281494Sandrew mov x8, #(KERNBASE & L2_BLOCK_MASK) 364281494Sandrew mov x9, x28 365289455Sandrew bl build_l2_block_pagetable 366281494Sandrew 367281494Sandrew /* Move to the l1 table */ 368281494Sandrew add x26, x26, #PAGE_SIZE 369281494Sandrew 370281494Sandrew /* Link the l1 -> l2 table */ 371281494Sandrew mov x9, x6 372281494Sandrew mov x6, x26 373281494Sandrew bl link_l1_pagetable 374281494Sandrew 375297446Sandrew /* Move to the l0 table */ 376297446Sandrew add x24, x26, #PAGE_SIZE 377281494Sandrew 378297446Sandrew /* Link the l0 -> l1 table */ 379297446Sandrew mov x9, x6 380297446Sandrew mov x6, x24 381297446Sandrew bl link_l0_pagetable 382297446Sandrew 383281494Sandrew /* 384281494Sandrew * Build the TTBR0 maps. 385281494Sandrew */ 386297446Sandrew add x27, x24, #PAGE_SIZE 387281494Sandrew 388289581Sandrew mov x6, x27 /* The initial page table */ 389281494Sandrew#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 390281494Sandrew /* Create a table for the UART */ 391281494Sandrew mov x7, #DEVICE_MEM 392281494Sandrew mov x8, #(SOCDEV_VA) /* VA start */ 393281494Sandrew mov x9, #(SOCDEV_PA) /* PA start */ 394289466Sandrew mov x10, #1 395289466Sandrew bl build_l1_block_pagetable 396281494Sandrew#endif 397281494Sandrew 398281494Sandrew /* Create the VA = PA map */ 399281494Sandrew mov x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */ 400281494Sandrew mov x9, x27 401281494Sandrew mov x8, x9 /* VA start (== PA start) */ 402289466Sandrew mov x10, #1 403289466Sandrew bl build_l1_block_pagetable 404281494Sandrew 405289581Sandrew /* Move to the l0 table */ 406289581Sandrew add x27, x27, #PAGE_SIZE 407289581Sandrew 408289581Sandrew /* Link the l0 -> l1 table */ 409289581Sandrew mov x9, x6 410289581Sandrew mov x6, x27 411289581Sandrew bl link_l0_pagetable 412289581Sandrew 413281494Sandrew /* Restore the Link register */ 414281494Sandrew mov x30, x5 415281494Sandrew ret 416281494Sandrew 417281494Sandrew/* 418289581Sandrew * Builds an L0 -> L1 table descriptor 419289581Sandrew * 420289581Sandrew * This is a link for a 512GiB block of memory with up to 1GiB regions mapped 421289581Sandrew * within it by build_l1_block_pagetable. 422289581Sandrew * 423289581Sandrew * x6 = L0 table 424289581Sandrew * x8 = Virtual Address 425289581Sandrew * x9 = L1 PA (trashed) 426289581Sandrew * x11, x12 and x13 are trashed 427289581Sandrew */ 428289581Sandrewlink_l0_pagetable: 429289581Sandrew /* 430289581Sandrew * Link an L0 -> L1 table entry. 431289581Sandrew */ 432289581Sandrew /* Find the table index */ 433289581Sandrew lsr x11, x8, #L0_SHIFT 434297446Sandrew and x11, x11, #L0_ADDR_MASK 435289581Sandrew 436289581Sandrew /* Build the L0 block entry */ 437289581Sandrew mov x12, #L0_TABLE 438289581Sandrew 439289581Sandrew /* Only use the output address bits */ 440289581Sandrew lsr x9, x9, #12 441289581Sandrew orr x12, x12, x9, lsl #12 442289581Sandrew 443289581Sandrew /* Store the entry */ 444289581Sandrew str x12, [x6, x11, lsl #3] 445289581Sandrew 446289581Sandrew ret 447289581Sandrew 448289581Sandrew/* 449289466Sandrew * Builds an L1 -> L2 table descriptor 450289466Sandrew * 451289466Sandrew * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 452289466Sandrew * within it by build_l2_block_pagetable. 453289466Sandrew * 454289466Sandrew * x6 = L1 table 455289466Sandrew * x8 = Virtual Address 456289466Sandrew * x9 = L2 PA (trashed) 457281494Sandrew * x11, x12 and x13 are trashed 458281494Sandrew */ 459289466Sandrewlink_l1_pagetable: 460281494Sandrew /* 461289466Sandrew * Link an L1 -> L2 table entry. 462281494Sandrew */ 463281494Sandrew /* Find the table index */ 464281494Sandrew lsr x11, x8, #L1_SHIFT 465281494Sandrew and x11, x11, #Ln_ADDR_MASK 466281494Sandrew 467281494Sandrew /* Build the L1 block entry */ 468289466Sandrew mov x12, #L1_TABLE 469281494Sandrew 470281494Sandrew /* Only use the output address bits */ 471289466Sandrew lsr x9, x9, #12 472289466Sandrew orr x12, x12, x9, lsl #12 473281494Sandrew 474281494Sandrew /* Store the entry */ 475281494Sandrew str x12, [x6, x11, lsl #3] 476281494Sandrew 477281494Sandrew ret 478281494Sandrew 479281494Sandrew/* 480289466Sandrew * Builds count 1 GiB page table entry 481281494Sandrew * x6 = L1 table 482289466Sandrew * x7 = Type (0 = Device, 1 = Normal) 483289466Sandrew * x8 = VA start 484289466Sandrew * x9 = PA start (trashed) 485289466Sandrew * x10 = Entry count (TODO) 486281494Sandrew * x11, x12 and x13 are trashed 487281494Sandrew */ 488289466Sandrewbuild_l1_block_pagetable: 489281494Sandrew /* 490289466Sandrew * Build the L1 table entry. 491281494Sandrew */ 492281494Sandrew /* Find the table index */ 493281494Sandrew lsr x11, x8, #L1_SHIFT 494281494Sandrew and x11, x11, #Ln_ADDR_MASK 495281494Sandrew 496281494Sandrew /* Build the L1 block entry */ 497289466Sandrew lsl x12, x7, #2 498289466Sandrew orr x12, x12, #L1_BLOCK 499289466Sandrew orr x12, x12, #(ATTR_AF) 500289466Sandrew#ifdef SMP 501289466Sandrew orr x12, x12, ATTR_SH(ATTR_SH_IS) 502289466Sandrew#endif 503281494Sandrew 504281494Sandrew /* Only use the output address bits */ 505289466Sandrew lsr x9, x9, #L1_SHIFT 506281494Sandrew 507289466Sandrew /* Set the physical address for this virtual address */ 508289466Sandrew1: orr x12, x12, x9, lsl #L1_SHIFT 509289466Sandrew 510281494Sandrew /* Store the entry */ 511281494Sandrew str x12, [x6, x11, lsl #3] 512281494Sandrew 513289466Sandrew /* Clear the address bits */ 514289466Sandrew and x12, x12, #ATTR_MASK_L 515281494Sandrew 516289466Sandrew sub x10, x10, #1 517289466Sandrew add x11, x11, #1 518289466Sandrew add x9, x9, #1 519289466Sandrew cbnz x10, 1b 520289466Sandrew 521289466Sandrew2: ret 522289466Sandrew 523281494Sandrew/* 524281494Sandrew * Builds count 2 MiB page table entry 525281494Sandrew * x6 = L2 table 526281494Sandrew * x7 = Type (0 = Device, 1 = Normal) 527281494Sandrew * x8 = VA start 528281494Sandrew * x9 = PA start (trashed) 529281494Sandrew * x10 = Entry count (TODO) 530281494Sandrew * x11, x12 and x13 are trashed 531281494Sandrew */ 532289455Sandrewbuild_l2_block_pagetable: 533281494Sandrew /* 534281494Sandrew * Build the L2 table entry. 535281494Sandrew */ 536281494Sandrew /* Find the table index */ 537281494Sandrew lsr x11, x8, #L2_SHIFT 538281494Sandrew and x11, x11, #Ln_ADDR_MASK 539281494Sandrew 540281494Sandrew /* Build the L2 block entry */ 541281494Sandrew lsl x12, x7, #2 542281494Sandrew orr x12, x12, #L2_BLOCK 543281494Sandrew orr x12, x12, #(ATTR_AF) 544285537Sandrew#ifdef SMP 545285537Sandrew orr x12, x12, ATTR_SH(ATTR_SH_IS) 546285537Sandrew#endif 547281494Sandrew 548281494Sandrew /* Only use the output address bits */ 549281494Sandrew lsr x9, x9, #L2_SHIFT 550281494Sandrew 551281494Sandrew /* Set the physical address for this virtual address */ 552281494Sandrew1: orr x12, x12, x9, lsl #L2_SHIFT 553281494Sandrew 554281494Sandrew /* Store the entry */ 555281494Sandrew str x12, [x6, x11, lsl #3] 556281494Sandrew 557281494Sandrew /* Clear the address bits */ 558281494Sandrew and x12, x12, #ATTR_MASK_L 559281494Sandrew 560281494Sandrew sub x10, x10, #1 561281494Sandrew add x11, x11, #1 562281494Sandrew add x9, x9, #1 563281494Sandrew cbnz x10, 1b 564281494Sandrew 565281494Sandrew2: ret 566281494Sandrew 567281494Sandrewstart_mmu: 568281494Sandrew dsb sy 569281494Sandrew 570281494Sandrew /* Load the exception vectors */ 571281494Sandrew ldr x2, =exception_vectors 572281494Sandrew msr vbar_el1, x2 573281494Sandrew 574281494Sandrew /* Load ttbr0 and ttbr1 */ 575281494Sandrew msr ttbr0_el1, x27 576297446Sandrew msr ttbr1_el1, x24 577281494Sandrew isb 578281494Sandrew 579281494Sandrew /* Clear the Monitor Debug System control register */ 580281494Sandrew msr mdscr_el1, xzr 581281494Sandrew 582281494Sandrew /* Invalidate the TLB */ 583281494Sandrew tlbi vmalle1is 584281494Sandrew 585281494Sandrew ldr x2, mair 586281494Sandrew msr mair_el1, x2 587281494Sandrew 588289581Sandrew /* 589289581Sandrew * Setup TCR according to PARange bits from ID_AA64MMFR0_EL1. 590289581Sandrew */ 591297446Sandrew ldr x2, tcr 592281494Sandrew mrs x3, id_aa64mmfr0_el1 593281494Sandrew bfi x2, x3, #32, #3 594281494Sandrew msr tcr_el1, x2 595281494Sandrew 596281494Sandrew /* Setup SCTLR */ 597281494Sandrew ldr x2, sctlr_set 598281494Sandrew ldr x3, sctlr_clear 599281494Sandrew mrs x1, sctlr_el1 600281494Sandrew bic x1, x1, x3 /* Clear the required bits */ 601281494Sandrew orr x1, x1, x2 /* Set the required bits */ 602281494Sandrew msr sctlr_el1, x1 603281494Sandrew isb 604281494Sandrew 605281494Sandrew ret 606281494Sandrew 607281494Sandrew .align 3 608281494Sandrewmair: 609281494Sandrew /* Device Normal, no cache Normal, write-back */ 610281494Sandrew .quad MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2) 611281494Sandrewtcr: 612285626Szbb .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K | \ 613285626Szbb TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 614281494Sandrewsctlr_set: 615281494Sandrew /* Bits to set */ 616281494Sandrew .quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 617295270Sandrew SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | SCTLR_M) 618281494Sandrewsctlr_clear: 619281494Sandrew /* Bits to clear */ 620281494Sandrew .quad (SCTLR_EE | SCTLR_EOE | SCTLR_WXN | SCTLR_UMA | SCTLR_ITD | \ 621295270Sandrew SCTLR_THEE | SCTLR_CP15BEN | SCTLR_A) 622281494Sandrew 623281494Sandrew .globl abort 624281494Sandrewabort: 625281494Sandrew b abort 626281494Sandrew 627281494Sandrew //.section .init_pagetable 628281494Sandrew .align 12 /* 4KiB aligned */ 629281494Sandrew /* 630281494Sandrew * 3 initial tables (in the following order): 631281494Sandrew * L2 for kernel (High addresses) 632281494Sandrew * L1 for kernel 633281494Sandrew * L1 for user (Low addresses) 634281494Sandrew */ 635281494Sandrewpagetable: 636281494Sandrew .space PAGE_SIZE 637281494Sandrewpagetable_l1_ttbr1: 638281494Sandrew .space PAGE_SIZE 639297446Sandrewpagetable_l0_ttbr1: 640297446Sandrew .space PAGE_SIZE 641281494Sandrewpagetable_l1_ttbr0: 642281494Sandrew .space PAGE_SIZE 643289581Sandrewpagetable_l0_ttbr0: 644289581Sandrew .space PAGE_SIZE 645281494Sandrewpagetable_end: 646281494Sandrew 647281494Sandrewel2_pagetable: 648281494Sandrew .space PAGE_SIZE 649281494Sandrew 650281494Sandrew .globl init_pt_va 651281494Sandrewinit_pt_va: 652281494Sandrew .quad pagetable /* XXX: Keep page tables VA */ 653281494Sandrew 654281494Sandrew .align 4 655281494Sandrewinitstack: 656281494Sandrew .space (PAGE_SIZE * KSTACK_PAGES) 657281494Sandrewinitstack_end: 658281494Sandrew 659281494Sandrew 660281494SandrewENTRY(sigcode) 661281494Sandrew mov x0, sp 662281494Sandrew add x0, x0, #SF_UC 663281494Sandrew 664281494Sandrew1: 665281494Sandrew mov x8, #SYS_sigreturn 666281494Sandrew svc 0 667281494Sandrew 668281494Sandrew /* sigreturn failed, exit */ 669281494Sandrew mov x8, #SYS_exit 670281494Sandrew svc 0 671281494Sandrew 672281494Sandrew b 1b 673281494SandrewEND(sigcode) 674281494Sandrew /* This may be copied to the stack, keep it 16-byte aligned */ 675281494Sandrew .align 3 676281494Sandrewesigcode: 677281494Sandrew 678281494Sandrew .data 679281494Sandrew .align 3 680281494Sandrew .global szsigcode 681281494Sandrewszsigcode: 682281494Sandrew .quad esigcode - sigcode 683