| locore.S (296256) | locore.S (297446) |
|---|---|
| 1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 9 unchanged lines hidden (view full) --- 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * | 1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 9 unchanged lines hidden (view full) --- 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * |
| 26 * $FreeBSD: head/sys/arm64/arm64/locore.S 296256 2016-03-01 08:15:00Z wma $ | 26 * $FreeBSD: head/sys/arm64/arm64/locore.S 297446 2016-03-31 11:07:24Z andrew $ |
| 27 */ 28 29#include "assym.s" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37 | 27 */ 28 29#include "assym.s" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37 |
| 38#define VIRT_BITS 39 | 38#define VIRT_BITS 48 |
| 39 40 .globl kernbase 41 .set kernbase, KERNBASE 42 43#define DEVICE_MEM 0 44#define NORMAL_UNCACHED 1 45#define NORMAL_MEM 2 46 --- 37 unchanged lines hidden (view full) --- 84 */ 85 86 /* Create the page tables */ 87 bl create_pagetables 88 89 /* 90 * At this point: 91 * x27 = TTBR0 table | 39 40 .globl kernbase 41 .set kernbase, KERNBASE 42 43#define DEVICE_MEM 0 44#define NORMAL_UNCACHED 1 45#define NORMAL_MEM 2 46 --- 37 unchanged lines hidden (view full) --- 84 */ 85 86 /* Create the page tables */ 87 bl create_pagetables 88 89 /* 90 * At this point: 91 * x27 = TTBR0 table |
| 92 * x26 = TTBR1 table | 92 * x26 = Kernel L1 table 93 * x24 = TTBR1 table |
| 93 */ 94 95 /* Enable the mmu */ 96 bl start_mmu 97 98 /* Jump to the virtual address space */ 99 ldr x15, .Lvirtdone 100 br x15 101 102virtdone: | 94 */ 95 96 /* Enable the mmu */ 97 bl start_mmu 98 99 /* Jump to the virtual address space */ 100 ldr x15, .Lvirtdone 101 br x15 102 103virtdone: |
| 103 /* 104 * Now that we are in virtual address space, 105 * we don't need the identity mapping in TTBR0 and 106 * can set the TCR to a more useful value. 107 */ 108 ldr x2, tcr 109 mrs x3, id_aa64mmfr0_el1 110 bfi x2, x3, #32, #3 111 msr tcr_el1, x2 112 | |
| 113 /* Set up the stack */ 114 adr x25, initstack_end 115 mov sp, x25 116 sub sp, sp, #PCB_SIZE 117 118 /* Zero the BSS */ 119 ldr x15, .Lbss 120 ldr x14, .Lend 1211: 122 str xzr, [x15], #8 123 cmp x15, x14 124 b.lo 1b 125 126 /* Backup the module pointer */ 127 mov x1, x0 128 129 /* Make the page table base a virtual address */ 130 sub x26, x26, x29 | 104 /* Set up the stack */ 105 adr x25, initstack_end 106 mov sp, x25 107 sub sp, sp, #PCB_SIZE 108 109 /* Zero the BSS */ 110 ldr x15, .Lbss 111 ldr x14, .Lend 1121: 113 str xzr, [x15], #8 114 cmp x15, x14 115 b.lo 1b 116 117 /* Backup the module pointer */ 118 mov x1, x0 119 120 /* Make the page table base a virtual address */ 121 sub x26, x26, x29 |
| 122 sub x24, x24, x29 |
|
| 131 132 sub sp, sp, #(64 * 4) 133 mov x0, sp 134 135 /* Degate the delda so it is VA -> PA */ 136 neg x29, x29 137 138 str x1, [x0] /* modulep */ 139 str x26, [x0, 8] /* kern_l1pt */ 140 str x29, [x0, 16] /* kern_delta */ 141 str x25, [x0, 24] /* kern_stack */ | 123 124 sub sp, sp, #(64 * 4) 125 mov x0, sp 126 127 /* Degate the delda so it is VA -> PA */ 128 neg x29, x29 129 130 str x1, [x0] /* modulep */ 131 str x26, [x0, 8] /* kern_l1pt */ 132 str x29, [x0, 16] /* kern_delta */ 133 str x25, [x0, 24] /* kern_stack */ |
| 134 str x24, [x0, 32] /* kern_l0pt */ |
|
| 142 143 /* trace back starts here */ 144 mov fp, #0 145 /* Branch to C code */ 146 bl initarm 147 bl mi_startup 148 149 /* We should not get here */ --- 20 unchanged lines hidden (view full) --- 170 171 /* Drop to EL1 */ 172 bl drop_to_el1 173 174 /* Set the context id */ 175 msr contextidr_el1, x1 176 177 /* Load the kernel page table */ | 135 136 /* trace back starts here */ 137 mov fp, #0 138 /* Branch to C code */ 139 bl initarm 140 bl mi_startup 141 142 /* We should not get here */ --- 20 unchanged lines hidden (view full) --- 163 164 /* Drop to EL1 */ 165 bl drop_to_el1 166 167 /* Set the context id */ 168 msr contextidr_el1, x1 169 170 /* Load the kernel page table */ |
| 178 adr x26, pagetable_l1_ttbr1 | 171 adr x24, pagetable_l0_ttbr1 |
| 179 /* Load the identity page table */ 180 adr x27, pagetable_l0_ttbr0 181 182 /* Enable the mmu */ 183 bl start_mmu 184 185 /* Jump to the virtual address space */ 186 ldr x15, =mp_virtdone 187 br x15 188 189mp_virtdone: | 172 /* Load the identity page table */ 173 adr x27, pagetable_l0_ttbr0 174 175 /* Enable the mmu */ 176 bl start_mmu 177 178 /* Jump to the virtual address space */ 179 ldr x15, =mp_virtdone 180 br x15 181 182mp_virtdone: |
| 190 /* 191 * Now that we are in virtual address space, 192 * we don't need the identity mapping in TTBR0 and 193 * can set the TCR to a more useful value. 194 */ 195 ldr x2, tcr 196 mrs x3, id_aa64mmfr0_el1 197 bfi x2, x3, #32, #3 198 msr tcr_el1, x2 199 | |
| 200 ldr x4, =secondary_stacks 201 mov x5, #(PAGE_SIZE * KSTACK_PAGES) 202 mul x5, x0, x5 203 add sp, x4, x5 204 205 b init_secondary 206END(mpentry) 207#endif --- 175 unchanged lines hidden (view full) --- 383 /* Move to the l1 table */ 384 add x26, x26, #PAGE_SIZE 385 386 /* Link the l1 -> l2 table */ 387 mov x9, x6 388 mov x6, x26 389 bl link_l1_pagetable 390 | 183 ldr x4, =secondary_stacks 184 mov x5, #(PAGE_SIZE * KSTACK_PAGES) 185 mul x5, x0, x5 186 add sp, x4, x5 187 188 b init_secondary 189END(mpentry) 190#endif --- 175 unchanged lines hidden (view full) --- 366 /* Move to the l1 table */ 367 add x26, x26, #PAGE_SIZE 368 369 /* Link the l1 -> l2 table */ 370 mov x9, x6 371 mov x6, x26 372 bl link_l1_pagetable 373 |
| 374 /* Move to the l0 table */ 375 add x24, x26, #PAGE_SIZE |
|
| 391 | 376 |
| 377 /* Link the l0 -> l1 table */ 378 mov x9, x6 379 mov x6, x24 380 bl link_l0_pagetable 381 |
|
| 392 /* 393 * Build the TTBR0 maps. 394 */ | 382 /* 383 * Build the TTBR0 maps. 384 */ |
| 395 add x27, x26, #PAGE_SIZE | 385 add x27, x24, #PAGE_SIZE |
| 396 397 mov x6, x27 /* The initial page table */ 398#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 399 /* Create a table for the UART */ 400 mov x7, #DEVICE_MEM 401 mov x8, #(SOCDEV_VA) /* VA start */ 402 mov x9, #(SOCDEV_PA) /* PA start */ 403 mov x10, #1 --- 31 unchanged lines hidden (view full) --- 435 * x11, x12 and x13 are trashed 436 */ 437link_l0_pagetable: 438 /* 439 * Link an L0 -> L1 table entry. 440 */ 441 /* Find the table index */ 442 lsr x11, x8, #L0_SHIFT | 386 387 mov x6, x27 /* The initial page table */ 388#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 389 /* Create a table for the UART */ 390 mov x7, #DEVICE_MEM 391 mov x8, #(SOCDEV_VA) /* VA start */ 392 mov x9, #(SOCDEV_PA) /* PA start */ 393 mov x10, #1 --- 31 unchanged lines hidden (view full) --- 425 * x11, x12 and x13 are trashed 426 */ 427link_l0_pagetable: 428 /* 429 * Link an L0 -> L1 table entry. 430 */ 431 /* Find the table index */ 432 lsr x11, x8, #L0_SHIFT |
| 443 and x11, x11, #Ln_ADDR_MASK | 433 and x11, x11, #L0_ADDR_MASK |
| 444 445 /* Build the L0 block entry */ 446 mov x12, #L0_TABLE 447 448 /* Only use the output address bits */ 449 lsr x9, x9, #12 450 orr x12, x12, x9, lsl #12 451 --- 125 unchanged lines hidden (view full) --- 577 dsb sy 578 579 /* Load the exception vectors */ 580 ldr x2, =exception_vectors 581 msr vbar_el1, x2 582 583 /* Load ttbr0 and ttbr1 */ 584 msr ttbr0_el1, x27 | 434 435 /* Build the L0 block entry */ 436 mov x12, #L0_TABLE 437 438 /* Only use the output address bits */ 439 lsr x9, x9, #12 440 orr x12, x12, x9, lsl #12 441 --- 125 unchanged lines hidden (view full) --- 567 dsb sy 568 569 /* Load the exception vectors */ 570 ldr x2, =exception_vectors 571 msr vbar_el1, x2 572 573 /* Load ttbr0 and ttbr1 */ 574 msr ttbr0_el1, x27 |
| 585 msr ttbr1_el1, x26 | 575 msr ttbr1_el1, x24 |
| 586 isb 587 588 /* Clear the Monitor Debug System control register */ 589 msr mdscr_el1, xzr 590 591 /* Invalidate the TLB */ 592 tlbi vmalle1is 593 594 ldr x2, mair 595 msr mair_el1, x2 596 597 /* 598 * Setup TCR according to PARange bits from ID_AA64MMFR0_EL1. | 576 isb 577 578 /* Clear the Monitor Debug System control register */ 579 msr mdscr_el1, xzr 580 581 /* Invalidate the TLB */ 582 tlbi vmalle1is 583 584 ldr x2, mair 585 msr mair_el1, x2 586 587 /* 588 * Setup TCR according to PARange bits from ID_AA64MMFR0_EL1. |
| 599 * Some machines have physical memory mapped >512GiB, which can not 600 * be identity-mapped using the default 39 VA bits. Thus, use 601 * 48 VA bits for now and switch back to 39 after the VA jump. | |
| 602 */ | 589 */ |
| 603 ldr x2, tcr_early | 590 ldr x2, tcr |
| 604 mrs x3, id_aa64mmfr0_el1 605 bfi x2, x3, #32, #3 606 msr tcr_el1, x2 607 608 /* Setup SCTLR */ 609 ldr x2, sctlr_set 610 ldr x3, sctlr_clear 611 mrs x1, sctlr_el1 --- 6 unchanged lines hidden (view full) --- 618 619 .align 3 620mair: 621 /* Device Normal, no cache Normal, write-back */ 622 .quad MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2) 623tcr: 624 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K | \ 625 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) | 591 mrs x3, id_aa64mmfr0_el1 592 bfi x2, x3, #32, #3 593 msr tcr_el1, x2 594 595 /* Setup SCTLR */ 596 ldr x2, sctlr_set 597 ldr x3, sctlr_clear 598 mrs x1, sctlr_el1 --- 6 unchanged lines hidden (view full) --- 605 606 .align 3 607mair: 608 /* Device Normal, no cache Normal, write-back */ 609 .quad MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2) 610tcr: 611 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K | \ 612 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) |
| 626tcr_early: 627 .quad (TCR_T1SZ(64 - VIRT_BITS) | TCR_T0SZ(64 - 48) | \ 628 TCR_ASID_16 | TCR_TG1_4K | TCR_CACHE_ATTRS | TCR_SMP_ATTRS) | |
| 629sctlr_set: 630 /* Bits to set */ 631 .quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 632 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | SCTLR_M) 633sctlr_clear: 634 /* Bits to clear */ 635 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_WXN | SCTLR_UMA | SCTLR_ITD | \ 636 SCTLR_THEE | SCTLR_CP15BEN | SCTLR_A) --- 9 unchanged lines hidden (view full) --- 646 * L2 for kernel (High addresses) 647 * L1 for kernel 648 * L1 for user (Low addresses) 649 */ 650pagetable: 651 .space PAGE_SIZE 652pagetable_l1_ttbr1: 653 .space PAGE_SIZE | 613sctlr_set: 614 /* Bits to set */ 615 .quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 616 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | SCTLR_M) 617sctlr_clear: 618 /* Bits to clear */ 619 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_WXN | SCTLR_UMA | SCTLR_ITD | \ 620 SCTLR_THEE | SCTLR_CP15BEN | SCTLR_A) --- 9 unchanged lines hidden (view full) --- 630 * L2 for kernel (High addresses) 631 * L1 for kernel 632 * L1 for user (Low addresses) 633 */ 634pagetable: 635 .space PAGE_SIZE 636pagetable_l1_ttbr1: 637 .space PAGE_SIZE |
| 638pagetable_l0_ttbr1: 639 .space PAGE_SIZE |
|
| 654pagetable_l1_ttbr0: 655 .space PAGE_SIZE 656pagetable_l0_ttbr0: 657 .space PAGE_SIZE 658pagetable_end: 659 660el2_pagetable: 661 .space PAGE_SIZE --- 34 unchanged lines hidden --- | 640pagetable_l1_ttbr0: 641 .space PAGE_SIZE 642pagetable_l0_ttbr0: 643 .space PAGE_SIZE 644pagetable_end: 645 646el2_pagetable: 647 .space PAGE_SIZE --- 34 unchanged lines hidden --- |