swtch.S revision 286225
1281494Sandrew/*- 2281494Sandrew * Copyright (c) 2014 Andrew Turner 3281494Sandrew * Copyright (c) 2014 The FreeBSD Foundation 4281494Sandrew * All rights reserved. 5281494Sandrew * 6281494Sandrew * This software was developed by Andrew Turner under sponsorship from 7281494Sandrew * the FreeBSD Foundation. 8281494Sandrew * 9281494Sandrew * Redistribution and use in source and binary forms, with or without 10281494Sandrew * modification, are permitted provided that the following conditions 11281494Sandrew * are met: 12281494Sandrew * 1. Redistributions of source code must retain the above copyright 13281494Sandrew * notice, this list of conditions and the following disclaimer. 14281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 15281494Sandrew * notice, this list of conditions and the following disclaimer in the 16281494Sandrew * documentation and/or other materials provided with the distribution. 17281494Sandrew * 18281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28281494Sandrew * SUCH DAMAGE. 29281494Sandrew * 30281494Sandrew */ 31281494Sandrew 32281494Sandrew#include "assym.s" 33285627Szbb#include "opt_kstack_pages.h" 34285316Sandrew#include "opt_sched.h" 35281494Sandrew 36281494Sandrew#include <machine/asm.h> 37281494Sandrew 38281494Sandrew__FBSDID("$FreeBSD: head/sys/arm64/arm64/swtch.S 286225 2015-08-03 11:05:02Z andrew $"); 39281494Sandrew 40281494Sandrew/* 41281494Sandrew * void cpu_throw(struct thread *old, struct thread *new) 42281494Sandrew */ 43281494SandrewENTRY(cpu_throw) 44281494Sandrew#ifdef VFP 45281494Sandrew /* Backup the new thread pointer around a call to C code */ 46281494Sandrew mov x19, x1 47281494Sandrew bl vfp_discard 48281494Sandrew mov x1, x19 49281494Sandrew#endif 50281494Sandrew 51281494Sandrew /* Store the new curthread */ 52281494Sandrew str x1, [x18, #PC_CURTHREAD] 53281494Sandrew /* And the new pcb */ 54281494Sandrew ldr x4, [x1, #TD_PCB] 55281494Sandrew str x4, [x18, #PC_CURPCB] 56281494Sandrew 57281494Sandrew /* 58281494Sandrew * TODO: We may need to flush the cache here. 59281494Sandrew */ 60281494Sandrew 61281494Sandrew /* Switch to the new pmap */ 62281494Sandrew ldr x5, [x4, #PCB_L1ADDR] 63281494Sandrew msr ttbr0_el1, x5 64281494Sandrew isb 65281494Sandrew 66281494Sandrew /* Invalidate the TLB */ 67281494Sandrew dsb sy 68281494Sandrew tlbi vmalle1is 69281494Sandrew dsb sy 70281494Sandrew isb 71281494Sandrew 72281494Sandrew /* Restore the registers */ 73281494Sandrew ldp x5, x6, [x4, #PCB_SP] 74281494Sandrew mov sp, x5 75281494Sandrew msr tpidr_el0, x6 76281494Sandrew ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 77281494Sandrew ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 78281494Sandrew ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 79281494Sandrew ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 80281494Sandrew ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 81281494Sandrew ldr x19, [x4, #PCB_REGS + 19 * 8] 82281494Sandrew ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 83281494Sandrew ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 84281494Sandrew ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 85281494Sandrew ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 86281494Sandrew ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 87281494Sandrew ldr x30, [x4, #PCB_REGS + 30 * 8] 88281494Sandrew 89281494Sandrew ret 90281494SandrewEND(cpu_throw) 91281494Sandrew 92281494Sandrew/* 93281494Sandrew * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 94281494Sandrew * 95281494Sandrew * x0 = old 96281494Sandrew * x1 = new 97281494Sandrew * x2 = mtx 98281494Sandrew * x3 to x7, x16 and x17 are caller saved 99281494Sandrew */ 100281494SandrewENTRY(cpu_switch) 101281494Sandrew /* Store the new curthread */ 102281494Sandrew str x1, [x18, #PC_CURTHREAD] 103281494Sandrew /* And the new pcb */ 104281494Sandrew ldr x4, [x1, #TD_PCB] 105281494Sandrew str x4, [x18, #PC_CURPCB] 106281494Sandrew 107281494Sandrew /* 108281494Sandrew * Save the old context. 109281494Sandrew */ 110281494Sandrew ldr x4, [x0, #TD_PCB] 111281494Sandrew 112281494Sandrew /* Store the callee-saved registers */ 113281494Sandrew stp x8, x9, [x4, #PCB_REGS + 8 * 8] 114281494Sandrew stp x10, x11, [x4, #PCB_REGS + 10 * 8] 115281494Sandrew stp x12, x13, [x4, #PCB_REGS + 12 * 8] 116281494Sandrew stp x14, x15, [x4, #PCB_REGS + 14 * 8] 117281494Sandrew stp x16, x17, [x4, #PCB_REGS + 16 * 8] 118281494Sandrew stp x18, x19, [x4, #PCB_REGS + 18 * 8] 119281494Sandrew stp x20, x21, [x4, #PCB_REGS + 20 * 8] 120281494Sandrew stp x22, x23, [x4, #PCB_REGS + 22 * 8] 121281494Sandrew stp x24, x25, [x4, #PCB_REGS + 24 * 8] 122281494Sandrew stp x26, x27, [x4, #PCB_REGS + 26 * 8] 123281494Sandrew stp x28, x29, [x4, #PCB_REGS + 28 * 8] 124281494Sandrew str x30, [x4, #PCB_REGS + 30 * 8] 125281494Sandrew /* And the old stack pointer */ 126281494Sandrew mov x5, sp 127281494Sandrew mrs x6, tpidr_el0 128281494Sandrew stp x5, x6, [x4, #PCB_SP] 129281494Sandrew 130281494Sandrew#ifdef VFP 131281494Sandrew mov x19, x0 132281494Sandrew mov x20, x1 133281494Sandrew mov x21, x2 134286225Sandrew /* Load the pcb address */ 135286225Sandrew mov x1, x4 136281494Sandrew bl vfp_save_state 137281494Sandrew mov x2, x21 138281494Sandrew mov x1, x20 139281494Sandrew mov x0, x19 140281494Sandrew#endif 141281494Sandrew 142281494Sandrew /* 143281494Sandrew * Restore the saved context. 144281494Sandrew */ 145281494Sandrew ldr x4, [x1, #TD_PCB] 146281494Sandrew 147281494Sandrew /* 148281494Sandrew * TODO: We may need to flush the cache here if switching 149281494Sandrew * to a user process. 150281494Sandrew */ 151281494Sandrew 152281494Sandrew /* Switch to the new pmap */ 153281494Sandrew ldr x5, [x4, #PCB_L1ADDR] 154281494Sandrew msr ttbr0_el1, x5 155281494Sandrew isb 156281494Sandrew 157281494Sandrew /* Invalidate the TLB */ 158281494Sandrew dsb sy 159281494Sandrew tlbi vmalle1is 160281494Sandrew dsb sy 161281494Sandrew isb 162281494Sandrew 163281494Sandrew /* Release the old thread */ 164281494Sandrew str x2, [x0, #TD_LOCK] 165281494Sandrew#if defined(SCHED_ULE) && defined(SMP) 166285316Sandrew /* Read the value in blocked_lock */ 167285316Sandrew ldr x0, =_C_LABEL(blocked_lock) 168285316Sandrew ldr x1, [x0] 169285316Sandrew /* Load curthread */ 170285316Sandrew ldr x2, [x18, #PC_CURTHREAD] 171285316Sandrew1: 172285316Sandrew ldr x3, [x2, #TD_LOCK] 173285316Sandrew cmp x3, x1 174285316Sandrew b.eq 1b 175281494Sandrew#endif 176281494Sandrew 177281494Sandrew /* Restore the registers */ 178281494Sandrew ldp x5, x6, [x4, #PCB_SP] 179281494Sandrew mov sp, x5 180281494Sandrew msr tpidr_el0, x6 181281494Sandrew ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 182281494Sandrew ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 183281494Sandrew ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 184281494Sandrew ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 185281494Sandrew ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 186281494Sandrew ldr x19, [x4, #PCB_REGS + 19 * 8] 187281494Sandrew ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 188281494Sandrew ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 189281494Sandrew ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 190281494Sandrew ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 191281494Sandrew ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 192281494Sandrew ldr x30, [x4, #PCB_REGS + 30 * 8] 193281494Sandrew 194281494Sandrew str xzr, [x4, #PCB_REGS + 18 * 8] 195281494Sandrew ret 196281494Sandrew.Lcpu_switch_panic_str: 197281494Sandrew .asciz "cpu_switch: %p\0" 198281494SandrewEND(cpu_switch) 199281494Sandrew 200281494SandrewENTRY(fork_trampoline) 201281494Sandrew mov x0, x8 202281494Sandrew mov x1, x9 203281494Sandrew mov x2, sp 204281494Sandrew mov fp, #0 /* Stack traceback stops here. */ 205281494Sandrew bl _C_LABEL(fork_exit) 206281494Sandrew 207281494Sandrew /* Restore sp and lr */ 208281494Sandrew ldp x0, x1, [sp] 209281494Sandrew msr sp_el0, x0 210281494Sandrew mov lr, x1 211281494Sandrew 212281494Sandrew /* Restore the registers other than x0 and x1 */ 213281494Sandrew ldp x2, x3, [sp, #TF_X + 2 * 8] 214281494Sandrew ldp x4, x5, [sp, #TF_X + 4 * 8] 215281494Sandrew ldp x6, x7, [sp, #TF_X + 6 * 8] 216281494Sandrew ldp x8, x9, [sp, #TF_X + 8 * 8] 217281494Sandrew ldp x10, x11, [sp, #TF_X + 10 * 8] 218281494Sandrew ldp x12, x13, [sp, #TF_X + 12 * 8] 219281494Sandrew ldp x14, x15, [sp, #TF_X + 14 * 8] 220281494Sandrew ldp x16, x17, [sp, #TF_X + 16 * 8] 221281494Sandrew ldr x19, [sp, #TF_X + 19 * 8] 222281494Sandrew ldp x20, x21, [sp, #TF_X + 20 * 8] 223281494Sandrew ldp x22, x23, [sp, #TF_X + 22 * 8] 224281494Sandrew ldp x24, x25, [sp, #TF_X + 24 * 8] 225281494Sandrew ldp x26, x27, [sp, #TF_X + 26 * 8] 226281494Sandrew ldp x28, x29, [sp, #TF_X + 28 * 8] 227281494Sandrew /* Skip x30 as it was restored above as lr */ 228281494Sandrew 229281494Sandrew /* 230281494Sandrew * Disable interrupts to avoid 231281494Sandrew * overwriting spsr_el1 by an IRQ exception. 232281494Sandrew */ 233281494Sandrew msr daifset, #2 234281494Sandrew 235281494Sandrew /* Restore elr and spsr */ 236281494Sandrew ldp x0, x1, [sp, #16] 237281494Sandrew msr elr_el1, x0 238281494Sandrew msr spsr_el1, x1 239281494Sandrew 240281494Sandrew /* Finally x0 and x1 */ 241281494Sandrew ldp x0, x1, [sp, #TF_X + 0 * 8] 242281494Sandrew ldr x18, [sp, #TF_X + 18 * 8] 243281494Sandrew 244281494Sandrew /* 245281494Sandrew * No need for interrupts reenabling since PSR 246281494Sandrew * will be set to the desired value anyway. 247281494Sandrew */ 248281494Sandrew eret 249281494Sandrew 250281494SandrewEND(fork_trampoline) 251281494Sandrew 252281494SandrewENTRY(savectx) 253285271Sandrew /* Store the callee-saved registers */ 254285271Sandrew stp x8, x9, [x0, #PCB_REGS + 8 * 8] 255285271Sandrew stp x10, x11, [x0, #PCB_REGS + 10 * 8] 256285271Sandrew stp x12, x13, [x0, #PCB_REGS + 12 * 8] 257285271Sandrew stp x14, x15, [x0, #PCB_REGS + 14 * 8] 258285271Sandrew stp x16, x17, [x0, #PCB_REGS + 16 * 8] 259285271Sandrew stp x18, x19, [x0, #PCB_REGS + 18 * 8] 260285271Sandrew stp x20, x21, [x0, #PCB_REGS + 20 * 8] 261285271Sandrew stp x22, x23, [x0, #PCB_REGS + 22 * 8] 262285271Sandrew stp x24, x25, [x0, #PCB_REGS + 24 * 8] 263285271Sandrew stp x26, x27, [x0, #PCB_REGS + 26 * 8] 264285271Sandrew stp x28, x29, [x0, #PCB_REGS + 28 * 8] 265285271Sandrew str x30, [x0, #PCB_REGS + 30 * 8] 266285271Sandrew /* And the old stack pointer */ 267285271Sandrew mov x5, sp 268285271Sandrew mrs x6, tpidr_el0 269285271Sandrew stp x5, x6, [x0, #PCB_SP] 270285271Sandrew 271285271Sandrew /* Store the VFP registers */ 272285271Sandrew#ifdef VFP 273286225Sandrew mov x28, lr 274286225Sandrew mov x1, x0 /* move pcb to the correct register */ 275286225Sandrew mov x0, xzr /* td = NULL */ 276285271Sandrew bl vfp_save_state 277286225Sandrew mov lr, x28 278285271Sandrew#endif 279285271Sandrew 280281494Sandrew ret 281281494SandrewEND(savectx) 282281494Sandrew 283