swtch.S revision 295142
1281494Sandrew/*- 2281494Sandrew * Copyright (c) 2014 Andrew Turner 3281494Sandrew * Copyright (c) 2014 The FreeBSD Foundation 4281494Sandrew * All rights reserved. 5281494Sandrew * 6281494Sandrew * This software was developed by Andrew Turner under sponsorship from 7281494Sandrew * the FreeBSD Foundation. 8281494Sandrew * 9281494Sandrew * Redistribution and use in source and binary forms, with or without 10281494Sandrew * modification, are permitted provided that the following conditions 11281494Sandrew * are met: 12281494Sandrew * 1. Redistributions of source code must retain the above copyright 13281494Sandrew * notice, this list of conditions and the following disclaimer. 14281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 15281494Sandrew * notice, this list of conditions and the following disclaimer in the 16281494Sandrew * documentation and/or other materials provided with the distribution. 17281494Sandrew * 18281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28281494Sandrew * SUCH DAMAGE. 29281494Sandrew * 30281494Sandrew */ 31281494Sandrew 32281494Sandrew#include "assym.s" 33285627Szbb#include "opt_kstack_pages.h" 34285316Sandrew#include "opt_sched.h" 35281494Sandrew 36281494Sandrew#include <machine/asm.h> 37281494Sandrew 38281494Sandrew__FBSDID("$FreeBSD: head/sys/arm64/arm64/swtch.S 295142 2016-02-02 10:28:56Z andrew $"); 39281494Sandrew 40295142Sandrew.macro clear_step_flag pcbflags, tmp 41295142Sandrew tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 42295142Sandrew mrs \tmp, mdscr_el1 43295142Sandrew bic \tmp, \tmp, #1 44295142Sandrew msr mdscr_el1, \tmp 45295142Sandrew isb 46295142Sandrew999: 47295142Sandrew.endm 48295142Sandrew 49295142Sandrew.macro set_step_flag pcbflags, tmp 50295142Sandrew tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 51295142Sandrew mrs \tmp, mdscr_el1 52295142Sandrew orr \tmp, \tmp, #1 53295142Sandrew msr mdscr_el1, \tmp 54295142Sandrew isb 55295142Sandrew999: 56295142Sandrew.endm 57295142Sandrew 58281494Sandrew/* 59281494Sandrew * void cpu_throw(struct thread *old, struct thread *new) 60281494Sandrew */ 61281494SandrewENTRY(cpu_throw) 62295142Sandrew /* Of old == NULL skip disabling stepping */ 63295142Sandrew cbz x0, 1f 64295142Sandrew 65295142Sandrew /* If we were single stepping, disable it */ 66295142Sandrew ldr x4, [x0, #TD_PCB] 67295142Sandrew ldr w5, [x4, #PCB_FLAGS] 68295142Sandrew clear_step_flag w5, x6 69295142Sandrew1: 70295142Sandrew 71281494Sandrew#ifdef VFP 72281494Sandrew /* Backup the new thread pointer around a call to C code */ 73281494Sandrew mov x19, x1 74281494Sandrew bl vfp_discard 75281494Sandrew mov x1, x19 76281494Sandrew#endif 77281494Sandrew 78281494Sandrew /* Store the new curthread */ 79281494Sandrew str x1, [x18, #PC_CURTHREAD] 80281494Sandrew /* And the new pcb */ 81281494Sandrew ldr x4, [x1, #TD_PCB] 82281494Sandrew str x4, [x18, #PC_CURPCB] 83281494Sandrew 84281494Sandrew /* 85281494Sandrew * TODO: We may need to flush the cache here. 86281494Sandrew */ 87281494Sandrew 88281494Sandrew /* Switch to the new pmap */ 89281494Sandrew ldr x5, [x4, #PCB_L1ADDR] 90281494Sandrew msr ttbr0_el1, x5 91281494Sandrew isb 92281494Sandrew 93281494Sandrew /* Invalidate the TLB */ 94281494Sandrew dsb sy 95281494Sandrew tlbi vmalle1is 96281494Sandrew dsb sy 97281494Sandrew isb 98281494Sandrew 99295142Sandrew /* If we are single stepping, enable it */ 100295142Sandrew ldr w5, [x4, #PCB_FLAGS] 101295142Sandrew set_step_flag w5, x6 102295142Sandrew 103281494Sandrew /* Restore the registers */ 104281494Sandrew ldp x5, x6, [x4, #PCB_SP] 105281494Sandrew mov sp, x5 106281494Sandrew msr tpidr_el0, x6 107281494Sandrew ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 108281494Sandrew ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 109281494Sandrew ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 110281494Sandrew ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 111281494Sandrew ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 112281494Sandrew ldr x19, [x4, #PCB_REGS + 19 * 8] 113281494Sandrew ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 114281494Sandrew ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 115281494Sandrew ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 116281494Sandrew ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 117281494Sandrew ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 118281494Sandrew ldr x30, [x4, #PCB_REGS + 30 * 8] 119281494Sandrew 120281494Sandrew ret 121281494SandrewEND(cpu_throw) 122281494Sandrew 123281494Sandrew/* 124281494Sandrew * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 125281494Sandrew * 126281494Sandrew * x0 = old 127281494Sandrew * x1 = new 128281494Sandrew * x2 = mtx 129281494Sandrew * x3 to x7, x16 and x17 are caller saved 130281494Sandrew */ 131281494SandrewENTRY(cpu_switch) 132281494Sandrew /* Store the new curthread */ 133281494Sandrew str x1, [x18, #PC_CURTHREAD] 134281494Sandrew /* And the new pcb */ 135281494Sandrew ldr x4, [x1, #TD_PCB] 136281494Sandrew str x4, [x18, #PC_CURPCB] 137281494Sandrew 138281494Sandrew /* 139281494Sandrew * Save the old context. 140281494Sandrew */ 141281494Sandrew ldr x4, [x0, #TD_PCB] 142281494Sandrew 143281494Sandrew /* Store the callee-saved registers */ 144281494Sandrew stp x8, x9, [x4, #PCB_REGS + 8 * 8] 145281494Sandrew stp x10, x11, [x4, #PCB_REGS + 10 * 8] 146281494Sandrew stp x12, x13, [x4, #PCB_REGS + 12 * 8] 147281494Sandrew stp x14, x15, [x4, #PCB_REGS + 14 * 8] 148281494Sandrew stp x16, x17, [x4, #PCB_REGS + 16 * 8] 149281494Sandrew stp x18, x19, [x4, #PCB_REGS + 18 * 8] 150281494Sandrew stp x20, x21, [x4, #PCB_REGS + 20 * 8] 151281494Sandrew stp x22, x23, [x4, #PCB_REGS + 22 * 8] 152281494Sandrew stp x24, x25, [x4, #PCB_REGS + 24 * 8] 153281494Sandrew stp x26, x27, [x4, #PCB_REGS + 26 * 8] 154281494Sandrew stp x28, x29, [x4, #PCB_REGS + 28 * 8] 155281494Sandrew str x30, [x4, #PCB_REGS + 30 * 8] 156281494Sandrew /* And the old stack pointer */ 157281494Sandrew mov x5, sp 158281494Sandrew mrs x6, tpidr_el0 159281494Sandrew stp x5, x6, [x4, #PCB_SP] 160281494Sandrew 161295142Sandrew /* If we were single stepping, disable it */ 162295142Sandrew ldr w5, [x4, #PCB_FLAGS] 163295142Sandrew clear_step_flag w5, x6 164295142Sandrew 165281494Sandrew#ifdef VFP 166281494Sandrew mov x19, x0 167281494Sandrew mov x20, x1 168281494Sandrew mov x21, x2 169286225Sandrew /* Load the pcb address */ 170286225Sandrew mov x1, x4 171281494Sandrew bl vfp_save_state 172281494Sandrew mov x2, x21 173281494Sandrew mov x1, x20 174281494Sandrew mov x0, x19 175281494Sandrew#endif 176281494Sandrew 177281494Sandrew /* 178281494Sandrew * Restore the saved context. 179281494Sandrew */ 180281494Sandrew ldr x4, [x1, #TD_PCB] 181281494Sandrew 182281494Sandrew /* 183281494Sandrew * TODO: We may need to flush the cache here if switching 184281494Sandrew * to a user process. 185281494Sandrew */ 186281494Sandrew 187281494Sandrew /* Switch to the new pmap */ 188281494Sandrew ldr x5, [x4, #PCB_L1ADDR] 189281494Sandrew msr ttbr0_el1, x5 190281494Sandrew isb 191281494Sandrew 192281494Sandrew /* Invalidate the TLB */ 193281494Sandrew dsb sy 194281494Sandrew tlbi vmalle1is 195281494Sandrew dsb sy 196281494Sandrew isb 197281494Sandrew 198287536Sandrew /* 199287536Sandrew * Release the old thread. This doesn't need to be a store-release 200287536Sandrew * as the above dsb instruction will provide release semantics. 201287536Sandrew */ 202281494Sandrew str x2, [x0, #TD_LOCK] 203281494Sandrew#if defined(SCHED_ULE) && defined(SMP) 204294979Swma /* Spin if TD_LOCK points to a blocked_lock */ 205294979Swma ldr x2, =_C_LABEL(blocked_lock) 206285316Sandrew1: 207287536Sandrew ldar x3, [x1, #TD_LOCK] 208287536Sandrew cmp x3, x2 209285316Sandrew b.eq 1b 210281494Sandrew#endif 211281494Sandrew 212295142Sandrew /* If we are single stepping, enable it */ 213295142Sandrew ldr w5, [x4, #PCB_FLAGS] 214295142Sandrew set_step_flag w5, x6 215295142Sandrew 216281494Sandrew /* Restore the registers */ 217281494Sandrew ldp x5, x6, [x4, #PCB_SP] 218281494Sandrew mov sp, x5 219281494Sandrew msr tpidr_el0, x6 220281494Sandrew ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 221281494Sandrew ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 222281494Sandrew ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 223281494Sandrew ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 224281494Sandrew ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 225281494Sandrew ldr x19, [x4, #PCB_REGS + 19 * 8] 226281494Sandrew ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 227281494Sandrew ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 228281494Sandrew ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 229281494Sandrew ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 230281494Sandrew ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 231281494Sandrew ldr x30, [x4, #PCB_REGS + 30 * 8] 232281494Sandrew 233281494Sandrew str xzr, [x4, #PCB_REGS + 18 * 8] 234281494Sandrew ret 235281494Sandrew.Lcpu_switch_panic_str: 236281494Sandrew .asciz "cpu_switch: %p\0" 237281494SandrewEND(cpu_switch) 238281494Sandrew 239281494SandrewENTRY(fork_trampoline) 240281494Sandrew mov x0, x8 241281494Sandrew mov x1, x9 242281494Sandrew mov x2, sp 243281494Sandrew mov fp, #0 /* Stack traceback stops here. */ 244281494Sandrew bl _C_LABEL(fork_exit) 245281494Sandrew 246281494Sandrew /* Restore sp and lr */ 247281494Sandrew ldp x0, x1, [sp] 248281494Sandrew msr sp_el0, x0 249281494Sandrew mov lr, x1 250281494Sandrew 251281494Sandrew /* Restore the registers other than x0 and x1 */ 252281494Sandrew ldp x2, x3, [sp, #TF_X + 2 * 8] 253281494Sandrew ldp x4, x5, [sp, #TF_X + 4 * 8] 254281494Sandrew ldp x6, x7, [sp, #TF_X + 6 * 8] 255281494Sandrew ldp x8, x9, [sp, #TF_X + 8 * 8] 256281494Sandrew ldp x10, x11, [sp, #TF_X + 10 * 8] 257281494Sandrew ldp x12, x13, [sp, #TF_X + 12 * 8] 258281494Sandrew ldp x14, x15, [sp, #TF_X + 14 * 8] 259281494Sandrew ldp x16, x17, [sp, #TF_X + 16 * 8] 260281494Sandrew ldr x19, [sp, #TF_X + 19 * 8] 261281494Sandrew ldp x20, x21, [sp, #TF_X + 20 * 8] 262281494Sandrew ldp x22, x23, [sp, #TF_X + 22 * 8] 263281494Sandrew ldp x24, x25, [sp, #TF_X + 24 * 8] 264281494Sandrew ldp x26, x27, [sp, #TF_X + 26 * 8] 265281494Sandrew ldp x28, x29, [sp, #TF_X + 28 * 8] 266281494Sandrew /* Skip x30 as it was restored above as lr */ 267281494Sandrew 268281494Sandrew /* 269281494Sandrew * Disable interrupts to avoid 270281494Sandrew * overwriting spsr_el1 by an IRQ exception. 271281494Sandrew */ 272281494Sandrew msr daifset, #2 273281494Sandrew 274281494Sandrew /* Restore elr and spsr */ 275281494Sandrew ldp x0, x1, [sp, #16] 276281494Sandrew msr elr_el1, x0 277281494Sandrew msr spsr_el1, x1 278281494Sandrew 279281494Sandrew /* Finally x0 and x1 */ 280281494Sandrew ldp x0, x1, [sp, #TF_X + 0 * 8] 281281494Sandrew ldr x18, [sp, #TF_X + 18 * 8] 282281494Sandrew 283281494Sandrew /* 284281494Sandrew * No need for interrupts reenabling since PSR 285281494Sandrew * will be set to the desired value anyway. 286281494Sandrew */ 287281494Sandrew eret 288281494Sandrew 289281494SandrewEND(fork_trampoline) 290281494Sandrew 291281494SandrewENTRY(savectx) 292285271Sandrew /* Store the callee-saved registers */ 293285271Sandrew stp x8, x9, [x0, #PCB_REGS + 8 * 8] 294285271Sandrew stp x10, x11, [x0, #PCB_REGS + 10 * 8] 295285271Sandrew stp x12, x13, [x0, #PCB_REGS + 12 * 8] 296285271Sandrew stp x14, x15, [x0, #PCB_REGS + 14 * 8] 297285271Sandrew stp x16, x17, [x0, #PCB_REGS + 16 * 8] 298285271Sandrew stp x18, x19, [x0, #PCB_REGS + 18 * 8] 299285271Sandrew stp x20, x21, [x0, #PCB_REGS + 20 * 8] 300285271Sandrew stp x22, x23, [x0, #PCB_REGS + 22 * 8] 301285271Sandrew stp x24, x25, [x0, #PCB_REGS + 24 * 8] 302285271Sandrew stp x26, x27, [x0, #PCB_REGS + 26 * 8] 303285271Sandrew stp x28, x29, [x0, #PCB_REGS + 28 * 8] 304285271Sandrew str x30, [x0, #PCB_REGS + 30 * 8] 305285271Sandrew /* And the old stack pointer */ 306285271Sandrew mov x5, sp 307285271Sandrew mrs x6, tpidr_el0 308285271Sandrew stp x5, x6, [x0, #PCB_SP] 309285271Sandrew 310285271Sandrew /* Store the VFP registers */ 311285271Sandrew#ifdef VFP 312286225Sandrew mov x28, lr 313286225Sandrew mov x1, x0 /* move pcb to the correct register */ 314286225Sandrew mov x0, xzr /* td = NULL */ 315285271Sandrew bl vfp_save_state 316286225Sandrew mov lr, x28 317285271Sandrew#endif 318285271Sandrew 319281494Sandrew ret 320281494SandrewEND(savectx) 321281494Sandrew 322