1281494Sandrew/*- 2281494Sandrew * Copyright (c) 2014 Andrew Turner 3281494Sandrew * Copyright (c) 2014 The FreeBSD Foundation 4281494Sandrew * All rights reserved. 5281494Sandrew * 6281494Sandrew * This software was developed by Andrew Turner under sponsorship from 7281494Sandrew * the FreeBSD Foundation. 8281494Sandrew * 9281494Sandrew * Redistribution and use in source and binary forms, with or without 10281494Sandrew * modification, are permitted provided that the following conditions 11281494Sandrew * are met: 12281494Sandrew * 1. Redistributions of source code must retain the above copyright 13281494Sandrew * notice, this list of conditions and the following disclaimer. 14281494Sandrew * 2. Redistributions in binary form must reproduce the above copyright 15281494Sandrew * notice, this list of conditions and the following disclaimer in the 16281494Sandrew * documentation and/or other materials provided with the distribution. 17281494Sandrew * 18281494Sandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19281494Sandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20281494Sandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21281494Sandrew * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22281494Sandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23281494Sandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24281494Sandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25281494Sandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26281494Sandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27281494Sandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28281494Sandrew * SUCH DAMAGE. 29281494Sandrew * 30281494Sandrew */ 31281494Sandrew 32281494Sandrew#include "assym.s" 33285627Szbb#include "opt_kstack_pages.h" 34285316Sandrew#include "opt_sched.h" 35281494Sandrew 36281494Sandrew#include <machine/asm.h> 37281494Sandrew 38281494Sandrew__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/swtch.S 317147 2017-04-19 15:59:16Z andrew $"); 39281494Sandrew 40295142Sandrew.macro clear_step_flag pcbflags, tmp 41295142Sandrew tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 42295142Sandrew mrs \tmp, mdscr_el1 43295142Sandrew bic \tmp, \tmp, #1 44295142Sandrew msr mdscr_el1, \tmp 45295142Sandrew isb 46295142Sandrew999: 47295142Sandrew.endm 48295142Sandrew 49295142Sandrew.macro set_step_flag pcbflags, tmp 50295142Sandrew tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 51295142Sandrew mrs \tmp, mdscr_el1 52295142Sandrew orr \tmp, \tmp, #1 53295142Sandrew msr mdscr_el1, \tmp 54295142Sandrew isb 55295142Sandrew999: 56295142Sandrew.endm 57295142Sandrew 58281494Sandrew/* 59281494Sandrew * void cpu_throw(struct thread *old, struct thread *new) 60281494Sandrew */ 61281494SandrewENTRY(cpu_throw) 62295142Sandrew /* Of old == NULL skip disabling stepping */ 63295142Sandrew cbz x0, 1f 64295142Sandrew 65295142Sandrew /* If we were single stepping, disable it */ 66295142Sandrew ldr x4, [x0, #TD_PCB] 67295142Sandrew ldr w5, [x4, #PCB_FLAGS] 68295142Sandrew clear_step_flag w5, x6 69295142Sandrew1: 70295142Sandrew 71281494Sandrew#ifdef VFP 72281494Sandrew /* Backup the new thread pointer around a call to C code */ 73281494Sandrew mov x19, x1 74281494Sandrew bl vfp_discard 75281494Sandrew mov x1, x19 76281494Sandrew#endif 77281494Sandrew 78281494Sandrew /* Store the new curthread */ 79281494Sandrew str x1, [x18, #PC_CURTHREAD] 80281494Sandrew /* And the new pcb */ 81281494Sandrew ldr x4, [x1, #TD_PCB] 82281494Sandrew str x4, [x18, #PC_CURPCB] 83281494Sandrew 84281494Sandrew /* 85281494Sandrew * TODO: We may need to flush the cache here. 86281494Sandrew */ 87281494Sandrew 88281494Sandrew /* Switch to the new pmap */ 89297446Sandrew ldr x5, [x4, #PCB_L0ADDR] 90281494Sandrew msr ttbr0_el1, x5 91281494Sandrew isb 92281494Sandrew 93281494Sandrew /* Invalidate the TLB */ 94281494Sandrew dsb sy 95281494Sandrew tlbi vmalle1is 96281494Sandrew dsb sy 97281494Sandrew isb 98281494Sandrew 99295142Sandrew /* If we are single stepping, enable it */ 100295142Sandrew ldr w5, [x4, #PCB_FLAGS] 101295142Sandrew set_step_flag w5, x6 102295142Sandrew 103281494Sandrew /* Restore the registers */ 104281494Sandrew ldp x5, x6, [x4, #PCB_SP] 105281494Sandrew mov sp, x5 106281494Sandrew msr tpidr_el0, x6 107281494Sandrew ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 108281494Sandrew ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 109281494Sandrew ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 110281494Sandrew ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 111281494Sandrew ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 112281494Sandrew ldr x19, [x4, #PCB_REGS + 19 * 8] 113281494Sandrew ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 114281494Sandrew ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 115281494Sandrew ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 116281494Sandrew ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 117281494Sandrew ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 118281494Sandrew ldr x30, [x4, #PCB_REGS + 30 * 8] 119281494Sandrew 120281494Sandrew ret 121281494SandrewEND(cpu_throw) 122281494Sandrew 123281494Sandrew/* 124281494Sandrew * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 125281494Sandrew * 126281494Sandrew * x0 = old 127281494Sandrew * x1 = new 128281494Sandrew * x2 = mtx 129281494Sandrew * x3 to x7, x16 and x17 are caller saved 130281494Sandrew */ 131281494SandrewENTRY(cpu_switch) 132281494Sandrew /* 133281494Sandrew * Save the old context. 134281494Sandrew */ 135281494Sandrew ldr x4, [x0, #TD_PCB] 136281494Sandrew 137281494Sandrew /* Store the callee-saved registers */ 138281494Sandrew stp x8, x9, [x4, #PCB_REGS + 8 * 8] 139281494Sandrew stp x10, x11, [x4, #PCB_REGS + 10 * 8] 140281494Sandrew stp x12, x13, [x4, #PCB_REGS + 12 * 8] 141281494Sandrew stp x14, x15, [x4, #PCB_REGS + 14 * 8] 142281494Sandrew stp x16, x17, [x4, #PCB_REGS + 16 * 8] 143281494Sandrew stp x18, x19, [x4, #PCB_REGS + 18 * 8] 144281494Sandrew stp x20, x21, [x4, #PCB_REGS + 20 * 8] 145281494Sandrew stp x22, x23, [x4, #PCB_REGS + 22 * 8] 146281494Sandrew stp x24, x25, [x4, #PCB_REGS + 24 * 8] 147281494Sandrew stp x26, x27, [x4, #PCB_REGS + 26 * 8] 148281494Sandrew stp x28, x29, [x4, #PCB_REGS + 28 * 8] 149281494Sandrew str x30, [x4, #PCB_REGS + 30 * 8] 150281494Sandrew /* And the old stack pointer */ 151281494Sandrew mov x5, sp 152281494Sandrew mrs x6, tpidr_el0 153281494Sandrew stp x5, x6, [x4, #PCB_SP] 154281494Sandrew 155295142Sandrew /* If we were single stepping, disable it */ 156295142Sandrew ldr w5, [x4, #PCB_FLAGS] 157295142Sandrew clear_step_flag w5, x6 158295142Sandrew 159281494Sandrew#ifdef VFP 160281494Sandrew mov x19, x0 161281494Sandrew mov x20, x1 162281494Sandrew mov x21, x2 163286225Sandrew /* Load the pcb address */ 164286225Sandrew mov x1, x4 165281494Sandrew bl vfp_save_state 166281494Sandrew mov x2, x21 167281494Sandrew mov x1, x20 168281494Sandrew mov x0, x19 169281494Sandrew#endif 170281494Sandrew 171295563Sandrew /* Store the new curthread */ 172295563Sandrew str x1, [x18, #PC_CURTHREAD] 173295563Sandrew 174281494Sandrew /* 175295563Sandrew * Restore the saved context and set it as curpcb. 176281494Sandrew */ 177281494Sandrew ldr x4, [x1, #TD_PCB] 178295563Sandrew str x4, [x18, #PC_CURPCB] 179281494Sandrew 180281494Sandrew /* 181281494Sandrew * TODO: We may need to flush the cache here if switching 182281494Sandrew * to a user process. 183281494Sandrew */ 184281494Sandrew 185281494Sandrew /* Switch to the new pmap */ 186297446Sandrew ldr x5, [x4, #PCB_L0ADDR] 187281494Sandrew msr ttbr0_el1, x5 188281494Sandrew isb 189281494Sandrew 190281494Sandrew /* Invalidate the TLB */ 191281494Sandrew dsb sy 192281494Sandrew tlbi vmalle1is 193281494Sandrew dsb sy 194281494Sandrew isb 195281494Sandrew 196287536Sandrew /* 197287536Sandrew * Release the old thread. This doesn't need to be a store-release 198287536Sandrew * as the above dsb instruction will provide release semantics. 199287536Sandrew */ 200281494Sandrew str x2, [x0, #TD_LOCK] 201281494Sandrew#if defined(SCHED_ULE) && defined(SMP) 202294979Swma /* Spin if TD_LOCK points to a blocked_lock */ 203294979Swma ldr x2, =_C_LABEL(blocked_lock) 204285316Sandrew1: 205287536Sandrew ldar x3, [x1, #TD_LOCK] 206287536Sandrew cmp x3, x2 207285316Sandrew b.eq 1b 208281494Sandrew#endif 209281494Sandrew 210295142Sandrew /* If we are single stepping, enable it */ 211295142Sandrew ldr w5, [x4, #PCB_FLAGS] 212295142Sandrew set_step_flag w5, x6 213295142Sandrew 214281494Sandrew /* Restore the registers */ 215281494Sandrew ldp x5, x6, [x4, #PCB_SP] 216281494Sandrew mov sp, x5 217281494Sandrew msr tpidr_el0, x6 218281494Sandrew ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 219281494Sandrew ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 220281494Sandrew ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 221281494Sandrew ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 222281494Sandrew ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 223281494Sandrew ldr x19, [x4, #PCB_REGS + 19 * 8] 224281494Sandrew ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 225281494Sandrew ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 226281494Sandrew ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 227281494Sandrew ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 228281494Sandrew ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 229281494Sandrew ldr x30, [x4, #PCB_REGS + 30 * 8] 230281494Sandrew 231281494Sandrew str xzr, [x4, #PCB_REGS + 18 * 8] 232281494Sandrew ret 233281494Sandrew.Lcpu_switch_panic_str: 234281494Sandrew .asciz "cpu_switch: %p\0" 235281494SandrewEND(cpu_switch) 236281494Sandrew 237281494SandrewENTRY(fork_trampoline) 238281494Sandrew mov x0, x8 239281494Sandrew mov x1, x9 240281494Sandrew mov x2, sp 241281494Sandrew mov fp, #0 /* Stack traceback stops here. */ 242281494Sandrew bl _C_LABEL(fork_exit) 243281494Sandrew 244281494Sandrew /* Restore the registers other than x0 and x1 */ 245281494Sandrew ldp x2, x3, [sp, #TF_X + 2 * 8] 246281494Sandrew ldp x4, x5, [sp, #TF_X + 4 * 8] 247281494Sandrew ldp x6, x7, [sp, #TF_X + 6 * 8] 248281494Sandrew ldp x8, x9, [sp, #TF_X + 8 * 8] 249281494Sandrew ldp x10, x11, [sp, #TF_X + 10 * 8] 250281494Sandrew ldp x12, x13, [sp, #TF_X + 12 * 8] 251281494Sandrew ldp x14, x15, [sp, #TF_X + 14 * 8] 252281494Sandrew ldp x16, x17, [sp, #TF_X + 16 * 8] 253281494Sandrew ldr x19, [sp, #TF_X + 19 * 8] 254281494Sandrew ldp x20, x21, [sp, #TF_X + 20 * 8] 255281494Sandrew ldp x22, x23, [sp, #TF_X + 22 * 8] 256281494Sandrew ldp x24, x25, [sp, #TF_X + 24 * 8] 257281494Sandrew ldp x26, x27, [sp, #TF_X + 26 * 8] 258281494Sandrew ldp x28, x29, [sp, #TF_X + 28 * 8] 259281494Sandrew 260281494Sandrew /* 261281494Sandrew * Disable interrupts to avoid 262317147Sandrew * overwriting spsr_el1 and sp_el0 by an IRQ exception. 263281494Sandrew */ 264281494Sandrew msr daifset, #2 265281494Sandrew 266317147Sandrew /* Restore sp and lr */ 267317147Sandrew ldp x0, x1, [sp] 268317147Sandrew msr sp_el0, x0 269317147Sandrew mov lr, x1 270317147Sandrew 271281494Sandrew /* Restore elr and spsr */ 272281494Sandrew ldp x0, x1, [sp, #16] 273281494Sandrew msr elr_el1, x0 274281494Sandrew msr spsr_el1, x1 275281494Sandrew 276281494Sandrew /* Finally x0 and x1 */ 277281494Sandrew ldp x0, x1, [sp, #TF_X + 0 * 8] 278281494Sandrew ldr x18, [sp, #TF_X + 18 * 8] 279281494Sandrew 280281494Sandrew /* 281281494Sandrew * No need for interrupts reenabling since PSR 282281494Sandrew * will be set to the desired value anyway. 283281494Sandrew */ 284281494Sandrew eret 285281494Sandrew 286281494SandrewEND(fork_trampoline) 287281494Sandrew 288281494SandrewENTRY(savectx) 289285271Sandrew /* Store the callee-saved registers */ 290285271Sandrew stp x8, x9, [x0, #PCB_REGS + 8 * 8] 291285271Sandrew stp x10, x11, [x0, #PCB_REGS + 10 * 8] 292285271Sandrew stp x12, x13, [x0, #PCB_REGS + 12 * 8] 293285271Sandrew stp x14, x15, [x0, #PCB_REGS + 14 * 8] 294285271Sandrew stp x16, x17, [x0, #PCB_REGS + 16 * 8] 295285271Sandrew stp x18, x19, [x0, #PCB_REGS + 18 * 8] 296285271Sandrew stp x20, x21, [x0, #PCB_REGS + 20 * 8] 297285271Sandrew stp x22, x23, [x0, #PCB_REGS + 22 * 8] 298285271Sandrew stp x24, x25, [x0, #PCB_REGS + 24 * 8] 299285271Sandrew stp x26, x27, [x0, #PCB_REGS + 26 * 8] 300285271Sandrew stp x28, x29, [x0, #PCB_REGS + 28 * 8] 301285271Sandrew str x30, [x0, #PCB_REGS + 30 * 8] 302285271Sandrew /* And the old stack pointer */ 303285271Sandrew mov x5, sp 304285271Sandrew mrs x6, tpidr_el0 305285271Sandrew stp x5, x6, [x0, #PCB_SP] 306285271Sandrew 307285271Sandrew /* Store the VFP registers */ 308285271Sandrew#ifdef VFP 309286225Sandrew mov x28, lr 310286225Sandrew mov x1, x0 /* move pcb to the correct register */ 311286225Sandrew mov x0, xzr /* td = NULL */ 312285271Sandrew bl vfp_save_state 313286225Sandrew mov lr, x28 314285271Sandrew#endif 315285271Sandrew 316281494Sandrew ret 317281494SandrewEND(savectx) 318281494Sandrew 319