1118611Snjl/*- 2118611Snjl * Copyright (c) 2014 Andrew Turner 3207344Sjkim * Copyright (c) 2014 The FreeBSD Foundation 4118611Snjl * All rights reserved. 5118611Snjl * 6118611Snjl * This software was developed by Andrew Turner under sponsorship from 7217365Sjkim * the FreeBSD Foundation. 8306536Sjkim * 9118611Snjl * Redistribution and use in source and binary forms, with or without 10118611Snjl * modification, are permitted provided that the following conditions 11217365Sjkim * are met: 12217365Sjkim * 1. Redistributions of source code must retain the above copyright 13217365Sjkim * notice, this list of conditions and the following disclaimer. 14217365Sjkim * 2. Redistributions in binary form must reproduce the above copyright 15217365Sjkim * notice, this list of conditions and the following disclaimer in the 16217365Sjkim * documentation and/or other materials provided with the distribution. 17217365Sjkim * 18217365Sjkim * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19217365Sjkim * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20217365Sjkim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21217365Sjkim * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22217365Sjkim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23217365Sjkim * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24217365Sjkim * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25118611Snjl * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26217365Sjkim * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27217365Sjkim * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28217365Sjkim * SUCH DAMAGE. 29118611Snjl * 30217365Sjkim */ 31217365Sjkim 32217365Sjkim#include "assym.inc" 33217365Sjkim#include "opt_kstack_pages.h" 34217365Sjkim#include "opt_sched.h" 35217365Sjkim 36217365Sjkim#include <sys/elf_common.h> 37217365Sjkim 38217365Sjkim#include <machine/asm.h> 39217365Sjkim#include <machine/armreg.h> 40217365Sjkim.macro clear_step_flag pcbflags, tmp 41217365Sjkim tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 42217365Sjkim mrs \tmp, mdscr_el1 43118611Snjl bic \tmp, \tmp, #MDSCR_SS 44151937Sjkim msr mdscr_el1, \tmp 45118611Snjl isb 46118611Snjl999: 47118611Snjl.endm 48118611Snjl 49118611Snjl.macro set_step_flag pcbflags, tmp 50207344Sjkim tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 51207344Sjkim mrs \tmp, mdscr_el1 52207344Sjkim orr \tmp, \tmp, #MDSCR_SS 53207344Sjkim msr mdscr_el1, \tmp 54207344Sjkim isb 55207344Sjkim999: 56207344Sjkim.endm 57207344Sjkim 58207344Sjkim/* 59207344Sjkim * void cpu_throw(struct thread *old, struct thread *new) 60207344Sjkim */ 61207344SjkimENTRY(cpu_throw) 62118611Snjl /* Of old == NULL skip disabling stepping */ 63118611Snjl cbz x0, 1f 64118611Snjl 65167802Sjkim /* If we were single stepping, disable it */ 66167802Sjkim ldr x4, [x0, #TD_PCB] 67281075Sdim ldr w5, [x4, #PCB_FLAGS] 68167802Sjkim clear_step_flag w5, x6 69167802Sjkim1: 70167802Sjkim 71167802Sjkim#ifdef VFP 72167802Sjkim /* Backup the new thread pointer around a call to C code */ 73167802Sjkim mov x19, x1 74167802Sjkim bl vfp_discard 75167802Sjkim mov x0, x19 76167802Sjkim#else 77281075Sdim mov x0, x1 78167802Sjkim#endif 79167802Sjkim 80167802Sjkim /* This returns the thread pointer so no need to save it */ 81167802Sjkim bl ptrauth_switch 82167802Sjkim#ifdef PERTHREAD_SSP 83167802Sjkim mov x19, x0 84167802Sjkim#endif 85167802Sjkim /* This returns the thread pcb */ 86167802Sjkim bl pmap_switch 87167802Sjkim mov x4, x0 88167802Sjkim#ifdef PERTHREAD_SSP 89167802Sjkim /* Update the per-thread stack canary pointer. */ 90167802Sjkim add x19, x19, #(TD_MD_CANARY) 91167802Sjkim msr sp_el0, x19 92167802Sjkim#endif 93167802Sjkim 94167802Sjkim /* If we are single stepping, enable it */ 95118611Snjl ldr w5, [x4, #PCB_FLAGS] 96118611Snjl set_step_flag w5, x6 97281075Sdim 98118611Snjl /* Restore the registers */ 99118611Snjl ldp x5, x6, [x4, #PCB_SP] 100118611Snjl mov sp, x5 101118611Snjl msr tpidr_el0, x6 102118611Snjl ldr x6, [x4, #PCB_TPIDRRO] 103118611Snjl msr tpidrro_el0, x6 104118611Snjl ldp x19, x20, [x4, #PCB_REGS + (PCB_X19 + 0) * 8] 105118611Snjl ldp x21, x22, [x4, #PCB_REGS + (PCB_X19 + 2) * 8] 106118611Snjl ldp x23, x24, [x4, #PCB_REGS + (PCB_X19 + 4) * 8] 107281075Sdim ldp x25, x26, [x4, #PCB_REGS + (PCB_X19 + 6) * 8] 108118611Snjl ldp x27, x28, [x4, #PCB_REGS + (PCB_X19 + 8) * 8] 109151937Sjkim ldp x29, lr, [x4, #PCB_REGS + (PCB_X19 + 10) * 8] 110118611Snjl 111118611Snjl ret 112118611SnjlEND(cpu_throw) 113151937Sjkim 114118611Snjl/* 115118611Snjl * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 116306536Sjkim * 117306536Sjkim * x0 = old 118118611Snjl * x1 = new 119118611Snjl * x2 = mtx 120118611Snjl * x3 to x7, x16 and x17 are caller saved 121118611Snjl */ 122118611SnjlENTRY(cpu_switch) 123118611Snjl /* 124118611Snjl * Save the old context. 125118611Snjl */ 126281075Sdim ldr x4, [x0, #TD_PCB] 127118611Snjl 128118611Snjl /* Store the callee-saved registers */ 129118611Snjl stp x19, x20, [x4, #PCB_REGS + (PCB_X19 + 0) * 8] 130118611Snjl stp x21, x22, [x4, #PCB_REGS + (PCB_X19 + 2) * 8] 131118611Snjl stp x23, x24, [x4, #PCB_REGS + (PCB_X19 + 4) * 8] 132118611Snjl stp x25, x26, [x4, #PCB_REGS + (PCB_X19 + 6) * 8] 133118611Snjl stp x27, x28, [x4, #PCB_REGS + (PCB_X19 + 8) * 8] 134118611Snjl stp x29, lr, [x4, #PCB_REGS + (PCB_X19 + 10) * 8] 135118611Snjl /* And the old stack pointer */ 136281075Sdim mov x5, sp 137118611Snjl mrs x6, tpidrro_el0 138151937Sjkim str x6, [x4, #PCB_TPIDRRO] 139118611Snjl mrs x6, tpidr_el0 140207344Sjkim stp x5, x6, [x4, #PCB_SP] 141207344Sjkim 142207344Sjkim /* If we were single stepping, disable it */ 143118611Snjl ldr w5, [x4, #PCB_FLAGS] 144281075Sdim clear_step_flag w5, x6 145118611Snjl 146118611Snjl mov x19, x0 147118611Snjl mov x20, x1 148281075Sdim mov x21, x2 149281075Sdim 150151937Sjkim#ifdef VFP 151118611Snjl /* Load the pcb address */ 152118611Snjl mov x1, x4 153306536Sjkim bl vfp_save_state 154151937Sjkim mov x0, x20 155118611Snjl#else 156151937Sjkim mov x0, x1 157151937Sjkim#endif 158118611Snjl 159118611Snjl /* This returns the thread pointer so no need to save it */ 160118611Snjl bl ptrauth_switch 161118611Snjl /* This returns the thread pcb */ 162118611Snjl bl pmap_switch 163118611Snjl /* Move the new pcb out of the way */ 164151937Sjkim mov x4, x0 165167802Sjkim 166151937Sjkim mov x2, x21 167118611Snjl mov x1, x20 168118611Snjl mov x0, x19 169118611Snjl#ifdef PERTHREAD_SSP 170118611Snjl /* Update the per-thread stack canary pointer. */ 171151937Sjkim add x20, x20, #(TD_MD_CANARY) 172228110Sjkim msr sp_el0, x20 173151937Sjkim#endif 174207344Sjkim 175118611Snjl /* 176118611Snjl * Release the old thread. 177118611Snjl */ 178118611Snjl stlr x2, [x0, #TD_LOCK] 179151937Sjkim#if defined(SCHED_ULE) && defined(SMP) 180228110Sjkim /* Spin if TD_LOCK points to a blocked_lock */ 181151937Sjkim ldr x2, =_C_LABEL(blocked_lock) 182207344Sjkim1: 183118611Snjl ldar x3, [x1, #TD_LOCK] 184118611Snjl cmp x3, x2 185118611Snjl b.eq 1b 186118611Snjl#endif 187151937Sjkim 188228110Sjkim /* If we are single stepping, enable it */ 189151937Sjkim ldr w5, [x4, #PCB_FLAGS] 190118611Snjl set_step_flag w5, x6 191118611Snjl 192118611Snjl /* Restore the registers */ 193118611Snjl ldp x5, x6, [x4, #PCB_SP] 194151937Sjkim mov sp, x5 195228110Sjkim msr tpidr_el0, x6 196151937Sjkim ldr x6, [x4, #PCB_TPIDRRO] 197207344Sjkim msr tpidrro_el0, x6 198118611Snjl ldp x19, x20, [x4, #PCB_REGS + (PCB_X19 + 0) * 8] 199118611Snjl ldp x21, x22, [x4, #PCB_REGS + (PCB_X19 + 2) * 8] 200118611Snjl ldp x23, x24, [x4, #PCB_REGS + (PCB_X19 + 4) * 8] 201118611Snjl ldp x25, x26, [x4, #PCB_REGS + (PCB_X19 + 6) * 8] 202281075Sdim ldp x27, x28, [x4, #PCB_REGS + (PCB_X19 + 8) * 8] 203118611Snjl ldp x29, lr, [x4, #PCB_REGS + (PCB_X19 + 10) * 8] 204118611Snjl 205118611Snjl ret 206118611SnjlEND(cpu_switch) 207118611Snjl 208118611SnjlENTRY(fork_trampoline) 209118611Snjl mov x0, x19 210118611Snjl mov x1, x20 211118611Snjl mov x2, sp 212118611Snjl mov fp, #0 /* Stack traceback stops here. */ 213118611Snjl bl _C_LABEL(fork_exit) 214207344Sjkim 215207344Sjkim /* 216207344Sjkim * Disable interrupts as we are setting userspace specific 217207344Sjkim * state that we won't handle correctly in an interrupt while 218207344Sjkim * in the kernel. 219207344Sjkim */ 220207344Sjkim msr daifset, #(DAIF_D | DAIF_INTR) 221281075Sdim 222207344Sjkim ldr x0, [x18, #PC_CURTHREAD] 223118611Snjl bl ptrauth_enter_el0 224118611Snjl 225118611Snjl /* Restore sp, lr, elr, and spsr */ 226118611Snjl ldp x18, lr, [sp, #TF_SP] 227118611Snjl ldp x10, x11, [sp, #TF_ELR] 228118611Snjl msr sp_el0, x18 229118611Snjl msr spsr_el1, x11 230118611Snjl msr elr_el1, x10 231281075Sdim 232118611Snjl /* Restore the CPU registers */ 233118611Snjl ldp x0, x1, [sp, #TF_X + 0 * 8] 234118611Snjl ldp x2, x3, [sp, #TF_X + 2 * 8] 235118611Snjl ldp x4, x5, [sp, #TF_X + 4 * 8] 236118611Snjl ldp x6, x7, [sp, #TF_X + 6 * 8] 237118611Snjl ldp x8, x9, [sp, #TF_X + 8 * 8] 238118611Snjl ldp x10, x11, [sp, #TF_X + 10 * 8] 239118611Snjl ldp x12, x13, [sp, #TF_X + 12 * 8] 240118611Snjl ldp x14, x15, [sp, #TF_X + 14 * 8] 241281075Sdim ldp x16, x17, [sp, #TF_X + 16 * 8] 242118611Snjl ldp x18, x19, [sp, #TF_X + 18 * 8] 243151937Sjkim ldp x20, x21, [sp, #TF_X + 20 * 8] 244118611Snjl ldp x22, x23, [sp, #TF_X + 22 * 8] 245207344Sjkim ldp x24, x25, [sp, #TF_X + 24 * 8] 246207344Sjkim ldp x26, x27, [sp, #TF_X + 26 * 8] 247207344Sjkim ldp x28, x29, [sp, #TF_X + 28 * 8] 248207344Sjkim 249118611Snjl /* 250281075Sdim * No need for interrupts reenabling since PSR 251118611Snjl * will be set to the desired value anyway. 252118611Snjl */ 253118611Snjl ERET 254281075Sdim 255281075SdimEND(fork_trampoline) 256151937Sjkim 257118611SnjlENTRY(savectx) 258118611Snjl /* Store the callee-saved registers */ 259306536Sjkim stp x19, x20, [x0, #PCB_REGS + (PCB_X19 + 0) * 8] 260151937Sjkim stp x21, x22, [x0, #PCB_REGS + (PCB_X19 + 2) * 8] 261118611Snjl stp x23, x24, [x0, #PCB_REGS + (PCB_X19 + 4) * 8] 262151937Sjkim stp x25, x26, [x0, #PCB_REGS + (PCB_X19 + 6) * 8] 263151937Sjkim stp x27, x28, [x0, #PCB_REGS + (PCB_X19 + 8) * 8] 264118611Snjl stp x29, lr, [x0, #PCB_REGS + (PCB_X19 + 10) * 8] 265118611Snjl /* And the old stack pointer */ 266118611Snjl mov x5, sp 267118611Snjl mrs x6, tpidrro_el0 268118611Snjl str x6, [x0, #PCB_TPIDRRO] 269118611Snjl mrs x6, tpidr_el0 270151937Sjkim stp x5, x6, [x0, #PCB_SP] 271167802Sjkim 272151937Sjkim /* Store the VFP registers */ 273118611Snjl#ifdef VFP 274118611Snjl mov x28, lr 275118611Snjl bl vfp_save_state_savectx 276118611Snjl mov lr, x28 277151937Sjkim#endif 278228110Sjkim 279151937Sjkim ret 280207344SjkimEND(savectx) 281118611Snjl 282118611SnjlGNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) 283118611Snjl