swtch.S revision 317147
1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by Andrew Turner under sponsorship from 7 * the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include "assym.s" 33#include "opt_kstack_pages.h" 34#include "opt_sched.h" 35 36#include <machine/asm.h> 37 38__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/swtch.S 317147 2017-04-19 15:59:16Z andrew $"); 39 40.macro clear_step_flag pcbflags, tmp 41 tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 42 mrs \tmp, mdscr_el1 43 bic \tmp, \tmp, #1 44 msr mdscr_el1, \tmp 45 isb 46999: 47.endm 48 49.macro set_step_flag pcbflags, tmp 50 tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f 51 mrs \tmp, mdscr_el1 52 orr \tmp, \tmp, #1 53 msr mdscr_el1, \tmp 54 isb 55999: 56.endm 57 58/* 59 * void cpu_throw(struct thread *old, struct thread *new) 60 */ 61ENTRY(cpu_throw) 62 /* Of old == NULL skip disabling stepping */ 63 cbz x0, 1f 64 65 /* If we were single stepping, disable it */ 66 ldr x4, [x0, #TD_PCB] 67 ldr w5, [x4, #PCB_FLAGS] 68 clear_step_flag w5, x6 691: 70 71#ifdef VFP 72 /* Backup the new thread pointer around a call to C code */ 73 mov x19, x1 74 bl vfp_discard 75 mov x1, x19 76#endif 77 78 /* Store the new curthread */ 79 str x1, [x18, #PC_CURTHREAD] 80 /* And the new pcb */ 81 ldr x4, [x1, #TD_PCB] 82 str x4, [x18, #PC_CURPCB] 83 84 /* 85 * TODO: We may need to flush the cache here. 86 */ 87 88 /* Switch to the new pmap */ 89 ldr x5, [x4, #PCB_L0ADDR] 90 msr ttbr0_el1, x5 91 isb 92 93 /* Invalidate the TLB */ 94 dsb sy 95 tlbi vmalle1is 96 dsb sy 97 isb 98 99 /* If we are single stepping, enable it */ 100 ldr w5, [x4, #PCB_FLAGS] 101 set_step_flag w5, x6 102 103 /* Restore the registers */ 104 ldp x5, x6, [x4, #PCB_SP] 105 mov sp, x5 106 msr tpidr_el0, x6 107 ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 108 ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 109 ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 110 ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 111 ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 112 ldr x19, [x4, #PCB_REGS + 19 * 8] 113 ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 114 ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 115 ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 116 ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 117 ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 118 ldr x30, [x4, #PCB_REGS + 30 * 8] 119 120 ret 121END(cpu_throw) 122 123/* 124 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 125 * 126 * x0 = old 127 * x1 = new 128 * x2 = mtx 129 * x3 to x7, x16 and x17 are caller saved 130 */ 131ENTRY(cpu_switch) 132 /* 133 * Save the old context. 134 */ 135 ldr x4, [x0, #TD_PCB] 136 137 /* Store the callee-saved registers */ 138 stp x8, x9, [x4, #PCB_REGS + 8 * 8] 139 stp x10, x11, [x4, #PCB_REGS + 10 * 8] 140 stp x12, x13, [x4, #PCB_REGS + 12 * 8] 141 stp x14, x15, [x4, #PCB_REGS + 14 * 8] 142 stp x16, x17, [x4, #PCB_REGS + 16 * 8] 143 stp x18, x19, [x4, #PCB_REGS + 18 * 8] 144 stp x20, x21, [x4, #PCB_REGS + 20 * 8] 145 stp x22, x23, [x4, #PCB_REGS + 22 * 8] 146 stp x24, x25, [x4, #PCB_REGS + 24 * 8] 147 stp x26, x27, [x4, #PCB_REGS + 26 * 8] 148 stp x28, x29, [x4, #PCB_REGS + 28 * 8] 149 str x30, [x4, #PCB_REGS + 30 * 8] 150 /* And the old stack pointer */ 151 mov x5, sp 152 mrs x6, tpidr_el0 153 stp x5, x6, [x4, #PCB_SP] 154 155 /* If we were single stepping, disable it */ 156 ldr w5, [x4, #PCB_FLAGS] 157 clear_step_flag w5, x6 158 159#ifdef VFP 160 mov x19, x0 161 mov x20, x1 162 mov x21, x2 163 /* Load the pcb address */ 164 mov x1, x4 165 bl vfp_save_state 166 mov x2, x21 167 mov x1, x20 168 mov x0, x19 169#endif 170 171 /* Store the new curthread */ 172 str x1, [x18, #PC_CURTHREAD] 173 174 /* 175 * Restore the saved context and set it as curpcb. 176 */ 177 ldr x4, [x1, #TD_PCB] 178 str x4, [x18, #PC_CURPCB] 179 180 /* 181 * TODO: We may need to flush the cache here if switching 182 * to a user process. 183 */ 184 185 /* Switch to the new pmap */ 186 ldr x5, [x4, #PCB_L0ADDR] 187 msr ttbr0_el1, x5 188 isb 189 190 /* Invalidate the TLB */ 191 dsb sy 192 tlbi vmalle1is 193 dsb sy 194 isb 195 196 /* 197 * Release the old thread. This doesn't need to be a store-release 198 * as the above dsb instruction will provide release semantics. 199 */ 200 str x2, [x0, #TD_LOCK] 201#if defined(SCHED_ULE) && defined(SMP) 202 /* Spin if TD_LOCK points to a blocked_lock */ 203 ldr x2, =_C_LABEL(blocked_lock) 2041: 205 ldar x3, [x1, #TD_LOCK] 206 cmp x3, x2 207 b.eq 1b 208#endif 209 210 /* If we are single stepping, enable it */ 211 ldr w5, [x4, #PCB_FLAGS] 212 set_step_flag w5, x6 213 214 /* Restore the registers */ 215 ldp x5, x6, [x4, #PCB_SP] 216 mov sp, x5 217 msr tpidr_el0, x6 218 ldp x8, x9, [x4, #PCB_REGS + 8 * 8] 219 ldp x10, x11, [x4, #PCB_REGS + 10 * 8] 220 ldp x12, x13, [x4, #PCB_REGS + 12 * 8] 221 ldp x14, x15, [x4, #PCB_REGS + 14 * 8] 222 ldp x16, x17, [x4, #PCB_REGS + 16 * 8] 223 ldr x19, [x4, #PCB_REGS + 19 * 8] 224 ldp x20, x21, [x4, #PCB_REGS + 20 * 8] 225 ldp x22, x23, [x4, #PCB_REGS + 22 * 8] 226 ldp x24, x25, [x4, #PCB_REGS + 24 * 8] 227 ldp x26, x27, [x4, #PCB_REGS + 26 * 8] 228 ldp x28, x29, [x4, #PCB_REGS + 28 * 8] 229 ldr x30, [x4, #PCB_REGS + 30 * 8] 230 231 str xzr, [x4, #PCB_REGS + 18 * 8] 232 ret 233.Lcpu_switch_panic_str: 234 .asciz "cpu_switch: %p\0" 235END(cpu_switch) 236 237ENTRY(fork_trampoline) 238 mov x0, x8 239 mov x1, x9 240 mov x2, sp 241 mov fp, #0 /* Stack traceback stops here. */ 242 bl _C_LABEL(fork_exit) 243 244 /* Restore the registers other than x0 and x1 */ 245 ldp x2, x3, [sp, #TF_X + 2 * 8] 246 ldp x4, x5, [sp, #TF_X + 4 * 8] 247 ldp x6, x7, [sp, #TF_X + 6 * 8] 248 ldp x8, x9, [sp, #TF_X + 8 * 8] 249 ldp x10, x11, [sp, #TF_X + 10 * 8] 250 ldp x12, x13, [sp, #TF_X + 12 * 8] 251 ldp x14, x15, [sp, #TF_X + 14 * 8] 252 ldp x16, x17, [sp, #TF_X + 16 * 8] 253 ldr x19, [sp, #TF_X + 19 * 8] 254 ldp x20, x21, [sp, #TF_X + 20 * 8] 255 ldp x22, x23, [sp, #TF_X + 22 * 8] 256 ldp x24, x25, [sp, #TF_X + 24 * 8] 257 ldp x26, x27, [sp, #TF_X + 26 * 8] 258 ldp x28, x29, [sp, #TF_X + 28 * 8] 259 260 /* 261 * Disable interrupts to avoid 262 * overwriting spsr_el1 and sp_el0 by an IRQ exception. 263 */ 264 msr daifset, #2 265 266 /* Restore sp and lr */ 267 ldp x0, x1, [sp] 268 msr sp_el0, x0 269 mov lr, x1 270 271 /* Restore elr and spsr */ 272 ldp x0, x1, [sp, #16] 273 msr elr_el1, x0 274 msr spsr_el1, x1 275 276 /* Finally x0 and x1 */ 277 ldp x0, x1, [sp, #TF_X + 0 * 8] 278 ldr x18, [sp, #TF_X + 18 * 8] 279 280 /* 281 * No need for interrupts reenabling since PSR 282 * will be set to the desired value anyway. 283 */ 284 eret 285 286END(fork_trampoline) 287 288ENTRY(savectx) 289 /* Store the callee-saved registers */ 290 stp x8, x9, [x0, #PCB_REGS + 8 * 8] 291 stp x10, x11, [x0, #PCB_REGS + 10 * 8] 292 stp x12, x13, [x0, #PCB_REGS + 12 * 8] 293 stp x14, x15, [x0, #PCB_REGS + 14 * 8] 294 stp x16, x17, [x0, #PCB_REGS + 16 * 8] 295 stp x18, x19, [x0, #PCB_REGS + 18 * 8] 296 stp x20, x21, [x0, #PCB_REGS + 20 * 8] 297 stp x22, x23, [x0, #PCB_REGS + 22 * 8] 298 stp x24, x25, [x0, #PCB_REGS + 24 * 8] 299 stp x26, x27, [x0, #PCB_REGS + 26 * 8] 300 stp x28, x29, [x0, #PCB_REGS + 28 * 8] 301 str x30, [x0, #PCB_REGS + 30 * 8] 302 /* And the old stack pointer */ 303 mov x5, sp 304 mrs x6, tpidr_el0 305 stp x5, x6, [x0, #PCB_SP] 306 307 /* Store the VFP registers */ 308#ifdef VFP 309 mov x28, lr 310 mov x1, x0 /* move pcb to the correct register */ 311 mov x0, xzr /* td = NULL */ 312 bl vfp_save_state 313 mov lr, x28 314#endif 315 316 ret 317END(savectx) 318 319