swtch.S revision 266311
1/* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ 2 3/*- 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37/*- 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 * 79 */ 80 81#include "assym.s" 82#include "opt_sched.h" 83 84#include <machine/asm.h> 85#include <machine/asmacros.h> 86#include <machine/armreg.h> 87#include <machine/vfp.h> 88 89__FBSDID("$FreeBSD: stable/10/sys/arm/arm/swtch.S 266311 2014-05-17 13:53:38Z ian $"); 90 91#define DOMAIN_CLIENT 0x01 92 93#if defined(_ARM_ARCH_6) && defined(SMP) 94#define GET_PCPU(tmp, tmp2) \ 95 mrc p15, 0, tmp, c0, c0, 5; \ 96 and tmp, tmp, #0xf; \ 97 ldr tmp2, .Lcurpcpu+4; \ 98 mul tmp, tmp, tmp2; \ 99 ldr tmp2, .Lcurpcpu; \ 100 add tmp, tmp, tmp2; 101#else 102 103#define GET_PCPU(tmp, tmp2) \ 104 ldr tmp, .Lcurpcpu 105#endif 106 107#ifdef VFP 108 .fpu vfp /* allow VFP instructions */ 109#endif 110 111.Lcurpcpu: 112 .word _C_LABEL(__pcpu) 113 .word PCPU_SIZE 114.Lcpufuncs: 115 .word _C_LABEL(cpufuncs) 116.Lblocked_lock: 117 .word _C_LABEL(blocked_lock) 118 119ENTRY(cpu_throw) 120 mov r5, r1 121 122 /* 123 * r0 = oldtd 124 * r5 = newtd 125 */ 126 127#ifdef VFP /* This thread is dying, disable */ 128 bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ 129#endif 130 131 GET_PCPU(r7, r9) 132 ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */ 133 134 /* Switch to lwp0 context */ 135 136 ldr r9, .Lcpufuncs 137#if !defined(CPU_ARM11) && !defined(CPU_CORTEXA) && !defined(CPU_MV_PJ4B) && !defined(CPU_KRAIT) 138 mov lr, pc 139 ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] 140#endif 141 ldr r0, [r7, #(PCB_PL1VEC)] 142 ldr r1, [r7, #(PCB_DACR)] 143 /* 144 * r0 = Pointer to L1 slot for vector_page (or NULL) 145 * r1 = lwp0's DACR 146 * r5 = lwp0 147 * r6 = exit func 148 * r7 = lwp0's PCB 149 * r9 = cpufuncs 150 */ 151 152 /* 153 * Ensure the vector table is accessible by fixing up lwp0's L1 154 */ 155 cmp r0, #0 /* No need to fixup vector table? */ 156 ldrne r3, [r0] /* But if yes, fetch current value */ 157 ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ 158 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ 159 cmpne r3, r2 /* Stuffing the same value? */ 160 strne r2, [r0] /* Store if not. */ 161 162#ifdef PMAP_INCLUDE_PTE_SYNC 163 /* 164 * Need to sync the cache to make sure that last store is 165 * visible to the MMU. 166 */ 167 movne r1, #4 168 movne lr, pc 169 ldrne pc, [r9, #CF_DCACHE_WB_RANGE] 170#endif /* PMAP_INCLUDE_PTE_SYNC */ 171 172 /* 173 * Note: We don't do the same optimisation as cpu_switch() with 174 * respect to avoiding flushing the TLB if we're switching to 175 * the same L1 since this process' VM space may be about to go 176 * away, so we don't want *any* turds left in the TLB. 177 */ 178 179 /* Switch the memory to the new process */ 180 ldr r0, [r7, #(PCB_PAGEDIR)] 181 mov lr, pc 182 ldr pc, [r9, #CF_CONTEXT_SWITCH] 183 184 /* Restore all the save registers */ 185#ifndef _ARM_ARCH_5E 186 add r1, r7, #PCB_R8 187 ldmia r1, {r8-r13} 188#else 189 ldr r8, [r7, #(PCB_R8)] 190 ldr r9, [r7, #(PCB_R9)] 191 ldr r10, [r7, #(PCB_R10)] 192 ldr r11, [r7, #(PCB_R11)] 193 ldr r12, [r7, #(PCB_R12)] 194 ldr r13, [r7, #(PCB_SP)] 195#endif 196 197 GET_PCPU(r6, r4) 198 /* Hook in a new pcb */ 199 str r7, [r6, #PC_CURPCB] 200 /* We have a new curthread now so make a note it */ 201 add r6, r6, #PC_CURTHREAD 202 str r5, [r6] 203#ifndef ARM_TP_ADDRESS 204 mcr p15, 0, r5, c13, c0, 4 205#endif 206 /* Set the new tp */ 207 ldr r6, [r5, #(TD_MD + MD_TP)] 208#ifdef ARM_TP_ADDRESS 209 ldr r4, =ARM_TP_ADDRESS 210 str r6, [r4] 211 ldr r6, [r5, #(TD_MD + MD_RAS_START)] 212 str r6, [r4, #4] /* ARM_RAS_START */ 213 ldr r6, [r5, #(TD_MD + MD_RAS_END)] 214 str r6, [r4, #8] /* ARM_RAS_END */ 215#else 216 mcr p15, 0, r6, c13, c0, 3 217#endif 218 219 add sp, sp, #4; 220 ldmfd sp!, {r4-r7, pc} 221END(cpu_throw) 222 223ENTRY(cpu_switch) 224 stmfd sp!, {r4-r7, lr} 225 sub sp, sp, #4; 226#ifdef __ARM_EABI__ 227 .save {r4-r7, lr} 228 .pad #4 229#endif 230 231 mov r6, r2 /* Save the mutex */ 232 233.Lswitch_resume: 234 /* rem: r0 = old lwp */ 235 /* rem: interrupts are disabled */ 236 237 /* Process is now on a processor. */ 238 /* We have a new curthread now so make a note it */ 239 GET_PCPU(r7, r2) 240 str r1, [r7, #PC_CURTHREAD] 241#ifndef ARM_TP_ADDRESS 242 mcr p15, 0, r1, c13, c0, 4 243#endif 244 245 /* Hook in a new pcb */ 246 ldr r2, [r1, #TD_PCB] 247 str r2, [r7, #PC_CURPCB] 248 249 /* rem: r1 = new process */ 250 /* rem: interrupts are enabled */ 251 252 /* Stage two : Save old context */ 253 254 /* Get the user structure for the old thread. */ 255 ldr r2, [r0, #(TD_PCB)] 256 mov r4, r0 /* Save the old thread. */ 257 258 /* Save all the registers in the old thread's pcb */ 259#ifndef _ARM_ARCH_5E 260 add r7, r2, #(PCB_R8) 261 stmia r7, {r8-r13} 262#else 263 strd r8, [r2, #(PCB_R8)] 264 strd r10, [r2, #(PCB_R10)] 265 strd r12, [r2, #(PCB_R12)] 266#endif 267 str pc, [r2, #(PCB_PC)] 268 269 /* 270 * NOTE: We can now use r8-r13 until it is time to restore 271 * them for the new process. 272 */ 273#ifdef ARM_TP_ADDRESS 274 /* Store the old tp */ 275 ldr r3, =ARM_TP_ADDRESS 276 ldr r9, [r3] 277 str r9, [r0, #(TD_MD + MD_TP)] 278 ldr r9, [r3, #4] 279 str r9, [r0, #(TD_MD + MD_RAS_START)] 280 ldr r9, [r3, #8] 281 str r9, [r0, #(TD_MD + MD_RAS_END)] 282 283 /* Set the new tp */ 284 ldr r9, [r1, #(TD_MD + MD_TP)] 285 str r9, [r3] 286 ldr r9, [r1, #(TD_MD + MD_RAS_START)] 287 str r9, [r3, #4] 288 ldr r9, [r1, #(TD_MD + MD_RAS_END)] 289 str r9, [r3, #8] 290#else 291 /* Store the old tp */ 292 mrc p15, 0, r9, c13, c0, 3 293 str r9, [r0, #(TD_MD + MD_TP)] 294 295 /* Set the new tp */ 296 ldr r9, [r1, #(TD_MD + MD_TP)] 297 mcr p15, 0, r9, c13, c0, 3 298#endif 299 300 /* Get the user structure for the new process in r9 */ 301 ldr r9, [r1, #(TD_PCB)] 302 303 /* rem: r2 = old PCB */ 304 /* rem: r9 = new PCB */ 305 /* rem: interrupts are enabled */ 306 307#ifdef VFP 308 fmrx r0, fpexc /* If the VFP is enabled */ 309 tst r0, #(VFPEXC_EN) /* the current thread has */ 310 movne r1, #1 /* used it, so go save */ 311 addne r0, r2, #(PCB_VFPSTATE) /* the state into the PCB */ 312 blne _C_LABEL(vfp_store) /* and disable the VFP. */ 313#endif 314 315 /* r0-r3 now free! */ 316 317 /* Third phase : restore saved context */ 318 319 /* rem: r2 = old PCB */ 320 /* rem: r9 = new PCB */ 321 /* rem: interrupts are enabled */ 322 323 ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */ 324 mov r2, #DOMAIN_CLIENT 325 cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 326 beq .Lcs_context_switched /* Yup. Don't flush cache */ 327 mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */ 328 /* 329 * Get the new L1 table pointer into r11. If we're switching to 330 * an LWP with the same address space as the outgoing one, we can 331 * skip the cache purge and the TTB load. 332 * 333 * To avoid data dep stalls that would happen anyway, we try 334 * and get some useful work done in the mean time. 335 */ 336 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 337 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 338 339 340 teq r10, r11 /* Same L1? */ 341 cmpeq r0, r5 /* Same DACR? */ 342 beq .Lcs_context_switched /* yes! */ 343 344#if !defined(CPU_ARM11) && !defined(CPU_CORTEXA) && !defined(CPU_MV_PJ4B) && !defined(CPU_KRAIT) 345 /* 346 * Definately need to flush the cache. 347 */ 348 349 ldr r1, .Lcpufuncs 350 mov lr, pc 351 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 352#endif 353.Lcs_cache_purge_skipped: 354 /* rem: r6 = lock */ 355 /* rem: r9 = new PCB */ 356 /* rem: r10 = old L1 */ 357 /* rem: r11 = new L1 */ 358 359 mov r2, #0x00000000 360 ldr r7, [r9, #(PCB_PL1VEC)] 361 362 /* 363 * Ensure the vector table is accessible by fixing up the L1 364 */ 365 cmp r7, #0 /* No need to fixup vector table? */ 366 ldrne r2, [r7] /* But if yes, fetch current value */ 367 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 368 mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */ 369 cmpne r2, r0 /* Stuffing the same value? */ 370#ifndef PMAP_INCLUDE_PTE_SYNC 371 strne r0, [r7] /* Nope, update it */ 372#else 373 beq .Lcs_same_vector 374 str r0, [r7] /* Otherwise, update it */ 375 376 /* 377 * Need to sync the cache to make sure that last store is 378 * visible to the MMU. 379 */ 380 ldr r2, .Lcpufuncs 381 mov r0, r7 382 mov r1, #4 383 mov lr, pc 384 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 385 386.Lcs_same_vector: 387#endif /* PMAP_INCLUDE_PTE_SYNC */ 388 389 cmp r10, r11 /* Switching to the same L1? */ 390 ldr r10, .Lcpufuncs 391 beq .Lcs_same_l1 /* Yup. */ 392 /* 393 * Do a full context switch, including full TLB flush. 394 */ 395 mov r0, r11 396 mov lr, pc 397 ldr pc, [r10, #CF_CONTEXT_SWITCH] 398 399 b .Lcs_context_switched 400 401 /* 402 * We're switching to a different process in the same L1. 403 * In this situation, we only need to flush the TLB for the 404 * vector_page mapping, and even then only if r7 is non-NULL. 405 */ 406.Lcs_same_l1: 407 cmp r7, #0 408 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 409 movne lr, pc 410 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 411 412.Lcs_context_switched: 413 414 /* Release the old thread */ 415 str r6, [r4, #TD_LOCK] 416#if defined(SCHED_ULE) && defined(SMP) 417 ldr r6, .Lblocked_lock 418 GET_CURTHREAD_PTR(r3) 4191: 420 ldr r4, [r3, #TD_LOCK] 421 cmp r4, r6 422 beq 1b 423#endif 424 425 /* XXXSCW: Safe to re-enable FIQs here */ 426 427 /* rem: r9 = new PCB */ 428 429 /* Restore all the save registers */ 430#ifndef _ARM_ARCH_5E 431 add r7, r9, #PCB_R8 432 ldmia r7, {r8-r13} 433 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 434#else 435 mov r7, r9 436 ldr r8, [r7, #(PCB_R8)] 437 ldr r9, [r7, #(PCB_R9)] 438 ldr r10, [r7, #(PCB_R10)] 439 ldr r11, [r7, #(PCB_R11)] 440 ldr r12, [r7, #(PCB_R12)] 441 ldr r13, [r7, #(PCB_SP)] 442#endif 443 444 /* rem: r5 = new lwp's proc */ 445 /* rem: r6 = lock */ 446 /* rem: r7 = new PCB */ 447 448.Lswitch_return: 449 450 /* 451 * Pull the registers that got pushed when either savectx() or 452 * cpu_switch() was called and return. 453 */ 454 add sp, sp, #4; 455 ldmfd sp!, {r4-r7, pc} 456#ifdef DIAGNOSTIC 457.Lswitch_bogons: 458 adr r0, .Lswitch_panic_str 459 bl _C_LABEL(panic) 4601: nop 461 b 1b 462 463.Lswitch_panic_str: 464 .asciz "cpu_switch: sched_qs empty with non-zero sched_whichqs!\n" 465#endif 466END(cpu_switch) 467 468ENTRY(savectx) 469 stmfd sp!, {r4-r7, lr} 470 sub sp, sp, #4 471 /* 472 * r0 = pcb 473 */ 474 /* Store all the registers in the process's pcb */ 475 add r2, r0, #(PCB_R8) 476 stmia r2, {r8-r13} 477#ifdef VFP 478 fmrx r2, fpexc /* If the VFP is enabled */ 479 tst r2, #(VFPEXC_EN) /* the current thread has */ 480 movne r1, #1 /* used it, so go save */ 481 addne r0, r0, #(PCB_VFPSTATE) /* the state into the PCB */ 482 blne _C_LABEL(vfp_store) /* and disable the VFP. */ 483#endif 484 add sp, sp, #4; 485 ldmfd sp!, {r4-r7, pc} 486END(savectx) 487 488ENTRY(fork_trampoline) 489 STOP_UNWINDING /* EABI: Don't unwind beyond the thread enty point. */ 490 mov fp, #0 /* OABI: Stack traceback via fp stops here. */ 491 mov r2, sp 492 mov r1, r5 493 mov r0, r4 494 ldr lr, =swi_exit /* Go finish forking, then return */ 495 b _C_LABEL(fork_exit) /* to userland via swi_exit code. */ 496END(fork_trampoline) 497 498