swtch-v4.S revision 254454
1/* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ 2 3/*- 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37/*- 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 * 79 */ 80 81#include "assym.s" 82#include "opt_sched.h" 83 84#include <machine/asm.h> 85#include <machine/asmacros.h> 86#include <machine/armreg.h> 87__FBSDID("$FreeBSD: head/sys/arm/arm/swtch.S 254454 2013-08-17 15:09:14Z andrew $"); 88 89#define DOMAIN_CLIENT 0x01 90 91#ifdef _ARM_ARCH_6 92#define GET_PCPU(tmp) \ 93 mrc p15, 0, tmp, c13, c0, 4; 94#else 95.Lcurpcpu: 96 .word _C_LABEL(__pcpu) 97 98#define GET_PCPU(tmp) \ 99 ldr tmp, .Lcurpcpu 100#endif 101 102.Lcpufuncs: 103 .word _C_LABEL(cpufuncs) 104.Lblocked_lock: 105 .word _C_LABEL(blocked_lock) 106 107ENTRY(cpu_throw) 108 mov r5, r1 109 110 /* 111 * r0 = oldtd 112 * r5 = newtd 113 */ 114 115 GET_PCPU(r7) 116 117#ifdef ARM_VFP_SUPPORT 118 /* 119 * vfp_discard will clear pcpu->pc_vfpcthread, and modify 120 * and modify the control as needed. 121 */ 122 ldr r4, [r7, #(PC_VFPCTHREAD)] /* this thread using vfp? */ 123 cmp r0, r4 124 bne 3f 125 bl _C_LABEL(vfp_discard) /* yes, shut down vfp */ 1263: 127#endif /* ARM_VFP_SUPPORT */ 128 129 ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */ 130 131 /* Switch to lwp0 context */ 132 133 ldr r9, .Lcpufuncs 134#if !defined(CPU_ARM11) && !defined(CPU_CORTEXA) && !defined(CPU_MV_PJ4B) 135 mov lr, pc 136 ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] 137#endif 138 ldr r0, [r7, #(PCB_PL1VEC)] 139 ldr r1, [r7, #(PCB_DACR)] 140 /* 141 * r0 = Pointer to L1 slot for vector_page (or NULL) 142 * r1 = lwp0's DACR 143 * r5 = lwp0 144 * r6 = exit func 145 * r7 = lwp0's PCB 146 * r9 = cpufuncs 147 */ 148 149 /* 150 * Ensure the vector table is accessible by fixing up lwp0's L1 151 */ 152 cmp r0, #0 /* No need to fixup vector table? */ 153 ldrne r3, [r0] /* But if yes, fetch current value */ 154 ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ 155 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ 156 cmpne r3, r2 /* Stuffing the same value? */ 157 strne r2, [r0] /* Store if not. */ 158 159#ifdef PMAP_INCLUDE_PTE_SYNC 160 /* 161 * Need to sync the cache to make sure that last store is 162 * visible to the MMU. 163 */ 164 movne r1, #4 165 movne lr, pc 166 ldrne pc, [r9, #CF_DCACHE_WB_RANGE] 167#endif /* PMAP_INCLUDE_PTE_SYNC */ 168 169 /* 170 * Note: We don't do the same optimisation as cpu_switch() with 171 * respect to avoiding flushing the TLB if we're switching to 172 * the same L1 since this process' VM space may be about to go 173 * away, so we don't want *any* turds left in the TLB. 174 */ 175 176 /* Switch the memory to the new process */ 177 ldr r0, [r7, #(PCB_PAGEDIR)] 178 mov lr, pc 179 ldr pc, [r9, #CF_CONTEXT_SWITCH] 180 181 /* Restore all the save registers */ 182#ifndef _ARM_ARCH_5E 183 add r1, r7, #PCB_R8 184 ldmia r1, {r8-r13} 185#else 186 ldr r8, [r7, #(PCB_R8)] 187 ldr r9, [r7, #(PCB_R9)] 188 ldr r10, [r7, #(PCB_R10)] 189 ldr r11, [r7, #(PCB_R11)] 190 ldr r12, [r7, #(PCB_R12)] 191 ldr r13, [r7, #(PCB_SP)] 192#endif 193 194 /* We have a new curthread now so make a note it */ 195 GET_CURTHREAD_PTR(r6) 196 str r5, [r6] 197 198 /* Set the new tp */ 199 ldr r6, [r5, #(TD_MD + MD_TP)] 200#ifdef ARM_TP_ADDRESS 201 ldr r4, =ARM_TP_ADDRESS 202 str r6, [r4] 203 ldr r6, [r5, #(TD_MD + MD_RAS_START)] 204 str r6, [r4, #4] /* ARM_RAS_START */ 205 ldr r6, [r5, #(TD_MD + MD_RAS_END)] 206 str r6, [r4, #8] /* ARM_RAS_END */ 207#else 208 mcr p15, 0, r6, c13, c0, 3 209#endif 210 /* Hook in a new pcb */ 211 GET_PCPU(r6) 212 str r7, [r6, #PC_CURPCB] 213 214 add sp, sp, #4; 215 ldmfd sp!, {r4-r7, pc} 216END(cpu_throw) 217 218ENTRY(cpu_switch) 219 stmfd sp!, {r4-r7, lr} 220 sub sp, sp, #4; 221 mov r6, r2 /* Save the mutex */ 222 223.Lswitch_resume: 224 /* rem: r0 = old lwp */ 225 /* rem: interrupts are disabled */ 226 227 /* Process is now on a processor. */ 228 /* We have a new curthread now so make a note it */ 229 GET_CURTHREAD_PTR(r7) 230 str r1, [r7] 231 232 /* Hook in a new pcb */ 233 GET_PCPU(r7) 234 ldr r2, [r1, #TD_PCB] 235 str r2, [r7, #PC_CURPCB] 236 237 /* rem: r1 = new process */ 238 /* rem: interrupts are enabled */ 239 240 /* Stage two : Save old context */ 241 242 /* Get the user structure for the old thread. */ 243 ldr r2, [r0, #(TD_PCB)] 244 mov r4, r0 /* Save the old thread. */ 245 246 /* Save all the registers in the old thread's pcb */ 247#ifndef _ARM_ARCH_5E 248 add r7, r2, #(PCB_R8) 249 stmia r7, {r8-r13} 250#else 251 strd r8, [r2, #(PCB_R8)] 252 strd r10, [r2, #(PCB_R10)] 253 strd r12, [r2, #(PCB_R12)] 254#endif 255 str pc, [r2, #(PCB_PC)] 256 257 /* 258 * NOTE: We can now use r8-r13 until it is time to restore 259 * them for the new process. 260 */ 261#ifdef ARM_TP_ADDRESS 262 /* Store the old tp */ 263 ldr r3, =ARM_TP_ADDRESS 264 ldr r9, [r3] 265 str r9, [r0, #(TD_MD + MD_TP)] 266 ldr r9, [r3, #4] 267 str r9, [r0, #(TD_MD + MD_RAS_START)] 268 ldr r9, [r3, #8] 269 str r9, [r0, #(TD_MD + MD_RAS_END)] 270 271 /* Set the new tp */ 272 ldr r9, [r1, #(TD_MD + MD_TP)] 273 str r9, [r3] 274 ldr r9, [r1, #(TD_MD + MD_RAS_START)] 275 str r9, [r3, #4] 276 ldr r9, [r1, #(TD_MD + MD_RAS_END)] 277 str r9, [r3, #8] 278#else 279 /* Store the old tp */ 280 mrc p15, 0, r9, c13, c0, 3 281 str r9, [r0, #(TD_MD + MD_TP)] 282 283 /* Set the new tp */ 284 ldr r9, [r1, #(TD_MD + MD_TP)] 285 mcr p15, 0, r9, c13, c0, 3 286#endif 287 288 /* Get the user structure for the new process in r9 */ 289 ldr r9, [r1, #(TD_PCB)] 290 291 mrs r3, cpsr 292 /* 293 * We can do that, since 294 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE 295 */ 296 orr r8, r3, #(PSR_UND32_MODE) 297 msr cpsr_c, r8 298 299 str sp, [r2, #(PCB_UND_SP)] 300 301 msr cpsr_c, r3 /* Restore the old mode */ 302 /* rem: r2 = old PCB */ 303 /* rem: r9 = new PCB */ 304 /* rem: interrupts are enabled */ 305 306#ifdef ARM_VFP_SUPPORT 307 /* 308 * vfp_store will clear pcpu->pc_vfpcthread, save 309 * registers and state, and modify the control as needed. 310 * a future exception will bounce the backup settings in the fp unit. 311 * XXX vfp_store can't change r4 312 */ 313 GET_PCPU(r7) 314 ldr r8, [r7, #(PC_VFPCTHREAD)] 315 cmp r4, r8 /* old thread used vfp? */ 316 bne 1f /* no, don't save */ 317 cmp r1, r4 /* same thread ? */ 318 beq 1f /* yes, skip vfp store */ 319#ifdef SMP 320 ldr r8, [r7, #(PC_CPU)] /* last used on this cpu? */ 321 ldr r3, [r2, #(PCB_VFPCPU)] 322 cmp r8, r3 /* last cpu to use these registers? */ 323 bne 1f /* no. these values are stale */ 324#endif 325 add r0, r2, #(PCB_VFPSTATE) 326 bl _C_LABEL(vfp_store) 3271: 328#endif /* ARM_VFP_SUPPORT */ 329 330 /* r1 now free! */ 331 332 /* Third phase : restore saved context */ 333 334 /* rem: r2 = old PCB */ 335 /* rem: r9 = new PCB */ 336 /* rem: interrupts are enabled */ 337 338 ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */ 339 mov r2, #DOMAIN_CLIENT 340 cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 341 beq .Lcs_context_switched /* Yup. Don't flush cache */ 342 mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */ 343 /* 344 * Get the new L1 table pointer into r11. If we're switching to 345 * an LWP with the same address space as the outgoing one, we can 346 * skip the cache purge and the TTB load. 347 * 348 * To avoid data dep stalls that would happen anyway, we try 349 * and get some useful work done in the mean time. 350 */ 351 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 352 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 353 354 355 teq r10, r11 /* Same L1? */ 356 cmpeq r0, r5 /* Same DACR? */ 357 beq .Lcs_context_switched /* yes! */ 358 359#if !defined(CPU_ARM11) && !defined(CPU_CORTEXA) && !defined(CPU_MV_PJ4B) 360 /* 361 * Definately need to flush the cache. 362 */ 363 364 ldr r1, .Lcpufuncs 365 mov lr, pc 366 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 367#endif 368.Lcs_cache_purge_skipped: 369 /* rem: r6 = lock */ 370 /* rem: r9 = new PCB */ 371 /* rem: r10 = old L1 */ 372 /* rem: r11 = new L1 */ 373 374 mov r2, #0x00000000 375 ldr r7, [r9, #(PCB_PL1VEC)] 376 377 /* 378 * Ensure the vector table is accessible by fixing up the L1 379 */ 380 cmp r7, #0 /* No need to fixup vector table? */ 381 ldrne r2, [r7] /* But if yes, fetch current value */ 382 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 383 mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */ 384 cmpne r2, r0 /* Stuffing the same value? */ 385#ifndef PMAP_INCLUDE_PTE_SYNC 386 strne r0, [r7] /* Nope, update it */ 387#else 388 beq .Lcs_same_vector 389 str r0, [r7] /* Otherwise, update it */ 390 391 /* 392 * Need to sync the cache to make sure that last store is 393 * visible to the MMU. 394 */ 395 ldr r2, .Lcpufuncs 396 mov r0, r7 397 mov r1, #4 398 mov lr, pc 399 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 400 401.Lcs_same_vector: 402#endif /* PMAP_INCLUDE_PTE_SYNC */ 403 404 cmp r10, r11 /* Switching to the same L1? */ 405 ldr r10, .Lcpufuncs 406 beq .Lcs_same_l1 /* Yup. */ 407 /* 408 * Do a full context switch, including full TLB flush. 409 */ 410 mov r0, r11 411 mov lr, pc 412 ldr pc, [r10, #CF_CONTEXT_SWITCH] 413 414 b .Lcs_context_switched 415 416 /* 417 * We're switching to a different process in the same L1. 418 * In this situation, we only need to flush the TLB for the 419 * vector_page mapping, and even then only if r7 is non-NULL. 420 */ 421.Lcs_same_l1: 422 cmp r7, #0 423 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 424 movne lr, pc 425 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 426 /* 427 * We can do that, since 428 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE 429 */ 430 431.Lcs_context_switched: 432 433 /* Release the old thread */ 434 str r6, [r4, #TD_LOCK] 435#if defined(SCHED_ULE) && defined(SMP) 436 ldr r6, .Lblocked_lock 437 GET_CURTHREAD_PTR(r3) 438 4391: 440 ldr r4, [r3, #TD_LOCK] 441 cmp r4, r6 442 beq 1b 443#endif 444 445 /* XXXSCW: Safe to re-enable FIQs here */ 446 447 /* rem: r9 = new PCB */ 448 449 mrs r3, cpsr 450 /* 451 * We can do that, since 452 * PSR_SVC32_MODE|PSR_UND32_MODE == MSR_UND32_MODE 453 */ 454 orr r2, r3, #(PSR_UND32_MODE) 455 msr cpsr_c, r2 456 457 ldr sp, [r9, #(PCB_UND_SP)] 458 459 msr cpsr_c, r3 /* Restore the old mode */ 460 /* Restore all the save registers */ 461#ifndef _ARM_ARCH_5E 462 add r7, r9, #PCB_R8 463 ldmia r7, {r8-r13} 464 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 465#else 466 mov r7, r9 467 ldr r8, [r7, #(PCB_R8)] 468 ldr r9, [r7, #(PCB_R9)] 469 ldr r10, [r7, #(PCB_R10)] 470 ldr r11, [r7, #(PCB_R11)] 471 ldr r12, [r7, #(PCB_R12)] 472 ldr r13, [r7, #(PCB_SP)] 473#endif 474 475 /* rem: r5 = new lwp's proc */ 476 /* rem: r6 = lock */ 477 /* rem: r7 = new PCB */ 478 479.Lswitch_return: 480 481 /* 482 * Pull the registers that got pushed when either savectx() or 483 * cpu_switch() was called and return. 484 */ 485 add sp, sp, #4; 486 ldmfd sp!, {r4-r7, pc} 487#ifdef DIAGNOSTIC 488.Lswitch_bogons: 489 adr r0, .Lswitch_panic_str 490 bl _C_LABEL(panic) 4911: nop 492 b 1b 493 494.Lswitch_panic_str: 495 .asciz "cpu_switch: sched_qs empty with non-zero sched_whichqs!\n" 496#endif 497END(cpu_switch) 498 499ENTRY(savectx) 500 stmfd sp!, {r4-r7, lr} 501 sub sp, sp, #4 502 /* 503 * r0 = pcb 504 */ 505 /* Store all the registers in the process's pcb */ 506 add r2, r0, #(PCB_R8) 507 stmia r2, {r8-r13} 508#ifdef ARM_VFP_SUPPORT 509 /* 510 * vfp_store will clear pcpu->pc_vfpcthread, save 511 * registers and state, and modify the control as needed. 512 * a future exception will bounce the backup settings in the fp unit. 513 */ 514 GET_PCPU(r7) 515 ldr r4, [r7, #(PC_VFPCTHREAD)] /* vfp thread */ 516 ldr r2, [r7, #(PC_CURTHREAD)] /* current thread */ 517 cmp r4, r2 518 bne 1f 519#ifdef SMP 520 ldr r2, [r7, #(PC_CPU)] /* last used on this cpu? */ 521 ldr r3, [r0, #(PCB_VFPCPU)] 522 cmp r2, r3 523 bne 1f /* no. these values are stale */ 524#endif 525 add r0, r0, #(PCB_VFPSTATE) 526 bl _C_LABEL(vfp_store) 5271: 528#endif /* ARM_VFP_SUPPORT */ 529 add sp, sp, #4; 530 ldmfd sp!, {r4-r7, pc} 531END(savectx) 532 533ENTRY(fork_trampoline) 534 STOP_UNWINDING /* Can't unwind beyond the thread enty point */ 535 mov r1, r5 536 mov r2, sp 537 mov r0, r4 538 mov fp, #0 539 bl _C_LABEL(fork_exit) 540 /* Kill irq"s */ 541 mrs r0, cpsr 542 orr r0, r0, #(I32_bit|F32_bit) 543 msr cpsr_c, r0 544 DO_AST 545 PULLFRAME 546 547 movs pc, lr /* Exit */ 548 549AST_LOCALS 550END(fork_trampoline) 551 552