swtch-v6.S revision 129198
1/* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ 2 3/* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37/* 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 * 79 */ 80 81#include <machine/asm.h> 82#include <machine/asmacros.h> 83#include <machine/armreg.h> 84__FBSDID("$FreeBSD: head/sys/arm/arm/swtch.S 129198 2004-05-14 11:46:45Z cognet $"); 85 86#include "assym.s" 87 88/* 89 * New experimental definitions of IRQdisable and IRQenable 90 * These keep FIQ's enabled since FIQ's are special. 91 */ 92 93#define DOMAIN_CLIENT 0x01 94#define IRQdisable \ 95 mrs r14, cpsr ; \ 96 orr r14, r14, #(I32_bit) ; \ 97 msr cpsr_c, r14 ; \ 98 99#define IRQenable \ 100 mrs r14, cpsr ; \ 101 bic r14, r14, #(I32_bit) ; \ 102 msr cpsr_c, r14 ; \ 103 104/* 105 * These are used for switching the translation table/DACR. 106 * Since the vector page can be invalid for a short time, we must 107 * disable both regular IRQs *and* FIQs. 108 * 109 * XXX: This is not necessary if the vector table is relocated. 110 */ 111#define IRQdisableALL \ 112 mrs r14, cpsr ; \ 113 orr r14, r14, #(I32_bit | F32_bit) ; \ 114 msr cpsr_c, r14 115 116#define IRQenableALL \ 117 mrs r14, cpsr ; \ 118 bic r14, r14, #(I32_bit | F32_bit) ; \ 119 msr cpsr_c, r14 120 121.Lpcpu: 122 .word _C_LABEL(__pcpu) 123.Lcurthread: 124 .word _C_LABEL(__pcpu) + PC_CURTHREAD 125.Lcurpcb: 126 .word _C_LABEL(__pcpu) + PC_CURPCB 127.Lcpufuncs: 128 .word _C_LABEL(cpufuncs) 129.Lblock_userspace_access: 130 .word _C_LABEL(block_userspace_access) 131 132.Lcpu_do_powersave: 133 .word _C_LABEL(cpu_do_powersave) 134 135.Lpmap_kernel_cstate: 136 .word (kernel_pmap_store + PMAP_CSTATE) 137 138.Llast_cache_state_ptr: 139 .word _C_LABEL(pmap_cache_state) 140 141/* XXX: wow */ 142ENTRY(cpu_throw) 143ENTRY(cpu_switch) 144 stmfd sp!, {r4-r7, lr} 145 mov r6, r1 146 mov r1, r0 147 148 .Lswitch_resume: 149 /* rem: r1 = old lwp */ 150 /* rem: r4 = return value [not used if came from cpu_switchto()] */ 151 /* rem: r6 = new process */ 152 /* rem: interrupts are disabled */ 153 154#ifdef MULTIPROCESSOR 155 /* XXX use curcpu() */ 156 ldr r0, .Lcpu_info_store 157 str r0, [r6, #(L_CPU)] 158#else 159 /* l->l_cpu initialized in fork1() for single-processor */ 160#endif 161 162 /* Process is now on a processor. */ 163 164 /* We have a new curlwp now so make a note it */ 165 ldr r7, .Lcurthread 166 str r6, [r7] 167 168 /* Hook in a new pcb */ 169 ldr r7, .Lcurpcb 170 ldr r0, [r6, #(TD_PCB)] 171 str r0, [r7] 172 173 /* At this point we can allow IRQ's again. */ 174 /* rem: r1 = old lwp */ 175 /* rem: r4 = return value */ 176 /* rem: r6 = new process */ 177 /* rem: interrupts are enabled */ 178 179 /* Remember the old lwp in r0 */ 180 mov r0, r1 181 182 /* 183 * If the old lwp on entry to cpu_switch was zero then the 184 * process that called it was exiting. This means that we do 185 * not need to save the current context. Instead we can jump 186 * straight to restoring the context for the new process. 187 */ 188 teq r0, #0x00000000 189 beq .Lswitch_exited 190 191 /* rem: r0 = old lwp */ 192 /* rem: r4 = return value */ 193 /* rem: r6 = new process */ 194 /* rem: interrupts are enabled */ 195 196 /* Stage two : Save old context */ 197 198 /* Get the user structure for the old lwp. */ 199 ldr r1, [r0, #(TD_PCB)] 200 201 /* Save all the registers in the old lwp's pcb */ 202#ifndef __XSCALE__ 203 add r7, r1, #(PCB_R8) 204 stmia r7, {r8-r13} 205#else 206 strd r8, [r1, #(PCB_R8)] 207 strd r10, [r1, #(PCB_R10)] 208 strd r12, [r1, #(PCB_R12)] 209#endif 210 211 /* 212 * NOTE: We can now use r8-r13 until it is time to restore 213 * them for the new process. 214 */ 215 216 /* Remember the old PCB. */ 217 mov r8, r1 218 219 /* r1 now free! */ 220 221 /* Get the user structure for the new process in r9 */ 222 ldr r9, [r6, #(TD_PCB)] 223 224 /* 225 * This can be optimised... We know we want to go from SVC32 226 * mode to UND32 mode 227 */ 228 mrs r3, cpsr 229 bic r2, r3, #(PSR_MODE) 230 orr r2, r2, #(PSR_UND32_MODE | I32_bit) 231 msr cpsr_c, r2 232 233 str sp, [r8, #(PCB_UND_SP)] 234 235 msr cpsr_c, r3 /* Restore the old mode */ 236 237 /* rem: r0 = old lwp */ 238 /* rem: r4 = return value */ 239 /* rem: r6 = new process */ 240 /* rem: r8 = old PCB */ 241 /* rem: r9 = new PCB */ 242 /* rem: interrupts are enabled */ 243 244 /* What else needs to be saved Only FPA stuff when that is supported */ 245 246 /* Third phase : restore saved context */ 247 248 /* rem: r0 = old lwp */ 249 /* rem: r4 = return value */ 250 /* rem: r6 = new lwp */ 251 /* rem: r8 = old PCB */ 252 /* rem: r9 = new PCB */ 253 /* rem: interrupts are enabled */ 254 255 /* 256 * Get the new L1 table pointer into r11. If we're switching to 257 * an LWP with the same address space as the outgoing one, we can 258 * skip the cache purge and the TTB load. 259 * 260 * To avoid data dep stalls that would happen anyway, we try 261 * and get some useful work done in the mean time. 262 */ 263 ldr r10, [r8, #(PCB_PAGEDIR)] /* r10 = old L1 */ 264 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 265 266 267 268 ldr r0, [r8, #(PCB_DACR)] /* r0 = old DACR */ 269 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 270 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = &new_pmap->pm_cstate */ 271 ldr r5, .Llast_cache_state_ptr /* Previous thread's cstate */ 272 273 teq r10, r11 /* Same L1? */ 274 ldr r5, [r5] 275 cmpeq r0, r1 /* Same DACR? */ 276 beq .Lcs_context_switched /* yes! */ 277 ldr r3, .Lblock_userspace_access 278 mov r12, #0 279 cmp r5, #0 /* No last vm? (switch_exit) */ 280 beq .Lcs_cache_purge_skipped /* No, we can skip cache flsh */ 281 282 mov r2, #DOMAIN_CLIENT 283 cmp r1, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 284 beq .Lcs_cache_purge_skipped /* Yup. Don't flush cache */ 285 286 cmp r5, r8 /* Same userland VM space? */ 287 ldrneb r12, [r5, #(CS_CACHE_ID)] /* Last VM space cache state */ 288 289 /* 290 * We're definately switching to a new userland VM space, 291 * and the previous userland VM space has yet to be flushed 292 * from the cache/tlb. 293 * 294 * r12 holds the previous VM space's cs_cache_id state 295 */ 296 tst r12, #0xff /* Test cs_cache_id */ 297 beq .Lcs_cache_purge_skipped /* VM space is not in cache */ 298 299 /* 300 * Definately need to flush the cache. 301 * Mark the old VM space as NOT being resident in the cache. 302 */ 303 mov r2, #0x00000000 304 strb r2, [r5, #(CS_CACHE_ID)] 305 strb r2, [r5, #(CS_CACHE_D)] 306 307 /* 308 * Don't allow user space access between the purge and the switch. 309 */ 310 mov r2, #0x00000001 311 str r2, [r3] 312 313 stmfd sp!, {r0-r3} 314 ldr r1, .Lcpufuncs 315 mov lr, pc 316 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 317 ldmfd sp!, {r0-r3} 318 319.Lcs_cache_purge_skipped: 320 /* rem: r1 = new DACR */ 321 /* rem: r3 = &block_userspace_access */ 322 /* rem: r4 = return value */ 323 /* rem: r5 = &old_pmap->pm_cstate (or NULL) */ 324 /* rem: r6 = new lwp */ 325 /* rem: r8 = &new_pmap->pm_cstate */ 326 /* rem: r9 = new PCB */ 327 /* rem: r10 = old L1 */ 328 /* rem: r11 = new L1 */ 329 330 mov r2, #0x00000000 331 ldr r7, [r9, #(PCB_PL1VEC)] 332 333 /* 334 * At this point we need to kill IRQ's again. 335 * 336 * XXXSCW: Don't need to block FIQs if vectors have been relocated 337 */ 338#if 0 339 IRQdisableALL 340#endif 341 342 /* 343 * Interrupts are disabled so we can allow user space accesses again 344 * as none will occur until interrupts are re-enabled after the 345 * switch. 346 */ 347 str r2, [r3] 348 349 /* 350 * Ensure the vector table is accessible by fixing up the L1 351 */ 352 cmp r7, #0 /* No need to fixup vector table? */ 353 ldrne r2, [r7] /* But if yes, fetch current value */ 354 ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 355 mcr p15, 0, r1, c3, c0, 0 /* Update DACR for new context */ 356 cmpne r2, r0 /* Stuffing the same value? */ 357#if 0 358 strne r0, [r7] /* Nope, update it */ 359#else 360 beq .Lcs_same_vector 361 str r0, [r7] /* Otherwise, update it */ 362 363 /* 364 * Need to sync the cache to make sure that last store is 365 * visible to the MMU. 366 */ 367 ldr r2, .Lcpufuncs 368 mov r0, r7 369 mov r1, #4 370 mov lr, pc 371 ldr pc, [r2, #CF_DCACHE_WB_RANGE] 372 373.Lcs_same_vector: 374#endif /* PMAP_INCLUDE_PTE_SYNC */ 375 376 cmp r10, r11 /* Switching to the same L1? */ 377 ldr r10, .Lcpufuncs 378 beq .Lcs_same_l1 /* Yup. */ 379 /* 380 * Do a full context switch, including full TLB flush. 381 */ 382 mov r0, r11 383 mov lr, pc 384 ldr pc, [r10, #CF_CONTEXT_SWITCH] 385 386 /* 387 * Mark the old VM space as NOT being resident in the TLB 388 */ 389 mov r2, #0x00000000 390 cmp r5, #0 391 strneh r2, [r5, #(CS_TLB_ID)] 392 b .Lcs_context_switched 393 394 /* 395 * We're switching to a different process in the same L1. 396 * In this situation, we only need to flush the TLB for the 397 * vector_page mapping, and even then only if r7 is non-NULL. 398 */ 399.Lcs_same_l1: 400 cmp r7, #0 401 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 402 movne lr, pc 403 ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 404 405.Lcs_context_switched: 406 /* rem: r8 = &new_pmap->pm_cstate */ 407 408 /* XXXSCW: Safe to re-enable FIQs here */ 409 410 /* 411 * The new VM space is live in the cache and TLB. 412 * Update its cache/tlb state, and if it's not the kernel 413 * pmap, update the 'last cache state' pointer. 414 */ 415 mov r2, #-1 416 ldr r5, .Lpmap_kernel_cstate 417 ldr r0, .Llast_cache_state_ptr 418 str r2, [r8, #(CS_ALL)] 419 cmp r5, r8 420 strne r8, [r0] 421 422 /* rem: r4 = return value */ 423 /* rem: r6 = new lwp */ 424 /* rem: r9 = new PCB */ 425 426 /* 427 * This can be optimised... We know we want to go from SVC32 428 * mode to UND32 mode 429 */ 430 mrs r3, cpsr 431 bic r2, r3, #(PSR_MODE) 432 orr r2, r2, #(PSR_UND32_MODE) 433 msr cpsr_c, r2 434 435 ldr sp, [r9, #(PCB_UND_SP)] 436 437 msr cpsr_c, r3 /* Restore the old mode */ 438 439 /* Restore all the save registers */ 440#ifndef __XSCALE__ 441 add r7, r9, #PCB_R8 442 ldmia r7, {r8-r13} 443 sub r7, r7, #PCB_R8 /* restore PCB pointer */ 444#else 445 mov r7, r9 446 ldr r8, [r7, #(PCB_R8)] 447 ldr r9, [r7, #(PCB_R9)] 448 ldr r10, [r7, #(PCB_R10)] 449 ldr r11, [r7, #(PCB_R11)] 450 ldr r12, [r7, #(PCB_R12)] 451 ldr r13, [r7, #(PCB_SP)] 452#endif 453 454 ldr r5, [r6, #(TD_PROC)] /* fetch the proc for below */ 455 456 /* rem: r4 = return value */ 457 /* rem: r5 = new lwp's proc */ 458 /* rem: r6 = new lwp */ 459 /* rem: r7 = new pcb */ 460 461#ifdef ARMFPE 462 add r0, r7, #(USER_SIZE) & 0x00ff 463 add r0, r0, #(USER_SIZE) & 0xff00 464 bl _C_LABEL(arm_fpe_core_changecontext) 465#endif 466 467 /* We can enable interrupts again */ 468#if 0 469 IRQenableALL 470#endif 471 /* rem: r4 = return value */ 472 /* rem: r5 = new lwp's proc */ 473 /* rem: r6 = new lwp */ 474 /* rem: r7 = new PCB */ 475 476.Lswitch_return: 477 478 /* 479 * Pull the registers that got pushed when either savectx() or 480 * cpu_switch() was called and return. 481 */ 482 ldmfd sp!, {r4-r7, pc} 483.Lswitch_exited: 484 /* 485 * We skip the cache purge because switch_exit() already did it. 486 * Load up registers the way .Lcs_cache_purge_skipped expects. 487 * Userpsace access already blocked by switch_exit(). 488 */ 489 ldr r9, [r6, #(TD_PCB)] /* r9 = new PCB */ 490 ldr r3, .Lblock_userspace_access 491 mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 492 mov r5, #0 /* No previous cache state */ 493 ldr r1, [r9, #(PCB_DACR)] /* r1 = new DACR */ 494 ldr r8, [r9, #(PCB_CSTATE)] /* r8 = new cache state */ 495 ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 496 b .Lcs_cache_purge_skipped 497#ifdef DIAGNOSTIC 498.Lswitch_bogons: 499 adr r0, .Lswitch_panic_str 500 bl _C_LABEL(panic) 5011: nop 502 b 1b 503 504.Lswitch_panic_str: 505 .asciz "cpu_switch: sched_qs empty with non-zero sched_whichqs!\n" 506#endif 507ENTRY(savectx) 508 mov pc, lr 509ENTRY(fork_trampoline) 510 mov r1, r5 511 mov r2, sp 512 mov r0, r4 513 mov lr, pc 514 #if 0 515 mov r2, sp 516 #endif 517 #if 0 518 mov pc, r4 519 #endif 520 bl _C_LABEL(fork_exit) 521 /* Kill irq's */ 522 mrs r0, cpsr 523 orr r0, r0, #(I32_bit) 524 msr cpsr_c, r0 525 526 PULLFRAME 527 528 movs pc, lr /* Exit */ 529 530#ifndef __XSCALE__ 531 .type .Lcpu_switch_ffs_table, _ASM_TYPE_OBJECT; 532.Lcpu_switch_ffs_table: 533/* same as ffs table but all nums are -1 from that */ 534/* 0 1 2 3 4 5 6 7 */ 535 .byte 0, 0, 1, 12, 2, 6, 0, 13 /* 0- 7 */ 536 .byte 3, 0, 7, 0, 0, 0, 0, 14 /* 8-15 */ 537 .byte 10, 4, 0, 0, 8, 0, 0, 25 /* 16-23 */ 538 .byte 0, 0, 0, 0, 0, 21, 27, 15 /* 24-31 */ 539 .byte 31, 11, 5, 0, 0, 0, 0, 0 /* 32-39 */ 540 .byte 9, 0, 0, 24, 0, 0, 20, 26 /* 40-47 */ 541 .byte 30, 0, 0, 0, 0, 23, 0, 19 /* 48-55 */ 542 .byte 29, 0, 22, 18, 28, 17, 16, 0 /* 56-63 */ 543#endif /* !__XSCALE_ */ 544