1/* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ 2 3/*- 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37/*- 38 * Copyright (c) 1994-1998 Mark Brinicombe. 39 * Copyright (c) 1994 Brini. 40 * All rights reserved. 41 * 42 * This code is derived from software written for Brini by Mark Brinicombe 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. All advertising materials mentioning features or use of this software 53 * must display the following acknowledgement: 54 * This product includes software developed by Brini. 55 * 4. The name of the company nor the name of the author may be used to 56 * endorse or promote products derived from this software without specific 57 * prior written permission. 58 * 59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69 * SUCH DAMAGE. 70 * 71 * RiscBSD kernel project 72 * 73 * cpuswitch.S 74 * 75 * cpu switching functions 76 * 77 * Created : 15/10/94 78 * 79 */ 80 81#include "assym.s" 82#include "opt_sched.h" 83 84#include <machine/asm.h> 85#include <machine/asmacros.h> 86#include <machine/armreg.h> 87#include <machine/sysreg.h> 88#include <machine/vfp.h> 89 90__FBSDID("$FreeBSD: stable/11/sys/arm/arm/swtch-v6.S 331988 2018-04-04 06:11:05Z mmel $"); 91 92#if defined(SMP) 93#define GET_PCPU(tmp, tmp2) \ 94 mrc CP15_MPIDR(tmp); \ 95 and tmp, tmp, #0xf; \ 96 ldr tmp2, .Lcurpcpu+4; \ 97 mul tmp, tmp, tmp2; \ 98 ldr tmp2, .Lcurpcpu; \ 99 add tmp, tmp, tmp2; 100#else 101 102#define GET_PCPU(tmp, tmp2) \ 103 ldr tmp, .Lcurpcpu 104#endif 105 106#ifdef VFP 107 .fpu vfp /* allow VFP instructions */ 108#endif 109 110.Lcurpcpu: 111 .word _C_LABEL(__pcpu) 112 .word PCPU_SIZE 113.Lblocked_lock: 114 .word _C_LABEL(blocked_lock) 115 116ENTRY(cpu_context_switch) 117 DSB 118 /* 119 * We can directly switch between translation tables only when the 120 * size of the mapping for any given virtual address is the same 121 * in the old and new translation tables. 122 * Thus, we must switch to kernel pmap translation table as 123 * intermediate mapping because all sizes of these mappings are same 124 * (or unmapped). The same is true for switch from kernel pmap 125 * translation table to new pmap one. 126 */ 127 mov r2, #(CPU_ASID_KERNEL) 128 ldr r1, =(_C_LABEL(pmap_kern_ttb)) 129 ldr r1, [r1] 130 mcr CP15_TTBR0(r1) /* switch to kernel TTB */ 131 ISB 132 mcr CP15_TLBIASID(r2) /* flush not global TLBs */ 133 DSB 134 mcr CP15_TTBR0(r0) /* switch to new TTB */ 135 ISB 136 /* 137 * We must flush not global TLBs again because PT2MAP mapping 138 * is different. 139 */ 140 mcr CP15_TLBIASID(r2) /* flush not global TLBs */ 141 /* 142 * Flush entire Branch Target Cache because of the branch predictor 143 * is not architecturally invisible. See ARM Architecture Reference 144 * Manual ARMv7-A and ARMv7-R edition, page B2-1264(65), Branch 145 * predictors and Requirements for branch predictor maintenance 146 * operations sections. 147 */ 148 /* 149 * Additionally, to mitigate mistrained branch predictor attack 150 * we must invalidate it on affected CPUs. Unfortunately, BPIALL 151 * is effectively NOP on Cortex-A15 so it needs special treatment. 152 */ 153 ldr r0, [r8, #PC_BP_HARDEN_KIND] 154 cmp r0, #PCPU_BP_HARDEN_KIND_ICIALLU 155 mcrne CP15_BPIALL /* Flush entire Branch Target Cache */ 156 mcreq CP15_ICIALLU /* This is the only way how to flush */ 157 /* Branch Target Cache on Cortex-A15. */ 158 DSB 159 mov pc, lr 160END(cpu_context_switch) 161 162/* 163 * cpu_throw(oldtd, newtd) 164 * 165 * Remove current thread state, then select the next thread to run 166 * and load its state. 167 * r0 = oldtd 168 * r1 = newtd 169 */ 170ENTRY(cpu_throw) 171 mov r10, r0 /* r10 = oldtd */ 172 mov r11, r1 /* r11 = newtd */ 173 174#ifdef VFP /* This thread is dying, disable */ 175 bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ 176#endif 177 GET_PCPU(r8, r9) /* r8 = current pcpu */ 178 ldr r4, [r8, #PC_CPUID] /* r4 = current cpu id */ 179 180 cmp r10, #0 /* old thread? */ 181 beq 2f /* no, skip */ 182 183 /* Remove this CPU from the active list. */ 184 ldr r5, [r8, #PC_CURPMAP] 185 mov r0, #(PM_ACTIVE) 186 add r5, r0 /* r5 = old pm_active */ 187 188 /* Compute position and mask. */ 189#if _NCPUWORDS > 1 190 lsr r0, r4, #3 191 bic r0, #3 192 add r5, r0 /* r5 = position in old pm_active */ 193 mov r2, #1 194 and r0, r4, #31 195 lsl r2, r0 /* r2 = mask */ 196#else 197 mov r2, #1 198 lsl r2, r4 /* r2 = mask */ 199#endif 200 /* Clear cpu from old active list. */ 201#ifdef SMP 2021: ldrex r0, [r5] 203 bic r0, r2 204 strex r1, r0, [r5] 205 teq r1, #0 206 bne 1b 207#else 208 ldr r0, [r5] 209 bic r0, r2 210 str r0, [r5] 211#endif 212 2132: 214#ifdef INVARIANTS 215 cmp r11, #0 /* new thread? */ 216 beq badsw1 /* no, panic */ 217#endif 218 ldr r7, [r11, #(TD_PCB)] /* r7 = new PCB */ 219 220 /* 221 * Registers at this point 222 * r4 = current cpu id 223 * r7 = new PCB 224 * r8 = current pcpu 225 * r11 = newtd 226 */ 227 228 /* MMU switch to new thread. */ 229 ldr r0, [r7, #(PCB_PAGEDIR)] 230#ifdef INVARIANTS 231 cmp r0, #0 /* new thread? */ 232 beq badsw4 /* no, panic */ 233#endif 234 bl _C_LABEL(cpu_context_switch) 235 236 /* 237 * Set new PMAP as current one. 238 * Insert cpu to new active list. 239 */ 240 241 ldr r6, [r11, #(TD_PROC)] /* newtd->proc */ 242 ldr r6, [r6, #(P_VMSPACE)] /* newtd->proc->vmspace */ 243 add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */ 244 str r6, [r8, #PC_CURPMAP] /* store to curpmap */ 245 246 mov r0, #PM_ACTIVE 247 add r6, r0 /* r6 = new pm_active */ 248 249 /* compute position and mask */ 250#if _NCPUWORDS > 1 251 lsr r0, r4, #3 252 bic r0, #3 253 add r6, r0 /* r6 = position in new pm_active */ 254 mov r2, #1 255 and r0, r4, #31 256 lsl r2, r0 /* r2 = mask */ 257#else 258 mov r2, #1 259 lsl r2, r4 /* r2 = mask */ 260#endif 261 /* Set cpu to new active list. */ 262#ifdef SMP 2631: ldrex r0, [r6] 264 orr r0, r2 265 strex r1, r0, [r6] 266 teq r1, #0 267 bne 1b 268#else 269 ldr r0, [r6] 270 orr r0, r2 271 str r0, [r6] 272#endif 273 /* 274 * Registers at this point. 275 * r7 = new PCB 276 * r8 = current pcpu 277 * r11 = newtd 278 * They must match the ones in sw1 position !!! 279 */ 280 DMB 281 b sw1 /* share new thread init with cpu_switch() */ 282END(cpu_throw) 283 284/* 285 * cpu_switch(oldtd, newtd, lock) 286 * 287 * Save the current thread state, then select the next thread to run 288 * and load its state. 289 * r0 = oldtd 290 * r1 = newtd 291 * r2 = lock (new lock for old thread) 292 */ 293ENTRY(cpu_switch) 294 /* Interrupts are disabled. */ 295#ifdef INVARIANTS 296 cmp r0, #0 /* old thread? */ 297 beq badsw2 /* no, panic */ 298#endif 299 /* Save all the registers in the old thread's pcb. */ 300 ldr r3, [r0, #(TD_PCB)] 301 add r3, #(PCB_R4) 302 stmia r3, {r4-r12, sp, lr, pc} 303 mrc CP15_TPIDRURW(r4) 304 str r4, [r3, #(PCB_TPIDRURW - PCB_R4)] 305 306#ifdef INVARIANTS 307 cmp r1, #0 /* new thread? */ 308 beq badsw3 /* no, panic */ 309#endif 310 /* 311 * Save arguments. Note that we can now use r0-r14 until 312 * it is time to restore them for the new thread. However, 313 * some registers are not safe over function call. 314 */ 315 mov r9, r2 /* r9 = lock */ 316 mov r10, r0 /* r10 = oldtd */ 317 mov r11, r1 /* r11 = newtd */ 318 319 GET_PCPU(r8, r3) /* r8 = current PCPU */ 320 ldr r7, [r11, #(TD_PCB)] /* r7 = newtd->td_pcb */ 321 322 323 324#ifdef VFP 325 ldr r3, [r10, #(TD_PCB)] 326 fmrx r0, fpexc /* If the VFP is enabled */ 327 tst r0, #(VFPEXC_EN) /* the current thread has */ 328 movne r1, #1 /* used it, so go save */ 329 addne r0, r3, #(PCB_VFPSTATE) /* the state into the PCB */ 330 blne _C_LABEL(vfp_store) /* and disable the VFP. */ 331#endif 332 333 /* 334 * MMU switch. If we're switching to a thread with the same 335 * address space as the outgoing one, we can skip the MMU switch. 336 */ 337 mrc CP15_TTBR0(r1) /* r1 = old TTB */ 338 ldr r0, [r7, #(PCB_PAGEDIR)] /* r0 = new TTB */ 339 cmp r0, r1 /* Switching to the TTB? */ 340 beq sw0 /* same TTB, skip */ 341 342#ifdef INVARIANTS 343 cmp r0, #0 /* new thread? */ 344 beq badsw4 /* no, panic */ 345#endif 346 347 bl cpu_context_switch /* new TTB as argument */ 348 349 /* 350 * Registers at this point 351 * r7 = new PCB 352 * r8 = current pcpu 353 * r9 = lock 354 * r10 = oldtd 355 * r11 = newtd 356 */ 357 358 /* 359 * Set new PMAP as current one. 360 * Update active list on PMAPs. 361 */ 362 ldr r6, [r11, #TD_PROC] /* newtd->proc */ 363 ldr r6, [r6, #P_VMSPACE] /* newtd->proc->vmspace */ 364 add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */ 365 366 ldr r5, [r8, #PC_CURPMAP] /* get old curpmap */ 367 str r6, [r8, #PC_CURPMAP] /* and save new one */ 368 369 mov r0, #PM_ACTIVE 370 add r5, r0 /* r5 = old pm_active */ 371 add r6, r0 /* r6 = new pm_active */ 372 373 /* Compute position and mask. */ 374 ldr r4, [r8, #PC_CPUID] 375#if _NCPUWORDS > 1 376 lsr r0, r4, #3 377 bic r0, #3 378 add r5, r0 /* r5 = position in old pm_active */ 379 add r6, r0 /* r6 = position in new pm_active */ 380 mov r2, #1 381 and r0, r4, #31 382 lsl r2, r0 /* r2 = mask */ 383#else 384 mov r2, #1 385 lsl r2, r4 /* r2 = mask */ 386#endif 387 /* Clear cpu from old active list. */ 388#ifdef SMP 3891: ldrex r0, [r5] 390 bic r0, r2 391 strex r1, r0, [r5] 392 teq r1, #0 393 bne 1b 394#else 395 ldr r0, [r5] 396 bic r0, r2 397 str r0, [r5] 398#endif 399 /* Set cpu to new active list. */ 400#ifdef SMP 4011: ldrex r0, [r6] 402 orr r0, r2 403 strex r1, r0, [r6] 404 teq r1, #0 405 bne 1b 406#else 407 ldr r0, [r6] 408 orr r0, r2 409 str r0, [r6] 410#endif 411 412sw0: 413 /* 414 * Registers at this point 415 * r7 = new PCB 416 * r8 = current pcpu 417 * r9 = lock 418 * r10 = oldtd 419 * r11 = newtd 420 */ 421 422 /* Change the old thread lock. */ 423 add r5, r10, #TD_LOCK 424 DMB 4251: ldrex r0, [r5] 426 strex r1, r9, [r5] 427 teq r1, #0 428 bne 1b 429 DMB 430 431sw1: 432 clrex 433 /* 434 * Registers at this point 435 * r7 = new PCB 436 * r8 = current pcpu 437 * r11 = newtd 438 */ 439 440#if defined(SMP) && defined(SCHED_ULE) 441 /* 442 * 386 and amd64 do the blocked lock test only for SMP and SCHED_ULE 443 * QQQ: What does it mean in reality and why is it done? 444 */ 445 ldr r6, =blocked_lock 4461: 447 ldr r3, [r11, #TD_LOCK] /* atomic write regular read */ 448 cmp r3, r6 449 beq 1b 450#endif 451 452 /* We have a new curthread now so make a note it */ 453 str r11, [r8, #PC_CURTHREAD] 454 mcr CP15_TPIDRPRW(r11) 455 456 /* store pcb in per cpu structure */ 457 str r7, [r8, #PC_CURPCB] 458 459 /* 460 * Restore all saved registers and return. Note that some saved 461 * registers can be changed when either cpu_fork(), cpu_copy_thread(), 462 * cpu_fork_kthread_handler(), or makectx() was called. 463 * 464 * The value of TPIDRURW is also written into TPIDRURO, as 465 * userspace still uses TPIDRURO, modifying it through 466 * sysarch(ARM_SET_TP, addr). 467 */ 468 ldr r3, [r7, #PCB_TPIDRURW] 469 mcr CP15_TPIDRURW(r3) /* write tls thread reg 2 */ 470 mcr CP15_TPIDRURO(r3) /* write tls thread reg 3 */ 471 add r3, r7, #PCB_R4 472 ldmia r3, {r4-r12, sp, pc} 473 474#ifdef INVARIANTS 475badsw1: 476 ldr r0, =sw1_panic_str 477 bl _C_LABEL(panic) 4781: nop 479 b 1b 480 481badsw2: 482 ldr r0, =sw2_panic_str 483 bl _C_LABEL(panic) 4841: nop 485 b 1b 486 487badsw3: 488 ldr r0, =sw3_panic_str 489 bl _C_LABEL(panic) 4901: nop 491 b 1b 492 493badsw4: 494 ldr r0, =sw4_panic_str 495 bl _C_LABEL(panic) 4961: nop 497 b 1b 498 499sw1_panic_str: 500 .asciz "cpu_throw: no newthread supplied.\n" 501sw2_panic_str: 502 .asciz "cpu_switch: no curthread supplied.\n" 503sw3_panic_str: 504 .asciz "cpu_switch: no newthread supplied.\n" 505sw4_panic_str: 506 .asciz "cpu_switch: new pagedir is NULL.\n" 507#endif 508END(cpu_switch) 509