1129198Scognet/* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ 2129198Scognet 3139735Simp/*- 4129198Scognet * Copyright 2003 Wasabi Systems, Inc. 5129198Scognet * All rights reserved. 6129198Scognet * 7129198Scognet * Written by Steve C. Woodford for Wasabi Systems, Inc. 8129198Scognet * 9129198Scognet * Redistribution and use in source and binary forms, with or without 10129198Scognet * modification, are permitted provided that the following conditions 11129198Scognet * are met: 12129198Scognet * 1. Redistributions of source code must retain the above copyright 13129198Scognet * notice, this list of conditions and the following disclaimer. 14129198Scognet * 2. Redistributions in binary form must reproduce the above copyright 15129198Scognet * notice, this list of conditions and the following disclaimer in the 16129198Scognet * documentation and/or other materials provided with the distribution. 17129198Scognet * 3. All advertising materials mentioning features or use of this software 18129198Scognet * must display the following acknowledgement: 19129198Scognet * This product includes software developed for the NetBSD Project by 20129198Scognet * Wasabi Systems, Inc. 21129198Scognet * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22129198Scognet * or promote products derived from this software without specific prior 23129198Scognet * written permission. 24129198Scognet * 25129198Scognet * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26129198Scognet * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27129198Scognet * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28129198Scognet * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29129198Scognet * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30129198Scognet * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31129198Scognet * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32129198Scognet * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33129198Scognet * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34129198Scognet * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35129198Scognet * POSSIBILITY OF SUCH DAMAGE. 36129198Scognet */ 37139735Simp/*- 38129198Scognet * Copyright (c) 1994-1998 Mark Brinicombe. 39129198Scognet * Copyright (c) 1994 Brini. 40129198Scognet * All rights reserved. 41129198Scognet * 42129198Scognet * This code is derived from software written for Brini by Mark Brinicombe 43129198Scognet * 44129198Scognet * Redistribution and use in source and binary forms, with or without 45129198Scognet * modification, are permitted provided that the following conditions 46129198Scognet * are met: 47129198Scognet * 1. Redistributions of source code must retain the above copyright 48129198Scognet * notice, this list of conditions and the following disclaimer. 49129198Scognet * 2. Redistributions in binary form must reproduce the above copyright 50129198Scognet * notice, this list of conditions and the following disclaimer in the 51129198Scognet * documentation and/or other materials provided with the distribution. 52129198Scognet * 3. All advertising materials mentioning features or use of this software 53129198Scognet * must display the following acknowledgement: 54129198Scognet * This product includes software developed by Brini. 55129198Scognet * 4. The name of the company nor the name of the author may be used to 56129198Scognet * endorse or promote products derived from this software without specific 57129198Scognet * prior written permission. 58129198Scognet * 59129198Scognet * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED 60129198Scognet * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 61129198Scognet * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 62129198Scognet * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 63129198Scognet * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 64129198Scognet * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 65129198Scognet * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 66129198Scognet * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 67129198Scognet * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 68129198Scognet * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 69129198Scognet * SUCH DAMAGE. 70129198Scognet * 71129198Scognet * RiscBSD kernel project 72129198Scognet * 73129198Scognet * cpuswitch.S 74129198Scognet * 75129198Scognet * cpu switching functions 76129198Scognet * 77129198Scognet * Created : 15/10/94 78129198Scognet * 79129198Scognet */ 80129198Scognet 81137274Scognet#include "assym.s" 82245477Scognet#include "opt_sched.h" 83137274Scognet 84129198Scognet#include <machine/asm.h> 85129198Scognet#include <machine/asmacros.h> 86129198Scognet#include <machine/armreg.h> 87262941Sian#include <machine/vfp.h> 88262941Sian 89129198Scognet__FBSDID("$FreeBSD$"); 90129198Scognet 91129198Scognet 92261415Scognet#define GET_PCPU(tmp, tmp2) \ 93239268Sgonzo ldr tmp, .Lcurpcpu 94129198Scognet 95262941Sian#ifdef VFP 96262941Sian .fpu vfp /* allow VFP instructions */ 97262941Sian#endif 98262941Sian 99261415Scognet.Lcurpcpu: 100295090Smmel .word _C_LABEL(__pcpu) 101171780Scognet.Lblocked_lock: 102171780Scognet .word _C_LABEL(blocked_lock) 103239268Sgonzo 104280712Sian 105280712Sian#define DOMAIN_CLIENT 0x01 106280712Sian 107280712Sian.Lcpufuncs: 108280712Sian .word _C_LABEL(cpufuncs) 109280712Sian 110276190Sian/* 111276190Sian * cpu_throw(oldtd, newtd) 112276190Sian * 113276190Sian * Remove current thread state, then select the next thread to run 114276190Sian * and load its state. 115276190Sian * r0 = oldtd 116276190Sian * r1 = newtd 117276190Sian */ 118135655ScognetENTRY(cpu_throw) 119135655Scognet mov r5, r1 120129198Scognet 121135655Scognet /* 122239268Sgonzo * r0 = oldtd 123137274Scognet * r5 = newtd 124135655Scognet */ 125129198Scognet 126262948Sian#ifdef VFP /* This thread is dying, disable */ 127262948Sian bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ 128262941Sian#endif 129239268Sgonzo 130262948Sian GET_PCPU(r7, r9) 131295090Smmel ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */ 132283366Sandrew 133135655Scognet /* Switch to lwp0 context */ 134135655Scognet 135135655Scognet ldr r9, .Lcpufuncs 136135655Scognet mov lr, pc 137135655Scognet ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] 138135655Scognet ldr r0, [r7, #(PCB_PL1VEC)] 139135655Scognet ldr r1, [r7, #(PCB_DACR)] 140135655Scognet /* 141135655Scognet * r0 = Pointer to L1 slot for vector_page (or NULL) 142135655Scognet * r1 = lwp0's DACR 143135655Scognet * r5 = lwp0 144135655Scognet * r7 = lwp0's PCB 145135655Scognet * r9 = cpufuncs 146135655Scognet */ 147135655Scognet 148135655Scognet /* 149135655Scognet * Ensure the vector table is accessible by fixing up lwp0's L1 150135655Scognet */ 151135655Scognet cmp r0, #0 /* No need to fixup vector table? */ 152135655Scognet ldrne r3, [r0] /* But if yes, fetch current value */ 153135655Scognet ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ 154135655Scognet mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ 155135655Scognet cmpne r3, r2 /* Stuffing the same value? */ 156135655Scognet strne r2, [r0] /* Store if not. */ 157135655Scognet 158135655Scognet#ifdef PMAP_INCLUDE_PTE_SYNC 159135655Scognet /* 160135655Scognet * Need to sync the cache to make sure that last store is 161135655Scognet * visible to the MMU. 162135655Scognet */ 163135655Scognet movne r1, #4 164135655Scognet movne lr, pc 165135655Scognet ldrne pc, [r9, #CF_DCACHE_WB_RANGE] 166135655Scognet#endif /* PMAP_INCLUDE_PTE_SYNC */ 167135655Scognet 168135655Scognet /* 169135655Scognet * Note: We don't do the same optimisation as cpu_switch() with 170135655Scognet * respect to avoiding flushing the TLB if we're switching to 171135655Scognet * the same L1 since this process' VM space may be about to go 172135655Scognet * away, so we don't want *any* turds left in the TLB. 173135655Scognet */ 174135655Scognet 175135655Scognet /* Switch the memory to the new process */ 176135655Scognet ldr r0, [r7, #(PCB_PAGEDIR)] 177135655Scognet mov lr, pc 178135655Scognet ldr pc, [r9, #CF_CONTEXT_SWITCH] 179135655Scognet 180261415Scognet GET_PCPU(r6, r4) 181261415Scognet /* Hook in a new pcb */ 182261415Scognet str r7, [r6, #PC_CURPCB] 183138751Scognet /* We have a new curthread now so make a note it */ 184276190Sian str r5, [r6, #PC_CURTHREAD] 185295066Smmel 186142570Scognet /* Set the new tp */ 187142955Scognet ldr r6, [r5, #(TD_MD + MD_TP)] 188188540Scognet ldr r4, =ARM_TP_ADDRESS 189188540Scognet str r6, [r4] 190188540Scognet ldr r6, [r5, #(TD_MD + MD_RAS_START)] 191188540Scognet str r6, [r4, #4] /* ARM_RAS_START */ 192188540Scognet ldr r6, [r5, #(TD_MD + MD_RAS_END)] 193188581Scognet str r6, [r4, #8] /* ARM_RAS_END */ 194295066Smmel 195276190Sian /* Restore all the saved registers and exit */ 196276190Sian add r3, r7, #PCB_R4 197276190Sian ldmia r3, {r4-r12, sp, pc} 198248361SandrewEND(cpu_throw) 199138751Scognet 200276190Sian/* 201276190Sian * cpu_switch(oldtd, newtd, lock) 202276190Sian * 203276190Sian * Save the current thread state, then select the next thread to run 204276190Sian * and load its state. 205276190Sian * r0 = oldtd 206276190Sian * r1 = newtd 207276190Sian * r2 = lock (new lock for old thread) 208276190Sian */ 209129198ScognetENTRY(cpu_switch) 210276190Sian /* Interrupts are disabled. */ 211276190Sian /* Save all the registers in the old thread's pcb. */ 212276190Sian ldr r3, [r0, #(TD_PCB)] 213254847Sandrew 214276190Sian /* Restore all the saved registers and exit */ 215276190Sian add r3, #(PCB_R4) 216276190Sian stmia r3, {r4-r12, sp, lr, pc} 217276190Sian 218171780Scognet mov r6, r2 /* Save the mutex */ 219129198Scognet 220137274Scognet /* rem: r0 = old lwp */ 221129198Scognet /* rem: interrupts are disabled */ 222129198Scognet 223129198Scognet /* Process is now on a processor. */ 224135655Scognet /* We have a new curthread now so make a note it */ 225261415Scognet GET_PCPU(r7, r2) 226261419Scognet str r1, [r7, #PC_CURTHREAD] 227129198Scognet 228129198Scognet /* Hook in a new pcb */ 229137274Scognet ldr r2, [r1, #TD_PCB] 230239268Sgonzo str r2, [r7, #PC_CURPCB] 231129198Scognet 232129198Scognet /* Stage two : Save old context */ 233129198Scognet 234171780Scognet /* Get the user structure for the old thread. */ 235137274Scognet ldr r2, [r0, #(TD_PCB)] 236171780Scognet mov r4, r0 /* Save the old thread. */ 237129198Scognet 238280402Sian /* Store the old tp; userland can change it on armv4. */ 239175982Sraj ldr r3, =ARM_TP_ADDRESS 240188540Scognet ldr r9, [r3] 241142570Scognet str r9, [r0, #(TD_MD + MD_TP)] 242188540Scognet ldr r9, [r3, #4] 243188540Scognet str r9, [r0, #(TD_MD + MD_RAS_START)] 244188540Scognet ldr r9, [r3, #8] 245188540Scognet str r9, [r0, #(TD_MD + MD_RAS_END)] 246129198Scognet 247142570Scognet /* Set the new tp */ 248142570Scognet ldr r9, [r1, #(TD_MD + MD_TP)] 249188540Scognet str r9, [r3] 250188540Scognet ldr r9, [r1, #(TD_MD + MD_RAS_START)] 251188540Scognet str r9, [r3, #4] 252188540Scognet ldr r9, [r1, #(TD_MD + MD_RAS_END)] 253188540Scognet str r9, [r3, #8] 254283366Sandrew 255129198Scognet /* Get the user structure for the new process in r9 */ 256137274Scognet ldr r9, [r1, #(TD_PCB)] 257129198Scognet 258239268Sgonzo /* rem: r2 = old PCB */ 259129198Scognet /* rem: r9 = new PCB */ 260129198Scognet /* rem: interrupts are enabled */ 261129198Scognet 262254461Sandrew#ifdef VFP 263262941Sian fmrx r0, fpexc /* If the VFP is enabled */ 264262941Sian tst r0, #(VFPEXC_EN) /* the current thread has */ 265262941Sian movne r1, #1 /* used it, so go save */ 266262941Sian addne r0, r2, #(PCB_VFPSTATE) /* the state into the PCB */ 267262941Sian blne _C_LABEL(vfp_store) /* and disable the VFP. */ 268239268Sgonzo#endif 269129198Scognet 270262941Sian /* r0-r3 now free! */ 271239268Sgonzo 272129198Scognet /* Third phase : restore saved context */ 273129198Scognet 274239268Sgonzo /* rem: r2 = old PCB */ 275129198Scognet /* rem: r9 = new PCB */ 276129198Scognet 277295090Smmel ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */ 278138414Scognet mov r2, #DOMAIN_CLIENT 279295090Smmel cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ 280295090Smmel beq .Lcs_context_switched /* Yup. Don't flush cache */ 281295090Smmel mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */ 282129198Scognet /* 283295090Smmel * Get the new L1 table pointer into r11. If we're switching to 284129198Scognet * an LWP with the same address space as the outgoing one, we can 285129198Scognet * skip the cache purge and the TTB load. 286129198Scognet * 287129198Scognet * To avoid data dep stalls that would happen anyway, we try 288129198Scognet * and get some useful work done in the mean time. 289129198Scognet */ 290295090Smmel mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ 291295090Smmel ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ 292129198Scognet 293295090Smmel teq r10, r11 /* Same L1? */ 294295090Smmel cmpeq r0, r5 /* Same DACR? */ 295295090Smmel beq .Lcs_context_switched /* yes! */ 296129198Scognet 297129198Scognet /* 298299069Spfg * Definitely need to flush the cache. 299129198Scognet */ 300129198Scognet 301129198Scognet ldr r1, .Lcpufuncs 302129198Scognet mov lr, pc 303129198Scognet ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] 304295066Smmel 305129198Scognet.Lcs_cache_purge_skipped: 306171780Scognet /* rem: r6 = lock */ 307129198Scognet /* rem: r9 = new PCB */ 308129198Scognet /* rem: r10 = old L1 */ 309129198Scognet /* rem: r11 = new L1 */ 310129198Scognet 311129198Scognet mov r2, #0x00000000 312129198Scognet ldr r7, [r9, #(PCB_PL1VEC)] 313129198Scognet 314129198Scognet /* 315129198Scognet * Ensure the vector table is accessible by fixing up the L1 316129198Scognet */ 317129198Scognet cmp r7, #0 /* No need to fixup vector table? */ 318129198Scognet ldrne r2, [r7] /* But if yes, fetch current value */ 319129198Scognet ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ 320137274Scognet mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */ 321129198Scognet cmpne r2, r0 /* Stuffing the same value? */ 322135655Scognet#ifndef PMAP_INCLUDE_PTE_SYNC 323129198Scognet strne r0, [r7] /* Nope, update it */ 324129198Scognet#else 325129198Scognet beq .Lcs_same_vector 326129198Scognet str r0, [r7] /* Otherwise, update it */ 327129198Scognet 328129198Scognet /* 329129198Scognet * Need to sync the cache to make sure that last store is 330129198Scognet * visible to the MMU. 331129198Scognet */ 332129198Scognet ldr r2, .Lcpufuncs 333129198Scognet mov r0, r7 334129198Scognet mov r1, #4 335129198Scognet mov lr, pc 336129198Scognet ldr pc, [r2, #CF_DCACHE_WB_RANGE] 337129198Scognet 338129198Scognet.Lcs_same_vector: 339129198Scognet#endif /* PMAP_INCLUDE_PTE_SYNC */ 340129198Scognet 341129198Scognet cmp r10, r11 /* Switching to the same L1? */ 342129198Scognet ldr r10, .Lcpufuncs 343129198Scognet beq .Lcs_same_l1 /* Yup. */ 344129198Scognet /* 345129198Scognet * Do a full context switch, including full TLB flush. 346129198Scognet */ 347129198Scognet mov r0, r11 348129198Scognet mov lr, pc 349129198Scognet ldr pc, [r10, #CF_CONTEXT_SWITCH] 350129198Scognet 351129198Scognet b .Lcs_context_switched 352129198Scognet 353129198Scognet /* 354129198Scognet * We're switching to a different process in the same L1. 355129198Scognet * In this situation, we only need to flush the TLB for the 356129198Scognet * vector_page mapping, and even then only if r7 is non-NULL. 357129198Scognet */ 358129198Scognet.Lcs_same_l1: 359129198Scognet cmp r7, #0 360129198Scognet movne r0, #0 /* We *know* vector_page's VA is 0x0 */ 361129198Scognet movne lr, pc 362129198Scognet ldrne pc, [r10, #CF_TLB_FLUSHID_SE] 363129198Scognet 364129198Scognet.Lcs_context_switched: 365129198Scognet 366171780Scognet /* Release the old thread */ 367171780Scognet str r6, [r4, #TD_LOCK] 368283366Sandrew 369129198Scognet /* XXXSCW: Safe to re-enable FIQs here */ 370129198Scognet 371129198Scognet /* rem: r9 = new PCB */ 372129198Scognet 373276190Sian /* Restore all the saved registers and exit */ 374276190Sian add r3, r9, #PCB_R4 375276190Sian ldmia r3, {r4-r12, sp, pc} 376248361SandrewEND(cpu_switch) 377