1/*- 2 * Copyright (c) 2001 Jake Burkholder. 3 * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <machine/asm.h> 29__FBSDID("$FreeBSD$"); 30 31#include <machine/asmacros.h> 32#include <machine/asi.h> 33#include <machine/fsr.h> 34#include <machine/ktr.h> 35#include <machine/pcb.h> 36#include <machine/tstate.h> 37 38#include "assym.s" 39#include "opt_sched.h" 40 41 .register %g2, #ignore 42 .register %g3, #ignore 43 44/* 45 * void cpu_throw(struct thread *old, struct thread *new) 46 */ 47ENTRY(cpu_throw) 48 save %sp, -CCFSZ, %sp 49 flushw 50 ba %xcc, .Lsw1 51 mov %g0, %i2 52END(cpu_throw) 53 54/* 55 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 56 */ 57ENTRY(cpu_switch) 58 save %sp, -CCFSZ, %sp 59 60 /* 61 * If the current thread was using floating point in the kernel, save 62 * its context. The userland floating point context has already been 63 * saved in that case. 64 */ 65 rd %fprs, %l2 66 andcc %l2, FPRS_FEF, %g0 67 bz,a,pt %xcc, 1f 68 nop 69 call savefpctx 70 add PCB_REG, PCB_KFP, %o0 71 ba,a,pt %xcc, 2f 72 nop 73 74 /* 75 * If the current thread was using floating point in userland, save 76 * its context. 77 */ 781: sub PCB_REG, TF_SIZEOF, %l2 79 ldx [%l2 + TF_FPRS], %l3 80 andcc %l3, FPRS_FEF, %g0 81 bz,a,pt %xcc, 2f 82 nop 83 call savefpctx 84 add PCB_REG, PCB_UFP, %o0 85 andn %l3, FPRS_FEF, %l3 86 stx %l3, [%l2 + TF_FPRS] 87 88 ldx [PCB_REG + PCB_FLAGS], %l3 89 or %l3, PCB_FEF, %l3 90 stx %l3, [PCB_REG + PCB_FLAGS] 91 92 /* 93 * Flush the windows out to the stack and save the current frame 94 * pointer and program counter. 95 */ 962: flushw 97 wrpr %g0, 0, %cleanwin 98 stx %fp, [PCB_REG + PCB_SP] 99 stx %i7, [PCB_REG + PCB_PC] 100 101 /* 102 * Load the new thread's frame pointer and program counter, and set 103 * the current thread and pcb. 104 */ 105.Lsw1: 106#if KTR_COMPILE & KTR_PROC 107 CATR(KTR_PROC, "cpu_switch: new td=%p pc=%#lx fp=%#lx" 108 , %g1, %g2, %g3, 8, 9, 10) 109 stx %i1, [%g1 + KTR_PARM1] 110 ldx [%i1 + TD_PCB], %g2 111 ldx [%g2 + PCB_PC], %g3 112 stx %g3, [%g1 + KTR_PARM2] 113 ldx [%g2 + PCB_SP], %g3 114 stx %g3, [%g1 + KTR_PARM3] 11510: 116#endif 117 ldx [%i1 + TD_PCB], %l0 118 119 stx %i1, [PCPU(CURTHREAD)] 120 stx %l0, [PCPU(CURPCB)] 121 122 wrpr %g0, PSTATE_NORMAL, %pstate 123 mov %l0, PCB_REG 124 wrpr %g0, PSTATE_ALT, %pstate 125 mov %l0, PCB_REG 126 wrpr %g0, PSTATE_KERNEL, %pstate 127 128 ldx [PCB_REG + PCB_SP], %fp 129 ldx [PCB_REG + PCB_PC], %i7 130 sub %fp, CCFSZ, %sp 131 132 /* 133 * Point to the pmaps of the new process, and of the last non-kernel 134 * process to run. 135 */ 136 ldx [%i1 + TD_PROC], %l1 137 ldx [PCPU(PMAP)], %l2 138 ldx [%l1 + P_VMSPACE], %i5 139 add %i5, VM_PMAP, %l1 140 141#if KTR_COMPILE & KTR_PROC 142 CATR(KTR_PROC, "cpu_switch: new pmap=%p old pmap=%p" 143 , %g1, %g2, %g3, 8, 9, 10) 144 stx %l1, [%g1 + KTR_PARM1] 145 stx %l2, [%g1 + KTR_PARM2] 14610: 147#endif 148 149 /* 150 * If they are the same we are done. 151 */ 152 cmp %l2, %l1 153 be,a,pn %xcc, 8f 154 nop 155 156 /* 157 * If the new process is a kernel thread we can just leave the old 158 * context active and avoid recycling its context number. 159 */ 160 SET(vmspace0, %i4, %i3) 161 cmp %i5, %i3 162 be,a,pn %xcc, 8f 163 nop 164 165 /* 166 * If there was no non-kernel pmap, don't try to deactivate it. 167 */ 168 brz,pn %l2, 3f 169 lduw [PCPU(CPUID)], %l3 170 171 /* 172 * Mark the pmap of the last non-kernel vmspace to run as no longer 173 * active on this CPU. 174 */ 175 mov _NCPUBITS, %l5 176 udivx %l3, %l5, %l6 177 srl %l6, 0, %l4 178 sllx %l4, PTR_SHIFT, %l4 179 add %l4, PM_ACTIVE, %l4 180 smul %l6, %l5, %l5 181 sub %l3, %l5, %l5 182 mov 1, %l6 183 sllx %l6, %l5, %l5 184#ifdef SMP 185 add %l2, %l4, %l4 186 membar #LoadStore | #StoreStore 187 ATOMIC_CLEAR_LONG(%l4, %l6, %l7, %l5) 188#else 189 ldx [%l2 + %l4], %l6 190 andn %l6, %l5, %l6 191 stx %l6, [%l2 + %l4] 192#endif 193 194 /* 195 * Take away its context number. 196 */ 197 sllx %l3, INT_SHIFT, %l3 198 add %l2, PM_CONTEXT, %l4 199 mov -1, %l5 200 stw %l5, [%l3 + %l4] 201 2023: cmp %i2, %g0 203 be,pn %xcc, 4f 204 add %i0, TD_LOCK, %l4 205#if defined(SCHED_ULE) && defined(SMP) 206 membar #LoadStore | #StoreStore 207 ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) 208#else 209 stx %i2, [%l4] 210#endif 211 212 /* 213 * Find a new TLB context. If we've run out we have to flush all 214 * user mappings from the TLB and reset the context numbers. 215 */ 2164: lduw [PCPU(TLB_CTX)], %i3 217 lduw [PCPU(TLB_CTX_MAX)], %i4 218 cmp %i3, %i4 219 bne,a,pt %xcc, 5f 220 nop 221 SET(tlb_flush_user, %i5, %i4) 222 ldx [%i4], %i5 223 call %i5 224 lduw [PCPU(TLB_CTX_MIN)], %i3 225 226 /* 227 * Advance next free context. 228 */ 2295: add %i3, 1, %i4 230 stw %i4, [PCPU(TLB_CTX)] 231 232 /* 233 * Set the new context number in the pmap. 234 */ 235 lduw [PCPU(CPUID)], %l3 236 sllx %l3, INT_SHIFT, %i4 237 add %l1, PM_CONTEXT, %i5 238 stw %i3, [%i4 + %i5] 239 240 /* 241 * Mark the pmap as active on this CPU. 242 */ 243 mov _NCPUBITS, %l5 244 udivx %l3, %l5, %l6 245 srl %l6, 0, %l4 246 sllx %l4, PTR_SHIFT, %l4 247 add %l4, PM_ACTIVE, %l4 248 smul %l6, %l5, %l5 249 sub %l3, %l5, %l5 250 mov 1, %l6 251 sllx %l6, %l5, %l5 252#ifdef SMP 253 add %l1, %l4, %l4 254 ATOMIC_SET_LONG(%l4, %l6, %l7, %l5) 255#else 256 ldx [%l1 + %l4], %l6 257 or %l6, %l5, %l6 258 stx %l6, [%l1 + %l4] 259#endif 260 261 /* 262 * Make note of the change in pmap. 263 */ 264#ifdef SMP 265 PCPU_ADDR(PMAP, %l4) 266 ATOMIC_STORE_LONG(%l4, %l5, %l6, %l1) 267#else 268 stx %l1, [PCPU(PMAP)] 269#endif 270 271 /* 272 * Fiddle the hardware bits. Set the TSB registers and install the 273 * new context number in the CPU. 274 */ 275 ldx [%l1 + PM_TSB], %i4 276 mov AA_DMMU_TSB, %i5 277 stxa %i4, [%i5] ASI_DMMU 278 mov AA_IMMU_TSB, %i5 279 stxa %i4, [%i5] ASI_IMMU 280 setx TLB_CXR_PGSZ_MASK, %i5, %i4 281 mov AA_DMMU_PCXR, %i5 282 ldxa [%i5] ASI_DMMU, %l1 283 and %l1, %i4, %l1 284 or %i3, %l1, %i3 285 sethi %hi(KERNBASE), %i4 286 stxa %i3, [%i5] ASI_DMMU 287 flush %i4 288 2896: 290#if defined(SCHED_ULE) && defined(SMP) 291 SET(blocked_lock, %l2, %l1) 292 add %i1, TD_LOCK, %l2 2937: 294 ATOMIC_LOAD_LONG(%l2, %l3) 295 cmp %l1, %l3 296 be,a,pn %xcc, 7b 297 nop 298#endif 299 300 /* 301 * Done, return and load the new process's window from the stack. 302 */ 303 ret 304 restore 305 3068: cmp %i2, %g0 307 be,pn %xcc, 6b 308 add %i0, TD_LOCK, %l4 309#if defined(SCHED_ULE) && defined(SMP) 310 membar #LoadStore | #StoreStore 311 ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) 312 ba,pt %xcc, 6b 313 nop 314#else 315 ba,pt %xcc, 6b 316 stx %i2, [%l4] 317#endif 318END(cpu_switch) 319 320ENTRY(savectx) 321 save %sp, -CCFSZ, %sp 322 flushw 323 call savefpctx 324 add %i0, PCB_UFP, %o0 325 stx %fp, [%i0 + PCB_SP] 326 stx %i7, [%i0 + PCB_PC] 327 ret 328 restore %g0, 0, %o0 329END(savectx) 330 331/* 332 * void savefpctx(uint32_t *); 333 */ 334ENTRY(savefpctx) 335 wr %g0, FPRS_FEF, %fprs 336 wr %g0, ASI_BLK_S, %asi 337 stda %f0, [%o0 + (0 * 64)] %asi 338 stda %f16, [%o0 + (1 * 64)] %asi 339 stda %f32, [%o0 + (2 * 64)] %asi 340 stda %f48, [%o0 + (3 * 64)] %asi 341 membar #Sync 342 retl 343 wr %g0, 0, %fprs 344END(savefpctx) 345