180709Sjake/*- 280709Sjake * Copyright (c) 2001 Jake Burkholder. 3230690Smarius * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org> 480709Sjake * All rights reserved. 580709Sjake * 680709Sjake * Redistribution and use in source and binary forms, with or without 780709Sjake * modification, are permitted provided that the following conditions 880709Sjake * are met: 980709Sjake * 1. Redistributions of source code must retain the above copyright 1080709Sjake * notice, this list of conditions and the following disclaimer. 1180709Sjake * 2. Redistributions in binary form must reproduce the above copyright 1280709Sjake * notice, this list of conditions and the following disclaimer in the 1380709Sjake * documentation and/or other materials provided with the distribution. 1480709Sjake * 1581337Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1680709Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1780709Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1881337Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1980709Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2080709Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2180709Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2280709Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2380709Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2480709Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2580709Sjake * SUCH DAMAGE. 2680709Sjake */ 2780709Sjake 28114188Sjake#include <machine/asm.h> 29114188Sjake__FBSDID("$FreeBSD$"); 30114188Sjake 3180709Sjake#include <machine/asmacros.h> 3281184Sjake#include <machine/asi.h> 33166105Smarius#include <machine/fsr.h> 3482909Sjake#include <machine/ktr.h> 35166105Smarius#include <machine/pcb.h> 3682909Sjake#include <machine/tstate.h> 3780709Sjake 38114188Sjake#include "assym.s" 39230690Smarius#include "opt_sched.h" 40114188Sjake 4191613Sjake .register %g2, #ignore 4291613Sjake .register %g3, #ignore 4391613Sjake 44112993Speter/* 45112993Speter * void cpu_throw(struct thread *old, struct thread *new) 46112993Speter */ 4782013SjakeENTRY(cpu_throw) 4882013Sjake save %sp, -CCFSZ, %sp 4982013Sjake flushw 50112993Speter ba %xcc, .Lsw1 51203185Smarius mov %g0, %i2 5282013SjakeEND(cpu_throw) 5382013Sjake 54112993Speter/* 55203185Smarius * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) 56112993Speter */ 5780709SjakeENTRY(cpu_switch) 5880709Sjake save %sp, -CCFSZ, %sp 5981184Sjake 6081184Sjake /* 61113024Sjake * If the current thread was using floating point in the kernel, save 62113024Sjake * its context. The userland floating point context has already been 63113453Sjake * saved in that case. 64113024Sjake */ 65113024Sjake rd %fprs, %l2 66113024Sjake andcc %l2, FPRS_FEF, %g0 67203185Smarius bz,a,pt %xcc, 1f 68113024Sjake nop 69113024Sjake call savefpctx 70113024Sjake add PCB_REG, PCB_KFP, %o0 71230690Smarius ba,a,pt %xcc, 2f 72113024Sjake nop 73113024Sjake 74113024Sjake /* 75113021Sjake * If the current thread was using floating point in userland, save 76113021Sjake * its context. 7781184Sjake */ 78113021Sjake1: sub PCB_REG, TF_SIZEOF, %l2 79105733Sjake ldx [%l2 + TF_FPRS], %l3 8088645Sjake andcc %l3, FPRS_FEF, %g0 81113021Sjake bz,a,pt %xcc, 2f 8282013Sjake nop 83113021Sjake call savefpctx 84113021Sjake add PCB_REG, PCB_UFP, %o0 8588645Sjake andn %l3, FPRS_FEF, %l3 86105733Sjake stx %l3, [%l2 + TF_FPRS] 87113021Sjake 88113021Sjake ldx [PCB_REG + PCB_FLAGS], %l3 89112924Sjake or %l3, PCB_FEF, %l3 90113021Sjake stx %l3, [PCB_REG + PCB_FLAGS] 9181184Sjake 9281184Sjake /* 9381184Sjake * Flush the windows out to the stack and save the current frame 9481184Sjake * pointer and program counter. 9581184Sjake */ 96113021Sjake2: flushw 9780709Sjake wrpr %g0, 0, %cleanwin 98113021Sjake stx %fp, [PCB_REG + PCB_SP] 99113021Sjake stx %i7, [PCB_REG + PCB_PC] 10081184Sjake 10181184Sjake /* 10288645Sjake * Load the new thread's frame pointer and program counter, and set 10388645Sjake * the current thread and pcb. 10481184Sjake */ 10588645Sjake.Lsw1: 10685239Sjake#if KTR_COMPILE & KTR_PROC 10791613Sjake CATR(KTR_PROC, "cpu_switch: new td=%p pc=%#lx fp=%#lx" 108203185Smarius , %g1, %g2, %g3, 8, 9, 10) 109203185Smarius stx %i1, [%g1 + KTR_PARM1] 110203185Smarius ldx [%i1 + TD_PCB], %g2 11191613Sjake ldx [%g2 + PCB_PC], %g3 11291613Sjake stx %g3, [%g1 + KTR_PARM2] 113112917Sjake ldx [%g2 + PCB_SP], %g3 11491613Sjake stx %g3, [%g1 + KTR_PARM3] 115203185Smarius10: 11682909Sjake#endif 117203185Smarius ldx [%i1 + TD_PCB], %l0 11881184Sjake 119203185Smarius stx %i1, [PCPU(CURTHREAD)] 120203185Smarius stx %l0, [PCPU(CURPCB)] 12191613Sjake 12289043Sjake wrpr %g0, PSTATE_NORMAL, %pstate 123203185Smarius mov %l0, PCB_REG 12488645Sjake wrpr %g0, PSTATE_ALT, %pstate 125203185Smarius mov %l0, PCB_REG 12688645Sjake wrpr %g0, PSTATE_KERNEL, %pstate 12788645Sjake 128113021Sjake ldx [PCB_REG + PCB_SP], %fp 129113021Sjake ldx [PCB_REG + PCB_PC], %i7 130113021Sjake sub %fp, CCFSZ, %sp 131113021Sjake 13281184Sjake /* 133129749Stmm * Point to the pmaps of the new process, and of the last non-kernel 13491781Sjake * process to run. 13581184Sjake */ 136203185Smarius ldx [%i1 + TD_PROC], %l1 137129749Stmm ldx [PCPU(PMAP)], %l2 138203185Smarius ldx [%l1 + P_VMSPACE], %i5 139203185Smarius add %i5, VM_PMAP, %l1 14081184Sjake 14191613Sjake#if KTR_COMPILE & KTR_PROC 142129749Stmm CATR(KTR_PROC, "cpu_switch: new pmap=%p old pmap=%p" 143203185Smarius , %g1, %g2, %g3, 8, 9, 10) 144203185Smarius stx %l1, [%g1 + KTR_PARM1] 14591613Sjake stx %l2, [%g1 + KTR_PARM2] 146203185Smarius10: 14791613Sjake#endif 14891613Sjake 14981184Sjake /* 15091613Sjake * If they are the same we are done. 15181184Sjake */ 152203185Smarius cmp %l2, %l1 153230690Smarius be,a,pn %xcc, 8f 15481184Sjake nop 15581184Sjake 15691781Sjake /* 157113453Sjake * If the new process is a kernel thread we can just leave the old 158113453Sjake * context active and avoid recycling its context number. 15991781Sjake */ 160113453Sjake SET(vmspace0, %i4, %i3) 161129749Stmm cmp %i5, %i3 162230690Smarius be,a,pn %xcc, 8f 16391781Sjake nop 16491781Sjake 16581184Sjake /* 166129749Stmm * If there was no non-kernel pmap, don't try to deactivate it. 16781184Sjake */ 168203185Smarius brz,pn %l2, 3f 169222813Sattilio lduw [PCPU(CPUID)], %l3 17081184Sjake 17181184Sjake /* 17291781Sjake * Mark the pmap of the last non-kernel vmspace to run as no longer 173181701Smarius * active on this CPU. 17488645Sjake */ 175222813Sattilio mov _NCPUBITS, %l5 176222813Sattilio mov %g0, %y 177222813Sattilio udiv %l3, %l5, %l6 178222813Sattilio srl %l6, 0, %l4 179222813Sattilio sllx %l4, PTR_SHIFT, %l4 180222813Sattilio add %l4, PM_ACTIVE, %l4 181222813Sattilio smul %l6, %l5, %l5 182222813Sattilio sub %l3, %l5, %l5 183222813Sattilio mov 1, %l6 184222813Sattilio sllx %l6, %l5, %l5 185230690Smarius#ifdef SMP 186230690Smarius add %l2, %l4, %l4 187230690Smarius membar #LoadStore | #StoreStore 188230690Smarius ATOMIC_CLEAR_LONG(%l4, %l6, %l7, %l5) 189230690Smarius#else 190222813Sattilio ldx [%l2 + %l4], %l6 191222813Sattilio andn %l6, %l5, %l6 192222813Sattilio stx %l6, [%l2 + %l4] 193230690Smarius#endif 19488645Sjake 19588645Sjake /* 196113453Sjake * Take away its context number. 19791613Sjake */ 19891613Sjake sllx %l3, INT_SHIFT, %l3 199129749Stmm add %l2, PM_CONTEXT, %l4 20091613Sjake mov -1, %l5 20191613Sjake stw %l5, [%l3 + %l4] 20291613Sjake 203203185Smarius3: cmp %i2, %g0 204203185Smarius be,pn %xcc, 4f 205230690Smarius add %i0, TD_LOCK, %l4 206230690Smarius#if defined(SCHED_ULE) && defined(SMP) 207230690Smarius membar #LoadStore | #StoreStore 208230690Smarius ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) 209230690Smarius#else 210230690Smarius stx %i2, [%l4] 211230690Smarius#endif 212203185Smarius 21391613Sjake /* 214181701Smarius * Find a new TLB context. If we've run out we have to flush all 215181701Smarius * user mappings from the TLB and reset the context numbers. 21681184Sjake */ 217203185Smarius4: lduw [PCPU(TLB_CTX)], %i3 218230690Smarius lduw [PCPU(TLB_CTX_MAX)], %i4 219113453Sjake cmp %i3, %i4 220203185Smarius bne,a,pt %xcc, 5f 221113453Sjake nop 222113453Sjake SET(tlb_flush_user, %i5, %i4) 223113453Sjake ldx [%i4], %i5 224113453Sjake call %i5 225203185Smarius lduw [PCPU(TLB_CTX_MIN)], %i3 22681184Sjake 22781184Sjake /* 228113453Sjake * Advance next free context. 22991613Sjake */ 230203185Smarius5: add %i3, 1, %i4 231113453Sjake stw %i4, [PCPU(TLB_CTX)] 23291613Sjake 23391613Sjake /* 234113453Sjake * Set the new context number in the pmap. 23591613Sjake */ 236222813Sattilio lduw [PCPU(CPUID)], %l3 237222813Sattilio sllx %l3, INT_SHIFT, %i4 238203185Smarius add %l1, PM_CONTEXT, %i5 239113453Sjake stw %i3, [%i4 + %i5] 24091613Sjake 24191613Sjake /* 242181701Smarius * Mark the pmap as active on this CPU. 24381184Sjake */ 244222813Sattilio mov _NCPUBITS, %l5 245222813Sattilio mov %g0, %y 246222813Sattilio udiv %l3, %l5, %l6 247222813Sattilio srl %l6, 0, %l4 248222813Sattilio sllx %l4, PTR_SHIFT, %l4 249222813Sattilio add %l4, PM_ACTIVE, %l4 250222813Sattilio smul %l6, %l5, %l5 251222813Sattilio sub %l3, %l5, %l5 252222813Sattilio mov 1, %l6 253222813Sattilio sllx %l6, %l5, %l5 254230690Smarius#ifdef SMP 255230690Smarius add %l1, %l4, %l4 256230690Smarius ATOMIC_SET_LONG(%l4, %l6, %l7, %l5) 257230690Smarius#else 258222813Sattilio ldx [%l1 + %l4], %l6 259222813Sattilio or %l6, %l5, %l6 260222813Sattilio stx %l6, [%l1 + %l4] 261230690Smarius#endif 26281184Sjake 26381184Sjake /* 264129749Stmm * Make note of the change in pmap. 26591781Sjake */ 266230690Smarius#ifdef SMP 267230690Smarius PCPU_ADDR(PMAP, %l4) 268230690Smarius ATOMIC_STORE_LONG(%l4, %l5, %l6, %l1) 269230690Smarius#else 270203185Smarius stx %l1, [PCPU(PMAP)] 271230690Smarius#endif 27291781Sjake 27391781Sjake /* 274181701Smarius * Fiddle the hardware bits. Set the TSB registers and install the 275181701Smarius * new context number in the CPU. 27681184Sjake */ 277203185Smarius ldx [%l1 + PM_TSB], %i4 278113453Sjake mov AA_DMMU_TSB, %i5 279113453Sjake stxa %i4, [%i5] ASI_DMMU 280113453Sjake mov AA_IMMU_TSB, %i5 281113453Sjake stxa %i4, [%i5] ASI_IMMU 282205258Smarius setx TLB_CXR_PGSZ_MASK, %i5, %i4 283113453Sjake mov AA_DMMU_PCXR, %i5 284203185Smarius ldxa [%i5] ASI_DMMU, %l1 285203185Smarius and %l1, %i4, %l1 286203185Smarius or %i3, %l1, %i3 287182877Smarius sethi %hi(KERNBASE), %i4 288113453Sjake stxa %i3, [%i5] ASI_DMMU 289182877Smarius flush %i4 29088645Sjake 291230690Smarius6: 292230690Smarius#if defined(SCHED_ULE) && defined(SMP) 293230690Smarius SET(blocked_lock, %l2, %l1) 294230690Smarius add %i1, TD_LOCK, %l2 295230690Smarius7: 296230690Smarius ATOMIC_LOAD_LONG(%l2, %l3) 297230690Smarius cmp %l1, %l3 298230690Smarius be,a,pn %xcc, 7b 299230690Smarius nop 300230690Smarius#endif 301230690Smarius 30288645Sjake /* 303181701Smarius * Done, return and load the new process's window from the stack. 30481184Sjake */ 305230690Smarius ret 30680709Sjake restore 307203185Smarius 308230690Smarius8: cmp %i2, %g0 309230690Smarius be,pn %xcc, 6b 310230690Smarius add %i0, TD_LOCK, %l4 311230690Smarius#if defined(SCHED_ULE) && defined(SMP) 312230690Smarius membar #LoadStore | #StoreStore 313230690Smarius ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) 314230690Smarius ba,pt %xcc, 6b 315203185Smarius nop 316230690Smarius#else 317230690Smarius ba,pt %xcc, 6b 318230690Smarius stx %i2, [%l4] 319230690Smarius#endif 32080709SjakeEND(cpu_switch) 32180709Sjake 32280709SjakeENTRY(savectx) 32380709Sjake save %sp, -CCFSZ, %sp 32480709Sjake flushw 32581135Stmm call savefpctx 326113019Sjake add %i0, PCB_UFP, %o0 327112917Sjake stx %fp, [%i0 + PCB_SP] 32888645Sjake stx %i7, [%i0 + PCB_PC] 32980709Sjake ret 33080709Sjake restore %g0, 0, %o0 33180709SjakeEND(savectx) 33281135Stmm 33391613Sjake/* 334112920Sjake * void savefpctx(uint32_t *); 33591613Sjake */ 33681135StmmENTRY(savefpctx) 33788645Sjake wr %g0, FPRS_FEF, %fprs 33888645Sjake wr %g0, ASI_BLK_S, %asi 339112920Sjake stda %f0, [%o0 + (0 * 64)] %asi 340112920Sjake stda %f16, [%o0 + (1 * 64)] %asi 341112920Sjake stda %f32, [%o0 + (2 * 64)] %asi 342112920Sjake stda %f48, [%o0 + (3 * 64)] %asi 34388645Sjake membar #Sync 34481135Stmm retl 34588645Sjake wr %g0, 0, %fprs 34681135StmmEND(savefpctx) 347