180709Sjake/*- 280709Sjake * Copyright (c) 2001 Jake Burkholder. 380709Sjake * All rights reserved. 480709Sjake * 580709Sjake * Redistribution and use in source and binary forms, with or without 680709Sjake * modification, are permitted provided that the following conditions 780709Sjake * are met: 880709Sjake * 1. Redistributions of source code must retain the above copyright 980709Sjake * notice, this list of conditions and the following disclaimer. 1080709Sjake * 2. Redistributions in binary form must reproduce the above copyright 1180709Sjake * notice, this list of conditions and the following disclaimer in the 1280709Sjake * documentation and/or other materials provided with the distribution. 1380709Sjake * 1481337Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1580709Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1680709Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1781337Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1880709Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1980709Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2080709Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2180709Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2280709Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2380709Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2480709Sjake * SUCH DAMAGE. 2580709Sjake */ 2680709Sjake 27114188Sjake#include <machine/asm.h> 28114188Sjake__FBSDID("$FreeBSD$"); 29114188Sjake 30166105Smarius#include <sys/errno.h> 31166105Smarius 3280709Sjake#include <machine/asi.h> 3380709Sjake#include <machine/asmacros.h> 34166105Smarius#include <machine/fsr.h> 35166105Smarius#include <machine/intr_machdep.h> 36166105Smarius#include <machine/pcb.h> 3782909Sjake#include <machine/pstate.h> 38205409Smarius#include <machine/wstate.h> 3980709Sjake 4080709Sjake#include "assym.s" 4180709Sjake 4288641Sjake .register %g2, #ignore 4388641Sjake .register %g3, #ignore 4488641Sjake .register %g6, #ignore 4588641Sjake 4684193Sjake/* 4784193Sjake * Common code for copy routines. 4884193Sjake * 4984193Sjake * We use large macros to generate functions for each of the copy routines. 5084193Sjake * This allows the load and store instructions to be generated for the right 5184193Sjake * operation, asi or not. It is possible to write an asi independent function 5284193Sjake * but this would require 2 expensive wrs in the main loop to switch %asi. 5384193Sjake * It would also screw up profiling (if we ever get it), but may save some I$. 5484193Sjake * We assume that either one of dasi and sasi is empty, or that they are both 5584193Sjake * the same (empty or non-empty). It is up to the caller to set %asi. 5684193Sjake */ 5784193Sjake 5884193Sjake/* 5984193Sjake * ASI independent implementation of copystr(9). 6084193Sjake * Used to implement copyinstr() and copystr(). 6184193Sjake * 6284193Sjake * Return value is in %g1. 6384193Sjake */ 6484193Sjake#define _COPYSTR(src, dst, len, done, sa, sasi, da, dasi) \ 6584193Sjake brz len, 4f ; \ 6684193Sjake mov src, %g2 ; \ 6784193Sjake1: deccc 1, len ; \ 6884193Sjake bl,a,pn %xcc, 3f ; \ 6984193Sjake nop ; \ 7084193Sjake LD(ub, sa) [src] sasi, %g1 ; \ 7184193Sjake ST(b, da) %g1, [dst] dasi ; \ 7284193Sjake brz,pn %g1, 3f ; \ 7384193Sjake inc src ; \ 74116589Sjake ba %xcc, 1b ; \ 7580709Sjake inc dst ; \ 7684193Sjake2: mov ENAMETOOLONG, %g1 ; \ 7784193Sjake3: sub src, %g2, %g2 ; \ 7884193Sjake brnz,a done, 4f ; \ 7984193Sjake stx %g2, [done] ; \ 8084193Sjake4: 8180709Sjake 8284193Sjake/* 8384193Sjake * ASI independent implementation of memset(3). 8486520Sjake * Used to implement bzero(), memset() and aszero(). 8584193Sjake * 8684193Sjake * If the pattern is non-zero, duplicate it to fill 64 bits. 8784193Sjake * Store bytes until dst is 8-byte aligned, then store 8 bytes. 8884193Sjake * It has yet to be determined how much unrolling is beneficial. 8984193Sjake * Could also read and compare before writing to minimize snoop traffic. 9084193Sjake * 9184193Sjake * XXX bzero() should be implemented as 9284193Sjake * #define bzero(dst, len) (void)memset((dst), 0, (len)) 9384193Sjake * if at all. 9484193Sjake */ 9584193Sjake#define _MEMSET(dst, pat, len, da, dasi) \ 9684193Sjake brlez,pn len, 5f ; \ 9784193Sjake and pat, 0xff, pat ; \ 9884193Sjake brz,pt pat, 1f ; \ 9984193Sjake sllx pat, 8, %g1 ; \ 10084193Sjake or pat, %g1, pat ; \ 10184193Sjake sllx pat, 16, %g1 ; \ 10284193Sjake or pat, %g1, pat ; \ 10384193Sjake sllx pat, 32, %g1 ; \ 10484193Sjake or pat, %g1, pat ; \ 105114189Sjake .align 16 ; \ 10684193Sjake1: deccc 1, len ; \ 10784193Sjake bl,pn %xcc, 5f ; \ 10884193Sjake btst 7, dst ; \ 10984193Sjake bz,a,pt %xcc, 2f ; \ 11084193Sjake inc 1, len ; \ 11184193Sjake ST(b, da) pat, [dst] dasi ; \ 112116589Sjake ba %xcc, 1b ; \ 11384193Sjake inc dst ; \ 114114189Sjake .align 16 ; \ 11584193Sjake2: deccc 32, len ; \ 11684193Sjake bl,a,pn %xcc, 3f ; \ 11784193Sjake inc 32, len ; \ 11884193Sjake ST(x, da) pat, [dst] dasi ; \ 11984193Sjake ST(x, da) pat, [dst + 8] dasi ; \ 12084193Sjake ST(x, da) pat, [dst + 16] dasi ; \ 12184193Sjake ST(x, da) pat, [dst + 24] dasi ; \ 122116589Sjake ba %xcc, 2b ; \ 12384193Sjake inc 32, dst ; \ 124114189Sjake .align 16 ; \ 12584193Sjake3: deccc 8, len ; \ 12684193Sjake bl,a,pn %xcc, 4f ; \ 12784193Sjake inc 8, len ; \ 12884193Sjake ST(x, da) pat, [dst] dasi ; \ 129116589Sjake ba %xcc, 3b ; \ 13084193Sjake inc 8, dst ; \ 131114189Sjake .align 16 ; \ 13284193Sjake4: deccc 1, len ; \ 13384193Sjake bl,a,pn %xcc, 5f ; \ 13484193Sjake nop ; \ 13584193Sjake ST(b, da) pat, [dst] dasi ; \ 136116589Sjake ba %xcc, 4b ; \ 13784193Sjake inc 1, dst ; \ 13884193Sjake5: 13980709Sjake 14084193Sjake/* 14184193Sjake * ASI independent implementation of memcpy(3). 14286520Sjake * Used to implement bcopy(), copyin(), copyout(), memcpy(), ascopy(), 14386520Sjake * ascopyfrom() and ascopyto(). 14484193Sjake * 14584193Sjake * Transfer bytes until dst is 8-byte aligned. If src is then also 8 byte 14684193Sjake * aligned, transfer 8 bytes, otherwise finish with bytes. The unaligned 14784193Sjake * case could be optimized, but it is expected that this is the uncommon 14884193Sjake * case and of questionable value. The code to do so is also rather large 14988641Sjake * and ugly. It has yet to be determined how much unrolling is beneficial. 15084193Sjake * 15184193Sjake * XXX bcopy() must also check for overlap. This is stupid. 15284193Sjake * XXX bcopy() should be implemented as 15384193Sjake * #define bcopy(src, dst, len) (void)memcpy((dst), (src), (len)) 15484193Sjake * if at all. 15584193Sjake */ 15684193Sjake#define _MEMCPY(dst, src, len, da, dasi, sa, sasi) \ 15784193Sjake1: deccc 1, len ; \ 15884193Sjake bl,pn %xcc, 6f ; \ 15984193Sjake btst 7, dst ; \ 16084193Sjake bz,a,pt %xcc, 2f ; \ 16184193Sjake inc 1, len ; \ 16284193Sjake LD(ub, sa) [src] sasi, %g1 ; \ 16380709Sjake ST(b, da) %g1, [dst] dasi ; \ 16484193Sjake inc 1, src ; \ 165116589Sjake ba %xcc, 1b ; \ 16684193Sjake inc 1, dst ; \ 167114189Sjake .align 16 ; \ 16884193Sjake2: btst 7, src ; \ 16984193Sjake bz,a,pt %xcc, 3f ; \ 17084193Sjake nop ; \ 171116589Sjake ba,a %xcc, 5f ; \ 172114189Sjake .align 16 ; \ 17384193Sjake3: deccc 32, len ; \ 17484193Sjake bl,a,pn %xcc, 4f ; \ 17584193Sjake inc 32, len ; \ 17684193Sjake LD(x, sa) [src] sasi, %g1 ; \ 17784193Sjake LD(x, sa) [src + 8] sasi, %g2 ; \ 17884193Sjake LD(x, sa) [src + 16] sasi, %g3 ; \ 17984193Sjake LD(x, sa) [src + 24] sasi, %g4 ; \ 18084193Sjake ST(x, da) %g1, [dst] dasi ; \ 18184193Sjake ST(x, da) %g2, [dst + 8] dasi ; \ 18284193Sjake ST(x, da) %g3, [dst + 16] dasi ; \ 18384193Sjake ST(x, da) %g4, [dst + 24] dasi ; \ 18484193Sjake inc 32, src ; \ 185116589Sjake ba %xcc, 3b ; \ 18684193Sjake inc 32, dst ; \ 187114189Sjake .align 16 ; \ 18884193Sjake4: deccc 8, len ; \ 18984193Sjake bl,a,pn %xcc, 5f ; \ 19084193Sjake inc 8, len ; \ 19184193Sjake LD(x, sa) [src] sasi, %g1 ; \ 19284193Sjake ST(x, da) %g1, [dst] dasi ; \ 19384193Sjake inc 8, src ; \ 194116589Sjake ba %xcc, 4b ; \ 19584193Sjake inc 8, dst ; \ 196114189Sjake .align 16 ; \ 19784193Sjake5: deccc 1, len ; \ 19884193Sjake bl,a,pn %xcc, 6f ; \ 19984193Sjake nop ; \ 20084193Sjake LD(ub, sa) [src] sasi, %g1 ; \ 20184193Sjake ST(b, da) %g1, [dst] dasi ; \ 20280709Sjake inc src ; \ 203116589Sjake ba %xcc, 5b ; \ 20480709Sjake inc dst ; \ 20584193Sjake6: 20680709Sjake 207101955Sjake/* 208101955Sjake * void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len) 209101955Sjake */ 210101955SjakeENTRY(ascopy) 211101955Sjake wr %o0, 0, %asi 212101955Sjake _MEMCPY(%o2, %o1, %o3, a, %asi, a, %asi) 213101955Sjake retl 214101955Sjake nop 215101955SjakeEND(ascopy) 21680709Sjake 217101955Sjake/* 218101955Sjake * void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len) 219101955Sjake */ 220101955SjakeENTRY(ascopyfrom) 221101955Sjake wr %o0, 0, %asi 222216802Smarius _MEMCPY(%o2, %o1, %o3, EMPTY, EMPTY, a, %asi) 223101955Sjake retl 224101955Sjake nop 225101955SjakeEND(ascopyfrom) 22680709Sjake 227101955Sjake/* 228101955Sjake * void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len) 229101955Sjake */ 230101955SjakeENTRY(ascopyto) 231101955Sjake wr %o1, 0, %asi 232216802Smarius _MEMCPY(%o2, %o0, %o3, a, %asi, EMPTY, EMPTY) 233101955Sjake retl 234101955Sjake nop 235101955SjakeEND(ascopyto) 23680709Sjake 237101955Sjake/* 238101955Sjake * void aszero(u_long asi, vm_offset_t pa, size_t len) 239101955Sjake */ 240101955SjakeENTRY(aszero) 241101955Sjake wr %o0, 0, %asi 242101955Sjake _MEMSET(%o1, %g0, %o2, a, %asi) 243101955Sjake retl 244101955Sjake nop 245101955SjakeEND(aszero) 24680709Sjake 24780709Sjake/* 24884193Sjake * int bcmp(const void *b1, const void *b2, size_t len) 24980709Sjake */ 25080709SjakeENTRY(bcmp) 25180709Sjake brz,pn %o2, 2f 25280709Sjake clr %o3 25380709Sjake1: ldub [%o0 + %o3], %o4 25480709Sjake ldub [%o1 + %o3], %o5 25580709Sjake cmp %o4, %o5 25684193Sjake bne,pn %xcc, 2f 25780709Sjake inc %o3 25880709Sjake deccc %o2 25980709Sjake bne,pt %xcc, 1b 26080709Sjake nop 26180709Sjake2: retl 26280709Sjake mov %o2, %o0 26380709SjakeEND(bcmp) 26480709Sjake 26580709Sjake/* 26680709Sjake * void bcopy(const void *src, void *dst, size_t len) 26780709Sjake */ 26886520SjakeENTRY(bcopy) 26984193Sjake /* 27084193Sjake * Check for overlap, and copy backwards if so. 27184193Sjake */ 27284193Sjake sub %o1, %o0, %g1 27384193Sjake cmp %g1, %o2 27484193Sjake bgeu,a,pt %xcc, 3f 27584193Sjake nop 27684193Sjake 27784193Sjake /* 27884193Sjake * Copy backwards. 27984193Sjake */ 28084193Sjake add %o0, %o2, %o0 28184193Sjake add %o1, %o2, %o1 28284193Sjake1: deccc 1, %o2 28384193Sjake bl,a,pn %xcc, 2f 28484193Sjake nop 28584193Sjake dec 1, %o0 28684193Sjake ldub [%o0], %g1 28784193Sjake dec 1, %o1 288116589Sjake ba %xcc, 1b 28984193Sjake stb %g1, [%o1] 29084193Sjake2: retl 29184193Sjake nop 29284193Sjake 29384193Sjake /* 29484193Sjake * Do the fast version. 29584193Sjake */ 296216802Smarius3: _MEMCPY(%o1, %o0, %o2, EMPTY, EMPTY, EMPTY, EMPTY) 29780709Sjake retl 29880709Sjake nop 29980709SjakeEND(bcopy) 30080709Sjake 30180709Sjake/* 30284193Sjake * void bzero(void *b, size_t len) 30380709Sjake */ 30484193SjakeENTRY(bzero) 305216802Smarius _MEMSET(%o0, %g0, %o1, EMPTY, EMPTY) 30680709Sjake retl 30780709Sjake nop 30884193SjakeEND(bzero) 30980709Sjake 31080709Sjake/* 311101955Sjake * int copystr(const void *src, void *dst, size_t len, size_t *done) 31280709Sjake */ 313101955SjakeENTRY(copystr) 314216802Smarius _COPYSTR(%o0, %o1, %o2, %o3, EMPTY, EMPTY, EMPTY, EMPTY) 31584193Sjake retl 316101955Sjake mov %g1, %o0 317101955SjakeEND(copystr) 31884193Sjake 31984193Sjake/* 32080709Sjake * void *memcpy(void *dst, const void *src, size_t len) 32180709Sjake */ 32280709SjakeENTRY(memcpy) 32384193Sjake mov %o0, %o3 324216802Smarius _MEMCPY(%o3, %o1, %o2, EMPTY, EMPTY, EMPTY, EMPTY) 32580709Sjake retl 32680709Sjake nop 32780709SjakeEND(memcpy) 32880709Sjake 32980709Sjake/* 33084193Sjake * void *memset(void *b, int c, size_t len) 33184193Sjake */ 33284193SjakeENTRY(memset) 33384193Sjake mov %o0, %o3 334216802Smarius _MEMSET(%o3, %o1, %o2, EMPTY, EMPTY) 33584193Sjake retl 33684193Sjake nop 33784193SjakeEND(memset) 33884193Sjake 339101955Sjake .globl copy_nofault_begin 340101955Sjakecopy_nofault_begin: 341101955Sjake nop 342101955Sjake 34384193Sjake/* 34480709Sjake * int copyin(const void *uaddr, void *kaddr, size_t len) 34580709Sjake */ 34680709SjakeENTRY(copyin) 34784193Sjake wr %g0, ASI_AIUP, %asi 348216802Smarius _MEMCPY(%o1, %o0, %o2, EMPTY, EMPTY, a, %asi) 34980709Sjake retl 35080709Sjake clr %o0 35180709SjakeEND(copyin) 35280709Sjake 35380709Sjake/* 35480709Sjake * int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 35580709Sjake */ 35680709SjakeENTRY(copyinstr) 35784193Sjake wr %g0, ASI_AIUP, %asi 358216802Smarius _COPYSTR(%o0, %o1, %o2, %o3, a, %asi, EMPTY, EMPTY) 35980709Sjake retl 36084193Sjake mov %g1, %o0 36180709SjakeEND(copyinstr) 36280709Sjake 36380709Sjake/* 36480709Sjake * int copyout(const void *kaddr, void *uaddr, size_t len) 36580709Sjake */ 36680709SjakeENTRY(copyout) 36784193Sjake wr %g0, ASI_AIUP, %asi 368216802Smarius _MEMCPY(%o1, %o0, %o2, a, %asi, EMPTY, EMPTY) 36980709Sjake retl 37080709Sjake clr %o0 37180709SjakeEND(copyout) 37280709Sjake 373101955Sjake .globl copy_nofault_end 374101955Sjakecopy_nofault_end: 375101955Sjake nop 376101955Sjake 377101955SjakeENTRY(copy_fault) 37880709Sjake retl 37980709Sjake mov EFAULT, %o0 380101955SjakeEND(copy_fault) 38180709Sjake 382101955Sjake .globl fs_nofault_begin 383101955Sjakefs_nofault_begin: 384101955Sjake nop 38580709Sjake 38680709Sjake/* 387101955Sjake * Chatty aliases for fetch, store functions. 38880709Sjake */ 389101955Sjake .globl fubyte, fusword, fuword, subyte, susword, suword 390101955Sjake .set fubyte, fuword8 391101955Sjake .set fusword, fuword16 392101955Sjake .set fuword, fuword64 393101955Sjake .set subyte, suword8 394101955Sjake .set susword, suword16 395101955Sjake .set suword, suword64 39680709Sjake 397163449Sdavidxu .globl casuword32, casuword, fuptr, suptr 398163449Sdavidxu .set casuword, casuword64 399112914Sjake .set fuptr, fuword64 400112914Sjake .set suptr, suword64 401112914Sjake 40280709Sjake/* 403112914Sjake * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s) 404112914Sjake */ 405112914SjakeENTRY(casuword32) 406112914Sjake casa [%o0] ASI_AIUP, %o1, %o2 407112914Sjake retl 408112914Sjake mov %o2, %o0 409112914SjakeEND(casuword32) 410112914Sjake 411112914Sjake/* 412112914Sjake * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s) 413112914Sjake */ 414112914SjakeENTRY(casuword64) 415112914Sjake casxa [%o0] ASI_AIUP, %o1, %o2 416112914Sjake retl 417112914Sjake mov %o2, %o0 418112914SjakeEND(casuword64) 419112914Sjake 420112914Sjake/* 421101955Sjake * int fuword8(const void *base) 42280709Sjake */ 423101955SjakeENTRY(fuword8) 424101955Sjake retl 425101955Sjake lduba [%o0] ASI_AIUP, %o0 426101955SjakeEND(fuword8) 42780709Sjake 42880709Sjake/* 429101955Sjake * int fuword16(const void *base) 43080709Sjake */ 43198511SjakeENTRY(fuword16) 432101955Sjake retl 433101955Sjake lduha [%o0] ASI_AIUP, %o0 43498511SjakeEND(fuword16) 43580709Sjake 43680709Sjake/* 43798511Sjake * int32_t fuword32(const void *base) 43897307Sdfr */ 43998511SjakeENTRY(fuword32) 440101955Sjake retl 441101955Sjake lduwa [%o0] ASI_AIUP, %o0 44298511SjakeEND(fuword32) 44398511Sjake 44498511Sjake/* 44598511Sjake * int64_t fuword64(const void *base) 44698511Sjake */ 44797307SdfrENTRY(fuword64) 448101955Sjake retl 449101955Sjake ldxa [%o0] ASI_AIUP, %o0 45097307SdfrEND(fuword64) 45197307Sdfr 45297307Sdfr/* 453101955Sjake * int suword8(const void *base, int word) 45480709Sjake */ 455101955SjakeENTRY(suword8) 456101955Sjake stba %o1, [%o0] ASI_AIUP 457101955Sjake retl 458101955Sjake clr %o0 459101955SjakeEND(suword8) 46080709Sjake 46180709Sjake/* 462101955Sjake * int suword16(const void *base, int word) 46380709Sjake */ 46498511SjakeENTRY(suword16) 465101955Sjake stha %o1, [%o0] ASI_AIUP 466101955Sjake retl 467101955Sjake clr %o0 46898511SjakeEND(suword16) 46980709Sjake 47097307Sdfr/* 47198511Sjake * int suword32(const void *base, int32_t word) 47297307Sdfr */ 47398511SjakeENTRY(suword32) 474101955Sjake stwa %o1, [%o0] ASI_AIUP 475101955Sjake retl 476101955Sjake clr %o0 47798511SjakeEND(suword32) 47898511Sjake 47998511Sjake/* 48098511Sjake * int suword64(const void *base, int64_t word) 48198511Sjake */ 48297307SdfrENTRY(suword64) 483101955Sjake stxa %o1, [%o0] ASI_AIUP 484101955Sjake retl 485101955Sjake clr %o0 48697307SdfrEND(suword64) 48797307Sdfr 488101955Sjake .globl fs_nofault_intr_begin 489101955Sjakefs_nofault_intr_begin: 490101955Sjake nop 491101955Sjake 492101955Sjake/* 493101955Sjake * int fuswintr(const void *base) 494101955Sjake */ 495101955SjakeENTRY(fuswintr) 49685238Sjake retl 497101955Sjake lduha [%o0] ASI_AIUP, %o0 498101955SjakeEND(fuswintr) 49985238Sjake 500101955Sjake/* 501101955Sjake * int suswintr(const void *base, int word) 502101955Sjake */ 503101955SjakeENTRY(suswintr) 504101955Sjake stha %o1, [%o0] ASI_AIUP 50580709Sjake retl 506101955Sjake clr %o0 507101955SjakeEND(suswintr) 50880709Sjake 509101955Sjake .globl fs_nofault_intr_end 510101955Sjakefs_nofault_intr_end: 511101955Sjake nop 512101955Sjake 513101955Sjake .globl fs_nofault_end 514101955Sjakefs_nofault_end: 515101955Sjake nop 516101955Sjake 517101955SjakeENTRY(fs_fault) 51885238Sjake retl 51985238Sjake mov -1, %o0 520101955SjakeEND(fsfault) 52185238Sjake 522116659Sjmg .globl fas_nofault_begin 523116659Sjmgfas_nofault_begin: 524116659Sjmg 525116659Sjmg/* 526116659Sjmg * int fasword8(u_long asi, uint64_t addr, uint8_t *val) 527116659Sjmg */ 528116659SjmgENTRY(fasword8) 529116659Sjmg wr %o0, 0, %asi 530116659Sjmg membar #Sync 531116659Sjmg lduba [%o1] %asi, %o3 532116659Sjmg membar #Sync 533116659Sjmg stb %o3, [%o2] 534116659Sjmg retl 535116659Sjmg clr %o0 536116659SjmgEND(fasword8) 537116659Sjmg 538116659Sjmg/* 539116659Sjmg * int fasword16(u_long asi, uint64_t addr, uint16_t *val) 540116659Sjmg */ 541116659SjmgENTRY(fasword16) 542116659Sjmg wr %o0, 0, %asi 543116659Sjmg membar #Sync 544116659Sjmg lduha [%o1] %asi, %o3 545116659Sjmg membar #Sync 546116659Sjmg sth %o3, [%o2] 547116659Sjmg retl 548116659Sjmg clr %o0 549116659SjmgEND(fasword16) 550116659Sjmg 551116659Sjmg/* 552116659Sjmg * int fasword32(u_long asi, uint64_t addr, uint32_t *val) 553116659Sjmg */ 554116659SjmgENTRY(fasword32) 555116659Sjmg wr %o0, 0, %asi 556116659Sjmg membar #Sync 557116659Sjmg lduwa [%o1] %asi, %o3 558116659Sjmg membar #Sync 559116659Sjmg stw %o3, [%o2] 560116659Sjmg retl 561116659Sjmg clr %o0 562116659SjmgEND(fasword32) 563116659Sjmg 564116659Sjmg .globl fas_nofault_end 565116659Sjmgfas_nofault_end: 566116659Sjmg nop 567116659Sjmg 568116659Sjmg .globl fas_fault 569116659SjmgENTRY(fas_fault) 570116659Sjmg retl 571116659Sjmg mov -1, %o0 572116659SjmgEND(fas_fault) 573116659Sjmg 574113027Sjake .globl fpu_fault_begin 575113027Sjakefpu_fault_begin: 576113027Sjake nop 577113027Sjake 578113027Sjake/* 579113027Sjake * void spitfire_block_copy(void *src, void *dst, size_t len) 580113027Sjake */ 581113027SjakeENTRY(spitfire_block_copy) 582230662Smarius rdpr %pstate, %o3 583230662Smarius wrpr %g0, PSTATE_NORMAL, %pstate 584113027Sjake 585113027Sjake wr %g0, ASI_BLK_S, %asi 586113027Sjake wr %g0, FPRS_FEF, %fprs 587113027Sjake 588113027Sjake sub PCB_REG, TF_SIZEOF, %o4 589113027Sjake ldx [%o4 + TF_FPRS], %o5 590113027Sjake andcc %o5, FPRS_FEF, %g0 591113027Sjake bz,a,pt %xcc, 1f 592113027Sjake nop 593239941Smarius stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi 594239941Smarius stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi 595239941Smarius stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi 596239941Smarius stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi 597113027Sjake membar #Sync 598113027Sjake 599113027Sjake andn %o5, FPRS_FEF, %o5 600113027Sjake stx %o5, [%o4 + TF_FPRS] 601113027Sjake ldx [PCB_REG + PCB_FLAGS], %o4 602113027Sjake or %o4, PCB_FEF, %o4 603113027Sjake stx %o4, [PCB_REG + PCB_FLAGS] 604113027Sjake 605230662Smarius1: wrpr %o3, 0, %pstate 606113027Sjake 607113027Sjake ldda [%o0] %asi, %f0 608239941Smarius add %o0, VIS_BLOCKSIZE, %o0 609239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 610113027Sjake 611113027Sjake2: ldda [%o0] %asi, %f16 612113027Sjake fsrc1 %f0, %f32 613113027Sjake fsrc1 %f2, %f34 614113027Sjake fsrc1 %f4, %f36 615113027Sjake fsrc1 %f6, %f38 616113027Sjake fsrc1 %f8, %f40 617113027Sjake fsrc1 %f10, %f42 618113027Sjake fsrc1 %f12, %f44 619113027Sjake fsrc1 %f14, %f46 620113027Sjake stda %f32, [%o1] %asi 621239941Smarius add %o0, VIS_BLOCKSIZE, %o0 622239941Smarius subcc %o2, VIS_BLOCKSIZE, %o2 623113027Sjake bz,pn %xcc, 3f 624239941Smarius add %o1, VIS_BLOCKSIZE, %o1 625113027Sjake ldda [%o0] %asi, %f0 626113027Sjake fsrc1 %f16, %f32 627113027Sjake fsrc1 %f18, %f34 628113027Sjake fsrc1 %f20, %f36 629113027Sjake fsrc1 %f22, %f38 630113027Sjake fsrc1 %f24, %f40 631113027Sjake fsrc1 %f26, %f42 632113027Sjake fsrc1 %f28, %f44 633113027Sjake fsrc1 %f30, %f46 634113027Sjake stda %f32, [%o1] %asi 635239941Smarius add %o0, VIS_BLOCKSIZE, %o0 636239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 637185006Smarius ba,pt %xcc, 2b 638239941Smarius add %o1, VIS_BLOCKSIZE, %o1 639113027Sjake 640113027Sjake3: membar #Sync 641113027Sjake 642113027Sjake stda %f16, [%o1] %asi 643113027Sjake membar #Sync 644113027Sjake 645113027Sjake retl 646185006Smarius wr %g0, 0, %fprs 647113027SjakeEND(spitfire_block_copy) 648113027Sjake 649113027Sjake/* 650212709Smarius * void zeus_block_copy(void *src, void *dst, size_t len) 651212709Smarius */ 652212709SmariusENTRY(zeus_block_copy) 653239941Smarius prefetch [%o0 + (0 * VIS_BLOCKSIZE)], 0 654212709Smarius 655230662Smarius rdpr %pstate, %o3 656230662Smarius wrpr %g0, PSTATE_NORMAL, %pstate 657212709Smarius 658212709Smarius wr %g0, ASI_BLK_S, %asi 659212709Smarius wr %g0, FPRS_FEF, %fprs 660212709Smarius 661212709Smarius sub PCB_REG, TF_SIZEOF, %o4 662212709Smarius ldx [%o4 + TF_FPRS], %o5 663212709Smarius andcc %o5, FPRS_FEF, %g0 664212709Smarius bz,a,pt %xcc, 1f 665212709Smarius nop 666239941Smarius stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi 667239941Smarius stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi 668239941Smarius stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi 669239941Smarius stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi 670212709Smarius membar #Sync 671212709Smarius 672212709Smarius andn %o5, FPRS_FEF, %o5 673212709Smarius stx %o5, [%o4 + TF_FPRS] 674212709Smarius ldx [PCB_REG + PCB_FLAGS], %o4 675212709Smarius or %o4, PCB_FEF, %o4 676212709Smarius stx %o4, [PCB_REG + PCB_FLAGS] 677212709Smarius 678230662Smarius1: wrpr %o3, 0, %pstate 679212709Smarius 680212709Smarius ldd [%o0 + (0 * 8)], %f0 681239941Smarius prefetch [%o0 + (1 * VIS_BLOCKSIZE)], 0 682212709Smarius ldd [%o0 + (1 * 8)], %f2 683239941Smarius prefetch [%o0 + (2 * VIS_BLOCKSIZE)], 0 684212709Smarius fmovd %f0, %f32 685212709Smarius ldd [%o0 + (2 * 8)], %f4 686239941Smarius prefetch [%o0 + (3 * VIS_BLOCKSIZE)], 0 687212709Smarius fmovd %f2, %f34 688212709Smarius ldd [%o0 + (3 * 8)], %f6 689239941Smarius prefetch [%o0 + (4 * VIS_BLOCKSIZE)], 1 690212709Smarius fmovd %f4, %f36 691212709Smarius ldd [%o0 + (4 * 8)], %f8 692239941Smarius prefetch [%o0 + (8 * VIS_BLOCKSIZE)], 1 693212709Smarius fmovd %f6, %f38 694212709Smarius ldd [%o0 + (5 * 8)], %f10 695239941Smarius prefetch [%o0 + (12 * VIS_BLOCKSIZE)], 1 696212709Smarius fmovd %f8, %f40 697212709Smarius ldd [%o0 + (6 * 8)], %f12 698239941Smarius prefetch [%o0 + (16 * VIS_BLOCKSIZE)], 1 699212709Smarius fmovd %f10, %f42 700212709Smarius ldd [%o0 + (7 * 8)], %f14 701212709Smarius ldd [%o0 + (8 * 8)], %f0 702239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 703239941Smarius add %o0, VIS_BLOCKSIZE, %o0 704239941Smarius prefetch [%o0 + (19 * VIS_BLOCKSIZE)], 1 705212709Smarius ba,pt %xcc, 2f 706239941Smarius prefetch [%o0 + (23 * VIS_BLOCKSIZE)], 1 707212709Smarius .align 32 708212709Smarius 709212709Smarius2: ldd [%o0 + (1 * 8)], %f2 710212709Smarius fmovd %f12, %f44 711212709Smarius ldd [%o0 + (2 * 8)], %f4 712212709Smarius fmovd %f14, %f46 713212709Smarius stda %f32, [%o1] %asi 714212709Smarius ldd [%o0 + (3 * 8)], %f6 715212709Smarius fmovd %f0, %f32 716212709Smarius ldd [%o0 + (4 * 8)], %f8 717212709Smarius fmovd %f2, %f34 718212709Smarius ldd [%o0 + (5 * 8)], %f10 719212709Smarius fmovd %f4, %f36 720212709Smarius ldd [%o0 + (6 * 8)], %f12 721212709Smarius fmovd %f6, %f38 722212709Smarius ldd [%o0 + (7 * 8)], %f14 723212709Smarius fmovd %f8, %f40 724212709Smarius ldd [%o0 + (8 * 8)], %f0 725212709Smarius fmovd %f10, %f42 726239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 727239941Smarius prefetch [%o0 + (3 * VIS_BLOCKSIZE)], 0 728239941Smarius add %o1, VIS_BLOCKSIZE, %o1 729239941Smarius prefetch [%o0 + (24 * VIS_BLOCKSIZE)], 1 730239941Smarius add %o0, VIS_BLOCKSIZE, %o0 731239941Smarius cmp %o2, VIS_BLOCKSIZE + 8 732212709Smarius bgu,pt %xcc, 2b 733239941Smarius prefetch [%o0 + (12 * VIS_BLOCKSIZE)], 1 734212709Smarius ldd [%o0 + (1 * 8)], %f2 735212709Smarius fsrc1 %f12, %f44 736212709Smarius ldd [%o0 + (2 * 8)], %f4 737212709Smarius fsrc1 %f14, %f46 738212709Smarius stda %f32, [%o1] %asi 739212709Smarius ldd [%o0 + (3 * 8)], %f6 740212709Smarius fsrc1 %f0, %f32 741212709Smarius ldd [%o0 + (4 * 8)], %f8 742212709Smarius fsrc1 %f2, %f34 743212709Smarius ldd [%o0 + (5 * 8)], %f10 744212709Smarius fsrc1 %f4, %f36 745212709Smarius ldd [%o0 + (6 * 8)], %f12 746212709Smarius fsrc1 %f6, %f38 747212709Smarius ldd [%o0 + (7 * 8)], %f14 748212709Smarius fsrc1 %f8, %f40 749239941Smarius add %o1, VIS_BLOCKSIZE, %o1 750212709Smarius fsrc1 %f10, %f42 751212709Smarius fsrc1 %f12, %f44 752212709Smarius fsrc1 %f14, %f46 753212709Smarius stda %f32, [%o1] %asi 754212709Smarius membar #Sync 755212709Smarius 756212709Smarius retl 757212709Smarius wr %g0, 0, %fprs 758212709SmariusEND(zeus_block_copy) 759212709Smarius 760212709Smarius/* 761113027Sjake * void spitfire_block_zero(void *dst, size_t len) 762212709Smarius * void zeus_block_zero(void *dst, size_t len) 763113027Sjake */ 764212709SmariusALTENTRY(zeus_block_zero) 765113027SjakeENTRY(spitfire_block_zero) 766230662Smarius rdpr %pstate, %o3 767230662Smarius wrpr %g0, PSTATE_NORMAL, %pstate 768113027Sjake 769113027Sjake wr %g0, ASI_BLK_S, %asi 770113027Sjake wr %g0, FPRS_FEF, %fprs 771113027Sjake 772113027Sjake sub PCB_REG, TF_SIZEOF, %o4 773113027Sjake ldx [%o4 + TF_FPRS], %o5 774113027Sjake andcc %o5, FPRS_FEF, %g0 775113027Sjake bz,a,pt %xcc, 1f 776113027Sjake nop 777239941Smarius stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi 778239941Smarius stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi 779239941Smarius stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi 780239941Smarius stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi 781113027Sjake membar #Sync 782113027Sjake 783113027Sjake andn %o5, FPRS_FEF, %o5 784113027Sjake stx %o5, [%o4 + TF_FPRS] 785113027Sjake ldx [PCB_REG + PCB_FLAGS], %o4 786113027Sjake or %o4, PCB_FEF, %o4 787113027Sjake stx %o4, [PCB_REG + PCB_FLAGS] 788113027Sjake 789230662Smarius1: wrpr %o3, 0, %pstate 790113027Sjake 791113027Sjake fzero %f0 792113027Sjake fzero %f2 793113027Sjake fzero %f4 794113027Sjake fzero %f6 795113027Sjake fzero %f8 796113027Sjake fzero %f10 797113027Sjake fzero %f12 798113027Sjake fzero %f14 799113027Sjake 800239941Smarius1: stda %f0, [%o0 + (0 * VIS_BLOCKSIZE)] %asi 801239941Smarius stda %f0, [%o0 + (1 * VIS_BLOCKSIZE)] %asi 802239941Smarius stda %f0, [%o0 + (2 * VIS_BLOCKSIZE)] %asi 803239941Smarius stda %f0, [%o0 + (3 * VIS_BLOCKSIZE)] %asi 804239941Smarius sub %o1, (4 * VIS_BLOCKSIZE), %o1 805185006Smarius brnz,pt %o1, 1b 806239941Smarius add %o0, (4 * VIS_BLOCKSIZE), %o0 807113027Sjake membar #Sync 808113027Sjake 809113027Sjake retl 810185006Smarius wr %g0, 0, %fprs 811113027SjakeEND(spitfire_block_zero) 812113027Sjake 813113027Sjake .globl fpu_fault_end 814113027Sjakefpu_fault_end: 815113027Sjake nop 816113027Sjake 817113027Sjake .globl fpu_fault_size 818113027Sjake .set fpu_fault_size, fpu_fault_end - fpu_fault_begin 819113027Sjake 82080709SjakeENTRY(longjmp) 82180709Sjake set 1, %g3 82280709Sjake movrz %o1, %o1, %g3 82380709Sjake mov %o0, %g1 82488641Sjake ldx [%g1 + _JB_FP], %g2 82580709Sjake1: cmp %fp, %g2 82680709Sjake bl,a,pt %xcc, 1b 82780709Sjake restore 82880709Sjake bne,pn %xcc, 2f 82988641Sjake ldx [%g1 + _JB_SP], %o2 83080709Sjake cmp %o2, %sp 83180709Sjake blt,pn %xcc, 2f 83280709Sjake movge %xcc, %o2, %sp 83388641Sjake ldx [%g1 + _JB_PC], %o7 83488641Sjake retl 83580709Sjake mov %g3, %o0 83680709Sjake2: PANIC("longjmp botch", %l1) 83780709SjakeEND(longjmp) 83880709Sjake 83980709SjakeENTRY(setjmp) 84088641Sjake stx %sp, [%o0 + _JB_SP] 84188641Sjake stx %o7, [%o0 + _JB_PC] 84288641Sjake stx %fp, [%o0 + _JB_FP] 84380709Sjake retl 84480709Sjake clr %o0 84580709SjakeEND(setjmp) 84680709Sjake 84780709Sjake/* 848186347Snwhitehorn * void ofw_entry(cell_t args[]) 84980709Sjake */ 850186347SnwhitehornENTRY(ofw_entry) 85184193Sjake save %sp, -CCFSZ, %sp 85286520Sjake SET(ofw_vec, %l7, %l6) 85381183Sjake ldx [%l6], %l6 854205409Smarius rdpr %pstate, %l7 855205409Smarius andn %l7, PSTATE_AM | PSTATE_IE, %l5 856205409Smarius wrpr %l5, 0, %pstate 857205409Smarius SET(tba_taken_over, %l5, %l4) 858205409Smarius brz,pn %l4, 1f 859205409Smarius rdpr %wstate, %l5 860205409Smarius andn %l5, WSTATE_PROM_MASK, %l3 861205409Smarius wrpr %l3, WSTATE_PROM_KMIX, %wstate 862205409Smarius1: call %l6 86380709Sjake mov %i0, %o0 864205409Smarius brz,pn %l4, 1f 865205409Smarius nop 866205409Smarius wrpr %g0, %l5, %wstate 867205409Smarius1: wrpr %l7, 0, %pstate 86884193Sjake ret 86984193Sjake restore %o0, %g0, %o0 870186347SnwhitehornEND(ofw_entry) 87186147Stmm 87286147Stmm/* 873186347Snwhitehorn * void ofw_exit(cell_t args[]) 87486147Stmm */ 875186347SnwhitehornENTRY(ofw_exit) 87686147Stmm save %sp, -CCFSZ, %sp 87786147Stmm flushw 87886520Sjake SET(ofw_tba, %l7, %l5) 87986147Stmm ldx [%l5], %l5 880205409Smarius rdpr %pstate, %l7 881205409Smarius andn %l7, PSTATE_AM | PSTATE_IE, %l7 882205409Smarius wrpr %l7, 0, %pstate 883205409Smarius rdpr %wstate, %l7 884205409Smarius andn %l7, WSTATE_PROM_MASK, %l7 885205409Smarius wrpr %l7, WSTATE_PROM_KMIX, %wstate 886181701Smarius wrpr %l5, 0, %tba ! restore the OFW trap table 88786520Sjake SET(ofw_vec, %l7, %l6) 88886147Stmm ldx [%l6], %l6 88986520Sjake SET(kstack0 + KSTACK_PAGES * PAGE_SIZE - PCB_SIZEOF, %l7, %l0) 89086147Stmm sub %l0, SPOFF, %fp ! setup a stack in a locked page 89186147Stmm sub %l0, SPOFF + CCFSZ, %sp 892181701Smarius mov AA_DMMU_PCXR, %l3 ! force primary DMMU context 0 893182877Smarius sethi %hi(KERNBASE), %l5 894181701Smarius stxa %g0, [%l3] ASI_DMMU 895182877Smarius flush %l5 89686147Stmm wrpr %g0, 0, %tl ! force trap level 0 89786147Stmm call %l6 89886147Stmm mov %i0, %o0 89986147Stmm ! never to return 900186347SnwhitehornEND(ofw_exit) 901100841Sjake 902100910Sjake#ifdef GPROF 903100841Sjake 904100910SjakeENTRY(user) 905100910Sjake nop 906100910Sjake 907100910SjakeENTRY(btrap) 908100910Sjake nop 909100910Sjake 910100910SjakeENTRY(etrap) 911100910Sjake nop 912100910Sjake 913100910SjakeENTRY(bintr) 914100910Sjake nop 915100910Sjake 916100910SjakeENTRY(eintr) 917100910Sjake nop 918100910Sjake 919100841Sjake/* 920100841Sjake * XXX including sys/gmon.h in genassym.c is not possible due to uintfptr_t 921100841Sjake * badness. 922100841Sjake */ 923100841Sjake#define GM_STATE 0x0 924100841Sjake#define GMON_PROF_OFF 3 925100841Sjake#define GMON_PROF_HIRES 4 926100841Sjake 927100910Sjake .globl _mcount 928100910Sjake .set _mcount, __cyg_profile_func_enter 929100910Sjake 930100910SjakeENTRY(__cyg_profile_func_enter) 931100841Sjake SET(_gmonparam, %o3, %o2) 932181701Smarius lduw [%o2 + GM_STATE], %o3 933181701Smarius cmp %o3, GMON_PROF_OFF 934100841Sjake be,a,pn %icc, 1f 935100841Sjake nop 936100841Sjake SET(mcount, %o3, %o2) 937100841Sjake jmpl %o2, %g0 938100841Sjake nop 939181701Smarius1: retl 940100841Sjake nop 941100910SjakeEND(__cyg_profile_func_enter) 942100841Sjake 943100910Sjake#ifdef GUPROF 944100910Sjake 945100910SjakeENTRY(__cyg_profile_func_exit) 946100841Sjake SET(_gmonparam, %o3, %o2) 947181701Smarius lduw [%o2 + GM_STATE], %o3 948181701Smarius cmp %o3, GMON_PROF_HIRES 949100841Sjake be,a,pn %icc, 1f 950100841Sjake nop 951100841Sjake SET(mexitcount, %o3, %o2) 952100841Sjake jmpl %o2, %g0 953100841Sjake nop 954181701Smarius1: retl 955100841Sjake nop 956100910SjakeEND(__cyg_profile_func_exit) 957100841Sjake 958100910Sjake#endif /* GUPROF */ 959100841Sjake 960100910Sjake#endif /* GPROF */ 961