180709Sjake/*- 280709Sjake * Copyright (c) 2001 Jake Burkholder. 380709Sjake * All rights reserved. 480709Sjake * 580709Sjake * Redistribution and use in source and binary forms, with or without 680709Sjake * modification, are permitted provided that the following conditions 780709Sjake * are met: 880709Sjake * 1. Redistributions of source code must retain the above copyright 980709Sjake * notice, this list of conditions and the following disclaimer. 1080709Sjake * 2. Redistributions in binary form must reproduce the above copyright 1180709Sjake * notice, this list of conditions and the following disclaimer in the 1280709Sjake * documentation and/or other materials provided with the distribution. 1380709Sjake * 1481337Sobrien * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1580709Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1680709Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1781337Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1880709Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1980709Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2080709Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2180709Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2280709Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2380709Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2480709Sjake * SUCH DAMAGE. 2580709Sjake */ 2680709Sjake 27114188Sjake#include <machine/asm.h> 28114188Sjake__FBSDID("$FreeBSD$"); 29114188Sjake 30285627Szbb#include "opt_kstack_pages.h" 31285627Szbb 32166105Smarius#include <sys/errno.h> 33166105Smarius 3480709Sjake#include <machine/asi.h> 3580709Sjake#include <machine/asmacros.h> 36166105Smarius#include <machine/fsr.h> 37166105Smarius#include <machine/intr_machdep.h> 38166105Smarius#include <machine/pcb.h> 3982909Sjake#include <machine/pstate.h> 40205409Smarius#include <machine/wstate.h> 4180709Sjake 4280709Sjake#include "assym.s" 4380709Sjake 4488641Sjake .register %g2, #ignore 4588641Sjake .register %g3, #ignore 4688641Sjake .register %g6, #ignore 4788641Sjake 4884193Sjake/* 4984193Sjake * Common code for copy routines. 5084193Sjake * 5184193Sjake * We use large macros to generate functions for each of the copy routines. 5284193Sjake * This allows the load and store instructions to be generated for the right 5384193Sjake * operation, asi or not. It is possible to write an asi independent function 5484193Sjake * but this would require 2 expensive wrs in the main loop to switch %asi. 5584193Sjake * It would also screw up profiling (if we ever get it), but may save some I$. 5684193Sjake * We assume that either one of dasi and sasi is empty, or that they are both 5784193Sjake * the same (empty or non-empty). It is up to the caller to set %asi. 5884193Sjake */ 5984193Sjake 6084193Sjake/* 6184193Sjake * ASI independent implementation of copystr(9). 6284193Sjake * Used to implement copyinstr() and copystr(). 6384193Sjake * 6484193Sjake * Return value is in %g1. 6584193Sjake */ 6684193Sjake#define _COPYSTR(src, dst, len, done, sa, sasi, da, dasi) \ 6784193Sjake brz len, 4f ; \ 6884193Sjake mov src, %g2 ; \ 6984193Sjake1: deccc 1, len ; \ 7084193Sjake bl,a,pn %xcc, 3f ; \ 7184193Sjake nop ; \ 7284193Sjake LD(ub, sa) [src] sasi, %g1 ; \ 7384193Sjake ST(b, da) %g1, [dst] dasi ; \ 7484193Sjake brz,pn %g1, 3f ; \ 7584193Sjake inc src ; \ 76116589Sjake ba %xcc, 1b ; \ 7780709Sjake inc dst ; \ 7884193Sjake2: mov ENAMETOOLONG, %g1 ; \ 7984193Sjake3: sub src, %g2, %g2 ; \ 8084193Sjake brnz,a done, 4f ; \ 8184193Sjake stx %g2, [done] ; \ 8284193Sjake4: 8380709Sjake 8484193Sjake/* 8584193Sjake * ASI independent implementation of memset(3). 8686520Sjake * Used to implement bzero(), memset() and aszero(). 8784193Sjake * 8884193Sjake * If the pattern is non-zero, duplicate it to fill 64 bits. 8984193Sjake * Store bytes until dst is 8-byte aligned, then store 8 bytes. 9084193Sjake * It has yet to be determined how much unrolling is beneficial. 9184193Sjake * Could also read and compare before writing to minimize snoop traffic. 9284193Sjake * 9384193Sjake * XXX bzero() should be implemented as 9484193Sjake * #define bzero(dst, len) (void)memset((dst), 0, (len)) 9584193Sjake * if at all. 9684193Sjake */ 9784193Sjake#define _MEMSET(dst, pat, len, da, dasi) \ 9884193Sjake brlez,pn len, 5f ; \ 9984193Sjake and pat, 0xff, pat ; \ 10084193Sjake brz,pt pat, 1f ; \ 10184193Sjake sllx pat, 8, %g1 ; \ 10284193Sjake or pat, %g1, pat ; \ 10384193Sjake sllx pat, 16, %g1 ; \ 10484193Sjake or pat, %g1, pat ; \ 10584193Sjake sllx pat, 32, %g1 ; \ 10684193Sjake or pat, %g1, pat ; \ 107114189Sjake .align 16 ; \ 10884193Sjake1: deccc 1, len ; \ 10984193Sjake bl,pn %xcc, 5f ; \ 11084193Sjake btst 7, dst ; \ 11184193Sjake bz,a,pt %xcc, 2f ; \ 11284193Sjake inc 1, len ; \ 11384193Sjake ST(b, da) pat, [dst] dasi ; \ 114116589Sjake ba %xcc, 1b ; \ 11584193Sjake inc dst ; \ 116114189Sjake .align 16 ; \ 11784193Sjake2: deccc 32, len ; \ 11884193Sjake bl,a,pn %xcc, 3f ; \ 11984193Sjake inc 32, len ; \ 12084193Sjake ST(x, da) pat, [dst] dasi ; \ 12184193Sjake ST(x, da) pat, [dst + 8] dasi ; \ 12284193Sjake ST(x, da) pat, [dst + 16] dasi ; \ 12384193Sjake ST(x, da) pat, [dst + 24] dasi ; \ 124116589Sjake ba %xcc, 2b ; \ 12584193Sjake inc 32, dst ; \ 126114189Sjake .align 16 ; \ 12784193Sjake3: deccc 8, len ; \ 12884193Sjake bl,a,pn %xcc, 4f ; \ 12984193Sjake inc 8, len ; \ 13084193Sjake ST(x, da) pat, [dst] dasi ; \ 131116589Sjake ba %xcc, 3b ; \ 13284193Sjake inc 8, dst ; \ 133114189Sjake .align 16 ; \ 13484193Sjake4: deccc 1, len ; \ 13584193Sjake bl,a,pn %xcc, 5f ; \ 13684193Sjake nop ; \ 13784193Sjake ST(b, da) pat, [dst] dasi ; \ 138116589Sjake ba %xcc, 4b ; \ 13984193Sjake inc 1, dst ; \ 14084193Sjake5: 14180709Sjake 14284193Sjake/* 14384193Sjake * ASI independent implementation of memcpy(3). 14486520Sjake * Used to implement bcopy(), copyin(), copyout(), memcpy(), ascopy(), 14586520Sjake * ascopyfrom() and ascopyto(). 14684193Sjake * 14784193Sjake * Transfer bytes until dst is 8-byte aligned. If src is then also 8 byte 14884193Sjake * aligned, transfer 8 bytes, otherwise finish with bytes. The unaligned 14984193Sjake * case could be optimized, but it is expected that this is the uncommon 15084193Sjake * case and of questionable value. The code to do so is also rather large 15188641Sjake * and ugly. It has yet to be determined how much unrolling is beneficial. 15284193Sjake * 15384193Sjake * XXX bcopy() must also check for overlap. This is stupid. 15484193Sjake * XXX bcopy() should be implemented as 15584193Sjake * #define bcopy(src, dst, len) (void)memcpy((dst), (src), (len)) 15684193Sjake * if at all. 15784193Sjake */ 15884193Sjake#define _MEMCPY(dst, src, len, da, dasi, sa, sasi) \ 15984193Sjake1: deccc 1, len ; \ 16084193Sjake bl,pn %xcc, 6f ; \ 16184193Sjake btst 7, dst ; \ 16284193Sjake bz,a,pt %xcc, 2f ; \ 16384193Sjake inc 1, len ; \ 16484193Sjake LD(ub, sa) [src] sasi, %g1 ; \ 16580709Sjake ST(b, da) %g1, [dst] dasi ; \ 16684193Sjake inc 1, src ; \ 167116589Sjake ba %xcc, 1b ; \ 16884193Sjake inc 1, dst ; \ 169114189Sjake .align 16 ; \ 17084193Sjake2: btst 7, src ; \ 17184193Sjake bz,a,pt %xcc, 3f ; \ 17284193Sjake nop ; \ 173116589Sjake ba,a %xcc, 5f ; \ 174114189Sjake .align 16 ; \ 17584193Sjake3: deccc 32, len ; \ 17684193Sjake bl,a,pn %xcc, 4f ; \ 17784193Sjake inc 32, len ; \ 17884193Sjake LD(x, sa) [src] sasi, %g1 ; \ 17984193Sjake LD(x, sa) [src + 8] sasi, %g2 ; \ 18084193Sjake LD(x, sa) [src + 16] sasi, %g3 ; \ 18184193Sjake LD(x, sa) [src + 24] sasi, %g4 ; \ 18284193Sjake ST(x, da) %g1, [dst] dasi ; \ 18384193Sjake ST(x, da) %g2, [dst + 8] dasi ; \ 18484193Sjake ST(x, da) %g3, [dst + 16] dasi ; \ 18584193Sjake ST(x, da) %g4, [dst + 24] dasi ; \ 18684193Sjake inc 32, src ; \ 187116589Sjake ba %xcc, 3b ; \ 18884193Sjake inc 32, dst ; \ 189114189Sjake .align 16 ; \ 19084193Sjake4: deccc 8, len ; \ 19184193Sjake bl,a,pn %xcc, 5f ; \ 19284193Sjake inc 8, len ; \ 19384193Sjake LD(x, sa) [src] sasi, %g1 ; \ 19484193Sjake ST(x, da) %g1, [dst] dasi ; \ 19584193Sjake inc 8, src ; \ 196116589Sjake ba %xcc, 4b ; \ 19784193Sjake inc 8, dst ; \ 198114189Sjake .align 16 ; \ 19984193Sjake5: deccc 1, len ; \ 20084193Sjake bl,a,pn %xcc, 6f ; \ 20184193Sjake nop ; \ 20284193Sjake LD(ub, sa) [src] sasi, %g1 ; \ 20384193Sjake ST(b, da) %g1, [dst] dasi ; \ 20480709Sjake inc src ; \ 205116589Sjake ba %xcc, 5b ; \ 20680709Sjake inc dst ; \ 20784193Sjake6: 20880709Sjake 209101955Sjake/* 210101955Sjake * void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len) 211101955Sjake */ 212101955SjakeENTRY(ascopy) 213101955Sjake wr %o0, 0, %asi 214101955Sjake _MEMCPY(%o2, %o1, %o3, a, %asi, a, %asi) 215101955Sjake retl 216101955Sjake nop 217101955SjakeEND(ascopy) 21880709Sjake 219101955Sjake/* 220101955Sjake * void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len) 221101955Sjake */ 222101955SjakeENTRY(ascopyfrom) 223101955Sjake wr %o0, 0, %asi 224216802Smarius _MEMCPY(%o2, %o1, %o3, EMPTY, EMPTY, a, %asi) 225101955Sjake retl 226101955Sjake nop 227101955SjakeEND(ascopyfrom) 22880709Sjake 229101955Sjake/* 230101955Sjake * void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len) 231101955Sjake */ 232101955SjakeENTRY(ascopyto) 233101955Sjake wr %o1, 0, %asi 234216802Smarius _MEMCPY(%o2, %o0, %o3, a, %asi, EMPTY, EMPTY) 235101955Sjake retl 236101955Sjake nop 237101955SjakeEND(ascopyto) 23880709Sjake 239101955Sjake/* 240101955Sjake * void aszero(u_long asi, vm_offset_t pa, size_t len) 241101955Sjake */ 242101955SjakeENTRY(aszero) 243101955Sjake wr %o0, 0, %asi 244101955Sjake _MEMSET(%o1, %g0, %o2, a, %asi) 245101955Sjake retl 246101955Sjake nop 247101955SjakeEND(aszero) 24880709Sjake 24980709Sjake/* 25084193Sjake * int bcmp(const void *b1, const void *b2, size_t len) 25180709Sjake */ 25280709SjakeENTRY(bcmp) 25380709Sjake brz,pn %o2, 2f 25480709Sjake clr %o3 25580709Sjake1: ldub [%o0 + %o3], %o4 25680709Sjake ldub [%o1 + %o3], %o5 25780709Sjake cmp %o4, %o5 25884193Sjake bne,pn %xcc, 2f 25980709Sjake inc %o3 26080709Sjake deccc %o2 26180709Sjake bne,pt %xcc, 1b 26280709Sjake nop 26380709Sjake2: retl 26480709Sjake mov %o2, %o0 26580709SjakeEND(bcmp) 26680709Sjake 26780709Sjake/* 26880709Sjake * void bcopy(const void *src, void *dst, size_t len) 26980709Sjake */ 27086520SjakeENTRY(bcopy) 27184193Sjake /* 27284193Sjake * Check for overlap, and copy backwards if so. 27384193Sjake */ 27484193Sjake sub %o1, %o0, %g1 27584193Sjake cmp %g1, %o2 27684193Sjake bgeu,a,pt %xcc, 3f 27784193Sjake nop 27884193Sjake 27984193Sjake /* 28084193Sjake * Copy backwards. 28184193Sjake */ 28284193Sjake add %o0, %o2, %o0 28384193Sjake add %o1, %o2, %o1 28484193Sjake1: deccc 1, %o2 28584193Sjake bl,a,pn %xcc, 2f 28684193Sjake nop 28784193Sjake dec 1, %o0 28884193Sjake ldub [%o0], %g1 28984193Sjake dec 1, %o1 290116589Sjake ba %xcc, 1b 29184193Sjake stb %g1, [%o1] 29284193Sjake2: retl 29384193Sjake nop 29484193Sjake 29584193Sjake /* 29684193Sjake * Do the fast version. 29784193Sjake */ 298216802Smarius3: _MEMCPY(%o1, %o0, %o2, EMPTY, EMPTY, EMPTY, EMPTY) 29980709Sjake retl 30080709Sjake nop 30180709SjakeEND(bcopy) 30280709Sjake 30380709Sjake/* 30484193Sjake * void bzero(void *b, size_t len) 30580709Sjake */ 30684193SjakeENTRY(bzero) 307216802Smarius _MEMSET(%o0, %g0, %o1, EMPTY, EMPTY) 30880709Sjake retl 30980709Sjake nop 31084193SjakeEND(bzero) 31180709Sjake 31280709Sjake/* 313101955Sjake * int copystr(const void *src, void *dst, size_t len, size_t *done) 31480709Sjake */ 315101955SjakeENTRY(copystr) 316216802Smarius _COPYSTR(%o0, %o1, %o2, %o3, EMPTY, EMPTY, EMPTY, EMPTY) 31784193Sjake retl 318101955Sjake mov %g1, %o0 319101955SjakeEND(copystr) 32084193Sjake 32184193Sjake/* 32280709Sjake * void *memcpy(void *dst, const void *src, size_t len) 32380709Sjake */ 32480709SjakeENTRY(memcpy) 32584193Sjake mov %o0, %o3 326216802Smarius _MEMCPY(%o3, %o1, %o2, EMPTY, EMPTY, EMPTY, EMPTY) 32780709Sjake retl 32880709Sjake nop 32980709SjakeEND(memcpy) 33080709Sjake 33180709Sjake/* 33284193Sjake * void *memset(void *b, int c, size_t len) 33384193Sjake */ 33484193SjakeENTRY(memset) 33584193Sjake mov %o0, %o3 336216802Smarius _MEMSET(%o3, %o1, %o2, EMPTY, EMPTY) 33784193Sjake retl 33884193Sjake nop 33984193SjakeEND(memset) 34084193Sjake 341101955Sjake .globl copy_nofault_begin 342101955Sjakecopy_nofault_begin: 343101955Sjake nop 344101955Sjake 34584193Sjake/* 34680709Sjake * int copyin(const void *uaddr, void *kaddr, size_t len) 34780709Sjake */ 34880709SjakeENTRY(copyin) 34984193Sjake wr %g0, ASI_AIUP, %asi 350216802Smarius _MEMCPY(%o1, %o0, %o2, EMPTY, EMPTY, a, %asi) 35180709Sjake retl 35280709Sjake clr %o0 35380709SjakeEND(copyin) 35480709Sjake 35580709Sjake/* 35680709Sjake * int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 35780709Sjake */ 35880709SjakeENTRY(copyinstr) 35984193Sjake wr %g0, ASI_AIUP, %asi 360216802Smarius _COPYSTR(%o0, %o1, %o2, %o3, a, %asi, EMPTY, EMPTY) 36180709Sjake retl 36284193Sjake mov %g1, %o0 36380709SjakeEND(copyinstr) 36480709Sjake 36580709Sjake/* 36680709Sjake * int copyout(const void *kaddr, void *uaddr, size_t len) 36780709Sjake */ 36880709SjakeENTRY(copyout) 36984193Sjake wr %g0, ASI_AIUP, %asi 370216802Smarius _MEMCPY(%o1, %o0, %o2, a, %asi, EMPTY, EMPTY) 37180709Sjake retl 37280709Sjake clr %o0 37380709SjakeEND(copyout) 37480709Sjake 375101955Sjake .globl copy_nofault_end 376101955Sjakecopy_nofault_end: 377101955Sjake nop 378101955Sjake 379101955SjakeENTRY(copy_fault) 38080709Sjake retl 38180709Sjake mov EFAULT, %o0 382101955SjakeEND(copy_fault) 38380709Sjake 384101955Sjake .globl fs_nofault_begin 385101955Sjakefs_nofault_begin: 386101955Sjake nop 38780709Sjake 38880709Sjake/* 389101955Sjake * Chatty aliases for fetch, store functions. 39080709Sjake */ 391101955Sjake .globl fubyte, fusword, fuword, subyte, susword, suword 392101955Sjake .set fubyte, fuword8 393101955Sjake .set fusword, fuword16 394101955Sjake .set fuword, fuword64 395101955Sjake .set subyte, suword8 396101955Sjake .set susword, suword16 397101955Sjake .set suword, suword64 39880709Sjake 399163449Sdavidxu .globl casuword32, casuword, fuptr, suptr 400163449Sdavidxu .set casuword, casuword64 401112914Sjake .set fuptr, fuword64 402112914Sjake .set suptr, suword64 403112914Sjake 40480709Sjake/* 405112914Sjake * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s) 406112914Sjake */ 407112914SjakeENTRY(casuword32) 408112914Sjake casa [%o0] ASI_AIUP, %o1, %o2 409112914Sjake retl 410112914Sjake mov %o2, %o0 411112914SjakeEND(casuword32) 412112914Sjake 413112914Sjake/* 414112914Sjake * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s) 415112914Sjake */ 416112914SjakeENTRY(casuword64) 417112914Sjake casxa [%o0] ASI_AIUP, %o1, %o2 418112914Sjake retl 419112914Sjake mov %o2, %o0 420112914SjakeEND(casuword64) 421112914Sjake 422112914Sjake/* 423101955Sjake * int fuword8(const void *base) 42480709Sjake */ 425101955SjakeENTRY(fuword8) 426101955Sjake retl 427101955Sjake lduba [%o0] ASI_AIUP, %o0 428101955SjakeEND(fuword8) 42980709Sjake 43080709Sjake/* 431101955Sjake * int fuword16(const void *base) 43280709Sjake */ 43398511SjakeENTRY(fuword16) 434101955Sjake retl 435101955Sjake lduha [%o0] ASI_AIUP, %o0 43698511SjakeEND(fuword16) 43780709Sjake 43880709Sjake/* 43998511Sjake * int32_t fuword32(const void *base) 44097307Sdfr */ 44198511SjakeENTRY(fuword32) 442101955Sjake retl 443101955Sjake lduwa [%o0] ASI_AIUP, %o0 44498511SjakeEND(fuword32) 44598511Sjake 44698511Sjake/* 44798511Sjake * int64_t fuword64(const void *base) 44898511Sjake */ 44997307SdfrENTRY(fuword64) 450101955Sjake retl 451101955Sjake ldxa [%o0] ASI_AIUP, %o0 45297307SdfrEND(fuword64) 45397307Sdfr 45497307Sdfr/* 455101955Sjake * int suword8(const void *base, int word) 45680709Sjake */ 457101955SjakeENTRY(suword8) 458101955Sjake stba %o1, [%o0] ASI_AIUP 459101955Sjake retl 460101955Sjake clr %o0 461101955SjakeEND(suword8) 46280709Sjake 46380709Sjake/* 464101955Sjake * int suword16(const void *base, int word) 46580709Sjake */ 46698511SjakeENTRY(suword16) 467101955Sjake stha %o1, [%o0] ASI_AIUP 468101955Sjake retl 469101955Sjake clr %o0 47098511SjakeEND(suword16) 47180709Sjake 47297307Sdfr/* 47398511Sjake * int suword32(const void *base, int32_t word) 47497307Sdfr */ 47598511SjakeENTRY(suword32) 476101955Sjake stwa %o1, [%o0] ASI_AIUP 477101955Sjake retl 478101955Sjake clr %o0 47998511SjakeEND(suword32) 48098511Sjake 48198511Sjake/* 48298511Sjake * int suword64(const void *base, int64_t word) 48398511Sjake */ 48497307SdfrENTRY(suword64) 485101955Sjake stxa %o1, [%o0] ASI_AIUP 486101955Sjake retl 487101955Sjake clr %o0 48897307SdfrEND(suword64) 48997307Sdfr 490101955Sjake .globl fs_nofault_intr_begin 491101955Sjakefs_nofault_intr_begin: 492101955Sjake nop 493101955Sjake 494101955Sjake/* 495101955Sjake * int fuswintr(const void *base) 496101955Sjake */ 497101955SjakeENTRY(fuswintr) 49885238Sjake retl 499101955Sjake lduha [%o0] ASI_AIUP, %o0 500101955SjakeEND(fuswintr) 50185238Sjake 502101955Sjake/* 503101955Sjake * int suswintr(const void *base, int word) 504101955Sjake */ 505101955SjakeENTRY(suswintr) 506101955Sjake stha %o1, [%o0] ASI_AIUP 50780709Sjake retl 508101955Sjake clr %o0 509101955SjakeEND(suswintr) 51080709Sjake 511101955Sjake .globl fs_nofault_intr_end 512101955Sjakefs_nofault_intr_end: 513101955Sjake nop 514101955Sjake 515101955Sjake .globl fs_nofault_end 516101955Sjakefs_nofault_end: 517101955Sjake nop 518101955Sjake 519101955SjakeENTRY(fs_fault) 52085238Sjake retl 52185238Sjake mov -1, %o0 522272895SbaptEND(fs_fault) 52385238Sjake 524116659Sjmg .globl fas_nofault_begin 525116659Sjmgfas_nofault_begin: 526116659Sjmg 527116659Sjmg/* 528116659Sjmg * int fasword8(u_long asi, uint64_t addr, uint8_t *val) 529116659Sjmg */ 530116659SjmgENTRY(fasword8) 531116659Sjmg wr %o0, 0, %asi 532116659Sjmg membar #Sync 533116659Sjmg lduba [%o1] %asi, %o3 534116659Sjmg membar #Sync 535116659Sjmg stb %o3, [%o2] 536116659Sjmg retl 537116659Sjmg clr %o0 538116659SjmgEND(fasword8) 539116659Sjmg 540116659Sjmg/* 541116659Sjmg * int fasword16(u_long asi, uint64_t addr, uint16_t *val) 542116659Sjmg */ 543116659SjmgENTRY(fasword16) 544116659Sjmg wr %o0, 0, %asi 545116659Sjmg membar #Sync 546116659Sjmg lduha [%o1] %asi, %o3 547116659Sjmg membar #Sync 548116659Sjmg sth %o3, [%o2] 549116659Sjmg retl 550116659Sjmg clr %o0 551116659SjmgEND(fasword16) 552116659Sjmg 553116659Sjmg/* 554116659Sjmg * int fasword32(u_long asi, uint64_t addr, uint32_t *val) 555116659Sjmg */ 556116659SjmgENTRY(fasword32) 557116659Sjmg wr %o0, 0, %asi 558116659Sjmg membar #Sync 559116659Sjmg lduwa [%o1] %asi, %o3 560116659Sjmg membar #Sync 561116659Sjmg stw %o3, [%o2] 562116659Sjmg retl 563116659Sjmg clr %o0 564116659SjmgEND(fasword32) 565116659Sjmg 566116659Sjmg .globl fas_nofault_end 567116659Sjmgfas_nofault_end: 568116659Sjmg nop 569116659Sjmg 570116659Sjmg .globl fas_fault 571116659SjmgENTRY(fas_fault) 572116659Sjmg retl 573116659Sjmg mov -1, %o0 574116659SjmgEND(fas_fault) 575116659Sjmg 576113027Sjake .globl fpu_fault_begin 577113027Sjakefpu_fault_begin: 578113027Sjake nop 579113027Sjake 580113027Sjake/* 581113027Sjake * void spitfire_block_copy(void *src, void *dst, size_t len) 582113027Sjake */ 583113027SjakeENTRY(spitfire_block_copy) 584230662Smarius rdpr %pstate, %o3 585230662Smarius wrpr %g0, PSTATE_NORMAL, %pstate 586113027Sjake 587113027Sjake wr %g0, ASI_BLK_S, %asi 588113027Sjake wr %g0, FPRS_FEF, %fprs 589113027Sjake 590113027Sjake sub PCB_REG, TF_SIZEOF, %o4 591113027Sjake ldx [%o4 + TF_FPRS], %o5 592113027Sjake andcc %o5, FPRS_FEF, %g0 593113027Sjake bz,a,pt %xcc, 1f 594113027Sjake nop 595239941Smarius stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi 596239941Smarius stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi 597239941Smarius stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi 598239941Smarius stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi 599113027Sjake membar #Sync 600113027Sjake 601113027Sjake andn %o5, FPRS_FEF, %o5 602113027Sjake stx %o5, [%o4 + TF_FPRS] 603113027Sjake ldx [PCB_REG + PCB_FLAGS], %o4 604113027Sjake or %o4, PCB_FEF, %o4 605113027Sjake stx %o4, [PCB_REG + PCB_FLAGS] 606113027Sjake 607230662Smarius1: wrpr %o3, 0, %pstate 608113027Sjake 609113027Sjake ldda [%o0] %asi, %f0 610239941Smarius add %o0, VIS_BLOCKSIZE, %o0 611239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 612113027Sjake 613113027Sjake2: ldda [%o0] %asi, %f16 614113027Sjake fsrc1 %f0, %f32 615113027Sjake fsrc1 %f2, %f34 616113027Sjake fsrc1 %f4, %f36 617113027Sjake fsrc1 %f6, %f38 618113027Sjake fsrc1 %f8, %f40 619113027Sjake fsrc1 %f10, %f42 620113027Sjake fsrc1 %f12, %f44 621113027Sjake fsrc1 %f14, %f46 622113027Sjake stda %f32, [%o1] %asi 623239941Smarius add %o0, VIS_BLOCKSIZE, %o0 624239941Smarius subcc %o2, VIS_BLOCKSIZE, %o2 625113027Sjake bz,pn %xcc, 3f 626239941Smarius add %o1, VIS_BLOCKSIZE, %o1 627113027Sjake ldda [%o0] %asi, %f0 628113027Sjake fsrc1 %f16, %f32 629113027Sjake fsrc1 %f18, %f34 630113027Sjake fsrc1 %f20, %f36 631113027Sjake fsrc1 %f22, %f38 632113027Sjake fsrc1 %f24, %f40 633113027Sjake fsrc1 %f26, %f42 634113027Sjake fsrc1 %f28, %f44 635113027Sjake fsrc1 %f30, %f46 636113027Sjake stda %f32, [%o1] %asi 637239941Smarius add %o0, VIS_BLOCKSIZE, %o0 638239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 639185006Smarius ba,pt %xcc, 2b 640239941Smarius add %o1, VIS_BLOCKSIZE, %o1 641113027Sjake 642113027Sjake3: membar #Sync 643113027Sjake 644113027Sjake stda %f16, [%o1] %asi 645113027Sjake membar #Sync 646113027Sjake 647113027Sjake retl 648185006Smarius wr %g0, 0, %fprs 649113027SjakeEND(spitfire_block_copy) 650113027Sjake 651113027Sjake/* 652212709Smarius * void zeus_block_copy(void *src, void *dst, size_t len) 653212709Smarius */ 654212709SmariusENTRY(zeus_block_copy) 655239941Smarius prefetch [%o0 + (0 * VIS_BLOCKSIZE)], 0 656212709Smarius 657230662Smarius rdpr %pstate, %o3 658230662Smarius wrpr %g0, PSTATE_NORMAL, %pstate 659212709Smarius 660212709Smarius wr %g0, ASI_BLK_S, %asi 661212709Smarius wr %g0, FPRS_FEF, %fprs 662212709Smarius 663212709Smarius sub PCB_REG, TF_SIZEOF, %o4 664212709Smarius ldx [%o4 + TF_FPRS], %o5 665212709Smarius andcc %o5, FPRS_FEF, %g0 666212709Smarius bz,a,pt %xcc, 1f 667212709Smarius nop 668239941Smarius stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi 669239941Smarius stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi 670239941Smarius stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi 671239941Smarius stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi 672212709Smarius membar #Sync 673212709Smarius 674212709Smarius andn %o5, FPRS_FEF, %o5 675212709Smarius stx %o5, [%o4 + TF_FPRS] 676212709Smarius ldx [PCB_REG + PCB_FLAGS], %o4 677212709Smarius or %o4, PCB_FEF, %o4 678212709Smarius stx %o4, [PCB_REG + PCB_FLAGS] 679212709Smarius 680230662Smarius1: wrpr %o3, 0, %pstate 681212709Smarius 682212709Smarius ldd [%o0 + (0 * 8)], %f0 683239941Smarius prefetch [%o0 + (1 * VIS_BLOCKSIZE)], 0 684212709Smarius ldd [%o0 + (1 * 8)], %f2 685239941Smarius prefetch [%o0 + (2 * VIS_BLOCKSIZE)], 0 686212709Smarius fmovd %f0, %f32 687212709Smarius ldd [%o0 + (2 * 8)], %f4 688239941Smarius prefetch [%o0 + (3 * VIS_BLOCKSIZE)], 0 689212709Smarius fmovd %f2, %f34 690212709Smarius ldd [%o0 + (3 * 8)], %f6 691239941Smarius prefetch [%o0 + (4 * VIS_BLOCKSIZE)], 1 692212709Smarius fmovd %f4, %f36 693212709Smarius ldd [%o0 + (4 * 8)], %f8 694239941Smarius prefetch [%o0 + (8 * VIS_BLOCKSIZE)], 1 695212709Smarius fmovd %f6, %f38 696212709Smarius ldd [%o0 + (5 * 8)], %f10 697239941Smarius prefetch [%o0 + (12 * VIS_BLOCKSIZE)], 1 698212709Smarius fmovd %f8, %f40 699212709Smarius ldd [%o0 + (6 * 8)], %f12 700239941Smarius prefetch [%o0 + (16 * VIS_BLOCKSIZE)], 1 701212709Smarius fmovd %f10, %f42 702212709Smarius ldd [%o0 + (7 * 8)], %f14 703212709Smarius ldd [%o0 + (8 * 8)], %f0 704239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 705239941Smarius add %o0, VIS_BLOCKSIZE, %o0 706239941Smarius prefetch [%o0 + (19 * VIS_BLOCKSIZE)], 1 707212709Smarius ba,pt %xcc, 2f 708239941Smarius prefetch [%o0 + (23 * VIS_BLOCKSIZE)], 1 709212709Smarius .align 32 710212709Smarius 711212709Smarius2: ldd [%o0 + (1 * 8)], %f2 712212709Smarius fmovd %f12, %f44 713212709Smarius ldd [%o0 + (2 * 8)], %f4 714212709Smarius fmovd %f14, %f46 715212709Smarius stda %f32, [%o1] %asi 716212709Smarius ldd [%o0 + (3 * 8)], %f6 717212709Smarius fmovd %f0, %f32 718212709Smarius ldd [%o0 + (4 * 8)], %f8 719212709Smarius fmovd %f2, %f34 720212709Smarius ldd [%o0 + (5 * 8)], %f10 721212709Smarius fmovd %f4, %f36 722212709Smarius ldd [%o0 + (6 * 8)], %f12 723212709Smarius fmovd %f6, %f38 724212709Smarius ldd [%o0 + (7 * 8)], %f14 725212709Smarius fmovd %f8, %f40 726212709Smarius ldd [%o0 + (8 * 8)], %f0 727212709Smarius fmovd %f10, %f42 728239941Smarius sub %o2, VIS_BLOCKSIZE, %o2 729239941Smarius prefetch [%o0 + (3 * VIS_BLOCKSIZE)], 0 730239941Smarius add %o1, VIS_BLOCKSIZE, %o1 731239941Smarius prefetch [%o0 + (24 * VIS_BLOCKSIZE)], 1 732239941Smarius add %o0, VIS_BLOCKSIZE, %o0 733239941Smarius cmp %o2, VIS_BLOCKSIZE + 8 734212709Smarius bgu,pt %xcc, 2b 735239941Smarius prefetch [%o0 + (12 * VIS_BLOCKSIZE)], 1 736212709Smarius ldd [%o0 + (1 * 8)], %f2 737212709Smarius fsrc1 %f12, %f44 738212709Smarius ldd [%o0 + (2 * 8)], %f4 739212709Smarius fsrc1 %f14, %f46 740212709Smarius stda %f32, [%o1] %asi 741212709Smarius ldd [%o0 + (3 * 8)], %f6 742212709Smarius fsrc1 %f0, %f32 743212709Smarius ldd [%o0 + (4 * 8)], %f8 744212709Smarius fsrc1 %f2, %f34 745212709Smarius ldd [%o0 + (5 * 8)], %f10 746212709Smarius fsrc1 %f4, %f36 747212709Smarius ldd [%o0 + (6 * 8)], %f12 748212709Smarius fsrc1 %f6, %f38 749212709Smarius ldd [%o0 + (7 * 8)], %f14 750212709Smarius fsrc1 %f8, %f40 751239941Smarius add %o1, VIS_BLOCKSIZE, %o1 752212709Smarius fsrc1 %f10, %f42 753212709Smarius fsrc1 %f12, %f44 754212709Smarius fsrc1 %f14, %f46 755212709Smarius stda %f32, [%o1] %asi 756212709Smarius membar #Sync 757212709Smarius 758212709Smarius retl 759212709Smarius wr %g0, 0, %fprs 760212709SmariusEND(zeus_block_copy) 761212709Smarius 762212709Smarius/* 763113027Sjake * void spitfire_block_zero(void *dst, size_t len) 764212709Smarius * void zeus_block_zero(void *dst, size_t len) 765113027Sjake */ 766212709SmariusALTENTRY(zeus_block_zero) 767113027SjakeENTRY(spitfire_block_zero) 768230662Smarius rdpr %pstate, %o3 769230662Smarius wrpr %g0, PSTATE_NORMAL, %pstate 770113027Sjake 771113027Sjake wr %g0, ASI_BLK_S, %asi 772113027Sjake wr %g0, FPRS_FEF, %fprs 773113027Sjake 774113027Sjake sub PCB_REG, TF_SIZEOF, %o4 775113027Sjake ldx [%o4 + TF_FPRS], %o5 776113027Sjake andcc %o5, FPRS_FEF, %g0 777113027Sjake bz,a,pt %xcc, 1f 778113027Sjake nop 779239941Smarius stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi 780239941Smarius stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi 781239941Smarius stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi 782239941Smarius stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi 783113027Sjake membar #Sync 784113027Sjake 785113027Sjake andn %o5, FPRS_FEF, %o5 786113027Sjake stx %o5, [%o4 + TF_FPRS] 787113027Sjake ldx [PCB_REG + PCB_FLAGS], %o4 788113027Sjake or %o4, PCB_FEF, %o4 789113027Sjake stx %o4, [PCB_REG + PCB_FLAGS] 790113027Sjake 791230662Smarius1: wrpr %o3, 0, %pstate 792113027Sjake 793113027Sjake fzero %f0 794113027Sjake fzero %f2 795113027Sjake fzero %f4 796113027Sjake fzero %f6 797113027Sjake fzero %f8 798113027Sjake fzero %f10 799113027Sjake fzero %f12 800113027Sjake fzero %f14 801113027Sjake 802239941Smarius1: stda %f0, [%o0 + (0 * VIS_BLOCKSIZE)] %asi 803239941Smarius stda %f0, [%o0 + (1 * VIS_BLOCKSIZE)] %asi 804239941Smarius stda %f0, [%o0 + (2 * VIS_BLOCKSIZE)] %asi 805239941Smarius stda %f0, [%o0 + (3 * VIS_BLOCKSIZE)] %asi 806239941Smarius sub %o1, (4 * VIS_BLOCKSIZE), %o1 807185006Smarius brnz,pt %o1, 1b 808239941Smarius add %o0, (4 * VIS_BLOCKSIZE), %o0 809113027Sjake membar #Sync 810113027Sjake 811113027Sjake retl 812185006Smarius wr %g0, 0, %fprs 813113027SjakeEND(spitfire_block_zero) 814113027Sjake 815113027Sjake .globl fpu_fault_end 816113027Sjakefpu_fault_end: 817113027Sjake nop 818113027Sjake 819113027Sjake .globl fpu_fault_size 820113027Sjake .set fpu_fault_size, fpu_fault_end - fpu_fault_begin 821113027Sjake 82280709SjakeENTRY(longjmp) 82380709Sjake set 1, %g3 82480709Sjake movrz %o1, %o1, %g3 82580709Sjake mov %o0, %g1 82688641Sjake ldx [%g1 + _JB_FP], %g2 82780709Sjake1: cmp %fp, %g2 82880709Sjake bl,a,pt %xcc, 1b 82980709Sjake restore 83080709Sjake bne,pn %xcc, 2f 83188641Sjake ldx [%g1 + _JB_SP], %o2 83280709Sjake cmp %o2, %sp 83380709Sjake blt,pn %xcc, 2f 83480709Sjake movge %xcc, %o2, %sp 83588641Sjake ldx [%g1 + _JB_PC], %o7 83688641Sjake retl 83780709Sjake mov %g3, %o0 83880709Sjake2: PANIC("longjmp botch", %l1) 83980709SjakeEND(longjmp) 84080709Sjake 84180709SjakeENTRY(setjmp) 84288641Sjake stx %sp, [%o0 + _JB_SP] 84388641Sjake stx %o7, [%o0 + _JB_PC] 84488641Sjake stx %fp, [%o0 + _JB_FP] 84580709Sjake retl 84680709Sjake clr %o0 84780709SjakeEND(setjmp) 84880709Sjake 84980709Sjake/* 850186347Snwhitehorn * void ofw_entry(cell_t args[]) 85180709Sjake */ 852186347SnwhitehornENTRY(ofw_entry) 85384193Sjake save %sp, -CCFSZ, %sp 85486520Sjake SET(ofw_vec, %l7, %l6) 85581183Sjake ldx [%l6], %l6 856205409Smarius rdpr %pstate, %l7 857205409Smarius andn %l7, PSTATE_AM | PSTATE_IE, %l5 858205409Smarius wrpr %l5, 0, %pstate 859205409Smarius SET(tba_taken_over, %l5, %l4) 860205409Smarius brz,pn %l4, 1f 861205409Smarius rdpr %wstate, %l5 862205409Smarius andn %l5, WSTATE_PROM_MASK, %l3 863205409Smarius wrpr %l3, WSTATE_PROM_KMIX, %wstate 864205409Smarius1: call %l6 86580709Sjake mov %i0, %o0 866205409Smarius brz,pn %l4, 1f 867205409Smarius nop 868205409Smarius wrpr %g0, %l5, %wstate 869205409Smarius1: wrpr %l7, 0, %pstate 87084193Sjake ret 87184193Sjake restore %o0, %g0, %o0 872186347SnwhitehornEND(ofw_entry) 87386147Stmm 87486147Stmm/* 875186347Snwhitehorn * void ofw_exit(cell_t args[]) 87686147Stmm */ 877186347SnwhitehornENTRY(ofw_exit) 87886147Stmm save %sp, -CCFSZ, %sp 87986147Stmm flushw 88086520Sjake SET(ofw_tba, %l7, %l5) 88186147Stmm ldx [%l5], %l5 882205409Smarius rdpr %pstate, %l7 883205409Smarius andn %l7, PSTATE_AM | PSTATE_IE, %l7 884205409Smarius wrpr %l7, 0, %pstate 885205409Smarius rdpr %wstate, %l7 886205409Smarius andn %l7, WSTATE_PROM_MASK, %l7 887205409Smarius wrpr %l7, WSTATE_PROM_KMIX, %wstate 888181701Smarius wrpr %l5, 0, %tba ! restore the OFW trap table 88986520Sjake SET(ofw_vec, %l7, %l6) 89086147Stmm ldx [%l6], %l6 89186520Sjake SET(kstack0 + KSTACK_PAGES * PAGE_SIZE - PCB_SIZEOF, %l7, %l0) 89286147Stmm sub %l0, SPOFF, %fp ! setup a stack in a locked page 89386147Stmm sub %l0, SPOFF + CCFSZ, %sp 894181701Smarius mov AA_DMMU_PCXR, %l3 ! force primary DMMU context 0 895182877Smarius sethi %hi(KERNBASE), %l5 896181701Smarius stxa %g0, [%l3] ASI_DMMU 897182877Smarius flush %l5 89886147Stmm wrpr %g0, 0, %tl ! force trap level 0 89986147Stmm call %l6 90086147Stmm mov %i0, %o0 90186147Stmm ! never to return 902186347SnwhitehornEND(ofw_exit) 903100841Sjake 904100910Sjake#ifdef GPROF 905100841Sjake 906100910SjakeENTRY(user) 907100910Sjake nop 908100910Sjake 909100910SjakeENTRY(btrap) 910100910Sjake nop 911100910Sjake 912100910SjakeENTRY(etrap) 913100910Sjake nop 914100910Sjake 915100910SjakeENTRY(bintr) 916100910Sjake nop 917100910Sjake 918100910SjakeENTRY(eintr) 919100910Sjake nop 920100910Sjake 921100841Sjake/* 922100841Sjake * XXX including sys/gmon.h in genassym.c is not possible due to uintfptr_t 923100841Sjake * badness. 924100841Sjake */ 925100841Sjake#define GM_STATE 0x0 926100841Sjake#define GMON_PROF_OFF 3 927100841Sjake#define GMON_PROF_HIRES 4 928100841Sjake 929100910Sjake .globl _mcount 930100910Sjake .set _mcount, __cyg_profile_func_enter 931100910Sjake 932100910SjakeENTRY(__cyg_profile_func_enter) 933100841Sjake SET(_gmonparam, %o3, %o2) 934181701Smarius lduw [%o2 + GM_STATE], %o3 935181701Smarius cmp %o3, GMON_PROF_OFF 936100841Sjake be,a,pn %icc, 1f 937100841Sjake nop 938100841Sjake SET(mcount, %o3, %o2) 939100841Sjake jmpl %o2, %g0 940100841Sjake nop 941181701Smarius1: retl 942100841Sjake nop 943100910SjakeEND(__cyg_profile_func_enter) 944100841Sjake 945100910Sjake#ifdef GUPROF 946100910Sjake 947100910SjakeENTRY(__cyg_profile_func_exit) 948100841Sjake SET(_gmonparam, %o3, %o2) 949181701Smarius lduw [%o2 + GM_STATE], %o3 950181701Smarius cmp %o3, GMON_PROF_HIRES 951100841Sjake be,a,pn %icc, 1f 952100841Sjake nop 953100841Sjake SET(mexitcount, %o3, %o2) 954100841Sjake jmpl %o2, %g0 955100841Sjake nop 956181701Smarius1: retl 957100841Sjake nop 958100910SjakeEND(__cyg_profile_func_exit) 959100841Sjake 960100910Sjake#endif /* GUPROF */ 961100841Sjake 962100910Sjake#endif /* GPROF */ 963