1139827Simp/*- 2165974Srwatson * Copyright (c) 2001 Jake Burkholder. 315885Sjulian * All rights reserved. 415885Sjulian * 515885Sjulian * Redistribution and use in source and binary forms, with or without 615885Sjulian * modification, are permitted provided that the following conditions 715885Sjulian * are met: 815885Sjulian * 1. Redistributions of source code must retain the above copyright 915885Sjulian * notice, this list of conditions and the following disclaimer. 1015885Sjulian * 2. Redistributions in binary form must reproduce the above copyright 1115885Sjulian * notice, this list of conditions and the following disclaimer in the 1215885Sjulian * documentation and/or other materials provided with the distribution. 1315885Sjulian * 1415885Sjulian * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1515885Sjulian * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1615885Sjulian * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1715885Sjulian * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1815885Sjulian * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1915885Sjulian * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2015885Sjulian * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2115885Sjulian * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2215885Sjulian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2315885Sjulian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2460889Sarchie * SUCH DAMAGE. 2560889Sarchie * 2615885Sjulian * $FreeBSD$ 2715885Sjulian */ 2815885Sjulian 2915885Sjulian#ifndef _MACHINE_CPUFUNC_H_ 3023396Sjulian#define _MACHINE_CPUFUNC_H_ 3115885Sjulian 3215885Sjulian#include <machine/asi.h> 3315885Sjulian#include <machine/pstate.h> 3415885Sjulian 3515885Sjulianstruct thread; 3615885Sjulian 3718207Sbde/* 3818207Sbde * Membar operand macros for use in other macros when # is a special 3918207Sbde * character. Keep these in sync with what the hardware expects. 4018207Sbde */ 4115885Sjulian#define C_Lookaside (0) 4215885Sjulian#define C_MemIssue (1) 43163606Srwatson#define C_Sync (2) 44163606Srwatson#define M_LoadLoad (0) 4515885Sjulian#define M_StoreLoad (1) 4615885Sjulian#define M_LoadStore (2) 4715885Sjulian#define M_StoreStore (3) 48127288Srwatson 4915885Sjulian#define CMASK_SHIFT (4) 50165974Srwatson#define MMASK_SHIFT (0) 51165974Srwatson 5215885Sjulian#define CMASK_GEN(bit) ((1 << (bit)) << CMASK_SHIFT) 53101937Srwatson#define MMASK_GEN(bit) ((1 << (bit)) << MMASK_SHIFT) 54172930Srwatson 55101937Srwatson#define Lookaside CMASK_GEN(C_Lookaside) 56101937Srwatson#define MemIssue CMASK_GEN(C_MemIssue) 57165974Srwatson#define Sync CMASK_GEN(C_Sync) 58165974Srwatson#define LoadLoad MMASK_GEN(M_LoadLoad) 59127288Srwatson#define StoreLoad MMASK_GEN(M_StoreLoad) 6015885Sjulian#define LoadStore MMASK_GEN(M_LoadStore) 61165974Srwatson#define StoreStore MMASK_GEN(M_StoreStore) 62165974Srwatson 63165974Srwatson#define casa(rs1, rs2, rd, asi) ({ \ 6415885Sjulian u_int __rd = (uint32_t)(rd); \ 65165974Srwatson __asm __volatile("casa [%2] %3, %4, %0" \ 6615885Sjulian : "+r" (__rd), "=m" (*rs1) \ 67165974Srwatson : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \ 68165974Srwatson __rd; \ 69165974Srwatson}) 70165974Srwatson 71165974Srwatson#define casxa(rs1, rs2, rd, asi) ({ \ 72165974Srwatson u_long __rd = (uint64_t)(rd); \ 7315885Sjulian __asm __volatile("casxa [%2] %3, %4, %0" \ 74165974Srwatson : "+r" (__rd), "=m" (*rs1) \ 75165974Srwatson : "r" (rs1), "n" (asi), "r" (rs2), "m" (*rs1)); \ 76165974Srwatson __rd; \ 77165974Srwatson}) 78165974Srwatson 79165974Srwatson#define flush(va) do { \ 80165974Srwatson __asm __volatile("flush %0" : : "r" (va)); \ 81165974Srwatson} while (0) 82165974Srwatson 8315885Sjulian#define flushw() do { \ 8438114Sjulian __asm __volatile("flushw" : :); \ 85165974Srwatson} while (0) 8638114Sjulian 8738114Sjulian#define mov(val, reg) do { \ 8838114Sjulian __asm __volatile("mov %0, %" __XSTRING(reg) : : "r" (val)); \ 89165974Srwatson} while (0) 9015885Sjulian 9115885Sjulian/* Generate ld*a/st*a functions for non-constant ASIs. */ 9215885Sjulian#define LDNC_GEN(tp, o) \ 93127288Srwatson static __inline tp \ 9415885Sjulian o ## _nc(caddr_t va, int asi) \ 95165974Srwatson { \ 96165974Srwatson tp r; \ 9715885Sjulian __asm __volatile("wr %2, 0, %%asi;" #o " [%1] %%asi, %0"\ 98165974Srwatson : "=r" (r) : "r" (va), "r" (asi)); \ 99165974Srwatson return (r); \ 100165974Srwatson } 101165974Srwatson 102165974SrwatsonLDNC_GEN(u_char, lduba); 103165974SrwatsonLDNC_GEN(u_short, lduha); 104165974SrwatsonLDNC_GEN(u_int, lduwa); 105165974SrwatsonLDNC_GEN(u_long, ldxa); 106165974Srwatson 107165974Srwatson#define LD_GENERIC(va, asi, op, type) ({ \ 108165974Srwatson type __r; \ 109165974Srwatson __asm __volatile(#op " [%1] %2, %0" \ 11015885Sjulian : "=r" (__r) : "r" (va), "n" (asi)); \ 11115885Sjulian __r; \ 112165974Srwatson}) 113165974Srwatson 114165974Srwatson#define lduba(va, asi) LD_GENERIC(va, asi, lduba, u_char) 11515885Sjulian#define lduha(va, asi) LD_GENERIC(va, asi, lduha, u_short) 11615885Sjulian#define lduwa(va, asi) LD_GENERIC(va, asi, lduwa, u_int) 11715885Sjulian#define ldxa(va, asi) LD_GENERIC(va, asi, ldxa, u_long) 118127288Srwatson 11915885Sjulian#define STNC_GEN(tp, o) \ 120165974Srwatson static __inline void \ 121165974Srwatson o ## _nc(caddr_t va, int asi, tp val) \ 122165974Srwatson { \ 123165974Srwatson __asm __volatile("wr %2, 0, %%asi;" #o " %0, [%1] %%asi"\ 124165974Srwatson : : "r" (val), "r" (va), "r" (asi)); \ 125165974Srwatson } 12615885Sjulian 12736908SjulianSTNC_GEN(u_char, stba); 128165974SrwatsonSTNC_GEN(u_short, stha); 129165974SrwatsonSTNC_GEN(u_int, stwa); 130165974SrwatsonSTNC_GEN(u_long, stxa); 131165974Srwatson 132165974Srwatson#define ST_GENERIC(va, asi, val, op) \ 13336908Sjulian __asm __volatile(#op " %0, [%1] %2" \ 13436908Sjulian : : "r" (val), "r" (va), "n" (asi)); \ 135165974Srwatson 136165974Srwatson#define stba(va, asi, val) ST_GENERIC(va, asi, val, stba) 137165974Srwatson#define stha(va, asi, val) ST_GENERIC(va, asi, val, stha) 138165974Srwatson#define stwa(va, asi, val) ST_GENERIC(va, asi, val, stwa) 139165974Srwatson#define stxa(va, asi, val) ST_GENERIC(va, asi, val, stxa) 140165974Srwatson 141165974Srwatson/* 142165974Srwatson * Attempt to read from addr, val. If a Data Access Error trap happens, 143165974Srwatson * they return -1 and the contents of val is undefined. A return of 0 144165974Srwatson * means no trap happened, and the contents of val is valid. 145194619Srwatson */ 146194913Srwatsonint fasword8(u_long asi, void *addr, uint8_t *val); 147165974Srwatsonint fasword16(u_long asi, void *addr, uint16_t *val); 148165974Srwatsonint fasword32(u_long asi, void *addr, uint32_t *val); 149165974Srwatson 150165974Srwatson#define membar(mask) do { \ 151165974Srwatson __asm __volatile("membar %0" : : "n" (mask) : "memory"); \ 152194619Srwatson} while (0) 153194619Srwatson 154194619Srwatson#define rd(name) ({ \ 155165974Srwatson uint64_t __sr; \ 156165974Srwatson __asm __volatile("rd %%" #name ", %0" : "=r" (__sr) :); \ 15738114Sjulian __sr; \ 158165974Srwatson}) 159165974Srwatson 160165974Srwatson#define wr(name, val, xorval) do { \ 161165974Srwatson __asm __volatile("wr %0, %1, %%" #name \ 162165974Srwatson : : "r" (val), "rI" (xorval)); \ 163165974Srwatson} while (0) 16438114Sjulian 165165974Srwatson#define rdpr(name) ({ \ 166165974Srwatson uint64_t __pr; \ 16728845Sjulian __asm __volatile("rdpr %%" #name", %0" : "=r" (__pr) :); \ 168165974Srwatson __pr; \ 16938114Sjulian}) 170165974Srwatson 171165974Srwatson#define wrpr(name, val, xorval) do { \ 17238114Sjulian __asm __volatile("wrpr %0, %1, %%" #name \ 173165974Srwatson : : "r" (val), "rI" (xorval)); \ 174165974Srwatson} while (0) 175165974Srwatson 17615885Sjulian/* 177165974Srwatson * Trick GAS/GCC into compiling access to TICK/(S)TICK_COMPARE independently 178165974Srwatson * of the selected instruction set. 179165974Srwatson */ 180165974Srwatson#define rdtickcmpr() rd(asr23) 181165974Srwatson#define rdstick() rd(asr24) 182127288Srwatson#define rdstickcmpr() rd(asr25) 183127288Srwatson#define wrtickcmpr(val, xorval) wr(asr23, (val), (xorval)) 184165974Srwatson#define wrstick(val, xorval) wr(asr24, (val), (xorval)) 185165974Srwatson#define wrstickcmpr(val, xorval) wr(asr25, (val), (xorval)) 186165974Srwatson 187165974Srwatson/* 18837521Sjulian * Macro intended to be used instead of wr(asr23, val, xorval) for writing to 189165974Srwatson * the TICK_COMPARE register in order to avoid a bug in BlackBird CPUs that 190165974Srwatson * can cause these writes to fail under certain conditions which in turn 191165974Srwatson * causes the hardclock to stop. The workaround is to read the TICK_COMPARE 192165974Srwatson * register back immediately after writing to it with these two instructions 193165974Srwatson * aligned to a quadword boundary in order to ensure that I$ misses won't 194173784Srwatson * split them up. 195173784Srwatson */ 196173784Srwatson#define wrtickcmpr_bbwar(val, xorval) ({ \ 197173784Srwatson __asm __volatile( \ 198173784Srwatson " ba,pt %%xcc, 1f ; " \ 199173784Srwatson " nop ; " \ 200173784Srwatson " .align 128 ; " \ 201173784Srwatson "1: wr %0, %1, %%asr23 ; " \ 202165974Srwatson " rd %%asr23, %%g0 ; " \ 203165974Srwatson : : "r" (val), "rI" (xorval)); \ 204165974Srwatson}) 205165974Srwatson 206194619Srwatsonstatic __inline void 207165974Srwatsonbreakpoint(void) 208165974Srwatson{ 209165974Srwatson 210165974Srwatson __asm __volatile("ta %%xcc, 1" : :); 211101937Srwatson} 212172930Srwatson 213101937Srwatsonstatic __inline register_t 214165974Srwatsonintr_disable(void) 215165974Srwatson{ 216165974Srwatson register_t s; 217165974Srwatson 21815885Sjulian s = rdpr(pstate); 219165974Srwatson wrpr(pstate, s & ~PSTATE_IE, 0); 220165974Srwatson return (s); 221165974Srwatson} 222165974Srwatson#define intr_restore(s) wrpr(pstate, (s), 0) 223165974Srwatson 224165974Srwatson/* 22515885Sjulian * In some places, it is required that the store is directly followed by a 22638114Sjulian * membar #Sync. Don't trust the compiler to not insert instructions in 227165974Srwatson * between. We also need to disable interrupts completely. 228165974Srwatson */ 229165974Srwatson#define stxa_sync(va, asi, val) do { \ 230165974Srwatson register_t s; \ 231165974Srwatson s = intr_disable(); \ 232165974Srwatson __asm __volatile("stxa %0, [%1] %2; membar #Sync" \ 23338114Sjulian : : "r" (val), "r" (va), "n" (asi)); \ 23438114Sjulian intr_restore(s); \ 235165974Srwatson} while (0) 236165974Srwatson 237165974Srwatsonvoid ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len); 238165974Srwatsonvoid ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len); 239194619Srwatsonvoid ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len); 240194619Srwatsonvoid aszero(u_long asi, vm_offset_t dst, size_t len); 241165974Srwatson 242194619Srwatson/* 243194619Srwatson * Ultrasparc II doesn't implement popc in hardware. 24438114Sjulian */ 245165974Srwatson#if 0 246165974Srwatson#define HAVE_INLINE_FFS 24715885Sjulian/* 248 * See page 202 of the SPARC v9 Architecture Manual. 249 */ 250static __inline int 251ffs(int mask) 252{ 253 int result; 254 int neg; 255 int tmp; 256 257 __asm __volatile( 258 " neg %3, %1 ; " 259 " xnor %3, %1, %2 ; " 260 " popc %2, %0 ; " 261 " movrz %3, %%g0, %0 ; " 262 : "=r" (result), "=r" (neg), "=r" (tmp) : "r" (mask)); 263 return (result); 264} 265#endif 266 267#undef LDNC_GEN 268#undef STNC_GEN 269 270#endif /* !_MACHINE_CPUFUNC_H_ */ 271