1227569Sphilip/*- 2301388Sarybchik * Copyright (c) 2010-2016 Solarflare Communications Inc. 3227569Sphilip * All rights reserved. 4227569Sphilip * 5227569Sphilip * This software was developed in part by Philip Paeps under contract for 6227569Sphilip * Solarflare Communications, Inc. 7227569Sphilip * 8227569Sphilip * Redistribution and use in source and binary forms, with or without 9284555Sarybchik * modification, are permitted provided that the following conditions are met: 10227569Sphilip * 11284555Sarybchik * 1. Redistributions of source code must retain the above copyright notice, 12284555Sarybchik * this list of conditions and the following disclaimer. 13284555Sarybchik * 2. Redistributions in binary form must reproduce the above copyright notice, 14284555Sarybchik * this list of conditions and the following disclaimer in the documentation 15284555Sarybchik * and/or other materials provided with the distribution. 16228078Sphilip * 17284555Sarybchik * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18284555Sarybchik * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19284555Sarybchik * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20284555Sarybchik * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21284555Sarybchik * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22284555Sarybchik * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23284555Sarybchik * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24284555Sarybchik * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25284555Sarybchik * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26284555Sarybchik * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27284555Sarybchik * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28284555Sarybchik * 29284555Sarybchik * The views and conclusions contained in the software and documentation are 30284555Sarybchik * those of the authors and should not be interpreted as representing official 31284555Sarybchik * policies, either expressed or implied, of the FreeBSD Project. 32284555Sarybchik * 33228078Sphilip * $FreeBSD: stable/10/sys/dev/sfxge/common/efsys.h 311500 2017-01-06 07:38:12Z arybchik $ 34227569Sphilip */ 35227569Sphilip 36227569Sphilip#ifndef _SYS_EFSYS_H 37227569Sphilip#define _SYS_EFSYS_H 38227569Sphilip 39227569Sphilip#ifdef __cplusplus 40227569Sphilipextern "C" { 41227569Sphilip#endif 42227569Sphilip 43227569Sphilip#include <sys/param.h> 44227569Sphilip#include <sys/bus.h> 45227569Sphilip#include <sys/endian.h> 46227569Sphilip#include <sys/lock.h> 47227569Sphilip#include <sys/malloc.h> 48227569Sphilip#include <sys/mbuf.h> 49227569Sphilip#include <sys/mutex.h> 50227569Sphilip#include <sys/rwlock.h> 51227569Sphilip#include <sys/sdt.h> 52227569Sphilip#include <sys/systm.h> 53227569Sphilip 54227569Sphilip#include <machine/bus.h> 55227569Sphilip#include <machine/endian.h> 56227569Sphilip 57227569Sphilip#define EFSYS_HAS_UINT64 1 58280525Sarybchik#if defined(__x86_64__) 59280525Sarybchik#define EFSYS_USE_UINT64 1 60280525Sarybchik#else 61227569Sphilip#define EFSYS_USE_UINT64 0 62280525Sarybchik#endif 63284555Sarybchik#define EFSYS_HAS_SSE2_M128 0 64227569Sphilip#if _BYTE_ORDER == _BIG_ENDIAN 65280501Sarybchik#define EFSYS_IS_BIG_ENDIAN 1 66280501Sarybchik#define EFSYS_IS_LITTLE_ENDIAN 0 67227569Sphilip#elif _BYTE_ORDER == _LITTLE_ENDIAN 68280501Sarybchik#define EFSYS_IS_BIG_ENDIAN 0 69280501Sarybchik#define EFSYS_IS_LITTLE_ENDIAN 1 70227569Sphilip#endif 71227569Sphilip#include "efx_types.h" 72227569Sphilip 73227569Sphilip/* Common code requires this */ 74227569Sphilip#if __FreeBSD_version < 800068 75280501Sarybchik#define memmove(d, s, l) bcopy(s, d, l) 76227569Sphilip#endif 77280501Sarybchik 78227569Sphilip/* FreeBSD equivalents of Solaris things */ 79227569Sphilip#ifndef _NOTE 80280501Sarybchik#define _NOTE(s) 81227569Sphilip#endif 82227569Sphilip 83227569Sphilip#ifndef B_FALSE 84280501Sarybchik#define B_FALSE FALSE 85227569Sphilip#endif 86227569Sphilip#ifndef B_TRUE 87280501Sarybchik#define B_TRUE TRUE 88227569Sphilip#endif 89227569Sphilip 90227569Sphilip#ifndef IS_P2ALIGNED 91280501Sarybchik#define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 92227569Sphilip#endif 93227569Sphilip 94227569Sphilip#ifndef P2ROUNDUP 95280501Sarybchik#define P2ROUNDUP(x, align) (-(-(x) & -(align))) 96227569Sphilip#endif 97227569Sphilip 98284555Sarybchik#ifndef P2ALIGN 99284555Sarybchik#define P2ALIGN(_x, _a) ((_x) & -(_a)) 100284555Sarybchik#endif 101284555Sarybchik 102227569Sphilip#ifndef IS2P 103280501Sarybchik#define ISP2(x) (((x) & ((x) - 1)) == 0) 104227569Sphilip#endif 105227569Sphilip 106280608Sarybchik#if defined(__x86_64__) && __FreeBSD_version >= 1000000 107280608Sarybchik 108280608Sarybchik#define SFXGE_USE_BUS_SPACE_8 1 109280608Sarybchik 110280580Sarybchik#if !defined(bus_space_read_stream_8) 111280608Sarybchik 112280580Sarybchik#define bus_space_read_stream_8(t, h, o) \ 113280580Sarybchik bus_space_read_8((t), (h), (o)) 114280608Sarybchik 115280580Sarybchik#define bus_space_write_stream_8(t, h, o, v) \ 116280580Sarybchik bus_space_write_8((t), (h), (o), (v)) 117280608Sarybchik 118280580Sarybchik#endif 119280608Sarybchik 120280580Sarybchik#endif 121280580Sarybchik 122280501Sarybchik#define ENOTACTIVE EINVAL 123227569Sphilip 124227569Sphilip/* Memory type to use on FreeBSD */ 125227569SphilipMALLOC_DECLARE(M_SFXGE); 126227569Sphilip 127227569Sphilip/* Machine dependend prefetch wrappers */ 128227641Smarius#if defined(__i386__) || defined(__amd64__) 129227569Sphilipstatic __inline void 130227569Sphilipprefetch_read_many(void *addr) 131227569Sphilip{ 132227641Smarius 133227569Sphilip __asm__( 134227569Sphilip "prefetcht0 (%0)" 135227569Sphilip : 136227569Sphilip : "r" (addr)); 137227569Sphilip} 138227569Sphilip 139227569Sphilipstatic __inline void 140227569Sphilipprefetch_read_once(void *addr) 141227569Sphilip{ 142227641Smarius 143227569Sphilip __asm__( 144227569Sphilip "prefetchnta (%0)" 145227569Sphilip : 146227569Sphilip : "r" (addr)); 147227569Sphilip} 148227641Smarius#elif defined(__sparc64__) 149227641Smariusstatic __inline void 150227641Smariusprefetch_read_many(void *addr) 151227641Smarius{ 152227641Smarius 153227641Smarius __asm__( 154227641Smarius "prefetch [%0], 0" 155227641Smarius : 156227641Smarius : "r" (addr)); 157227641Smarius} 158227641Smarius 159227641Smariusstatic __inline void 160227641Smariusprefetch_read_once(void *addr) 161227641Smarius{ 162227641Smarius 163227641Smarius __asm__( 164227641Smarius "prefetch [%0], 1" 165227641Smarius : 166227641Smarius : "r" (addr)); 167227641Smarius} 168227641Smarius#else 169227641Smariusstatic __inline void 170227641Smariusprefetch_read_many(void *addr) 171227641Smarius{ 172227641Smarius 173227641Smarius} 174227641Smarius 175227641Smariusstatic __inline void 176227641Smariusprefetch_read_once(void *addr) 177227641Smarius{ 178227641Smarius 179227641Smarius} 180227569Sphilip#endif 181227569Sphilip 182227569Sphilip#if defined(__i386__) || defined(__amd64__) 183227569Sphilip#include <vm/vm.h> 184227569Sphilip#include <vm/pmap.h> 185227569Sphilip#endif 186227569Sphilipstatic __inline void 187227569Sphilipsfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 188284555Sarybchik struct mbuf *m, bus_dma_segment_t *seg) 189227569Sphilip{ 190227569Sphilip#if defined(__i386__) || defined(__amd64__) 191227569Sphilip seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 192227569Sphilip seg->ds_len = m->m_len; 193227569Sphilip#else 194227569Sphilip int nsegstmp; 195227569Sphilip 196227569Sphilip bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 197227569Sphilip#endif 198227569Sphilip} 199227569Sphilip 200227569Sphilip/* Modifiers used for Windows builds */ 201227569Sphilip#define __in 202227569Sphilip#define __in_opt 203227569Sphilip#define __in_ecount(_n) 204227569Sphilip#define __in_ecount_opt(_n) 205227569Sphilip#define __in_bcount(_n) 206227569Sphilip#define __in_bcount_opt(_n) 207227569Sphilip 208227569Sphilip#define __out 209227569Sphilip#define __out_opt 210227569Sphilip#define __out_ecount(_n) 211227569Sphilip#define __out_ecount_opt(_n) 212227569Sphilip#define __out_bcount(_n) 213227569Sphilip#define __out_bcount_opt(_n) 214294386Sarybchik#define __out_bcount_part(_n, _l) 215294386Sarybchik#define __out_bcount_part_opt(_n, _l) 216227569Sphilip 217227569Sphilip#define __deref_out 218227569Sphilip 219227569Sphilip#define __inout 220227569Sphilip#define __inout_opt 221227569Sphilip#define __inout_ecount(_n) 222227569Sphilip#define __inout_ecount_opt(_n) 223227569Sphilip#define __inout_bcount(_n) 224227569Sphilip#define __inout_bcount_opt(_n) 225227569Sphilip#define __inout_bcount_full_opt(_n) 226227569Sphilip 227227569Sphilip#define __deref_out_bcount_opt(n) 228227569Sphilip 229227569Sphilip#define __checkReturn 230293927Sarybchik#define __success(_x) 231227569Sphilip 232227569Sphilip#define __drv_when(_p, _c) 233227569Sphilip 234227569Sphilip/* Code inclusion options */ 235227569Sphilip 236227569Sphilip 237227569Sphilip#define EFSYS_OPT_NAMES 1 238227569Sphilip 239227569Sphilip#define EFSYS_OPT_SIENA 1 240284555Sarybchik#define EFSYS_OPT_HUNTINGTON 1 241301390Sarybchik#define EFSYS_OPT_MEDFORD 1 242227569Sphilip#ifdef DEBUG 243227569Sphilip#define EFSYS_OPT_CHECK_REG 1 244227569Sphilip#else 245227569Sphilip#define EFSYS_OPT_CHECK_REG 0 246227569Sphilip#endif 247227569Sphilip 248227569Sphilip#define EFSYS_OPT_MCDI 1 249293939Sarybchik#define EFSYS_OPT_MCDI_LOGGING 0 250293958Sarybchik#define EFSYS_OPT_MCDI_PROXY_AUTH 0 251227569Sphilip 252227569Sphilip#define EFSYS_OPT_MAC_STATS 1 253227569Sphilip 254227569Sphilip#define EFSYS_OPT_LOOPBACK 0 255227569Sphilip 256284555Sarybchik#define EFSYS_OPT_MON_MCDI 0 257227569Sphilip#define EFSYS_OPT_MON_STATS 0 258227569Sphilip 259227569Sphilip#define EFSYS_OPT_PHY_STATS 1 260284555Sarybchik#define EFSYS_OPT_BIST 1 261227569Sphilip#define EFSYS_OPT_PHY_LED_CONTROL 1 262280501Sarybchik#define EFSYS_OPT_PHY_FLAGS 0 263227569Sphilip 264227569Sphilip#define EFSYS_OPT_VPD 1 265227569Sphilip#define EFSYS_OPT_NVRAM 1 266227569Sphilip#define EFSYS_OPT_BOOTCFG 0 267227569Sphilip 268227569Sphilip#define EFSYS_OPT_DIAG 0 269227569Sphilip#define EFSYS_OPT_RX_SCALE 1 270227569Sphilip#define EFSYS_OPT_QSTATS 1 271284555Sarybchik#define EFSYS_OPT_FILTER 1 272280501Sarybchik#define EFSYS_OPT_RX_SCATTER 0 273227569Sphilip 274227569Sphilip#define EFSYS_OPT_EV_PREFETCH 0 275227569Sphilip 276227569Sphilip#define EFSYS_OPT_DECODE_INTR_FATAL 1 277227569Sphilip 278294386Sarybchik#define EFSYS_OPT_LICENSING 0 279294386Sarybchik 280301387Sarybchik#define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0 281301387Sarybchik 282227569Sphilip/* ID */ 283227569Sphilip 284227569Sphiliptypedef struct __efsys_identifier_s efsys_identifier_t; 285227569Sphilip 286227569Sphilip/* PROBE */ 287227569Sphilip 288227569Sphilip#ifndef KDTRACE_HOOKS 289227569Sphilip 290280501Sarybchik#define EFSYS_PROBE(_name) 291227569Sphilip 292227569Sphilip#define EFSYS_PROBE1(_name, _type1, _arg1) 293227569Sphilip 294227569Sphilip#define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 295227569Sphilip 296227569Sphilip#define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 297227569Sphilip _type3, _arg3) 298227569Sphilip 299227569Sphilip#define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 300227569Sphilip _type3, _arg3, _type4, _arg4) 301227569Sphilip 302227569Sphilip#define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 303227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5) 304227569Sphilip 305227569Sphilip#define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 306227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 307227569Sphilip _type6, _arg6) 308227569Sphilip 309227569Sphilip#define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 310227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 311227569Sphilip _type6, _arg6, _type7, _arg7) 312227569Sphilip 313227569Sphilip#else /* KDTRACE_HOOKS */ 314227569Sphilip 315227569Sphilip#define EFSYS_PROBE(_name) \ 316227569Sphilip DTRACE_PROBE(_name) 317227569Sphilip 318227569Sphilip#define EFSYS_PROBE1(_name, _type1, _arg1) \ 319227569Sphilip DTRACE_PROBE1(_name, _type1, _arg1) 320227569Sphilip 321227569Sphilip#define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 322227569Sphilip DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 323227569Sphilip 324227569Sphilip#define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 325227569Sphilip _type3, _arg3) \ 326227569Sphilip DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 327227569Sphilip _type3, _arg3) 328227569Sphilip 329227569Sphilip#define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 330227569Sphilip _type3, _arg3, _type4, _arg4) \ 331227569Sphilip DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 332227569Sphilip _type3, _arg3, _type4, _arg4) 333227569Sphilip 334227569Sphilip#ifdef DTRACE_PROBE5 335227569Sphilip#define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 336227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 337227569Sphilip DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 338227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5) 339227569Sphilip#else 340227569Sphilip#define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 341227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 342227569Sphilip DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 343227569Sphilip _type3, _arg3, _type4, _arg4) 344227569Sphilip#endif 345227569Sphilip 346227569Sphilip#ifdef DTRACE_PROBE6 347227569Sphilip#define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 348227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 349227569Sphilip _type6, _arg6) \ 350227569Sphilip DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 351227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 352227569Sphilip _type6, _arg6) 353227569Sphilip#else 354227569Sphilip#define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 355227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 356227569Sphilip _type6, _arg6) \ 357227569Sphilip EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 358227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5) 359227569Sphilip#endif 360227569Sphilip 361227569Sphilip#ifdef DTRACE_PROBE7 362227569Sphilip#define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 363227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 364227569Sphilip _type6, _arg6, _type7, _arg7) \ 365227569Sphilip DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 366227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 367227569Sphilip _type6, _arg6, _type7, _arg7) 368227569Sphilip#else 369227569Sphilip#define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 370227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 371227569Sphilip _type6, _arg6, _type7, _arg7) \ 372227569Sphilip EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 373227569Sphilip _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 374227569Sphilip _type6, _arg6) 375227569Sphilip#endif 376227569Sphilip 377227569Sphilip#endif /* KDTRACE_HOOKS */ 378227569Sphilip 379227569Sphilip/* DMA */ 380227569Sphilip 381227569Sphiliptypedef uint64_t efsys_dma_addr_t; 382227569Sphilip 383227569Sphiliptypedef struct efsys_mem_s { 384227569Sphilip bus_dma_tag_t esm_tag; 385227569Sphilip bus_dmamap_t esm_map; 386227569Sphilip caddr_t esm_base; 387227569Sphilip efsys_dma_addr_t esm_addr; 388227569Sphilip} efsys_mem_t; 389227569Sphilip 390227569Sphilip 391227569Sphilip#define EFSYS_MEM_ZERO(_esmp, _size) \ 392227569Sphilip do { \ 393227569Sphilip (void) memset((_esmp)->esm_base, 0, (_size)); \ 394227569Sphilip \ 395227569Sphilip _NOTE(CONSTANTCONDITION) \ 396227569Sphilip } while (B_FALSE) 397227569Sphilip 398227569Sphilip#define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 399227569Sphilip do { \ 400227569Sphilip uint32_t *addr; \ 401227569Sphilip \ 402227569Sphilip _NOTE(CONSTANTCONDITION) \ 403227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 404227569Sphilip ("not power of 2 aligned")); \ 405227569Sphilip \ 406227569Sphilip addr = (void *)((_esmp)->esm_base + (_offset)); \ 407227569Sphilip \ 408227569Sphilip (_edp)->ed_u32[0] = *addr; \ 409227569Sphilip \ 410227569Sphilip EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 411227569Sphilip uint32_t, (_edp)->ed_u32[0]); \ 412227569Sphilip \ 413227569Sphilip _NOTE(CONSTANTCONDITION) \ 414227569Sphilip } while (B_FALSE) 415227569Sphilip 416280525Sarybchik#if defined(__x86_64__) 417227569Sphilip#define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 418227569Sphilip do { \ 419280525Sarybchik uint64_t *addr; \ 420280525Sarybchik \ 421280525Sarybchik _NOTE(CONSTANTCONDITION) \ 422280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 423280525Sarybchik ("not power of 2 aligned")); \ 424280525Sarybchik \ 425280525Sarybchik addr = (void *)((_esmp)->esm_base + (_offset)); \ 426280525Sarybchik \ 427280525Sarybchik (_eqp)->eq_u64[0] = *addr; \ 428280525Sarybchik \ 429280525Sarybchik EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 430280525Sarybchik uint32_t, (_eqp)->eq_u32[1], \ 431280525Sarybchik uint32_t, (_eqp)->eq_u32[0]); \ 432280525Sarybchik \ 433280525Sarybchik _NOTE(CONSTANTCONDITION) \ 434280525Sarybchik } while (B_FALSE) 435280525Sarybchik#else 436280525Sarybchik#define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 437280525Sarybchik do { \ 438227569Sphilip uint32_t *addr; \ 439227569Sphilip \ 440227569Sphilip _NOTE(CONSTANTCONDITION) \ 441227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 442227569Sphilip ("not power of 2 aligned")); \ 443227569Sphilip \ 444227569Sphilip addr = (void *)((_esmp)->esm_base + (_offset)); \ 445227569Sphilip \ 446227569Sphilip (_eqp)->eq_u32[0] = *addr++; \ 447227569Sphilip (_eqp)->eq_u32[1] = *addr; \ 448227569Sphilip \ 449227569Sphilip EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 450227569Sphilip uint32_t, (_eqp)->eq_u32[1], \ 451227569Sphilip uint32_t, (_eqp)->eq_u32[0]); \ 452227569Sphilip \ 453227569Sphilip _NOTE(CONSTANTCONDITION) \ 454227569Sphilip } while (B_FALSE) 455280525Sarybchik#endif 456227569Sphilip 457280525Sarybchik#if defined(__x86_64__) 458227569Sphilip#define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 459227569Sphilip do { \ 460280525Sarybchik uint64_t *addr; \ 461280525Sarybchik \ 462280525Sarybchik _NOTE(CONSTANTCONDITION) \ 463280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 464280525Sarybchik ("not power of 2 aligned")); \ 465280525Sarybchik \ 466280525Sarybchik addr = (void *)((_esmp)->esm_base + (_offset)); \ 467280525Sarybchik \ 468280525Sarybchik (_eop)->eo_u64[0] = *addr++; \ 469280525Sarybchik (_eop)->eo_u64[1] = *addr; \ 470280525Sarybchik \ 471280525Sarybchik EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 472280525Sarybchik uint32_t, (_eop)->eo_u32[3], \ 473280525Sarybchik uint32_t, (_eop)->eo_u32[2], \ 474280525Sarybchik uint32_t, (_eop)->eo_u32[1], \ 475280525Sarybchik uint32_t, (_eop)->eo_u32[0]); \ 476280525Sarybchik \ 477280525Sarybchik _NOTE(CONSTANTCONDITION) \ 478280525Sarybchik } while (B_FALSE) 479280525Sarybchik#else 480280525Sarybchik#define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 481280525Sarybchik do { \ 482227569Sphilip uint32_t *addr; \ 483227569Sphilip \ 484227569Sphilip _NOTE(CONSTANTCONDITION) \ 485227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 486227569Sphilip ("not power of 2 aligned")); \ 487227569Sphilip \ 488227569Sphilip addr = (void *)((_esmp)->esm_base + (_offset)); \ 489227569Sphilip \ 490227569Sphilip (_eop)->eo_u32[0] = *addr++; \ 491227569Sphilip (_eop)->eo_u32[1] = *addr++; \ 492227569Sphilip (_eop)->eo_u32[2] = *addr++; \ 493227569Sphilip (_eop)->eo_u32[3] = *addr; \ 494227569Sphilip \ 495227569Sphilip EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 496227569Sphilip uint32_t, (_eop)->eo_u32[3], \ 497227569Sphilip uint32_t, (_eop)->eo_u32[2], \ 498227569Sphilip uint32_t, (_eop)->eo_u32[1], \ 499227569Sphilip uint32_t, (_eop)->eo_u32[0]); \ 500227569Sphilip \ 501227569Sphilip _NOTE(CONSTANTCONDITION) \ 502227569Sphilip } while (B_FALSE) 503280525Sarybchik#endif 504227569Sphilip 505227569Sphilip#define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 506227569Sphilip do { \ 507227569Sphilip uint32_t *addr; \ 508227569Sphilip \ 509227569Sphilip _NOTE(CONSTANTCONDITION) \ 510227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 511227569Sphilip ("not power of 2 aligned")); \ 512227569Sphilip \ 513227569Sphilip EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 514227569Sphilip uint32_t, (_edp)->ed_u32[0]); \ 515227569Sphilip \ 516227569Sphilip addr = (void *)((_esmp)->esm_base + (_offset)); \ 517227569Sphilip \ 518227569Sphilip *addr = (_edp)->ed_u32[0]; \ 519227569Sphilip \ 520227569Sphilip _NOTE(CONSTANTCONDITION) \ 521227569Sphilip } while (B_FALSE) 522227569Sphilip 523280525Sarybchik#if defined(__x86_64__) 524227569Sphilip#define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 525227569Sphilip do { \ 526280525Sarybchik uint64_t *addr; \ 527280525Sarybchik \ 528280525Sarybchik _NOTE(CONSTANTCONDITION) \ 529280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 530280525Sarybchik ("not power of 2 aligned")); \ 531280525Sarybchik \ 532280525Sarybchik EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 533280525Sarybchik uint32_t, (_eqp)->eq_u32[1], \ 534280525Sarybchik uint32_t, (_eqp)->eq_u32[0]); \ 535280525Sarybchik \ 536280525Sarybchik addr = (void *)((_esmp)->esm_base + (_offset)); \ 537280525Sarybchik \ 538280525Sarybchik *addr = (_eqp)->eq_u64[0]; \ 539280525Sarybchik \ 540280525Sarybchik _NOTE(CONSTANTCONDITION) \ 541280525Sarybchik } while (B_FALSE) 542280525Sarybchik 543280525Sarybchik#else 544280525Sarybchik#define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 545280525Sarybchik do { \ 546227569Sphilip uint32_t *addr; \ 547227569Sphilip \ 548227569Sphilip _NOTE(CONSTANTCONDITION) \ 549227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 550227569Sphilip ("not power of 2 aligned")); \ 551227569Sphilip \ 552227569Sphilip EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 553227569Sphilip uint32_t, (_eqp)->eq_u32[1], \ 554227569Sphilip uint32_t, (_eqp)->eq_u32[0]); \ 555227569Sphilip \ 556227569Sphilip addr = (void *)((_esmp)->esm_base + (_offset)); \ 557227569Sphilip \ 558227569Sphilip *addr++ = (_eqp)->eq_u32[0]; \ 559227569Sphilip *addr = (_eqp)->eq_u32[1]; \ 560227569Sphilip \ 561227569Sphilip _NOTE(CONSTANTCONDITION) \ 562227569Sphilip } while (B_FALSE) 563280525Sarybchik#endif 564227569Sphilip 565280525Sarybchik#if defined(__x86_64__) 566227569Sphilip#define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 567227569Sphilip do { \ 568280525Sarybchik uint64_t *addr; \ 569280525Sarybchik \ 570280525Sarybchik _NOTE(CONSTANTCONDITION) \ 571280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 572280525Sarybchik ("not power of 2 aligned")); \ 573280525Sarybchik \ 574280525Sarybchik EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 575280525Sarybchik uint32_t, (_eop)->eo_u32[3], \ 576280525Sarybchik uint32_t, (_eop)->eo_u32[2], \ 577280525Sarybchik uint32_t, (_eop)->eo_u32[1], \ 578280525Sarybchik uint32_t, (_eop)->eo_u32[0]); \ 579280525Sarybchik \ 580280525Sarybchik addr = (void *)((_esmp)->esm_base + (_offset)); \ 581280525Sarybchik \ 582280525Sarybchik *addr++ = (_eop)->eo_u64[0]; \ 583280525Sarybchik *addr = (_eop)->eo_u64[1]; \ 584280525Sarybchik \ 585280525Sarybchik _NOTE(CONSTANTCONDITION) \ 586280525Sarybchik } while (B_FALSE) 587280525Sarybchik#else 588280525Sarybchik#define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 589280525Sarybchik do { \ 590227569Sphilip uint32_t *addr; \ 591227569Sphilip \ 592227569Sphilip _NOTE(CONSTANTCONDITION) \ 593227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 594227569Sphilip ("not power of 2 aligned")); \ 595227569Sphilip \ 596227569Sphilip EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 597227569Sphilip uint32_t, (_eop)->eo_u32[3], \ 598227569Sphilip uint32_t, (_eop)->eo_u32[2], \ 599227569Sphilip uint32_t, (_eop)->eo_u32[1], \ 600227569Sphilip uint32_t, (_eop)->eo_u32[0]); \ 601227569Sphilip \ 602227569Sphilip addr = (void *)((_esmp)->esm_base + (_offset)); \ 603227569Sphilip \ 604227569Sphilip *addr++ = (_eop)->eo_u32[0]; \ 605227569Sphilip *addr++ = (_eop)->eo_u32[1]; \ 606227569Sphilip *addr++ = (_eop)->eo_u32[2]; \ 607227569Sphilip *addr = (_eop)->eo_u32[3]; \ 608227569Sphilip \ 609227569Sphilip _NOTE(CONSTANTCONDITION) \ 610227569Sphilip } while (B_FALSE) 611280525Sarybchik#endif 612227569Sphilip 613227569Sphilip#define EFSYS_MEM_ADDR(_esmp) \ 614227569Sphilip ((_esmp)->esm_addr) 615227569Sphilip 616284555Sarybchik#define EFSYS_MEM_IS_NULL(_esmp) \ 617284555Sarybchik ((_esmp)->esm_base == NULL) 618284555Sarybchik 619227569Sphilip/* BAR */ 620227569Sphilip 621280524Sarybchik#define SFXGE_LOCK_NAME_MAX 16 622280524Sarybchik 623227569Sphiliptypedef struct efsys_bar_s { 624227569Sphilip struct mtx esb_lock; 625280524Sarybchik char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 626227569Sphilip bus_space_tag_t esb_tag; 627227569Sphilip bus_space_handle_t esb_handle; 628227569Sphilip int esb_rid; 629227569Sphilip struct resource *esb_res; 630227569Sphilip} efsys_bar_t; 631227569Sphilip 632280524Sarybchik#define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 633280524Sarybchik do { \ 634280524Sarybchik snprintf((_esbp)->esb_lock_name, \ 635280524Sarybchik sizeof((_esbp)->esb_lock_name), \ 636280524Sarybchik "%s:bar", (_ifname)); \ 637280524Sarybchik mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 638280524Sarybchik NULL, MTX_DEF); \ 639280524Sarybchik _NOTE(CONSTANTCONDITION) \ 640280524Sarybchik } while (B_FALSE) 641280522Sarybchik#define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 642280522Sarybchik mtx_destroy(&(_esbp)->esb_lock) 643280522Sarybchik#define SFXGE_BAR_LOCK(_esbp) \ 644280522Sarybchik mtx_lock(&(_esbp)->esb_lock) 645280522Sarybchik#define SFXGE_BAR_UNLOCK(_esbp) \ 646280522Sarybchik mtx_unlock(&(_esbp)->esb_lock) 647280522Sarybchik 648227569Sphilip#define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 649227569Sphilip do { \ 650227569Sphilip _NOTE(CONSTANTCONDITION) \ 651227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 652227569Sphilip ("not power of 2 aligned")); \ 653227569Sphilip \ 654227569Sphilip _NOTE(CONSTANTCONDITION) \ 655227569Sphilip if (_lock) \ 656280522Sarybchik SFXGE_BAR_LOCK(_esbp); \ 657227569Sphilip \ 658280580Sarybchik (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 659280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 660280580Sarybchik (_offset)); \ 661227569Sphilip \ 662227569Sphilip EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 663227569Sphilip uint32_t, (_edp)->ed_u32[0]); \ 664227569Sphilip \ 665227569Sphilip _NOTE(CONSTANTCONDITION) \ 666227569Sphilip if (_lock) \ 667280522Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 668227569Sphilip _NOTE(CONSTANTCONDITION) \ 669227569Sphilip } while (B_FALSE) 670227569Sphilip 671280608Sarybchik#if defined(SFXGE_USE_BUS_SPACE_8) 672227569Sphilip#define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 673227569Sphilip do { \ 674227569Sphilip _NOTE(CONSTANTCONDITION) \ 675227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 676227569Sphilip ("not power of 2 aligned")); \ 677227569Sphilip \ 678280522Sarybchik SFXGE_BAR_LOCK(_esbp); \ 679227569Sphilip \ 680280580Sarybchik (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 681280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 682280580Sarybchik (_offset)); \ 683280525Sarybchik \ 684280525Sarybchik EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 685280525Sarybchik uint32_t, (_eqp)->eq_u32[1], \ 686280525Sarybchik uint32_t, (_eqp)->eq_u32[0]); \ 687280525Sarybchik \ 688280567Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 689280525Sarybchik _NOTE(CONSTANTCONDITION) \ 690280525Sarybchik } while (B_FALSE) 691280525Sarybchik 692280525Sarybchik#define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 693280525Sarybchik do { \ 694280525Sarybchik _NOTE(CONSTANTCONDITION) \ 695280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 696280525Sarybchik ("not power of 2 aligned")); \ 697280525Sarybchik \ 698280525Sarybchik _NOTE(CONSTANTCONDITION) \ 699280525Sarybchik if (_lock) \ 700280567Sarybchik SFXGE_BAR_LOCK(_esbp); \ 701280525Sarybchik \ 702280580Sarybchik (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 703280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 704280580Sarybchik (_offset)); \ 705280580Sarybchik (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 706280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 707280580Sarybchik (_offset) + 8); \ 708280525Sarybchik \ 709280525Sarybchik EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 710280525Sarybchik uint32_t, (_eop)->eo_u32[3], \ 711280525Sarybchik uint32_t, (_eop)->eo_u32[2], \ 712280525Sarybchik uint32_t, (_eop)->eo_u32[1], \ 713280525Sarybchik uint32_t, (_eop)->eo_u32[0]); \ 714280525Sarybchik \ 715280525Sarybchik _NOTE(CONSTANTCONDITION) \ 716280525Sarybchik if (_lock) \ 717280567Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 718280525Sarybchik _NOTE(CONSTANTCONDITION) \ 719280525Sarybchik } while (B_FALSE) 720280525Sarybchik 721280525Sarybchik#else 722280525Sarybchik#define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 723280525Sarybchik do { \ 724280525Sarybchik _NOTE(CONSTANTCONDITION) \ 725280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 726280525Sarybchik ("not power of 2 aligned")); \ 727280525Sarybchik \ 728280567Sarybchik SFXGE_BAR_LOCK(_esbp); \ 729280525Sarybchik \ 730280580Sarybchik (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 731280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 732280580Sarybchik (_offset)); \ 733280580Sarybchik (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 734280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 735280580Sarybchik (_offset) + 4); \ 736227569Sphilip \ 737227569Sphilip EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 738227569Sphilip uint32_t, (_eqp)->eq_u32[1], \ 739227569Sphilip uint32_t, (_eqp)->eq_u32[0]); \ 740227569Sphilip \ 741280522Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 742227569Sphilip _NOTE(CONSTANTCONDITION) \ 743227569Sphilip } while (B_FALSE) 744227569Sphilip 745227569Sphilip#define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 746227569Sphilip do { \ 747227569Sphilip _NOTE(CONSTANTCONDITION) \ 748227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 749227569Sphilip ("not power of 2 aligned")); \ 750227569Sphilip \ 751227569Sphilip _NOTE(CONSTANTCONDITION) \ 752227569Sphilip if (_lock) \ 753280522Sarybchik SFXGE_BAR_LOCK(_esbp); \ 754227569Sphilip \ 755280580Sarybchik (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 756280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 757280580Sarybchik (_offset)); \ 758280580Sarybchik (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 759280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 760280580Sarybchik (_offset) + 4); \ 761280580Sarybchik (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 762280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 763280580Sarybchik (_offset) + 8); \ 764280580Sarybchik (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 765280580Sarybchik (_esbp)->esb_tag, (_esbp)->esb_handle, \ 766280580Sarybchik (_offset) + 12); \ 767227569Sphilip \ 768227569Sphilip EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 769227569Sphilip uint32_t, (_eop)->eo_u32[3], \ 770227569Sphilip uint32_t, (_eop)->eo_u32[2], \ 771227569Sphilip uint32_t, (_eop)->eo_u32[1], \ 772227569Sphilip uint32_t, (_eop)->eo_u32[0]); \ 773227569Sphilip \ 774227569Sphilip _NOTE(CONSTANTCONDITION) \ 775227569Sphilip if (_lock) \ 776280522Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 777227569Sphilip _NOTE(CONSTANTCONDITION) \ 778227569Sphilip } while (B_FALSE) 779280525Sarybchik#endif 780227569Sphilip 781227569Sphilip#define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 782227569Sphilip do { \ 783227569Sphilip _NOTE(CONSTANTCONDITION) \ 784227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 785227569Sphilip ("not power of 2 aligned")); \ 786227569Sphilip \ 787227569Sphilip _NOTE(CONSTANTCONDITION) \ 788227569Sphilip if (_lock) \ 789280567Sarybchik SFXGE_BAR_LOCK(_esbp); \ 790227569Sphilip \ 791227569Sphilip EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 792227569Sphilip uint32_t, (_edp)->ed_u32[0]); \ 793227569Sphilip \ 794280605Sarybchik /* \ 795280605Sarybchik * Make sure that previous writes to the dword have \ 796280605Sarybchik * been done. It should be cheaper than barrier just \ 797280605Sarybchik * after the write below. \ 798280605Sarybchik */ \ 799280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 800280605Sarybchik (_offset), sizeof (efx_dword_t), \ 801280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 802280580Sarybchik bus_space_write_stream_4((_esbp)->esb_tag, \ 803280580Sarybchik (_esbp)->esb_handle, \ 804227569Sphilip (_offset), (_edp)->ed_u32[0]); \ 805227569Sphilip \ 806227569Sphilip _NOTE(CONSTANTCONDITION) \ 807227569Sphilip if (_lock) \ 808280567Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 809227569Sphilip _NOTE(CONSTANTCONDITION) \ 810227569Sphilip } while (B_FALSE) 811227569Sphilip 812280608Sarybchik#if defined(SFXGE_USE_BUS_SPACE_8) 813227569Sphilip#define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 814227569Sphilip do { \ 815227569Sphilip _NOTE(CONSTANTCONDITION) \ 816227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 817227569Sphilip ("not power of 2 aligned")); \ 818227569Sphilip \ 819280522Sarybchik SFXGE_BAR_LOCK(_esbp); \ 820227569Sphilip \ 821227569Sphilip EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 822227569Sphilip uint32_t, (_eqp)->eq_u32[1], \ 823227569Sphilip uint32_t, (_eqp)->eq_u32[0]); \ 824227569Sphilip \ 825280605Sarybchik /* \ 826280605Sarybchik * Make sure that previous writes to the qword have \ 827280605Sarybchik * been done. It should be cheaper than barrier just \ 828280605Sarybchik * after the write below. \ 829280605Sarybchik */ \ 830280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 831280605Sarybchik (_offset), sizeof (efx_qword_t), \ 832280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 833280580Sarybchik bus_space_write_stream_8((_esbp)->esb_tag, \ 834280580Sarybchik (_esbp)->esb_handle, \ 835280525Sarybchik (_offset), (_eqp)->eq_u64[0]); \ 836280525Sarybchik \ 837280567Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 838280525Sarybchik _NOTE(CONSTANTCONDITION) \ 839280525Sarybchik } while (B_FALSE) 840280525Sarybchik#else 841280525Sarybchik#define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 842280525Sarybchik do { \ 843280525Sarybchik _NOTE(CONSTANTCONDITION) \ 844280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 845280525Sarybchik ("not power of 2 aligned")); \ 846280525Sarybchik \ 847280567Sarybchik SFXGE_BAR_LOCK(_esbp); \ 848280525Sarybchik \ 849280525Sarybchik EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 850280525Sarybchik uint32_t, (_eqp)->eq_u32[1], \ 851280525Sarybchik uint32_t, (_eqp)->eq_u32[0]); \ 852280525Sarybchik \ 853280605Sarybchik /* \ 854280605Sarybchik * Make sure that previous writes to the qword have \ 855280605Sarybchik * been done. It should be cheaper than barrier just \ 856280605Sarybchik * after the last write below. \ 857280605Sarybchik */ \ 858280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 859280605Sarybchik (_offset), sizeof (efx_qword_t), \ 860280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 861280580Sarybchik bus_space_write_stream_4((_esbp)->esb_tag, \ 862280580Sarybchik (_esbp)->esb_handle, \ 863227569Sphilip (_offset), (_eqp)->eq_u32[0]); \ 864280605Sarybchik /* \ 865280605Sarybchik * It should be guaranteed that the last dword comes \ 866280605Sarybchik * the last, so barrier entire qword to be sure that \ 867280605Sarybchik * neither above nor below writes are reordered. \ 868280605Sarybchik */ \ 869280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 870280605Sarybchik (_offset), sizeof (efx_qword_t), \ 871280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 872280580Sarybchik bus_space_write_stream_4((_esbp)->esb_tag, \ 873280580Sarybchik (_esbp)->esb_handle, \ 874280580Sarybchik (_offset) + 4, (_eqp)->eq_u32[1]); \ 875227569Sphilip \ 876280522Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 877227569Sphilip _NOTE(CONSTANTCONDITION) \ 878227569Sphilip } while (B_FALSE) 879280525Sarybchik#endif 880227569Sphilip 881284555Sarybchik/* 882284555Sarybchik * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 883284555Sarybchik * (required by PIO hardware) 884284555Sarybchik */ 885284555Sarybchik#define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 886284555Sarybchik do { \ 887284555Sarybchik _NOTE(CONSTANTCONDITION) \ 888284555Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 889284555Sarybchik ("not power of 2 aligned")); \ 890284555Sarybchik \ 891284555Sarybchik (void) (_esbp); \ 892284555Sarybchik \ 893284555Sarybchik /* FIXME: Perform a 64-bit write */ \ 894284555Sarybchik KASSERT(0, ("not implemented")); \ 895284555Sarybchik \ 896284555Sarybchik _NOTE(CONSTANTCONDITION) \ 897284555Sarybchik } while (B_FALSE) 898284555Sarybchik 899280608Sarybchik#if defined(SFXGE_USE_BUS_SPACE_8) 900227569Sphilip#define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 901227569Sphilip do { \ 902227569Sphilip _NOTE(CONSTANTCONDITION) \ 903227569Sphilip KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 904227569Sphilip ("not power of 2 aligned")); \ 905227569Sphilip \ 906227569Sphilip _NOTE(CONSTANTCONDITION) \ 907227569Sphilip if (_lock) \ 908280522Sarybchik SFXGE_BAR_LOCK(_esbp); \ 909227569Sphilip \ 910227569Sphilip EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 911227569Sphilip uint32_t, (_eop)->eo_u32[3], \ 912227569Sphilip uint32_t, (_eop)->eo_u32[2], \ 913227569Sphilip uint32_t, (_eop)->eo_u32[1], \ 914227569Sphilip uint32_t, (_eop)->eo_u32[0]); \ 915227569Sphilip \ 916280605Sarybchik /* \ 917280605Sarybchik * Make sure that previous writes to the oword have \ 918280605Sarybchik * been done. It should be cheaper than barrier just \ 919280605Sarybchik * after the last write below. \ 920280605Sarybchik */ \ 921280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 922280605Sarybchik (_offset), sizeof (efx_oword_t), \ 923280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 924280580Sarybchik bus_space_write_stream_8((_esbp)->esb_tag, \ 925280580Sarybchik (_esbp)->esb_handle, \ 926280525Sarybchik (_offset), (_eop)->eo_u64[0]); \ 927280605Sarybchik /* \ 928280605Sarybchik * It should be guaranteed that the last qword comes \ 929280605Sarybchik * the last, so barrier entire oword to be sure that \ 930280605Sarybchik * neither above nor below writes are reordered. \ 931280605Sarybchik */ \ 932280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 933280605Sarybchik (_offset), sizeof (efx_oword_t), \ 934280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 935280580Sarybchik bus_space_write_stream_8((_esbp)->esb_tag, \ 936280580Sarybchik (_esbp)->esb_handle, \ 937280580Sarybchik (_offset) + 8, (_eop)->eo_u64[1]); \ 938280525Sarybchik \ 939280525Sarybchik _NOTE(CONSTANTCONDITION) \ 940280525Sarybchik if (_lock) \ 941280567Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 942280525Sarybchik _NOTE(CONSTANTCONDITION) \ 943280525Sarybchik } while (B_FALSE) 944280525Sarybchik 945280525Sarybchik#else 946280525Sarybchik#define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 947280525Sarybchik do { \ 948280525Sarybchik _NOTE(CONSTANTCONDITION) \ 949280525Sarybchik KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 950280525Sarybchik ("not power of 2 aligned")); \ 951280525Sarybchik \ 952280525Sarybchik _NOTE(CONSTANTCONDITION) \ 953280525Sarybchik if (_lock) \ 954280567Sarybchik SFXGE_BAR_LOCK(_esbp); \ 955280525Sarybchik \ 956280525Sarybchik EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 957280525Sarybchik uint32_t, (_eop)->eo_u32[3], \ 958280525Sarybchik uint32_t, (_eop)->eo_u32[2], \ 959280525Sarybchik uint32_t, (_eop)->eo_u32[1], \ 960280525Sarybchik uint32_t, (_eop)->eo_u32[0]); \ 961280525Sarybchik \ 962280605Sarybchik /* \ 963280605Sarybchik * Make sure that previous writes to the oword have \ 964280605Sarybchik * been done. It should be cheaper than barrier just \ 965280605Sarybchik * after the last write below. \ 966280605Sarybchik */ \ 967280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 968280605Sarybchik (_offset), sizeof (efx_oword_t), \ 969280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 970280580Sarybchik bus_space_write_stream_4((_esbp)->esb_tag, \ 971280580Sarybchik (_esbp)->esb_handle, \ 972227569Sphilip (_offset), (_eop)->eo_u32[0]); \ 973280580Sarybchik bus_space_write_stream_4((_esbp)->esb_tag, \ 974280580Sarybchik (_esbp)->esb_handle, \ 975280580Sarybchik (_offset) + 4, (_eop)->eo_u32[1]); \ 976280580Sarybchik bus_space_write_stream_4((_esbp)->esb_tag, \ 977280580Sarybchik (_esbp)->esb_handle, \ 978280580Sarybchik (_offset) + 8, (_eop)->eo_u32[2]); \ 979280605Sarybchik /* \ 980280605Sarybchik * It should be guaranteed that the last dword comes \ 981280605Sarybchik * the last, so barrier entire oword to be sure that \ 982280605Sarybchik * neither above nor below writes are reordered. \ 983280605Sarybchik */ \ 984280605Sarybchik bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 985280605Sarybchik (_offset), sizeof (efx_oword_t), \ 986280605Sarybchik BUS_SPACE_BARRIER_WRITE); \ 987280580Sarybchik bus_space_write_stream_4((_esbp)->esb_tag, \ 988280580Sarybchik (_esbp)->esb_handle, \ 989280580Sarybchik (_offset) + 12, (_eop)->eo_u32[3]); \ 990227569Sphilip \ 991227569Sphilip _NOTE(CONSTANTCONDITION) \ 992227569Sphilip if (_lock) \ 993280522Sarybchik SFXGE_BAR_UNLOCK(_esbp); \ 994227569Sphilip _NOTE(CONSTANTCONDITION) \ 995227569Sphilip } while (B_FALSE) 996280525Sarybchik#endif 997227569Sphilip 998284555Sarybchik/* Use the standard octo-word write for doorbell writes */ 999284555Sarybchik#define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 1000284555Sarybchik do { \ 1001284555Sarybchik EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 1002284555Sarybchik _NOTE(CONSTANTCONDITION) \ 1003284555Sarybchik } while (B_FALSE) 1004284555Sarybchik 1005227569Sphilip/* SPIN */ 1006227569Sphilip 1007227569Sphilip#define EFSYS_SPIN(_us) \ 1008227569Sphilip do { \ 1009227569Sphilip DELAY(_us); \ 1010227569Sphilip _NOTE(CONSTANTCONDITION) \ 1011227569Sphilip } while (B_FALSE) 1012227569Sphilip 1013227569Sphilip#define EFSYS_SLEEP EFSYS_SPIN 1014227569Sphilip 1015227569Sphilip/* BARRIERS */ 1016227569Sphilip 1017280521Sarybchik#define EFSYS_MEM_READ_BARRIER() rmb() 1018227569Sphilip#define EFSYS_PIO_WRITE_BARRIER() 1019227569Sphilip 1020284555Sarybchik/* DMA SYNC */ 1021284555Sarybchik#define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 1022284555Sarybchik do { \ 1023284555Sarybchik bus_dmamap_sync((_esmp)->esm_tag, \ 1024284555Sarybchik (_esmp)->esm_map, \ 1025284555Sarybchik BUS_DMASYNC_POSTREAD); \ 1026284555Sarybchik _NOTE(CONSTANTCONDITION) \ 1027284555Sarybchik } while (B_FALSE) 1028284555Sarybchik 1029284555Sarybchik#define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 1030284555Sarybchik do { \ 1031284555Sarybchik bus_dmamap_sync((_esmp)->esm_tag, \ 1032284555Sarybchik (_esmp)->esm_map, \ 1033284555Sarybchik BUS_DMASYNC_PREWRITE); \ 1034284555Sarybchik _NOTE(CONSTANTCONDITION) \ 1035284555Sarybchik } while (B_FALSE) 1036284555Sarybchik 1037227569Sphilip/* TIMESTAMP */ 1038227569Sphilip 1039227569Sphiliptypedef clock_t efsys_timestamp_t; 1040227569Sphilip 1041227569Sphilip#define EFSYS_TIMESTAMP(_usp) \ 1042227569Sphilip do { \ 1043227569Sphilip clock_t now; \ 1044227569Sphilip \ 1045227569Sphilip now = ticks; \ 1046227569Sphilip *(_usp) = now * hz / 1000000; \ 1047227569Sphilip _NOTE(CONSTANTCONDITION) \ 1048227569Sphilip } while (B_FALSE) 1049227569Sphilip 1050227569Sphilip/* KMEM */ 1051227569Sphilip 1052227569Sphilip#define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1053227569Sphilip do { \ 1054227569Sphilip (_esip) = (_esip); \ 1055280584Sarybchik /* \ 1056280584Sarybchik * The macro is used in non-sleepable contexts, for \ 1057280584Sarybchik * example, holding a mutex. \ 1058280584Sarybchik */ \ 1059280584Sarybchik (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1060227569Sphilip _NOTE(CONSTANTCONDITION) \ 1061227569Sphilip } while (B_FALSE) 1062227569Sphilip 1063227569Sphilip#define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1064227569Sphilip do { \ 1065227569Sphilip (void) (_esip); \ 1066227569Sphilip (void) (_size); \ 1067227569Sphilip free((_p), M_SFXGE); \ 1068227569Sphilip _NOTE(CONSTANTCONDITION) \ 1069227569Sphilip } while (B_FALSE) 1070227569Sphilip 1071227569Sphilip/* LOCK */ 1072227569Sphilip 1073280524Sarybchiktypedef struct efsys_lock_s { 1074280524Sarybchik struct mtx lock; 1075280524Sarybchik char lock_name[SFXGE_LOCK_NAME_MAX]; 1076280524Sarybchik} efsys_lock_t; 1077227569Sphilip 1078280524Sarybchik#define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 1079280524Sarybchik do { \ 1080280524Sarybchik efsys_lock_t *__eslp = (_eslp); \ 1081280524Sarybchik \ 1082280524Sarybchik snprintf((__eslp)->lock_name, \ 1083280524Sarybchik sizeof((__eslp)->lock_name), \ 1084280524Sarybchik "%s:%s", (_ifname), (_label)); \ 1085280524Sarybchik mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 1086280524Sarybchik NULL, MTX_DEF); \ 1087280524Sarybchik } while (B_FALSE) 1088280524Sarybchik#define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 1089280524Sarybchik mtx_destroy(&(_eslp)->lock) 1090280524Sarybchik#define SFXGE_EFSYS_LOCK(_eslp) \ 1091280524Sarybchik mtx_lock(&(_eslp)->lock) 1092280524Sarybchik#define SFXGE_EFSYS_UNLOCK(_eslp) \ 1093280524Sarybchik mtx_unlock(&(_eslp)->lock) 1094280524Sarybchik#define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 1095280524Sarybchik mtx_assert(&(_eslp)->lock, MA_OWNED) 1096280524Sarybchik 1097311095Sarybchiktypedef int efsys_lock_state_t; 1098311095Sarybchik 1099227569Sphilip#define EFSYS_LOCK_MAGIC 0x000010c4 1100227569Sphilip 1101227569Sphilip#define EFSYS_LOCK(_lockp, _state) \ 1102227569Sphilip do { \ 1103280524Sarybchik SFXGE_EFSYS_LOCK(_lockp); \ 1104227569Sphilip (_state) = EFSYS_LOCK_MAGIC; \ 1105227569Sphilip _NOTE(CONSTANTCONDITION) \ 1106227569Sphilip } while (B_FALSE) 1107227569Sphilip 1108227569Sphilip#define EFSYS_UNLOCK(_lockp, _state) \ 1109227569Sphilip do { \ 1110227569Sphilip if ((_state) != EFSYS_LOCK_MAGIC) \ 1111227569Sphilip KASSERT(B_FALSE, ("not locked")); \ 1112280524Sarybchik SFXGE_EFSYS_UNLOCK(_lockp); \ 1113227569Sphilip _NOTE(CONSTANTCONDITION) \ 1114227569Sphilip } while (B_FALSE) 1115227569Sphilip 1116227569Sphilip/* STAT */ 1117227569Sphilip 1118227569Sphiliptypedef uint64_t efsys_stat_t; 1119227569Sphilip 1120227569Sphilip#define EFSYS_STAT_INCR(_knp, _delta) \ 1121227569Sphilip do { \ 1122227569Sphilip *(_knp) += (_delta); \ 1123227569Sphilip _NOTE(CONSTANTCONDITION) \ 1124227569Sphilip } while (B_FALSE) 1125227569Sphilip 1126227569Sphilip#define EFSYS_STAT_DECR(_knp, _delta) \ 1127227569Sphilip do { \ 1128227569Sphilip *(_knp) -= (_delta); \ 1129227569Sphilip _NOTE(CONSTANTCONDITION) \ 1130227569Sphilip } while (B_FALSE) 1131227569Sphilip 1132227569Sphilip#define EFSYS_STAT_SET(_knp, _val) \ 1133227569Sphilip do { \ 1134227569Sphilip *(_knp) = (_val); \ 1135227569Sphilip _NOTE(CONSTANTCONDITION) \ 1136227569Sphilip } while (B_FALSE) 1137227569Sphilip 1138227569Sphilip#define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1139227569Sphilip do { \ 1140227569Sphilip *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1141227569Sphilip _NOTE(CONSTANTCONDITION) \ 1142227569Sphilip } while (B_FALSE) 1143227569Sphilip 1144227569Sphilip#define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1145227569Sphilip do { \ 1146227569Sphilip *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1147227569Sphilip _NOTE(CONSTANTCONDITION) \ 1148227569Sphilip } while (B_FALSE) 1149227569Sphilip 1150227569Sphilip#define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1151227569Sphilip do { \ 1152227569Sphilip *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1153227569Sphilip _NOTE(CONSTANTCONDITION) \ 1154227569Sphilip } while (B_FALSE) 1155227569Sphilip 1156227569Sphilip#define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1157227569Sphilip do { \ 1158227569Sphilip *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1159227569Sphilip _NOTE(CONSTANTCONDITION) \ 1160227569Sphilip } while (B_FALSE) 1161227569Sphilip 1162227569Sphilip/* ERR */ 1163227569Sphilip 1164227569Sphilipextern void sfxge_err(efsys_identifier_t *, unsigned int, 1165227569Sphilip uint32_t, uint32_t); 1166227569Sphilip 1167227569Sphilip#if EFSYS_OPT_DECODE_INTR_FATAL 1168227569Sphilip#define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1169227569Sphilip do { \ 1170227569Sphilip sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1171227569Sphilip _NOTE(CONSTANTCONDITION) \ 1172227569Sphilip } while (B_FALSE) 1173227569Sphilip#endif 1174227569Sphilip 1175227569Sphilip/* ASSERT */ 1176227569Sphilip 1177227569Sphilip#define EFSYS_ASSERT(_exp) do { \ 1178227569Sphilip if (!(_exp)) \ 1179284555Sarybchik panic("%s", #_exp); \ 1180227569Sphilip } while (0) 1181227569Sphilip 1182280501Sarybchik#define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1183227569Sphilip const _t __x = (_t)(_x); \ 1184227569Sphilip const _t __y = (_t)(_y); \ 1185227569Sphilip if (!(__x _op __y)) \ 1186280501Sarybchik panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1187227569Sphilip } while(0) 1188227569Sphilip 1189280501Sarybchik#define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1190280501Sarybchik#define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1191280501Sarybchik#define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1192227569Sphilip 1193284555Sarybchik/* ROTATE */ 1194284555Sarybchik 1195284555Sarybchik#define EFSYS_HAS_ROTL_DWORD 0 1196284555Sarybchik 1197227569Sphilip#ifdef __cplusplus 1198227569Sphilip} 1199227569Sphilip#endif 1200227569Sphilip 1201227569Sphilip#endif /* _SYS_EFSYS_H */ 1202