1/*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32#ifndef _SYS_EFSYS_H 33#define _SYS_EFSYS_H 34 35#ifdef __cplusplus 36extern "C" { 37#endif 38 39#include <sys/param.h> 40#include <sys/bus.h> 41#include <sys/endian.h> 42#include <sys/lock.h> 43#include <sys/malloc.h> 44#include <sys/mbuf.h> 45#include <sys/mutex.h> 46#include <sys/rwlock.h> 47#include <sys/sdt.h> 48#include <sys/systm.h> 49 50#include <machine/bus.h> 51#include <machine/endian.h> 52 53#define EFSYS_HAS_UINT64 1 54#define EFSYS_USE_UINT64 0 55#if _BYTE_ORDER == _BIG_ENDIAN 56#define EFSYS_IS_BIG_ENDIAN 1 57#define EFSYS_IS_LITTLE_ENDIAN 0 58#elif _BYTE_ORDER == _LITTLE_ENDIAN 59#define EFSYS_IS_BIG_ENDIAN 0 60#define EFSYS_IS_LITTLE_ENDIAN 1 61#endif 62#include "efx_types.h" 63 64/* Common code requires this */ 65#if __FreeBSD_version < 800068 66#define memmove(d, s, l) bcopy(s, d, l) 67#endif 68 69/* FreeBSD equivalents of Solaris things */ 70#ifndef _NOTE 71#define _NOTE(s) 72#endif 73 74#ifndef B_FALSE 75#define B_FALSE FALSE 76#endif 77#ifndef B_TRUE 78#define B_TRUE TRUE 79#endif 80 81#ifndef IS_P2ALIGNED 82#define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 83#endif 84 85#ifndef P2ROUNDUP 86#define P2ROUNDUP(x, align) (-(-(x) & -(align))) 87#endif 88 89#ifndef IS2P 90#define ISP2(x) (((x) & ((x) - 1)) == 0) 91#endif 92 93#define ENOTACTIVE EINVAL 94 95/* Memory type to use on FreeBSD */ 96MALLOC_DECLARE(M_SFXGE); 97 98/* Machine dependend prefetch wrappers */ 99#if defined(__i386__) || defined(__amd64__) 100static __inline void 101prefetch_read_many(void *addr) 102{ 103 104 __asm__( 105 "prefetcht0 (%0)" 106 : 107 : "r" (addr)); 108} 109 110static __inline void 111prefetch_read_once(void *addr) 112{ 113 114 __asm__( 115 "prefetchnta (%0)" 116 : 117 : "r" (addr)); 118} 119#elif defined(__sparc64__) 120static __inline void 121prefetch_read_many(void *addr) 122{ 123 124 __asm__( 125 "prefetch [%0], 0" 126 : 127 : "r" (addr)); 128} 129 130static __inline void 131prefetch_read_once(void *addr) 132{ 133 134 __asm__( 135 "prefetch [%0], 1" 136 : 137 : "r" (addr)); 138} 139#else 140static __inline void 141prefetch_read_many(void *addr) 142{ 143 144} 145 146static __inline void 147prefetch_read_once(void *addr) 148{ 149 150} 151#endif 152 153#if defined(__i386__) || defined(__amd64__) 154#include <vm/vm.h> 155#include <vm/pmap.h> 156#endif 157static __inline void 158sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 159 struct mbuf *m, bus_dma_segment_t *seg) 160{ 161#if defined(__i386__) || defined(__amd64__) 162 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 163 seg->ds_len = m->m_len; 164#else 165 int nsegstmp; 166 167 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 168#endif 169} 170 171/* Modifiers used for DOS builds */ 172#define __cs 173#define __far 174 175/* Modifiers used for Windows builds */ 176#define __in 177#define __in_opt 178#define __in_ecount(_n) 179#define __in_ecount_opt(_n) 180#define __in_bcount(_n) 181#define __in_bcount_opt(_n) 182 183#define __out 184#define __out_opt 185#define __out_ecount(_n) 186#define __out_ecount_opt(_n) 187#define __out_bcount(_n) 188#define __out_bcount_opt(_n) 189 190#define __deref_out 191 192#define __inout 193#define __inout_opt 194#define __inout_ecount(_n) 195#define __inout_ecount_opt(_n) 196#define __inout_bcount(_n) 197#define __inout_bcount_opt(_n) 198#define __inout_bcount_full_opt(_n) 199 200#define __deref_out_bcount_opt(n) 201 202#define __checkReturn 203 204#define __drv_when(_p, _c) 205 206/* Code inclusion options */ 207 208 209#define EFSYS_OPT_NAMES 1 210 211#define EFSYS_OPT_FALCON 0 212#define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 213#define EFSYS_OPT_SIENA 1 214#ifdef DEBUG 215#define EFSYS_OPT_CHECK_REG 1 216#else 217#define EFSYS_OPT_CHECK_REG 0 218#endif 219 220#define EFSYS_OPT_MCDI 1 221 222#define EFSYS_OPT_MAC_FALCON_GMAC 0 223#define EFSYS_OPT_MAC_FALCON_XMAC 0 224#define EFSYS_OPT_MAC_STATS 1 225 226#define EFSYS_OPT_LOOPBACK 0 227 228#define EFSYS_OPT_MON_NULL 0 229#define EFSYS_OPT_MON_LM87 0 230#define EFSYS_OPT_MON_MAX6647 0 231#define EFSYS_OPT_MON_SIENA 0 232#define EFSYS_OPT_MON_STATS 0 233 234#define EFSYS_OPT_PHY_NULL 0 235#define EFSYS_OPT_PHY_QT2022C2 0 236#define EFSYS_OPT_PHY_SFX7101 0 237#define EFSYS_OPT_PHY_TXC43128 0 238#define EFSYS_OPT_PHY_PM8358 0 239#define EFSYS_OPT_PHY_SFT9001 0 240#define EFSYS_OPT_PHY_QT2025C 0 241#define EFSYS_OPT_PHY_STATS 1 242#define EFSYS_OPT_PHY_PROPS 0 243#define EFSYS_OPT_PHY_BIST 1 244#define EFSYS_OPT_PHY_LED_CONTROL 1 245#define EFSYS_OPT_PHY_FLAGS 0 246 247#define EFSYS_OPT_VPD 1 248#define EFSYS_OPT_NVRAM 1 249#define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 250#define EFSYS_OPT_NVRAM_SFT9001 0 251#define EFSYS_OPT_NVRAM_SFX7101 0 252#define EFSYS_OPT_BOOTCFG 0 253 254#define EFSYS_OPT_PCIE_TUNE 0 255#define EFSYS_OPT_DIAG 0 256#define EFSYS_OPT_WOL 1 257#define EFSYS_OPT_RX_SCALE 1 258#define EFSYS_OPT_QSTATS 1 259#define EFSYS_OPT_FILTER 0 260#define EFSYS_OPT_RX_SCATTER 0 261#define EFSYS_OPT_RX_HDR_SPLIT 0 262 263#define EFSYS_OPT_EV_PREFETCH 0 264 265#define EFSYS_OPT_DECODE_INTR_FATAL 1 266 267/* ID */ 268 269typedef struct __efsys_identifier_s efsys_identifier_t; 270 271/* PROBE */ 272 273#ifndef KDTRACE_HOOKS 274 275#define EFSYS_PROBE(_name) 276 277#define EFSYS_PROBE1(_name, _type1, _arg1) 278 279#define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 280 281#define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 282 _type3, _arg3) 283 284#define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 285 _type3, _arg3, _type4, _arg4) 286 287#define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 288 _type3, _arg3, _type4, _arg4, _type5, _arg5) 289 290#define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 291 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 292 _type6, _arg6) 293 294#define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 295 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 296 _type6, _arg6, _type7, _arg7) 297 298#else /* KDTRACE_HOOKS */ 299 300#define EFSYS_PROBE(_name) \ 301 DTRACE_PROBE(_name) 302 303#define EFSYS_PROBE1(_name, _type1, _arg1) \ 304 DTRACE_PROBE1(_name, _type1, _arg1) 305 306#define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 307 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 308 309#define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 310 _type3, _arg3) \ 311 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 312 _type3, _arg3) 313 314#define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 315 _type3, _arg3, _type4, _arg4) \ 316 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 317 _type3, _arg3, _type4, _arg4) 318 319#ifdef DTRACE_PROBE5 320#define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 321 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 322 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 323 _type3, _arg3, _type4, _arg4, _type5, _arg5) 324#else 325#define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 326 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 327 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 328 _type3, _arg3, _type4, _arg4) 329#endif 330 331#ifdef DTRACE_PROBE6 332#define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 333 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 334 _type6, _arg6) \ 335 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 336 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 337 _type6, _arg6) 338#else 339#define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 340 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 341 _type6, _arg6) \ 342 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 343 _type3, _arg3, _type4, _arg4, _type5, _arg5) 344#endif 345 346#ifdef DTRACE_PROBE7 347#define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 348 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 349 _type6, _arg6, _type7, _arg7) \ 350 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 351 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 352 _type6, _arg6, _type7, _arg7) 353#else 354#define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 355 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 356 _type6, _arg6, _type7, _arg7) \ 357 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 358 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 359 _type6, _arg6) 360#endif 361 362#endif /* KDTRACE_HOOKS */ 363 364/* DMA */ 365 366typedef uint64_t efsys_dma_addr_t; 367 368typedef struct efsys_mem_s { 369 bus_dma_tag_t esm_tag; 370 bus_dmamap_t esm_map; 371 caddr_t esm_base; 372 efsys_dma_addr_t esm_addr; 373 size_t esm_size; 374} efsys_mem_t; 375 376 377#define EFSYS_MEM_ZERO(_esmp, _size) \ 378 do { \ 379 (void) memset((_esmp)->esm_base, 0, (_size)); \ 380 \ 381 _NOTE(CONSTANTCONDITION) \ 382 } while (B_FALSE) 383 384#define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 385 do { \ 386 uint32_t *addr; \ 387 \ 388 _NOTE(CONSTANTCONDITION) \ 389 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 390 ("not power of 2 aligned")); \ 391 \ 392 addr = (void *)((_esmp)->esm_base + (_offset)); \ 393 \ 394 (_edp)->ed_u32[0] = *addr; \ 395 \ 396 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 397 uint32_t, (_edp)->ed_u32[0]); \ 398 \ 399 _NOTE(CONSTANTCONDITION) \ 400 } while (B_FALSE) 401 402#define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 403 do { \ 404 uint32_t *addr; \ 405 \ 406 _NOTE(CONSTANTCONDITION) \ 407 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 408 ("not power of 2 aligned")); \ 409 \ 410 addr = (void *)((_esmp)->esm_base + (_offset)); \ 411 \ 412 (_eqp)->eq_u32[0] = *addr++; \ 413 (_eqp)->eq_u32[1] = *addr; \ 414 \ 415 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 416 uint32_t, (_eqp)->eq_u32[1], \ 417 uint32_t, (_eqp)->eq_u32[0]); \ 418 \ 419 _NOTE(CONSTANTCONDITION) \ 420 } while (B_FALSE) 421 422#define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 423 do { \ 424 uint32_t *addr; \ 425 \ 426 _NOTE(CONSTANTCONDITION) \ 427 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 428 ("not power of 2 aligned")); \ 429 \ 430 addr = (void *)((_esmp)->esm_base + (_offset)); \ 431 \ 432 (_eop)->eo_u32[0] = *addr++; \ 433 (_eop)->eo_u32[1] = *addr++; \ 434 (_eop)->eo_u32[2] = *addr++; \ 435 (_eop)->eo_u32[3] = *addr; \ 436 \ 437 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 438 uint32_t, (_eop)->eo_u32[3], \ 439 uint32_t, (_eop)->eo_u32[2], \ 440 uint32_t, (_eop)->eo_u32[1], \ 441 uint32_t, (_eop)->eo_u32[0]); \ 442 \ 443 _NOTE(CONSTANTCONDITION) \ 444 } while (B_FALSE) 445 446#define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 447 do { \ 448 uint32_t *addr; \ 449 \ 450 _NOTE(CONSTANTCONDITION) \ 451 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 452 ("not power of 2 aligned")); \ 453 \ 454 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 455 uint32_t, (_edp)->ed_u32[0]); \ 456 \ 457 addr = (void *)((_esmp)->esm_base + (_offset)); \ 458 \ 459 *addr = (_edp)->ed_u32[0]; \ 460 \ 461 _NOTE(CONSTANTCONDITION) \ 462 } while (B_FALSE) 463 464#define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 465 do { \ 466 uint32_t *addr; \ 467 \ 468 _NOTE(CONSTANTCONDITION) \ 469 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 470 ("not power of 2 aligned")); \ 471 \ 472 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 473 uint32_t, (_eqp)->eq_u32[1], \ 474 uint32_t, (_eqp)->eq_u32[0]); \ 475 \ 476 addr = (void *)((_esmp)->esm_base + (_offset)); \ 477 \ 478 *addr++ = (_eqp)->eq_u32[0]; \ 479 *addr = (_eqp)->eq_u32[1]; \ 480 \ 481 _NOTE(CONSTANTCONDITION) \ 482 } while (B_FALSE) 483 484#define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 485 do { \ 486 uint32_t *addr; \ 487 \ 488 _NOTE(CONSTANTCONDITION) \ 489 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 490 ("not power of 2 aligned")); \ 491 \ 492 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 493 uint32_t, (_eop)->eo_u32[3], \ 494 uint32_t, (_eop)->eo_u32[2], \ 495 uint32_t, (_eop)->eo_u32[1], \ 496 uint32_t, (_eop)->eo_u32[0]); \ 497 \ 498 addr = (void *)((_esmp)->esm_base + (_offset)); \ 499 \ 500 *addr++ = (_eop)->eo_u32[0]; \ 501 *addr++ = (_eop)->eo_u32[1]; \ 502 *addr++ = (_eop)->eo_u32[2]; \ 503 *addr = (_eop)->eo_u32[3]; \ 504 \ 505 _NOTE(CONSTANTCONDITION) \ 506 } while (B_FALSE) 507 508#define EFSYS_MEM_ADDR(_esmp) \ 509 ((_esmp)->esm_addr) 510 511/* BAR */ 512 513typedef struct efsys_bar_s { 514 struct mtx esb_lock; 515 bus_space_tag_t esb_tag; 516 bus_space_handle_t esb_handle; 517 int esb_rid; 518 struct resource *esb_res; 519} efsys_bar_t; 520 521#define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 522 do { \ 523 _NOTE(CONSTANTCONDITION) \ 524 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 525 ("not power of 2 aligned")); \ 526 \ 527 _NOTE(CONSTANTCONDITION) \ 528 if (_lock) \ 529 mtx_lock(&((_esbp)->esb_lock)); \ 530 \ 531 (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 532 (_esbp)->esb_handle, (_offset)); \ 533 \ 534 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 535 uint32_t, (_edp)->ed_u32[0]); \ 536 \ 537 _NOTE(CONSTANTCONDITION) \ 538 if (_lock) \ 539 mtx_unlock(&((_esbp)->esb_lock)); \ 540 _NOTE(CONSTANTCONDITION) \ 541 } while (B_FALSE) 542 543#define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 544 do { \ 545 _NOTE(CONSTANTCONDITION) \ 546 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 547 ("not power of 2 aligned")); \ 548 \ 549 mtx_lock(&((_esbp)->esb_lock)); \ 550 \ 551 (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 552 (_esbp)->esb_handle, (_offset)); \ 553 (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 554 (_esbp)->esb_handle, (_offset+4)); \ 555 \ 556 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 557 uint32_t, (_eqp)->eq_u32[1], \ 558 uint32_t, (_eqp)->eq_u32[0]); \ 559 \ 560 mtx_unlock(&((_esbp)->esb_lock)); \ 561 _NOTE(CONSTANTCONDITION) \ 562 } while (B_FALSE) 563 564#define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 565 do { \ 566 _NOTE(CONSTANTCONDITION) \ 567 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 568 ("not power of 2 aligned")); \ 569 \ 570 _NOTE(CONSTANTCONDITION) \ 571 if (_lock) \ 572 mtx_lock(&((_esbp)->esb_lock)); \ 573 \ 574 (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 575 (_esbp)->esb_handle, (_offset)); \ 576 (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 577 (_esbp)->esb_handle, (_offset+4)); \ 578 (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag, \ 579 (_esbp)->esb_handle, (_offset+8)); \ 580 (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag, \ 581 (_esbp)->esb_handle, (_offset+12)); \ 582 \ 583 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 584 uint32_t, (_eop)->eo_u32[3], \ 585 uint32_t, (_eop)->eo_u32[2], \ 586 uint32_t, (_eop)->eo_u32[1], \ 587 uint32_t, (_eop)->eo_u32[0]); \ 588 \ 589 _NOTE(CONSTANTCONDITION) \ 590 if (_lock) \ 591 mtx_unlock(&((_esbp)->esb_lock)); \ 592 _NOTE(CONSTANTCONDITION) \ 593 } while (B_FALSE) 594 595#define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 596 do { \ 597 _NOTE(CONSTANTCONDITION) \ 598 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 599 ("not power of 2 aligned")); \ 600 \ 601 _NOTE(CONSTANTCONDITION) \ 602 if (_lock) \ 603 mtx_lock(&((_esbp)->esb_lock)); \ 604 \ 605 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 606 uint32_t, (_edp)->ed_u32[0]); \ 607 \ 608 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 609 (_offset), (_edp)->ed_u32[0]); \ 610 \ 611 _NOTE(CONSTANTCONDITION) \ 612 if (_lock) \ 613 mtx_unlock(&((_esbp)->esb_lock)); \ 614 _NOTE(CONSTANTCONDITION) \ 615 } while (B_FALSE) 616 617#define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 618 do { \ 619 _NOTE(CONSTANTCONDITION) \ 620 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 621 ("not power of 2 aligned")); \ 622 \ 623 mtx_lock(&((_esbp)->esb_lock)); \ 624 \ 625 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 626 uint32_t, (_eqp)->eq_u32[1], \ 627 uint32_t, (_eqp)->eq_u32[0]); \ 628 \ 629 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 630 (_offset), (_eqp)->eq_u32[0]); \ 631 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 632 (_offset+4), (_eqp)->eq_u32[1]); \ 633 \ 634 mtx_unlock(&((_esbp)->esb_lock)); \ 635 _NOTE(CONSTANTCONDITION) \ 636 } while (B_FALSE) 637 638#define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 639 do { \ 640 _NOTE(CONSTANTCONDITION) \ 641 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 642 ("not power of 2 aligned")); \ 643 \ 644 _NOTE(CONSTANTCONDITION) \ 645 if (_lock) \ 646 mtx_lock(&((_esbp)->esb_lock)); \ 647 \ 648 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 649 uint32_t, (_eop)->eo_u32[3], \ 650 uint32_t, (_eop)->eo_u32[2], \ 651 uint32_t, (_eop)->eo_u32[1], \ 652 uint32_t, (_eop)->eo_u32[0]); \ 653 \ 654 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 655 (_offset), (_eop)->eo_u32[0]); \ 656 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 657 (_offset+4), (_eop)->eo_u32[1]); \ 658 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 659 (_offset+8), (_eop)->eo_u32[2]); \ 660 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 661 (_offset+12), (_eop)->eo_u32[3]); \ 662 \ 663 _NOTE(CONSTANTCONDITION) \ 664 if (_lock) \ 665 mtx_unlock(&((_esbp)->esb_lock)); \ 666 _NOTE(CONSTANTCONDITION) \ 667 } while (B_FALSE) 668 669/* SPIN */ 670 671#define EFSYS_SPIN(_us) \ 672 do { \ 673 DELAY(_us); \ 674 _NOTE(CONSTANTCONDITION) \ 675 } while (B_FALSE) 676 677#define EFSYS_SLEEP EFSYS_SPIN 678 679/* BARRIERS */ 680 681/* Strict ordering guaranteed by devacc.devacc_attr_dataorder */ 682#define EFSYS_MEM_READ_BARRIER() 683#define EFSYS_PIO_WRITE_BARRIER() 684 685/* TIMESTAMP */ 686 687typedef clock_t efsys_timestamp_t; 688 689#define EFSYS_TIMESTAMP(_usp) \ 690 do { \ 691 clock_t now; \ 692 \ 693 now = ticks; \ 694 *(_usp) = now * hz / 1000000; \ 695 _NOTE(CONSTANTCONDITION) \ 696 } while (B_FALSE) 697 698/* KMEM */ 699 700#define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 701 do { \ 702 (_esip) = (_esip); \ 703 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO); \ 704 _NOTE(CONSTANTCONDITION) \ 705 } while (B_FALSE) 706 707#define EFSYS_KMEM_FREE(_esip, _size, _p) \ 708 do { \ 709 (void) (_esip); \ 710 (void) (_size); \ 711 free((_p), M_SFXGE); \ 712 _NOTE(CONSTANTCONDITION) \ 713 } while (B_FALSE) 714 715/* LOCK */ 716 717typedef struct mtx efsys_lock_t; 718 719#define EFSYS_LOCK_MAGIC 0x000010c4 720 721#define EFSYS_LOCK(_lockp, _state) \ 722 do { \ 723 mtx_lock(_lockp); \ 724 (_state) = EFSYS_LOCK_MAGIC; \ 725 _NOTE(CONSTANTCONDITION) \ 726 } while (B_FALSE) 727 728#define EFSYS_UNLOCK(_lockp, _state) \ 729 do { \ 730 if ((_state) != EFSYS_LOCK_MAGIC) \ 731 KASSERT(B_FALSE, ("not locked")); \ 732 mtx_unlock(_lockp); \ 733 _NOTE(CONSTANTCONDITION) \ 734 } while (B_FALSE) 735 736/* PREEMPT */ 737 738#define EFSYS_PREEMPT_DISABLE(_state) \ 739 do { \ 740 (_state) = (_state); \ 741 critical_enter(); \ 742 _NOTE(CONSTANTCONDITION) \ 743 } while (B_FALSE) 744 745#define EFSYS_PREEMPT_ENABLE(_state) \ 746 do { \ 747 (_state) = (_state); \ 748 critical_exit(_state); \ 749 _NOTE(CONSTANTCONDITION) \ 750 } while (B_FALSE) 751 752/* STAT */ 753 754typedef uint64_t efsys_stat_t; 755 756#define EFSYS_STAT_INCR(_knp, _delta) \ 757 do { \ 758 *(_knp) += (_delta); \ 759 _NOTE(CONSTANTCONDITION) \ 760 } while (B_FALSE) 761 762#define EFSYS_STAT_DECR(_knp, _delta) \ 763 do { \ 764 *(_knp) -= (_delta); \ 765 _NOTE(CONSTANTCONDITION) \ 766 } while (B_FALSE) 767 768#define EFSYS_STAT_SET(_knp, _val) \ 769 do { \ 770 *(_knp) = (_val); \ 771 _NOTE(CONSTANTCONDITION) \ 772 } while (B_FALSE) 773 774#define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 775 do { \ 776 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 777 _NOTE(CONSTANTCONDITION) \ 778 } while (B_FALSE) 779 780#define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 781 do { \ 782 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 783 _NOTE(CONSTANTCONDITION) \ 784 } while (B_FALSE) 785 786#define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 787 do { \ 788 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 789 _NOTE(CONSTANTCONDITION) \ 790 } while (B_FALSE) 791 792#define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 793 do { \ 794 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 795 _NOTE(CONSTANTCONDITION) \ 796 } while (B_FALSE) 797 798/* ERR */ 799 800extern void sfxge_err(efsys_identifier_t *, unsigned int, 801 uint32_t, uint32_t); 802 803#if EFSYS_OPT_DECODE_INTR_FATAL 804#define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 805 do { \ 806 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 807 _NOTE(CONSTANTCONDITION) \ 808 } while (B_FALSE) 809#endif 810 811/* ASSERT */ 812 813#define EFSYS_ASSERT(_exp) do { \ 814 if (!(_exp)) \ 815 panic(#_exp); \ 816 } while (0) 817 818#define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 819 const _t __x = (_t)(_x); \ 820 const _t __y = (_t)(_y); \ 821 if (!(__x _op __y)) \ 822 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 823 } while(0) 824 825#define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 826#define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 827#define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 828 829#ifdef __cplusplus 830} 831#endif 832 833#endif /* _SYS_EFSYS_H */ 834