1/* 2 * Generic Broadcom Home Networking Division (HND) DMA module. 3 * This supports the following chips: BCM42xx, 44xx, 47xx . 4 * 5 * Copyright (C) 2015, Broadcom Corporation. All Rights Reserved. 6 * 7 * Permission to use, copy, modify, and/or distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 * 19 * $Id: hnddma.c 497985 2014-08-21 10:43:51Z $ 20 */ 21 22#include <bcm_cfg.h> 23#include <typedefs.h> 24#include <bcmdefs.h> 25#include <bcmdevs.h> 26#include <osl.h> 27#include <bcmendian.h> 28#include <hndsoc.h> 29#include <bcmutils.h> 30#include <siutils.h> 31 32#include <sbhnddma.h> 33#include <hnddma.h> 34 35/* debug/trace */ 36#ifdef BCMDBG 37#define DMA_ERROR(args) if (!(*di->msg_level & 1)); else printf args 38#define DMA_TRACE(args) if (!(*di->msg_level & 2)); else printf args 39#elif defined(BCMDBG_ERR) 40#define DMA_ERROR(args) if (!(*di->msg_level & 1)); else printf args 41#define DMA_TRACE(args) 42#else 43#define DMA_ERROR(args) 44#define DMA_TRACE(args) 45#endif /* BCMDBG */ 46 47#define DMA_NONE(args) 48 49 50#define d32txregs dregs.d32_u.txregs_32 51#define d32rxregs dregs.d32_u.rxregs_32 52#define txd32 dregs.d32_u.txd_32 53#define rxd32 dregs.d32_u.rxd_32 54 55#define d64txregs dregs.d64_u.txregs_64 56#define d64rxregs dregs.d64_u.rxregs_64 57#define txd64 dregs.d64_u.txd_64 58#define rxd64 dregs.d64_u.rxd_64 59 60/* default dma message level (if input msg_level pointer is null in dma_attach()) */ 61static uint dma_msg_level = 62#ifdef BCMDBG_ERR 63 1; 64#else 65 0; 66#endif /* BCMDBG_ERR */ 67 68#define MAXNAMEL 8 /* 8 char names */ 69 70#define DI_INFO(dmah) ((dma_info_t *)dmah) 71 72/** dma engine software state */ 73typedef struct dma_info { 74 struct hnddma_pub hnddma; /* exported structure, don't use hnddma_t, 75 * which could be const 76 */ 77 uint *msg_level; /* message level pointer */ 78 char name[MAXNAMEL]; /* callers name for diag msgs */ 79 80 void *osh; /* os handle */ 81 si_t *sih; /* sb handle */ 82 83 bool dma64; /* this dma engine is operating in 64-bit mode */ 84 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */ 85 86 union { 87 struct { 88 dma32regs_t *txregs_32; /* 32-bit dma tx engine registers */ 89 dma32regs_t *rxregs_32; /* 32-bit dma rx engine registers */ 90 dma32dd_t *txd_32; /* pointer to dma32 tx descriptor ring */ 91 dma32dd_t *rxd_32; /* pointer to dma32 rx descriptor ring */ 92 } d32_u; 93 struct { 94 dma64regs_t *txregs_64; /* 64-bit dma tx engine registers */ 95 dma64regs_t *rxregs_64; /* 64-bit dma rx engine registers */ 96 dma64dd_t *txd_64; /* pointer to dma64 tx descriptor ring */ 97 dma64dd_t *rxd_64; /* pointer to dma64 rx descriptor ring */ 98 } d64_u; 99 } dregs; 100 101 uint16 dmadesc_align; /* alignment requirement for dma descriptors */ 102 103 uint16 ntxd; /* # tx descriptors tunable */ 104 uint16 txin; /* index of next descriptor to reclaim */ 105 uint16 txout; /* index of next descriptor to post */ 106 void **txp; /* pointer to parallel array of pointers to packets */ 107 osldma_t *tx_dmah; /* DMA TX descriptor ring handle */ 108 hnddma_seg_map_t *txp_dmah; /* DMA MAP meta-data handle */ 109 dmaaddr_t txdpa; /* Aligned physical address of descriptor ring */ 110 dmaaddr_t txdpaorig; /* Original physical address of descriptor ring */ 111 uint16 txdalign; /* #bytes added to alloc'd mem to align txd */ 112 uint32 txdalloc; /* #bytes allocated for the ring */ 113 uint32 xmtptrbase; /* When using unaligned descriptors, the ptr register 114 * is not just an index, it needs all 13 bits to be 115 * an offset from the addr register. 116 */ 117 118 uint16 nrxd; /* # rx descriptors tunable */ 119 uint16 rxin; /* index of next descriptor to reclaim */ 120 uint16 rxout; /* index of next descriptor to post */ 121 void **rxp; /* pointer to parallel array of pointers to packets */ 122 osldma_t *rx_dmah; /* DMA RX descriptor ring handle */ 123 hnddma_seg_map_t *rxp_dmah; /* DMA MAP meta-data handle */ 124 dmaaddr_t rxdpa; /* Aligned physical address of descriptor ring */ 125 dmaaddr_t rxdpaorig; /* Original physical address of descriptor ring */ 126 uint16 rxdalign; /* #bytes added to alloc'd mem to align rxd */ 127 uint32 rxdalloc; /* #bytes allocated for the ring */ 128 uint32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */ 129 130 /* tunables */ 131 uint16 rxbufsize; /* rx buffer size in bytes, 132 * not including the extra headroom 133 */ 134 uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper stack 135 * e.g. some rx pkt buffers will be bridged to tx side 136 * without byte copying. The extra headroom needs to be 137 * large enough to fit txheader needs. 138 * Some dongle driver may not need it. 139 */ 140 uint nrxpost; /* # rx buffers to keep posted */ 141 uint rxoffset; /* rxcontrol offset */ 142 uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */ 143 uint ddoffsethigh; /* high 32 bits */ 144 uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */ 145 uint dataoffsethigh; /* high 32 bits */ 146 bool aligndesc_4k; /* descriptor base need to be aligned or not */ 147 uint8 rxburstlen; /* burstlen field for rx (for cores supporting burstlen) */ 148 uint8 txburstlen; /* burstlen field for tx (for cores supporting burstlen) */ 149 uint8 txmultioutstdrd; /* tx multiple outstanding reads */ 150 uint8 txprefetchctl; /* prefetch control for tx */ 151 uint8 txprefetchthresh; /* prefetch threshold for tx */ 152 uint8 rxprefetchctl; /* prefetch control for rx */ 153 uint8 rxprefetchthresh; /* prefetch threshold for rx */ 154 pktpool_t *pktpool; /* pktpool */ 155 uint dma_avoidance_cnt; 156 157 uint32 d64_xs0_cd_mask; /* tx current descriptor pointer mask */ 158 uint32 d64_xs1_ad_mask; /* tx active descriptor mask */ 159 uint32 d64_rs0_cd_mask; /* rx current descriptor pointer mask */ 160 uint16 rs0cd; /* cached value of rcvstatus0 currdescr */ 161 uint16 xs0cd; /* cached value of xmtstatus0 currdescr */ 162 uint16 xs0cd_snapshot; /* snapshot of xmtstatus0 currdescr */ 163#ifdef BCM_SECURE_DMA 164 struct sec_cma_info sec_cma_info_rx; 165 struct sec_cma_info sec_cma_info_tx; 166#endif 167} dma_info_t; 168 169/* 170 * If BCMDMA32 is defined, hnddma will support both 32-bit and 64-bit DMA engines. 171 * Otherwise it will support only 64-bit. 172 * 173 * DMA32_ENAB indicates whether hnddma is compiled with support for 32-bit DMA engines. 174 * DMA64_ENAB indicates whether hnddma is compiled with support for 64-bit DMA engines. 175 * 176 * DMA64_MODE indicates whether the current DMA engine is running as 64-bit. 177 */ 178#ifdef BCMDMA32 179#define DMA32_ENAB(di) 1 180#define DMA64_ENAB(di) 1 181#define DMA64_MODE(di) ((di)->dma64) 182#else /* !BCMDMA32 */ 183#define DMA32_ENAB(di) 0 184#define DMA64_ENAB(di) 1 185#define DMA64_MODE(di) 1 186#endif /* !BCMDMA32 */ 187 188/* DMA Scatter-gather list is supported. Note this is limited to TX direction only */ 189#ifdef BCMDMASGLISTOSL 190#define DMASGLIST_ENAB TRUE 191#else 192#define DMASGLIST_ENAB FALSE 193#endif /* BCMDMASGLISTOSL */ 194 195/* descriptor bumping macros */ 196#define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */ 197#define TXD(x) XXD((x), di->ntxd) 198#define RXD(x) XXD((x), di->nrxd) 199#define NEXTTXD(i) TXD((i) + 1) 200#define PREVTXD(i) TXD((i) - 1) 201#define NEXTRXD(i) RXD((i) + 1) 202#define PREVRXD(i) RXD((i) - 1) 203 204#define NTXDACTIVE(h, t) TXD((t) - (h)) 205#define NRXDACTIVE(h, t) RXD((t) - (h)) 206 207/* macros to convert between byte offsets and indexes */ 208#define B2I(bytes, type) ((uint16)((bytes) / sizeof(type))) 209#define I2B(index, type) ((index) * sizeof(type)) 210 211#define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */ 212#define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */ 213 214#define PCI64ADDR_HIGH 0x80000000 /* address[63] */ 215#define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */ 216 217/* Common prototypes */ 218static bool _dma_isaddrext(dma_info_t *di); 219static bool _dma_descriptor_align(dma_info_t *di); 220static bool _dma_alloc(dma_info_t *di, uint direction); 221static void _dma_detach(dma_info_t *di); 222static void _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa); 223static void _dma_rxinit(dma_info_t *di); 224static void *_dma_rx(dma_info_t *di); 225static bool _dma_rxfill(dma_info_t *di); 226static void _dma_rxreclaim(dma_info_t *di); 227static void _dma_rxenable(dma_info_t *di); 228static void *_dma_getnextrxp(dma_info_t *di, bool forceall); 229static void _dma_rx_param_get(dma_info_t *di, uint16 *rxoffset, uint16 *rxbufsize); 230 231static void _dma_txblock(dma_info_t *di); 232static void _dma_txunblock(dma_info_t *di); 233static uint _dma_txactive(dma_info_t *di); 234static uint _dma_rxactive(dma_info_t *di); 235static uint _dma_activerxbuf(dma_info_t *di); 236static uint _dma_txpending(dma_info_t *di); 237static uint _dma_txcommitted(dma_info_t *di); 238 239static void *_dma_peeknexttxp(dma_info_t *di); 240static int _dma_peekntxp(dma_info_t *di, int *len, void *txps[], txd_range_t range); 241static void *_dma_peeknextrxp(dma_info_t *di); 242static uintptr _dma_getvar(dma_info_t *di, const char *name); 243static void _dma_counterreset(dma_info_t *di); 244static void _dma_fifoloopbackenable(dma_info_t *di); 245static uint _dma_ctrlflags(dma_info_t *di, uint mask, uint flags); 246static uint8 dma_align_sizetobits(uint size); 247static void *dma_ringalloc(osl_t *osh, uint32 boundary, uint size, uint16 *alignbits, uint* alloced, 248 dmaaddr_t *descpa, osldma_t **dmah); 249static int _dma_pktpool_set(dma_info_t *di, pktpool_t *pool); 250static bool _dma_rxtx_error(dma_info_t *di, bool istx); 251static void _dma_burstlen_set(dma_info_t *di, uint8 rxburstlen, uint8 txburstlen); 252static uint _dma_avoidancecnt(dma_info_t *di); 253static void _dma_param_set(dma_info_t *di, uint16 paramid, uint16 paramval); 254static bool _dma_glom_enable(dma_info_t *di, uint32 val); 255 256 257/* Prototypes for 32-bit routines */ 258static bool dma32_alloc(dma_info_t *di, uint direction); 259static bool dma32_txreset(dma_info_t *di); 260static bool dma32_rxreset(dma_info_t *di); 261static bool dma32_txsuspendedidle(dma_info_t *di); 262static int dma32_txfast(dma_info_t *di, void *p0, bool commit); 263static void *dma32_getnexttxp(dma_info_t *di, txd_range_t range); 264static void *dma32_getnextrxp(dma_info_t *di, bool forceall); 265static void dma32_txrotate(dma_info_t *di); 266static bool dma32_rxidle(dma_info_t *di); 267static void dma32_txinit(dma_info_t *di); 268static bool dma32_txenabled(dma_info_t *di); 269static void dma32_txsuspend(dma_info_t *di); 270static void dma32_txresume(dma_info_t *di); 271static bool dma32_txsuspended(dma_info_t *di); 272static void dma32_txflush(dma_info_t *di); 273static void dma32_txflush_clear(dma_info_t *di); 274static void dma32_txreclaim(dma_info_t *di, txd_range_t range); 275static bool dma32_txstopped(dma_info_t *di); 276static bool dma32_rxstopped(dma_info_t *di); 277static bool dma32_rxenabled(dma_info_t *di); 278#if defined(BCMDBG) 279static void dma32_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma32dd_t *ring, uint start, 280 uint end, uint max_num); 281static void dma32_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 282static void dma32_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 283static void dma32_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 284#endif 285 286static bool _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs); 287 288/* Prototypes for 64-bit routines */ 289static bool dma64_alloc(dma_info_t *di, uint direction); 290static bool dma64_txreset(dma_info_t *di); 291static bool dma64_rxreset(dma_info_t *di); 292static bool dma64_txsuspendedidle(dma_info_t *di); 293static int dma64_txfast(dma_info_t *di, void *p0, bool commit); 294static int dma64_txunframed(dma_info_t *di, void *p0, uint len, bool commit); 295static void *dma64_getpos(dma_info_t *di, bool direction); 296static void *dma64_getnexttxp(dma_info_t *di, txd_range_t range); 297static void *dma64_getnextrxp(dma_info_t *di, bool forceall); 298static void dma64_txrotate(dma_info_t *di); 299 300static bool dma64_rxidle(dma_info_t *di); 301static void dma64_txinit(dma_info_t *di); 302static bool dma64_txenabled(dma_info_t *di); 303static void dma64_txsuspend(dma_info_t *di); 304static void dma64_txresume(dma_info_t *di); 305static bool dma64_txsuspended(dma_info_t *di); 306static void dma64_txflush(dma_info_t *di); 307static void dma64_txflush_clear(dma_info_t *di); 308static void dma64_txreclaim(dma_info_t *di, txd_range_t range); 309static bool dma64_txstopped(dma_info_t *di); 310static bool dma64_rxstopped(dma_info_t *di); 311static bool dma64_rxenabled(dma_info_t *di); 312static bool _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs); 313 314 315STATIC INLINE uint32 parity32(uint32 data); 316 317#if defined(BCMDBG) 318static void dma64_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma64dd_t *ring, uint start, 319 uint end, uint max_num); 320static void dma64_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 321static void dma64_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 322static void dma64_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 323#endif 324 325 326const di_fcn_t dma64proc = { 327 (di_detach_t)_dma_detach, 328 (di_txinit_t)dma64_txinit, 329 (di_txreset_t)dma64_txreset, 330 (di_txenabled_t)dma64_txenabled, 331 (di_txsuspend_t)dma64_txsuspend, 332 (di_txresume_t)dma64_txresume, 333 (di_txsuspended_t)dma64_txsuspended, 334 (di_txsuspendedidle_t)dma64_txsuspendedidle, 335 (di_txflush_t)dma64_txflush, 336 (di_txflush_clear_t)dma64_txflush_clear, 337 (di_txfast_t)dma64_txfast, 338 (di_txunframed_t)dma64_txunframed, 339 (di_getpos_t)dma64_getpos, 340 (di_txstopped_t)dma64_txstopped, 341 (di_txreclaim_t)dma64_txreclaim, 342 (di_getnexttxp_t)dma64_getnexttxp, 343 (di_peeknexttxp_t)_dma_peeknexttxp, 344 (di_peekntxp_t)_dma_peekntxp, 345 (di_txblock_t)_dma_txblock, 346 (di_txunblock_t)_dma_txunblock, 347 (di_txactive_t)_dma_txactive, 348 (di_txrotate_t)dma64_txrotate, 349 350 (di_rxinit_t)_dma_rxinit, 351 (di_rxreset_t)dma64_rxreset, 352 (di_rxidle_t)dma64_rxidle, 353 (di_rxstopped_t)dma64_rxstopped, 354 (di_rxenable_t)_dma_rxenable, 355 (di_rxenabled_t)dma64_rxenabled, 356 (di_rx_t)_dma_rx, 357 (di_rxfill_t)_dma_rxfill, 358 (di_rxreclaim_t)_dma_rxreclaim, 359 (di_getnextrxp_t)_dma_getnextrxp, 360 (di_peeknextrxp_t)_dma_peeknextrxp, 361 (di_rxparam_get_t)_dma_rx_param_get, 362 363 (di_fifoloopbackenable_t)_dma_fifoloopbackenable, 364 (di_getvar_t)_dma_getvar, 365 (di_counterreset_t)_dma_counterreset, 366 (di_ctrlflags_t)_dma_ctrlflags, 367 368#if defined(BCMDBG) 369 (di_dump_t)dma64_dump, 370 (di_dumptx_t)dma64_dumptx, 371 (di_dumprx_t)dma64_dumprx, 372#else 373 NULL, 374 NULL, 375 NULL, 376#endif 377 (di_rxactive_t)_dma_rxactive, 378 (di_txpending_t)_dma_txpending, 379 (di_txcommitted_t)_dma_txcommitted, 380 (di_pktpool_set_t)_dma_pktpool_set, 381 (di_rxtxerror_t)_dma_rxtx_error, 382 (di_burstlen_set_t)_dma_burstlen_set, 383 (di_avoidancecnt_t)_dma_avoidancecnt, 384 (di_param_set_t)_dma_param_set, 385 (dma_glom_enable_t)_dma_glom_enable, 386 (dma_active_rxbuf_t)_dma_activerxbuf, 387 40 388}; 389 390static const di_fcn_t dma32proc = { 391 (di_detach_t)_dma_detach, 392 (di_txinit_t)dma32_txinit, 393 (di_txreset_t)dma32_txreset, 394 (di_txenabled_t)dma32_txenabled, 395 (di_txsuspend_t)dma32_txsuspend, 396 (di_txresume_t)dma32_txresume, 397 (di_txsuspended_t)dma32_txsuspended, 398 (di_txsuspendedidle_t)dma32_txsuspendedidle, 399 (di_txflush_t)dma32_txflush, 400 (di_txflush_clear_t)dma32_txflush_clear, 401 (di_txfast_t)dma32_txfast, 402 NULL, 403 NULL, 404 (di_txstopped_t)dma32_txstopped, 405 (di_txreclaim_t)dma32_txreclaim, 406 (di_getnexttxp_t)dma32_getnexttxp, 407 (di_peeknexttxp_t)_dma_peeknexttxp, 408 (di_peekntxp_t)_dma_peekntxp, 409 (di_txblock_t)_dma_txblock, 410 (di_txunblock_t)_dma_txunblock, 411 (di_txactive_t)_dma_txactive, 412 (di_txrotate_t)dma32_txrotate, 413 414 (di_rxinit_t)_dma_rxinit, 415 (di_rxreset_t)dma32_rxreset, 416 (di_rxidle_t)dma32_rxidle, 417 (di_rxstopped_t)dma32_rxstopped, 418 (di_rxenable_t)_dma_rxenable, 419 (di_rxenabled_t)dma32_rxenabled, 420 (di_rx_t)_dma_rx, 421 (di_rxfill_t)_dma_rxfill, 422 (di_rxreclaim_t)_dma_rxreclaim, 423 (di_getnextrxp_t)_dma_getnextrxp, 424 (di_peeknextrxp_t)_dma_peeknextrxp, 425 (di_rxparam_get_t)_dma_rx_param_get, 426 427 (di_fifoloopbackenable_t)_dma_fifoloopbackenable, 428 (di_getvar_t)_dma_getvar, 429 (di_counterreset_t)_dma_counterreset, 430 (di_ctrlflags_t)_dma_ctrlflags, 431 432#if defined(BCMDBG) 433 (di_dump_t)dma32_dump, 434 (di_dumptx_t)dma32_dumptx, 435 (di_dumprx_t)dma32_dumprx, 436#else 437 NULL, 438 NULL, 439 NULL, 440#endif 441 (di_rxactive_t)_dma_rxactive, 442 (di_txpending_t)_dma_txpending, 443 (di_txcommitted_t)_dma_txcommitted, 444 (di_pktpool_set_t)_dma_pktpool_set, 445 (di_rxtxerror_t)_dma_rxtx_error, 446 (di_burstlen_set_t)_dma_burstlen_set, 447 (di_avoidancecnt_t)_dma_avoidancecnt, 448 (di_param_set_t)_dma_param_set, 449 NULL, 450 NULL, 451 40 452}; 453 454hnddma_t * 455dma_attach(osl_t *osh, const char *name, si_t *sih, 456 volatile void *dmaregstx, volatile void *dmaregsrx, 457 uint ntxd, uint nrxd, uint rxbufsize, int rxextheadroom, uint nrxpost, uint rxoffset, 458 uint *msg_level) 459{ 460 dma_info_t *di; 461 uint size; 462 uint32 mask; 463 464 /* allocate private info structure */ 465 if ((di = MALLOC(osh, sizeof (dma_info_t))) == NULL) { 466#ifdef BCMDBG 467 DMA_ERROR(("%s: out of memory, malloced %d bytes\n", __FUNCTION__, MALLOCED(osh))); 468#endif 469 return (NULL); 470 } 471 472 bzero(di, sizeof(dma_info_t)); 473 474 di->msg_level = msg_level ? msg_level : &dma_msg_level; 475 476 /* old chips w/o sb is no longer supported */ 477 ASSERT(sih != NULL); 478 479 if (DMA64_ENAB(di)) 480 di->dma64 = ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64); 481 else 482 di->dma64 = 0; 483 484 /* check arguments */ 485 ASSERT(ISPOWEROF2(ntxd)); 486 ASSERT(ISPOWEROF2(nrxd)); 487 488 if (nrxd == 0) 489 ASSERT(dmaregsrx == NULL); 490 if (ntxd == 0) 491 ASSERT(dmaregstx == NULL); 492 493 /* init dma reg pointer */ 494 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 495 di->d64txregs = (dma64regs_t *)dmaregstx; 496 di->d64rxregs = (dma64regs_t *)dmaregsrx; 497 di->hnddma.di_fn = (const di_fcn_t *)&dma64proc; 498 } else if (DMA32_ENAB(di)) { 499 ASSERT(ntxd <= D32MAXDD); 500 ASSERT(nrxd <= D32MAXDD); 501 di->d32txregs = (dma32regs_t *)dmaregstx; 502 di->d32rxregs = (dma32regs_t *)dmaregsrx; 503 di->hnddma.di_fn = (const di_fcn_t *)&dma32proc; 504 } else { 505 DMA_ERROR(("%s: driver doesn't support 32-bit DMA\n", __FUNCTION__)); 506 ASSERT(0); 507 goto fail; 508 } 509 510 /* Default flags (which can be changed by the driver calling dma_ctrlflags 511 * before enable): For backwards compatibility both Rx Overflow Continue 512 * and Parity are DISABLED. 513 * supports it. 514 */ 515 di->hnddma.di_fn->ctrlflags(&di->hnddma, DMA_CTRL_ROC | DMA_CTRL_PEN, 0); 516 517 DMA_TRACE(("%s: %s: %s osh %p flags 0x%x ntxd %d nrxd %d rxbufsize %d " 518 "rxextheadroom %d nrxpost %d rxoffset %d dmaregstx %p dmaregsrx %p\n", 519 name, __FUNCTION__, (DMA64_MODE(di) ? "DMA64" : "DMA32"), 520 osh, di->hnddma.dmactrlflags, ntxd, nrxd, 521 rxbufsize, rxextheadroom, nrxpost, rxoffset, dmaregstx, dmaregsrx)); 522 523 /* make a private copy of our callers name */ 524 strncpy(di->name, name, MAXNAMEL); 525 di->name[MAXNAMEL-1] = '\0'; 526 527 di->osh = osh; 528 di->sih = sih; 529 530 /* save tunables */ 531 di->ntxd = (uint16)ntxd; 532 di->nrxd = (uint16)nrxd; 533 534 /* the actual dma size doesn't include the extra headroom */ 535 di->rxextrahdrroom = (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom; 536 if (rxbufsize > BCMEXTRAHDROOM) 537 di->rxbufsize = (uint16)(rxbufsize - di->rxextrahdrroom); 538 else 539 di->rxbufsize = (uint16)rxbufsize; 540 541 di->nrxpost = (uint16)nrxpost; 542 di->rxoffset = (uint8)rxoffset; 543 544 /* Get the default values (POR) of the burstlen. This can be overridden by the modules 545 * if this has to be different. Otherwise this value will be used to program the control 546 * register after the reset or during the init. 547 */ 548 if (dmaregsrx) { 549 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 550 /* detect the dma descriptor address mask, 551 * should be 0x1fff before 4360B0, 0xffff start from 4360B0 552 */ 553 W_REG(di->osh, &di->d64rxregs->addrlow, 0xffffffff); 554 mask = R_REG(di->osh, &di->d64rxregs->addrlow); 555 556 if (mask & 0xfff) 557 mask = R_REG(di->osh, &di->d64rxregs->ptr) | 0xf; 558 else 559 mask = 0x1fff; 560 561 DMA_TRACE(("%s: dma_rx_mask: %08x\n", di->name, mask)); 562 di->d64_rs0_cd_mask = mask; 563 564 if (mask == 0x1fff) 565 ASSERT(nrxd <= D64MAXDD); 566 else 567 ASSERT(nrxd <= D64MAXDD_LARGE); 568 569 di->rxburstlen = (R_REG(di->osh, 570 &di->d64rxregs->control) & D64_RC_BL_MASK) >> D64_RC_BL_SHIFT; 571 di->rxprefetchctl = (R_REG(di->osh, 572 &di->d64rxregs->control) & D64_RC_PC_MASK) >> 573 D64_RC_PC_SHIFT; 574 di->rxprefetchthresh = (R_REG(di->osh, 575 &di->d64rxregs->control) & D64_RC_PT_MASK) >> 576 D64_RC_PT_SHIFT; 577 } else if (DMA32_ENAB(di)) { 578 di->rxburstlen = (R_REG(di->osh, 579 &di->d32rxregs->control) & RC_BL_MASK) >> RC_BL_SHIFT; 580 di->rxprefetchctl = (R_REG(di->osh, 581 &di->d32rxregs->control) & RC_PC_MASK) >> RC_PC_SHIFT; 582 di->rxprefetchthresh = (R_REG(di->osh, 583 &di->d32rxregs->control) & RC_PT_MASK) >> RC_PT_SHIFT; 584 } 585 } 586 if (dmaregstx) { 587 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 588 589 /* detect the dma descriptor address mask, 590 * should be 0x1fff before 4360B0, 0xffff start from 4360B0 591 */ 592 W_REG(di->osh, &di->d64txregs->addrlow, 0xffffffff); 593 mask = R_REG(di->osh, &di->d64txregs->addrlow); 594 595 if (mask & 0xfff) 596 mask = R_REG(di->osh, &di->d64txregs->ptr) | 0xf; 597 else 598 mask = 0x1fff; 599 600 DMA_TRACE(("%s: dma_tx_mask: %08x\n", di->name, mask)); 601 di->d64_xs0_cd_mask = mask; 602 di->d64_xs1_ad_mask = mask; 603 604 if (mask == 0x1fff) 605 ASSERT(ntxd <= D64MAXDD); 606 else 607 ASSERT(ntxd <= D64MAXDD_LARGE); 608 609 di->txburstlen = (R_REG(di->osh, 610 &di->d64txregs->control) & D64_XC_BL_MASK) >> D64_XC_BL_SHIFT; 611 di->txmultioutstdrd = (R_REG(di->osh, 612 &di->d64txregs->control) & D64_XC_MR_MASK) >> D64_XC_MR_SHIFT; 613 di->txprefetchctl = (R_REG(di->osh, 614 &di->d64txregs->control) & D64_XC_PC_MASK) >> D64_XC_PC_SHIFT; 615 di->txprefetchthresh = (R_REG(di->osh, 616 &di->d64txregs->control) & D64_XC_PT_MASK) >> D64_XC_PT_SHIFT; 617 } else if (DMA32_ENAB(di)) { 618 di->txburstlen = (R_REG(di->osh, 619 &di->d32txregs->control) & XC_BL_MASK) >> XC_BL_SHIFT; 620 di->txmultioutstdrd = (R_REG(di->osh, 621 &di->d32txregs->control) & XC_MR_MASK) >> XC_MR_SHIFT; 622 di->txprefetchctl = (R_REG(di->osh, 623 &di->d32txregs->control) & XC_PC_MASK) >> XC_PC_SHIFT; 624 di->txprefetchthresh = (R_REG(di->osh, 625 &di->d32txregs->control) & XC_PT_MASK) >> XC_PT_SHIFT; 626 } 627 } 628 629 /* 630 * figure out the DMA physical address offset for dd and data 631 * PCI/PCIE: they map silicon backplace address to zero based memory, need offset 632 * Other bus: use zero 633 * SI_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor 634 */ 635 di->ddoffsetlow = 0; 636 di->dataoffsetlow = 0; 637 /* for pci bus, add offset */ 638 if (sih->bustype == PCI_BUS) { 639 if ((sih->buscoretype == PCIE_CORE_ID || 640 sih->buscoretype == PCIE2_CORE_ID) && 641 DMA64_MODE(di)) { 642 /* pcie with DMA64 */ 643 di->ddoffsetlow = 0; 644 di->ddoffsethigh = SI_PCIE_DMA_H32; 645 } else { 646 /* pci(DMA32/DMA64) or pcie with DMA32 */ 647 if ((CHIPID(sih->chip) == BCM4322_CHIP_ID) || 648 (CHIPID(sih->chip) == BCM4342_CHIP_ID) || 649 (CHIPID(sih->chip) == BCM43221_CHIP_ID) || 650 (CHIPID(sih->chip) == BCM43231_CHIP_ID) || 651 (CHIPID(sih->chip) == BCM43111_CHIP_ID) || 652 (CHIPID(sih->chip) == BCM43112_CHIP_ID) || 653 (CHIPID(sih->chip) == BCM43222_CHIP_ID)) 654 di->ddoffsetlow = SI_PCI_DMA2; 655 else 656 di->ddoffsetlow = SI_PCI_DMA; 657 658 di->ddoffsethigh = 0; 659 } 660 di->dataoffsetlow = di->ddoffsetlow; 661 di->dataoffsethigh = di->ddoffsethigh; 662 } 663 664#if defined(__mips__) && defined(IL_BIGENDIAN) 665 di->dataoffsetlow = di->dataoffsetlow + SI_SDRAM_SWAPPED; 666#endif /* defined(__mips__) && defined(IL_BIGENDIAN) */ 667 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */ 668 if ((si_coreid(sih) == SDIOD_CORE_ID) && ((si_corerev(sih) > 0) && (si_corerev(sih) <= 2))) 669 di->addrext = 0; 670 else if ((si_coreid(sih) == I2S_CORE_ID) && 671 ((si_corerev(sih) == 0) || (si_corerev(sih) == 1))) 672 di->addrext = 0; 673 else 674 di->addrext = _dma_isaddrext(di); 675 676 /* does the descriptors need to be aligned and if yes, on 4K/8K or not */ 677 di->aligndesc_4k = _dma_descriptor_align(di); 678 if (di->aligndesc_4k) { 679 if (DMA64_MODE(di)) { 680 di->dmadesc_align = D64RINGALIGN_BITS; 681 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) { 682 /* for smaller dd table, HW relax the alignment requirement */ 683 di->dmadesc_align = D64RINGALIGN_BITS - 1; 684 } 685 } else 686 di->dmadesc_align = D32RINGALIGN_BITS; 687 } else { 688 /* The start address of descriptor table should be algined to cache line size, 689 * or other structure may share a cache line with it, which can lead to memory 690 * overlapping due to cache write-back operation. In the case of MIPS 74k, the 691 * cache line size is 32 bytes. 692 */ 693#ifdef __mips__ 694 di->dmadesc_align = 5; /* 32 byte alignment */ 695#else 696 di->dmadesc_align = 4; /* 16 byte alignment */ 697#endif 698 } 699 700 DMA_NONE(("DMA descriptor align_needed %d, align %d\n", 701 di->aligndesc_4k, di->dmadesc_align)); 702 703 /* allocate tx packet pointer vector */ 704 if (ntxd) { 705 size = ntxd * sizeof(void *); 706 if ((di->txp = MALLOC(osh, size)) == NULL) { 707 DMA_ERROR(("%s: %s: out of tx memory, malloced %d bytes\n", 708 di->name, __FUNCTION__, MALLOCED(osh))); 709 goto fail; 710 } 711 bzero(di->txp, size); 712 } 713 714 /* allocate rx packet pointer vector */ 715 if (nrxd) { 716 size = nrxd * sizeof(void *); 717 if ((di->rxp = MALLOC(osh, size)) == NULL) { 718 DMA_ERROR(("%s: %s: out of rx memory, malloced %d bytes\n", 719 di->name, __FUNCTION__, MALLOCED(osh))); 720 goto fail; 721 } 722 bzero(di->rxp, size); 723 } 724 725 /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */ 726 if (ntxd) { 727 if (!_dma_alloc(di, DMA_TX)) 728 goto fail; 729 } 730 731 /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */ 732 if (nrxd) { 733 if (!_dma_alloc(di, DMA_RX)) 734 goto fail; 735 } 736 737 if ((di->ddoffsetlow != 0) && !di->addrext) { 738 if (PHYSADDRLO(di->txdpa) > SI_PCI_DMA_SZ) { 739 DMA_ERROR(("%s: %s: txdpa 0x%x: addrext not supported\n", 740 di->name, __FUNCTION__, (uint32)PHYSADDRLO(di->txdpa))); 741 goto fail; 742 } 743 if (PHYSADDRLO(di->rxdpa) > SI_PCI_DMA_SZ) { 744 DMA_ERROR(("%s: %s: rxdpa 0x%x: addrext not supported\n", 745 di->name, __FUNCTION__, (uint32)PHYSADDRLO(di->rxdpa))); 746 goto fail; 747 } 748 } 749 750 DMA_TRACE(("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh " 751 "0x%x addrext %d\n", di->ddoffsetlow, di->ddoffsethigh, di->dataoffsetlow, 752 di->dataoffsethigh, di->addrext)); 753 754 /* allocate DMA mapping vectors */ 755 if (DMASGLIST_ENAB) { 756 if (ntxd) { 757 size = ntxd * sizeof(hnddma_seg_map_t); 758 if ((di->txp_dmah = (hnddma_seg_map_t *)MALLOC(osh, size)) == NULL) 759 goto fail; 760 bzero(di->txp_dmah, size); 761 } 762 763 if (nrxd) { 764 size = nrxd * sizeof(hnddma_seg_map_t); 765 if ((di->rxp_dmah = (hnddma_seg_map_t *)MALLOC(osh, size)) == NULL) 766 goto fail; 767 bzero(di->rxp_dmah, size); 768 } 769 } 770 771 return ((hnddma_t *)di); 772 773fail: 774 _dma_detach(di); 775 return (NULL); 776} 777 778/** init the tx or rx descriptor */ 779static INLINE void 780dma32_dd_upd(dma_info_t *di, dma32dd_t *ddring, dmaaddr_t pa, uint outidx, uint32 *flags, 781 uint32 bufcount) 782{ 783 /* dma32 uses 32-bit control to fit both flags and bufcounter */ 784 *flags = *flags | (bufcount & CTRL_BC_MASK); 785 786 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 787 W_SM(&ddring[outidx].addr, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 788 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags)); 789 } else { 790 /* address extension */ 791 uint32 ae; 792 ASSERT(di->addrext); 793 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 794 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 795 796 *flags |= (ae << CTRL_AE_SHIFT); 797 W_SM(&ddring[outidx].addr, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 798 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags)); 799 } 800} 801 802/** Check for odd number of 1's */ 803STATIC INLINE uint32 parity32(uint32 data) 804{ 805 data ^= data >> 16; 806 data ^= data >> 8; 807 data ^= data >> 4; 808 data ^= data >> 2; 809 data ^= data >> 1; 810 811 return (data & 1); 812} 813 814#define DMA64_DD_PARITY(dd) parity32((dd)->addrlow ^ (dd)->addrhigh ^ (dd)->ctrl1 ^ (dd)->ctrl2) 815 816static INLINE void 817dma64_dd_upd(dma_info_t *di, dma64dd_t *ddring, dmaaddr_t pa, uint outidx, uint32 *flags, 818 uint32 bufcount) 819{ 820 uint32 ctrl2 = bufcount & D64_CTRL2_BC_MASK; 821 822 /* PCI bus with big(>1G) physical address, use address extension */ 823#if defined(__mips__) && defined(IL_BIGENDIAN) 824 if ((di->dataoffsetlow == SI_SDRAM_SWAPPED) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 825#else 826 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 827#endif /* defined(__mips__) && defined(IL_BIGENDIAN) */ 828 ASSERT((PHYSADDRHI(pa) & PCI64ADDR_HIGH) == 0); 829 830 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 831 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(PHYSADDRHI(pa) + di->dataoffsethigh)); 832 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags)); 833 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2)); 834 } else { 835 /* address extension for 32-bit PCI */ 836 uint32 ae; 837 ASSERT(di->addrext); 838 839 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 840 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 841 ASSERT(PHYSADDRHI(pa) == 0); 842 843 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE; 844 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 845 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(0 + di->dataoffsethigh)); 846 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags)); 847 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2)); 848 } 849 if (di->hnddma.dmactrlflags & DMA_CTRL_PEN) { 850 if (DMA64_DD_PARITY(&ddring[outidx])) { 851 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2 | D64_CTRL2_PARITY)); 852 } 853 } 854 855#if defined(BCM47XX_CA9) && !defined(__NetBSD__) 856#ifndef BCM_SECURE_DMA 857 DMA_MAP(di->osh, (void *)(((uint)(&ddring[outidx])) & ~0x1f), 32, DMA_TX, NULL, NULL); 858#else 859 SECURE_DMA_DD_MAP(di->osh, (void *)(((uint)(&ddring[outidx])) & ~0x1f), 860 32, DMA_TX, NULL, NULL); 861#endif 862#endif /* BCM47XX_CA9 && !__NetBSD__ */ 863} 864 865static bool 866_dma32_addrext(osl_t *osh, dma32regs_t *dma32regs) 867{ 868 uint32 w; 869 870 OR_REG(osh, &dma32regs->control, XC_AE); 871 w = R_REG(osh, &dma32regs->control); 872 AND_REG(osh, &dma32regs->control, ~XC_AE); 873 return ((w & XC_AE) == XC_AE); 874} 875 876static bool 877_dma_alloc(dma_info_t *di, uint direction) 878{ 879 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 880 return dma64_alloc(di, direction); 881 } else if (DMA32_ENAB(di)) { 882 return dma32_alloc(di, direction); 883 } else 884 ASSERT(0); 885} 886 887/** !! may be called with core in reset */ 888static void 889_dma_detach(dma_info_t *di) 890{ 891 892 DMA_TRACE(("%s: dma_detach\n", di->name)); 893 894 /* shouldn't be here if descriptors are unreclaimed */ 895 ASSERT(di->txin == di->txout); 896 ASSERT(di->rxin == di->rxout); 897 898 /* free dma descriptor rings */ 899 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 900 if (di->txd64) 901 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->txd64 - di->txdalign), 902 di->txdalloc, (di->txdpaorig), di->tx_dmah); 903 if (di->rxd64) 904 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->rxd64 - di->rxdalign), 905 di->rxdalloc, (di->rxdpaorig), di->rx_dmah); 906 } else if (DMA32_ENAB(di)) { 907 if (di->txd32) 908 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->txd32 - di->txdalign), 909 di->txdalloc, (di->txdpaorig), di->tx_dmah); 910 if (di->rxd32) 911 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->rxd32 - di->rxdalign), 912 di->rxdalloc, (di->rxdpaorig), di->rx_dmah); 913 } else 914 ASSERT(0); 915 916 /* free packet pointer vectors */ 917 if (di->txp) 918 MFREE(di->osh, (void *)di->txp, (di->ntxd * sizeof(void *))); 919 if (di->rxp) 920 MFREE(di->osh, (void *)di->rxp, (di->nrxd * sizeof(void *))); 921 922 /* free tx packet DMA handles */ 923 if (di->txp_dmah) 924 MFREE(di->osh, (void *)di->txp_dmah, di->ntxd * sizeof(hnddma_seg_map_t)); 925 926 /* free rx packet DMA handles */ 927 if (di->rxp_dmah) 928 MFREE(di->osh, (void *)di->rxp_dmah, di->nrxd * sizeof(hnddma_seg_map_t)); 929 930 /* free our private info structure */ 931 MFREE(di->osh, (void *)di, sizeof(dma_info_t)); 932 933} 934 935static bool 936_dma_descriptor_align(dma_info_t *di) 937{ 938 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 939 uint32 addrl; 940 941 /* Check to see if the descriptors need to be aligned on 4K/8K or not */ 942 if (di->d64txregs != NULL) { 943 W_REG(di->osh, &di->d64txregs->addrlow, 0xff0); 944 addrl = R_REG(di->osh, &di->d64txregs->addrlow); 945 if (addrl != 0) 946 return FALSE; 947 } else if (di->d64rxregs != NULL) { 948 W_REG(di->osh, &di->d64rxregs->addrlow, 0xff0); 949 addrl = R_REG(di->osh, &di->d64rxregs->addrlow); 950 if (addrl != 0) 951 return FALSE; 952 } 953 } 954 return TRUE; 955} 956 957/** return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */ 958static bool 959_dma_isaddrext(dma_info_t *di) 960{ 961 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 962 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */ 963 964 /* not all tx or rx channel are available */ 965 if (di->d64txregs != NULL) { 966 if (!_dma64_addrext(di->osh, di->d64txregs)) { 967 DMA_ERROR(("%s: _dma_isaddrext: DMA64 tx doesn't have AE set\n", 968 di->name)); 969 ASSERT(0); 970 } 971 return TRUE; 972 } else if (di->d64rxregs != NULL) { 973 if (!_dma64_addrext(di->osh, di->d64rxregs)) { 974 DMA_ERROR(("%s: _dma_isaddrext: DMA64 rx doesn't have AE set\n", 975 di->name)); 976 ASSERT(0); 977 } 978 return TRUE; 979 } 980 return FALSE; 981 } else if (DMA32_ENAB(di)) { 982 if (di->d32txregs) 983 return (_dma32_addrext(di->osh, di->d32txregs)); 984 else if (di->d32rxregs) 985 return (_dma32_addrext(di->osh, di->d32rxregs)); 986 } else 987 ASSERT(0); 988 989 return FALSE; 990} 991 992/** initialize descriptor table base address */ 993static void 994_dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa) 995{ 996 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 997 if (!di->aligndesc_4k) { 998 if (direction == DMA_TX) 999 di->xmtptrbase = PHYSADDRLO(pa); 1000 else 1001 di->rcvptrbase = PHYSADDRLO(pa); 1002 } 1003 1004 if ((di->ddoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 1005 if (direction == DMA_TX) { 1006 W_REG(di->osh, &di->d64txregs->addrlow, (PHYSADDRLO(pa) + 1007 di->ddoffsetlow)); 1008 W_REG(di->osh, &di->d64txregs->addrhigh, (PHYSADDRHI(pa) + 1009 di->ddoffsethigh)); 1010 } else { 1011 W_REG(di->osh, &di->d64rxregs->addrlow, (PHYSADDRLO(pa) + 1012 di->ddoffsetlow)); 1013 W_REG(di->osh, &di->d64rxregs->addrhigh, (PHYSADDRHI(pa) + 1014 di->ddoffsethigh)); 1015 } 1016 } else { 1017 /* DMA64 32bits address extension */ 1018 uint32 ae; 1019 ASSERT(di->addrext); 1020 ASSERT(PHYSADDRHI(pa) == 0); 1021 1022 /* shift the high bit(s) from pa to ae */ 1023 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 1024 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 1025 1026 if (direction == DMA_TX) { 1027 W_REG(di->osh, &di->d64txregs->addrlow, (PHYSADDRLO(pa) + 1028 di->ddoffsetlow)); 1029 W_REG(di->osh, &di->d64txregs->addrhigh, di->ddoffsethigh); 1030 SET_REG(di->osh, &di->d64txregs->control, D64_XC_AE, 1031 (ae << D64_XC_AE_SHIFT)); 1032 } else { 1033 W_REG(di->osh, &di->d64rxregs->addrlow, (PHYSADDRLO(pa) + 1034 di->ddoffsetlow)); 1035 W_REG(di->osh, &di->d64rxregs->addrhigh, di->ddoffsethigh); 1036 SET_REG(di->osh, &di->d64rxregs->control, D64_RC_AE, 1037 (ae << D64_RC_AE_SHIFT)); 1038 } 1039 } 1040 1041 } else if (DMA32_ENAB(di)) { 1042 ASSERT(PHYSADDRHI(pa) == 0); 1043 if ((di->ddoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 1044 if (direction == DMA_TX) 1045 W_REG(di->osh, &di->d32txregs->addr, (PHYSADDRLO(pa) + 1046 di->ddoffsetlow)); 1047 else 1048 W_REG(di->osh, &di->d32rxregs->addr, (PHYSADDRLO(pa) + 1049 di->ddoffsetlow)); 1050 } else { 1051 /* dma32 address extension */ 1052 uint32 ae; 1053 ASSERT(di->addrext); 1054 1055 /* shift the high bit(s) from pa to ae */ 1056 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 1057 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 1058 1059 if (direction == DMA_TX) { 1060 W_REG(di->osh, &di->d32txregs->addr, (PHYSADDRLO(pa) + 1061 di->ddoffsetlow)); 1062 SET_REG(di->osh, &di->d32txregs->control, XC_AE, ae <<XC_AE_SHIFT); 1063 } else { 1064 W_REG(di->osh, &di->d32rxregs->addr, (PHYSADDRLO(pa) + 1065 di->ddoffsetlow)); 1066 SET_REG(di->osh, &di->d32rxregs->control, RC_AE, ae <<RC_AE_SHIFT); 1067 } 1068 } 1069 } else 1070 ASSERT(0); 1071} 1072 1073static void 1074_dma_fifoloopbackenable(dma_info_t *di) 1075{ 1076 DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name)); 1077 1078 if (DMA64_ENAB(di) && DMA64_MODE(di)) 1079 OR_REG(di->osh, &di->d64txregs->control, D64_XC_LE); 1080 else if (DMA32_ENAB(di)) 1081 OR_REG(di->osh, &di->d32txregs->control, XC_LE); 1082 else 1083 ASSERT(0); 1084} 1085 1086static void 1087_dma_rxinit(dma_info_t *di) 1088{ 1089 DMA_TRACE(("%s: dma_rxinit\n", di->name)); 1090 1091 if (di->nrxd == 0) 1092 return; 1093 1094 /* During the reset procedure, the active rxd may not be zero if pktpool is 1095 * enabled, we need to reclaim active rxd to avoid rxd being leaked. 1096 */ 1097 if ((POOL_ENAB(di->pktpool)) && (NRXDACTIVE(di->rxin, di->rxout))) { 1098 _dma_rxreclaim(di); 1099 } 1100 1101 ASSERT(di->rxin == di->rxout); 1102 di->rxin = di->rxout = di->rs0cd = 0; 1103 1104 /* clear rx descriptor ring */ 1105 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1106 BZERO_SM((void *)(uintptr)di->rxd64, (di->nrxd * sizeof(dma64dd_t))); 1107 1108 /* DMA engine with out alignment requirement requires table to be inited 1109 * before enabling the engine 1110 */ 1111 if (!di->aligndesc_4k) 1112 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 1113 1114 _dma_rxenable(di); 1115 1116 if (di->aligndesc_4k) 1117 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 1118 } else if (DMA32_ENAB(di)) { 1119 BZERO_SM((void *)(uintptr)di->rxd32, (di->nrxd * sizeof(dma32dd_t))); 1120 _dma_rxenable(di); 1121 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 1122 } else 1123 ASSERT(0); 1124} 1125 1126static void 1127_dma_rxenable(dma_info_t *di) 1128{ 1129 uint dmactrlflags = di->hnddma.dmactrlflags; 1130 1131 DMA_TRACE(("%s: dma_rxenable\n", di->name)); 1132 1133 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1134 uint32 control = (R_REG(di->osh, &di->d64rxregs->control) & D64_RC_AE) | D64_RC_RE; 1135 1136 if ((dmactrlflags & DMA_CTRL_PEN) == 0) 1137 control |= D64_RC_PD; 1138 1139 if (dmactrlflags & DMA_CTRL_ROC) 1140 control |= D64_RC_OC; 1141 1142 /* These bits 20:18 (burstLen) of control register can be written but will take 1143 * effect only if these bits are valid. So this will not affect previous versions 1144 * of the DMA. They will continue to have those bits set to 0. 1145 */ 1146 control &= ~D64_RC_BL_MASK; 1147 control |= (di->rxburstlen << D64_RC_BL_SHIFT); 1148 1149 control &= ~D64_RC_PC_MASK; 1150 control |= (di->rxprefetchctl << D64_RC_PC_SHIFT); 1151 1152 control &= ~D64_RC_PT_MASK; 1153 control |= (di->rxprefetchthresh << D64_RC_PT_SHIFT); 1154 1155 W_REG(di->osh, &di->d64rxregs->control, 1156 ((di->rxoffset << D64_RC_RO_SHIFT) | control)); 1157 } else if (DMA32_ENAB(di)) { 1158 uint32 control = (R_REG(di->osh, &di->d32rxregs->control) & RC_AE) | RC_RE; 1159 1160 if ((dmactrlflags & DMA_CTRL_PEN) == 0) 1161 control |= RC_PD; 1162 1163 if (dmactrlflags & DMA_CTRL_ROC) 1164 control |= RC_OC; 1165 1166 /* These bits 20:18 (burstLen) of control register can be written but will take 1167 * effect only if these bits are valid. So this will not affect previous versions 1168 * of the DMA. They will continue to have those bits set to 0. 1169 */ 1170 control &= ~RC_BL_MASK; 1171 control |= (di->rxburstlen << RC_BL_SHIFT); 1172 1173 control &= ~RC_PC_MASK; 1174 control |= (di->rxprefetchctl << RC_PC_SHIFT); 1175 1176 control &= ~RC_PT_MASK; 1177 control |= (di->rxprefetchthresh << RC_PT_SHIFT); 1178 1179 W_REG(di->osh, &di->d32rxregs->control, 1180 ((di->rxoffset << RC_RO_SHIFT) | control)); 1181 } else 1182 ASSERT(0); 1183} 1184 1185static void 1186_dma_rx_param_get(dma_info_t *di, uint16 *rxoffset, uint16 *rxbufsize) 1187{ 1188 /* the normal values fit into 16 bits */ 1189 *rxoffset = (uint16)di->rxoffset; 1190 *rxbufsize = (uint16)di->rxbufsize; 1191} 1192 1193/** 1194 * !! rx entry routine 1195 * returns a pointer to the next frame received, or NULL if there are no more 1196 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is supported 1197 * with pkts chain 1198 * otherwise, it's treated as giant pkt and will be tossed. 1199 * The DMA scattering starts with normal DMA header, followed by first buffer data. 1200 * After it reaches the max size of buffer, the data continues in next DMA descriptor 1201 * buffer WITHOUT DMA header 1202 */ 1203static void * BCMFASTPATH 1204_dma_rx(dma_info_t *di) 1205{ 1206 void *p, *head, *tail; 1207 uint len; 1208 uint pkt_len; 1209 int resid = 0; 1210#if defined(BCM4335) || defined(BCM4345) || defined(BCM4350) || defined(BCM43602) 1211 dma64regs_t *dregs = di->d64rxregs; 1212#endif 1213 1214next_frame: 1215 head = _dma_getnextrxp(di, FALSE); 1216 if (head == NULL) 1217 return (NULL); 1218 1219#if (!defined(__mips__) && !defined(BCM47XX_CA9) && !defined(__NetBSD__)) 1220#if defined(BCM4335) || defined(BCM4345) || defined(BCM4350) || defined(BCM43602) 1221 if ((R_REG(osh, &dregs->control) & D64_RC_GE)) { 1222 /* In case of glommed pkt get length from hwheader */ 1223 len = ltoh16(*((uint16 *)(PKTDATA(di->osh, head)) + di->rxoffset/2 + 2)) + 4; 1224 1225 *(uint16 *)(PKTDATA(di->osh, head)) = len; 1226 } else { 1227 len = ltoh16(*(uint16 *)(PKTDATA(di->osh, head))); 1228 } 1229#else 1230 len = ltoh16(*(uint16 *)(PKTDATA(di->osh, head))); 1231#endif 1232#else 1233 { 1234 int read_count = 0; 1235 for (read_count = 200; read_count; read_count--) { 1236 len = ltoh16(*(uint16 *)PKTDATA(di->osh, head)); 1237 if (len != 0) 1238 break; 1239 DMA_MAP(di->osh, PKTDATA(di->osh, head), sizeof(uint16), DMA_RX, NULL, NULL); 1240 OSL_DELAY(1); 1241 } 1242 1243 if (!len) { 1244 DMA_ERROR(("%s: dma_rx: frame length (%d)\n", di->name, len)); 1245 PKTFREE(di->osh, head, FALSE); 1246 goto next_frame; 1247 } 1248 1249 } 1250#endif /* defined(__mips__) */ 1251 DMA_TRACE(("%s: dma_rx len %d\n", di->name, len)); 1252 1253 /* set actual length */ 1254 pkt_len = MIN((di->rxoffset + len), di->rxbufsize); 1255 PKTSETLEN(di->osh, head, pkt_len); 1256 resid = len - (di->rxbufsize - di->rxoffset); 1257 1258 if (resid <= 0) { 1259 /* Single frame, all good */ 1260 } else if (di->hnddma.dmactrlflags & DMA_CTRL_RXSINGLE) { 1261 DMA_TRACE(("%s: dma_rx: corrupted length (%d)\n", di->name, len)); 1262 PKTFREE(di->osh, head, FALSE); 1263 di->hnddma.rxgiants++; 1264 goto next_frame; 1265 } else { 1266 /* multi-buffer rx */ 1267#ifdef BCMDBG 1268 /* get rid of compiler warning */ 1269 p = NULL; 1270#endif /* BCMDBG */ 1271 tail = head; 1272 while ((resid > 0) && (p = _dma_getnextrxp(di, FALSE))) { 1273 PKTSETNEXT(di->osh, tail, p); 1274 pkt_len = MIN(resid, (int)di->rxbufsize); 1275 PKTSETLEN(di->osh, p, pkt_len); 1276 1277 tail = p; 1278 resid -= di->rxbufsize; 1279 } 1280 1281#ifdef BCMDBG 1282 if (resid > 0) { 1283 uint16 cur; 1284 ASSERT(p == NULL); 1285 cur = (DMA64_ENAB(di) && DMA64_MODE(di)) ? 1286 B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 1287 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t) : 1288 B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, 1289 dma32dd_t); 1290 DMA_ERROR(("_dma_rx, rxin %d rxout %d, hw_curr %d\n", 1291 di->rxin, di->rxout, cur)); 1292 } 1293#endif /* BCMDBG */ 1294 1295 if ((di->hnddma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) { 1296 DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n", di->name, len)); 1297 PKTFREE(di->osh, head, FALSE); 1298 di->hnddma.rxgiants++; 1299 goto next_frame; 1300 } 1301 } 1302 1303 return (head); 1304} 1305 1306/** 1307 * post receive buffers 1308 * return FALSE is refill failed completely and ring is empty 1309 * this will stall the rx dma and user might want to call rxfill again asap 1310 * This unlikely happens on memory-rich NIC, but often on memory-constrained dongle 1311 */ 1312static bool BCMFASTPATH 1313_dma_rxfill(dma_info_t *di) 1314{ 1315 void *p; 1316 uint16 rxin, rxout; 1317 uint32 flags = 0; 1318 uint n; 1319 uint i; 1320 dmaaddr_t pa; 1321 uint extra_offset = 0, extra_pad; 1322 bool ring_empty; 1323 uint alignment_req = (di->hnddma.dmactrlflags & DMA_CTRL_USB_BOUNDRY4KB_WAR) ? 1324 16 : 1; /* MUST BE POWER of 2 */ 1325 1326 ring_empty = FALSE; 1327 1328 /* 1329 * Determine how many receive buffers we're lacking 1330 * from the full complement, allocate, initialize, 1331 * and post them, then update the chip rx lastdscr. 1332 */ 1333 1334 rxin = di->rxin; 1335 rxout = di->rxout; 1336 1337 n = di->nrxpost - NRXDACTIVE(rxin, rxout); 1338 1339 if (di->rxbufsize > BCMEXTRAHDROOM) 1340 extra_offset = di->rxextrahdrroom; 1341 1342 DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n)); 1343 1344 for (i = 0; i < n; i++) { 1345 /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the 1346 size to be allocated 1347 */ 1348 if (POOL_ENAB(di->pktpool)) { 1349 ASSERT(di->pktpool); 1350 p = pktpool_get(di->pktpool); 1351#ifdef BCMDBG_POOL 1352 if (p) 1353 PKTPOOLSETSTATE(p, POOL_RXFILL); 1354#endif /* BCMDBG_POOL */ 1355 } 1356 else { 1357 p = PKTGET(di->osh, (di->rxbufsize + extra_offset + alignment_req - 1), 1358 FALSE); 1359 } 1360 if (p == NULL) { 1361 DMA_TRACE(("%s: dma_rxfill: out of rxbufs\n", di->name)); 1362 if (i == 0) { 1363 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1364 if (dma64_rxidle(di)) { 1365 DMA_TRACE(("%s: rxfill64: ring is empty !\n", 1366 di->name)); 1367 ring_empty = TRUE; 1368 } 1369 } else if (DMA32_ENAB(di)) { 1370 if (dma32_rxidle(di)) { 1371 DMA_TRACE(("%s: rxfill32: ring is empty !\n", 1372 di->name)); 1373 ring_empty = TRUE; 1374 } 1375 } else 1376 ASSERT(0); 1377 } 1378 di->hnddma.rxnobuf++; 1379 break; 1380 } 1381 /* reserve an extra headroom, if applicable */ 1382 if (di->hnddma.dmactrlflags & DMA_CTRL_USB_BOUNDRY4KB_WAR) { 1383 extra_pad = ((alignment_req - (uint)(((unsigned long)PKTDATA(di->osh, p) - 1384 (unsigned long)(uchar *)0))) & (alignment_req - 1)); 1385 } else 1386 extra_pad = 0; 1387 1388 if (extra_offset + extra_pad) 1389 PKTPULL(di->osh, p, extra_offset + extra_pad); 1390 1391#ifdef CTFMAP 1392 /* mark as ctf buffer for fast mapping */ 1393 if (CTF_ENAB(kcih)) { 1394 ASSERT((((uint32)PKTDATA(di->osh, p)) & 31) == 0); 1395 PKTSETCTF(di->osh, p); 1396 } 1397#endif /* CTFMAP */ 1398 1399 /* Do a cached write instead of uncached write since DMA_MAP 1400 * will flush the cache. 1401 */ 1402 *(uint32 *)(PKTDATA(di->osh, p)) = 0; 1403#if defined(linux) && (defined(BCM47XX_CA9) || defined(__mips__)) 1404 DMA_MAP(di->osh, PKTDATA(di->osh, p), sizeof(uint16), DMA_TX, NULL, NULL); 1405#endif 1406 1407 if (DMASGLIST_ENAB) 1408 bzero(&di->rxp_dmah[rxout], sizeof(hnddma_seg_map_t)); 1409#ifdef BCM_SECURE_DMA 1410 pa = SECURE_DMA_MAP(di->osh, PKTDATA(di->osh, p), di->rxbufsize, DMA_RX, 1411 NULL, NULL, &di->sec_cma_info_rx, 0); 1412#else 1413 pa = DMA_MAP(di->osh, PKTDATA(di->osh, p), 1414 di->rxbufsize, DMA_RX, p, 1415 &di->rxp_dmah[rxout]); 1416#endif 1417 ASSERT(ISALIGNED(PHYSADDRLO(pa), 4)); 1418 1419 /* save the free packet pointer */ 1420 ASSERT(di->rxp[rxout] == NULL); 1421 di->rxp[rxout] = p; 1422 1423 /* reset flags for each descriptor */ 1424 flags = 0; 1425 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1426 if (rxout == (di->nrxd - 1)) 1427 flags = D64_CTRL1_EOT; 1428 1429 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags, di->rxbufsize); 1430 } else if (DMA32_ENAB(di)) { 1431 if (rxout == (di->nrxd - 1)) 1432 flags = CTRL_EOT; 1433 1434 ASSERT(PHYSADDRHI(pa) == 0); 1435 dma32_dd_upd(di, di->rxd32, pa, rxout, &flags, di->rxbufsize); 1436 } else 1437 ASSERT(0); 1438 rxout = NEXTRXD(rxout); 1439 } 1440 1441 di->rxout = rxout; 1442 1443 /* update the chip lastdscr pointer */ 1444 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1445 W_REG(di->osh, &di->d64rxregs->ptr, di->rcvptrbase + I2B(rxout, dma64dd_t)); 1446 } else if (DMA32_ENAB(di)) { 1447 W_REG(di->osh, &di->d32rxregs->ptr, I2B(rxout, dma32dd_t)); 1448 } else 1449 ASSERT(0); 1450 1451 return ring_empty; 1452} 1453 1454/** like getnexttxp but no reclaim */ 1455static void * 1456_dma_peeknexttxp(dma_info_t *di) 1457{ 1458 uint16 end, i; 1459 1460 if (di->ntxd == 0) 1461 return (NULL); 1462 1463 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1464 end = (uint16)B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 1465 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 1466 di->xs0cd = end; 1467 } else if (DMA32_ENAB(di)) { 1468 end = (uint16)B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t); 1469 di->xs0cd = end; 1470 } else 1471 ASSERT(0); 1472 1473 for (i = di->txin; i != end; i = NEXTTXD(i)) 1474 if (di->txp[i]) 1475 return (di->txp[i]); 1476 1477 return (NULL); 1478} 1479 1480int 1481_dma_peekntxp(dma_info_t *di, int *len, void *txps[], txd_range_t range) 1482{ 1483 uint16 start, end, i; 1484 uint act; 1485 void *txp = NULL; 1486 int k, len_max; 1487 1488 DMA_TRACE(("%s: dma_peekntxp\n", di->name)); 1489 1490 ASSERT(len); 1491 ASSERT(txps); 1492 ASSERT(di); 1493 if (di->ntxd == 0) { 1494 *len = 0; 1495 return BCME_ERROR; 1496 } 1497 1498 len_max = *len; 1499 *len = 0; 1500 1501 start = di->txin; 1502 1503 if (range == HNDDMA_RANGE_ALL) 1504 end = di->txout; 1505 else { 1506 if (DMA64_ENAB(di)) { 1507 end = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 1508 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 1509 1510 act = (uint)(R_REG(di->osh, &di->d64txregs->status1) & D64_XS1_AD_MASK); 1511 act = (act - di->xmtptrbase) & D64_XS0_CD_MASK; 1512 act = (uint)B2I(act, dma64dd_t); 1513 } else { 1514 end = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t); 1515 1516 act = (uint)((R_REG(di->osh, &di->d32txregs->status) & XS_AD_MASK) >> 1517 XS_AD_SHIFT); 1518 act = (uint)B2I(act, dma32dd_t); 1519 } 1520 1521 di->xs0cd = end; 1522 if (end != act) 1523 end = PREVTXD(act); 1524 } 1525 1526 if ((start == 0) && (end > di->txout)) 1527 return BCME_ERROR; 1528 1529 k = 0; 1530 for (i = start; i != end; i = NEXTTXD(i)) { 1531 txp = di->txp[i]; 1532 if (txp != NULL) { 1533 if (k < len_max) 1534 txps[k++] = txp; 1535 else 1536 break; 1537 } 1538 } 1539 *len = k; 1540 1541 return BCME_OK; 1542} 1543 1544/** like getnextrxp but not take off the ring */ 1545static void * 1546_dma_peeknextrxp(dma_info_t *di) 1547{ 1548 uint16 end, i; 1549 1550 if (di->nrxd == 0) 1551 return (NULL); 1552 1553 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1554 end = (uint16)B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 1555 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 1556 di->rs0cd = end; 1557 } else if (DMA32_ENAB(di)) { 1558 end = (uint16)B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t); 1559 di->rs0cd = end; 1560 } else 1561 ASSERT(0); 1562 1563 for (i = di->rxin; i != end; i = NEXTRXD(i)) 1564 if (di->rxp[i]) 1565 return (di->rxp[i]); 1566 1567 return (NULL); 1568} 1569 1570static void 1571_dma_rxreclaim(dma_info_t *di) 1572{ 1573 void *p; 1574 bool origcb = TRUE; 1575 1576#ifndef EFI 1577 /* "unused local" warning suppression for OSLs that 1578 * define PKTFREE() without using the di->osh arg 1579 */ 1580 di = di; 1581#endif /* EFI */ 1582 1583 DMA_TRACE(("%s: dma_rxreclaim\n", di->name)); 1584 1585 if (POOL_ENAB(di->pktpool) && 1586 ((origcb = pktpool_emptycb_disabled(di->pktpool)) == FALSE)) 1587 pktpool_emptycb_disable(di->pktpool, TRUE); 1588 1589 while ((p = _dma_getnextrxp(di, TRUE))) 1590 PKTFREE(di->osh, p, FALSE); 1591 1592 if (origcb == FALSE) 1593 pktpool_emptycb_disable(di->pktpool, FALSE); 1594} 1595 1596static void * BCMFASTPATH 1597_dma_getnextrxp(dma_info_t *di, bool forceall) 1598{ 1599 if (di->nrxd == 0) 1600 return (NULL); 1601 1602 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1603 return dma64_getnextrxp(di, forceall); 1604 } else if (DMA32_ENAB(di)) { 1605 return dma32_getnextrxp(di, forceall); 1606 } else 1607 ASSERT(0); 1608} 1609 1610static void 1611_dma_txblock(dma_info_t *di) 1612{ 1613 di->hnddma.txavail = 0; 1614} 1615 1616static void 1617_dma_txunblock(dma_info_t *di) 1618{ 1619 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 1620} 1621 1622static uint 1623_dma_txactive(dma_info_t *di) 1624{ 1625 return NTXDACTIVE(di->txin, di->txout); 1626} 1627 1628static uint 1629_dma_txpending(dma_info_t *di) 1630{ 1631 uint16 curr; 1632 1633 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1634 curr = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 1635 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 1636 di->xs0cd = curr; 1637 } else if (DMA32_ENAB(di)) { 1638 curr = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t); 1639 di->xs0cd = curr; 1640 } else 1641 ASSERT(0); 1642 1643 return NTXDACTIVE(curr, di->txout); 1644} 1645 1646static uint 1647_dma_txcommitted(dma_info_t *di) 1648{ 1649 uint16 ptr; 1650 uint txin = di->txin; 1651 1652 if (txin == di->txout) 1653 return 0; 1654 1655 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1656 ptr = B2I(R_REG(di->osh, &di->d64txregs->ptr), dma64dd_t); 1657 } else if (DMA32_ENAB(di)) { 1658 ptr = B2I(R_REG(di->osh, &di->d32txregs->ptr), dma32dd_t); 1659 } else 1660 ASSERT(0); 1661 1662 return NTXDACTIVE(di->txin, ptr); 1663} 1664 1665static uint 1666_dma_rxactive(dma_info_t *di) 1667{ 1668 return NRXDACTIVE(di->rxin, di->rxout); 1669} 1670 1671static uint 1672_dma_activerxbuf(dma_info_t *di) 1673{ 1674 uint16 curr, ptr; 1675 curr = B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 1676 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 1677 ptr = B2I(((R_REG(di->osh, &di->d64rxregs->ptr) & D64_RS0_CD_MASK) - 1678 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 1679 return NRXDACTIVE(curr, ptr); 1680} 1681 1682 1683static void 1684_dma_counterreset(dma_info_t *di) 1685{ 1686 /* reset all software counter */ 1687 di->hnddma.rxgiants = 0; 1688 di->hnddma.rxnobuf = 0; 1689 di->hnddma.txnobuf = 0; 1690} 1691 1692static uint 1693_dma_ctrlflags(dma_info_t *di, uint mask, uint flags) 1694{ 1695 uint dmactrlflags; 1696 1697 if (!di) { 1698 DMA_ERROR(("_dma_ctrlflags: NULL dma handle\n")); 1699 return (0); 1700 } 1701 1702 dmactrlflags = di->hnddma.dmactrlflags; 1703 ASSERT((flags & ~mask) == 0); 1704 1705 dmactrlflags &= ~mask; 1706 dmactrlflags |= flags; 1707 1708 /* If trying to enable parity, check if parity is actually supported */ 1709 if (dmactrlflags & DMA_CTRL_PEN) { 1710 uint32 control; 1711 1712 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1713 control = R_REG(di->osh, &di->d64txregs->control); 1714 W_REG(di->osh, &di->d64txregs->control, control | D64_XC_PD); 1715 if (R_REG(di->osh, &di->d64txregs->control) & D64_XC_PD) { 1716 /* We *can* disable it so it is supported, 1717 * restore control register 1718 */ 1719 W_REG(di->osh, &di->d64txregs->control, control); 1720 } else { 1721 /* Not supported, don't allow it to be enabled */ 1722 dmactrlflags &= ~DMA_CTRL_PEN; 1723 } 1724 } else if (DMA32_ENAB(di)) { 1725 control = R_REG(di->osh, &di->d32txregs->control); 1726 W_REG(di->osh, &di->d32txregs->control, control | XC_PD); 1727 if (R_REG(di->osh, &di->d32txregs->control) & XC_PD) { 1728 W_REG(di->osh, &di->d32txregs->control, control); 1729 } else { 1730 /* Not supported, don't allow it to be enabled */ 1731 dmactrlflags &= ~DMA_CTRL_PEN; 1732 } 1733 } else 1734 ASSERT(0); 1735 } 1736 1737 di->hnddma.dmactrlflags = dmactrlflags; 1738 1739 return (dmactrlflags); 1740} 1741 1742/** get the address of the var in order to change later */ 1743static uintptr 1744_dma_getvar(dma_info_t *di, const char *name) 1745{ 1746 if (!strcmp(name, "&txavail")) 1747 return ((uintptr) &(di->hnddma.txavail)); 1748 else { 1749 ASSERT(0); 1750 } 1751 return (0); 1752} 1753 1754static uint 1755_dma_avoidancecnt(dma_info_t *di) 1756{ 1757 return (di->dma_avoidance_cnt); 1758} 1759 1760void 1761dma_txpioloopback(osl_t *osh, dma32regs_t *regs) 1762{ 1763 OR_REG(osh, ®s->control, XC_LE); 1764} 1765 1766static 1767uint8 dma_align_sizetobits(uint size) 1768{ 1769 uint8 bitpos = 0; 1770 ASSERT(size); 1771 ASSERT(!(size & (size-1))); 1772 while (size >>= 1) { 1773 bitpos ++; 1774 } 1775 return (bitpos); 1776} 1777 1778/** 1779 * This function ensures that the DMA descriptor ring will not get allocated 1780 * across Page boundary. If the allocation is done across the page boundary 1781 * at the first time, then it is freed and the allocation is done at 1782 * descriptor ring size aligned location. This will ensure that the ring will 1783 * not cross page boundary 1784 */ 1785static void * 1786dma_ringalloc(osl_t *osh, uint32 boundary, uint size, uint16 *alignbits, uint* alloced, 1787 dmaaddr_t *descpa, osldma_t **dmah) 1788{ 1789 void * va; 1790 uint32 desc_strtaddr; 1791 uint32 alignbytes = 1 << *alignbits; 1792 1793 if ((va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced, descpa, dmah)) == NULL) 1794 return NULL; 1795 1796 desc_strtaddr = (uint32)ROUNDUP((uint)PHYSADDRLO(*descpa), alignbytes); 1797 if (((desc_strtaddr + size - 1) & boundary) != 1798 (desc_strtaddr & boundary)) { 1799 *alignbits = dma_align_sizetobits(size); 1800 DMA_FREE_CONSISTENT(osh, va, 1801 size, *descpa, *dmah); 1802 va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced, descpa, dmah); 1803 } 1804 return va; 1805} 1806 1807#if defined(BCMDBG) 1808static void 1809dma32_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma32dd_t *ring, uint start, uint end, 1810 uint max_num) 1811{ 1812 uint i; 1813 1814 for (i = start; i != end; i = XXD((i + 1), max_num)) { 1815 /* in the format of high->low 8 bytes */ 1816 bcm_bprintf(b, "ring index %d: 0x%x %x\n", 1817 i, R_SM(&ring[i].addr), R_SM(&ring[i].ctrl)); 1818 } 1819} 1820 1821static void 1822dma32_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1823{ 1824 if (di->ntxd == 0) 1825 return; 1826 1827 bcm_bprintf(b, "DMA32: txd32 %p txdpa 0x%lx txp %p txin %d txout %d " 1828 "txavail %d txnodesc %d\n", di->txd32, PHYSADDRLO(di->txdpa), di->txp, di->txin, 1829 di->txout, di->hnddma.txavail, di->hnddma.txnodesc); 1830 1831 bcm_bprintf(b, "xmtcontrol 0x%x xmtaddr 0x%x xmtptr 0x%x xmtstatus 0x%x\n", 1832 R_REG(di->osh, &di->d32txregs->control), 1833 R_REG(di->osh, &di->d32txregs->addr), 1834 R_REG(di->osh, &di->d32txregs->ptr), 1835 R_REG(di->osh, &di->d32txregs->status)); 1836 1837 if (dumpring && di->txd32) 1838 dma32_dumpring(di, b, di->txd32, di->txin, di->txout, di->ntxd); 1839} 1840 1841static void 1842dma32_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1843{ 1844 if (di->nrxd == 0) 1845 return; 1846 1847 bcm_bprintf(b, "DMA32: rxd32 %p rxdpa 0x%lx rxp %p rxin %d rxout %d\n", 1848 di->rxd32, PHYSADDRLO(di->rxdpa), di->rxp, di->rxin, di->rxout); 1849 1850 bcm_bprintf(b, "rcvcontrol 0x%x rcvaddr 0x%x rcvptr 0x%x rcvstatus 0x%x\n", 1851 R_REG(di->osh, &di->d32rxregs->control), 1852 R_REG(di->osh, &di->d32rxregs->addr), 1853 R_REG(di->osh, &di->d32rxregs->ptr), 1854 R_REG(di->osh, &di->d32rxregs->status)); 1855 if (di->rxd32 && dumpring) 1856 dma32_dumpring(di, b, di->rxd32, di->rxin, di->rxout, di->nrxd); 1857} 1858 1859static void 1860dma32_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1861{ 1862 dma32_dumptx(di, b, dumpring); 1863 dma32_dumprx(di, b, dumpring); 1864} 1865 1866static void 1867dma64_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma64dd_t *ring, uint start, uint end, 1868 uint max_num) 1869{ 1870 uint i; 1871 1872 for (i = start; i != end; i = XXD((i + 1), max_num)) { 1873 /* in the format of high->low 16 bytes */ 1874 bcm_bprintf(b, "ring index %d: 0x%x %x %x %x\n", 1875 i, R_SM(&ring[i].addrhigh), R_SM(&ring[i].addrlow), 1876 R_SM(&ring[i].ctrl2), R_SM(&ring[i].ctrl1)); 1877 } 1878} 1879 1880static void 1881dma64_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1882{ 1883 if (di->ntxd == 0) 1884 return; 1885 1886 bcm_bprintf(b, "DMA64: txd64 %p txdpa 0x%lx txdpahi 0x%lx txp %p txin %d txout %d " 1887 "txavail %d txnodesc %d\n", di->txd64, PHYSADDRLO(di->txdpa), 1888 PHYSADDRHI(di->txdpaorig), di->txp, di->txin, di->txout, di->hnddma.txavail, 1889 di->hnddma.txnodesc); 1890 1891 bcm_bprintf(b, "xmtcontrol 0x%x xmtaddrlow 0x%x xmtaddrhigh 0x%x " 1892 "xmtptr 0x%x xmtstatus0 0x%x xmtstatus1 0x%x\n", 1893 R_REG(di->osh, &di->d64txregs->control), 1894 R_REG(di->osh, &di->d64txregs->addrlow), 1895 R_REG(di->osh, &di->d64txregs->addrhigh), 1896 R_REG(di->osh, &di->d64txregs->ptr), 1897 R_REG(di->osh, &di->d64txregs->status0), 1898 R_REG(di->osh, &di->d64txregs->status1)); 1899 1900 bcm_bprintf(b, "DMA64: DMA avoidance applied %d\n", di->dma_avoidance_cnt); 1901 1902 if (dumpring && di->txd64) { 1903 dma64_dumpring(di, b, di->txd64, di->txin, di->txout, di->ntxd); 1904 } 1905} 1906 1907static void 1908dma64_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1909{ 1910 if (di->nrxd == 0) 1911 return; 1912 1913 bcm_bprintf(b, "DMA64: rxd64 %p rxdpa 0x%lx rxdpahi 0x%lx rxp %p rxin %d rxout %d\n", 1914 di->rxd64, PHYSADDRLO(di->rxdpa), PHYSADDRHI(di->rxdpaorig), di->rxp, 1915 di->rxin, di->rxout); 1916 1917 bcm_bprintf(b, "rcvcontrol 0x%x rcvaddrlow 0x%x rcvaddrhigh 0x%x rcvptr " 1918 "0x%x rcvstatus0 0x%x rcvstatus1 0x%x\n", 1919 R_REG(di->osh, &di->d64rxregs->control), 1920 R_REG(di->osh, &di->d64rxregs->addrlow), 1921 R_REG(di->osh, &di->d64rxregs->addrhigh), 1922 R_REG(di->osh, &di->d64rxregs->ptr), 1923 R_REG(di->osh, &di->d64rxregs->status0), 1924 R_REG(di->osh, &di->d64rxregs->status1)); 1925 if (di->rxd64 && dumpring) { 1926 dma64_dumpring(di, b, di->rxd64, di->rxin, di->rxout, di->nrxd); 1927 } 1928} 1929 1930static void 1931dma64_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1932{ 1933 dma64_dumptx(di, b, dumpring); 1934 dma64_dumprx(di, b, dumpring); 1935} 1936 1937#endif 1938 1939 1940/* 32-bit DMA functions */ 1941 1942static void 1943dma32_txinit(dma_info_t *di) 1944{ 1945 uint32 control = XC_XE; 1946 1947 DMA_TRACE(("%s: dma_txinit\n", di->name)); 1948 1949 if (di->ntxd == 0) 1950 return; 1951 1952 di->txin = di->txout = di->xs0cd = 0; 1953 di->hnddma.txavail = di->ntxd - 1; 1954 1955 /* clear tx descriptor ring */ 1956 BZERO_SM(DISCARD_QUAL(di->txd32, void), (di->ntxd * sizeof(dma32dd_t))); 1957 1958 /* These bits 20:18 (burstLen) of control register can be written but will take 1959 * effect only if these bits are valid. So this will not affect previous versions 1960 * of the DMA. They will continue to have those bits set to 0. 1961 */ 1962 control |= (di->txburstlen << XC_BL_SHIFT); 1963 control |= (di->txmultioutstdrd << XC_MR_SHIFT); 1964 control |= (di->txprefetchctl << XC_PC_SHIFT); 1965 control |= (di->txprefetchthresh << XC_PT_SHIFT); 1966 1967 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0) 1968 control |= XC_PD; 1969 W_REG(di->osh, &di->d32txregs->control, control); 1970 _dma_ddtable_init(di, DMA_TX, di->txdpa); 1971} 1972 1973static bool 1974dma32_txenabled(dma_info_t *di) 1975{ 1976 uint32 xc; 1977 1978 /* If the chip is dead, it is not enabled :-) */ 1979 xc = R_REG(di->osh, &di->d32txregs->control); 1980 return ((xc != 0xffffffff) && (xc & XC_XE)); 1981} 1982 1983static void 1984dma32_txsuspend(dma_info_t *di) 1985{ 1986 DMA_TRACE(("%s: dma_txsuspend\n", di->name)); 1987 1988 if (di->ntxd == 0) 1989 return; 1990 1991 OR_REG(di->osh, &di->d32txregs->control, XC_SE); 1992} 1993 1994static void 1995dma32_txresume(dma_info_t *di) 1996{ 1997 DMA_TRACE(("%s: dma_txresume\n", di->name)); 1998 1999 if (di->ntxd == 0) 2000 return; 2001 2002 AND_REG(di->osh, &di->d32txregs->control, ~XC_SE); 2003} 2004 2005static bool 2006dma32_txsuspended(dma_info_t *di) 2007{ 2008 return (di->ntxd == 0) || ((R_REG(di->osh, &di->d32txregs->control) & XC_SE) == XC_SE); 2009} 2010 2011static void 2012dma32_txflush(dma_info_t *di) 2013{ 2014 DMA_TRACE(("%s: dma_txflush\n", di->name)); 2015 2016 if (di->ntxd == 0) 2017 return; 2018 2019 OR_REG(di->osh, &di->d32txregs->control, XC_SE | XC_FL); 2020} 2021 2022static void 2023dma32_txflush_clear(dma_info_t *di) 2024{ 2025 uint32 status; 2026 2027 DMA_TRACE(("%s: dma_txflush_clear\n", di->name)); 2028 2029 if (di->ntxd == 0) 2030 return; 2031 2032 SPINWAIT(((status = (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK)) 2033 != XS_XS_DISABLED) && 2034 (status != XS_XS_IDLE) && 2035 (status != XS_XS_STOPPED), 2036 (10000)); 2037 AND_REG(di->osh, &di->d32txregs->control, ~XC_FL); 2038} 2039 2040static void 2041dma32_txreclaim(dma_info_t *di, txd_range_t range) 2042{ 2043 void *p; 2044 2045 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, 2046 (range == HNDDMA_RANGE_ALL) ? "all" : 2047 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 2048 2049 if (di->txin == di->txout) 2050 return; 2051 2052 while ((p = dma32_getnexttxp(di, range))) 2053 PKTFREE(di->osh, p, TRUE); 2054} 2055 2056static bool 2057dma32_txstopped(dma_info_t *di) 2058{ 2059 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_STOPPED); 2060} 2061 2062static bool 2063dma32_rxstopped(dma_info_t *di) 2064{ 2065 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_RS_MASK) == RS_RS_STOPPED); 2066} 2067 2068static bool 2069dma32_alloc(dma_info_t *di, uint direction) 2070{ 2071 uint size; 2072 uint ddlen; 2073 void *va; 2074 uint alloced; 2075 uint16 align; 2076 uint16 align_bits; 2077 2078 ddlen = sizeof(dma32dd_t); 2079 2080 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen); 2081 2082 alloced = 0; 2083 align_bits = di->dmadesc_align; 2084 align = (1 << align_bits); 2085 2086 if (direction == DMA_TX) { 2087 if ((va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits, &alloced, 2088 &di->txdpaorig, &di->tx_dmah)) == NULL) { 2089 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", 2090 di->name)); 2091 return FALSE; 2092 } 2093 2094 PHYSADDRHISET(di->txdpa, 0); 2095 ASSERT(PHYSADDRHI(di->txdpaorig) == 0); 2096 di->txd32 = (dma32dd_t *)ROUNDUP((uintptr)va, align); 2097 di->txdalign = (uint)((int8 *)(uintptr)di->txd32 - (int8 *)va); 2098 2099 PHYSADDRLOSET(di->txdpa, PHYSADDRLO(di->txdpaorig) + di->txdalign); 2100 /* Make sure that alignment didn't overflow */ 2101 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig)); 2102 2103 di->txdalloc = alloced; 2104 ASSERT(ISALIGNED(di->txd32, align)); 2105 } else { 2106 if ((va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits, &alloced, 2107 &di->rxdpaorig, &di->rx_dmah)) == NULL) { 2108 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", 2109 di->name)); 2110 return FALSE; 2111 } 2112 2113 PHYSADDRHISET(di->rxdpa, 0); 2114 ASSERT(PHYSADDRHI(di->rxdpaorig) == 0); 2115 di->rxd32 = (dma32dd_t *)ROUNDUP((uintptr)va, align); 2116 di->rxdalign = (uint)((int8 *)(uintptr)di->rxd32 - (int8 *)va); 2117 2118 PHYSADDRLOSET(di->rxdpa, PHYSADDRLO(di->rxdpaorig) + di->rxdalign); 2119 /* Make sure that alignment didn't overflow */ 2120 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig)); 2121 di->rxdalloc = alloced; 2122 ASSERT(ISALIGNED(di->rxd32, align)); 2123 } 2124 2125 return TRUE; 2126} 2127 2128static bool 2129dma32_txreset(dma_info_t *di) 2130{ 2131 uint32 status; 2132 2133 if (di->ntxd == 0) 2134 return TRUE; 2135 2136 /* suspend tx DMA first */ 2137 W_REG(di->osh, &di->d32txregs->control, XC_SE); 2138 SPINWAIT(((status = (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK)) 2139 != XS_XS_DISABLED) && 2140 (status != XS_XS_IDLE) && 2141 (status != XS_XS_STOPPED), 2142 (10000)); 2143 2144 W_REG(di->osh, &di->d32txregs->control, 0); 2145 SPINWAIT(((status = (R_REG(di->osh, 2146 &di->d32txregs->status) & XS_XS_MASK)) != XS_XS_DISABLED), 2147 10000); 2148 2149 /* We should be disabled at this point */ 2150 if (status != XS_XS_DISABLED) { 2151 DMA_ERROR(("%s: status != D64_XS0_XS_DISABLED 0x%x\n", __FUNCTION__, status)); 2152 ASSERT(status == XS_XS_DISABLED); 2153 OSL_DELAY(300); 2154 } 2155 2156 return (status == XS_XS_DISABLED); 2157} 2158 2159static bool 2160dma32_rxidle(dma_info_t *di) 2161{ 2162 DMA_TRACE(("%s: dma_rxidle\n", di->name)); 2163 2164 if (di->nrxd == 0) 2165 return TRUE; 2166 2167 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK) == 2168 R_REG(di->osh, &di->d32rxregs->ptr)); 2169} 2170 2171static bool 2172dma32_rxreset(dma_info_t *di) 2173{ 2174 uint32 status; 2175 2176 if (di->nrxd == 0) 2177 return TRUE; 2178 2179 W_REG(di->osh, &di->d32rxregs->control, 0); 2180 SPINWAIT(((status = (R_REG(di->osh, 2181 &di->d32rxregs->status) & RS_RS_MASK)) != RS_RS_DISABLED), 2182 10000); 2183 2184 return (status == RS_RS_DISABLED); 2185} 2186 2187static bool 2188dma32_rxenabled(dma_info_t *di) 2189{ 2190 uint32 rc; 2191 2192 rc = R_REG(di->osh, &di->d32rxregs->control); 2193 return ((rc != 0xffffffff) && (rc & RC_RE)); 2194} 2195 2196static bool 2197dma32_txsuspendedidle(dma_info_t *di) 2198{ 2199 if (di->ntxd == 0) 2200 return TRUE; 2201 2202 if (!(R_REG(di->osh, &di->d32txregs->control) & XC_SE)) 2203 return 0; 2204 2205 if ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) != XS_XS_IDLE) 2206 return 0; 2207 2208 OSL_DELAY(2); 2209 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_IDLE); 2210} 2211 2212/** 2213 * !! tx entry routine 2214 * supports full 32bit dma engine buffer addressing so 2215 * dma buffers can cross 4 Kbyte page boundaries. 2216 * 2217 * WARNING: call must check the return value for error. 2218 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems 2219 */ 2220static int 2221dma32_txfast(dma_info_t *di, void *p0, bool commit) 2222{ 2223 void *p, *next; 2224 uchar *data; 2225 uint len; 2226 uint16 txout; 2227 uint32 flags = 0; 2228 dmaaddr_t pa; 2229 2230 DMA_TRACE(("%s: dma_txfast\n", di->name)); 2231 2232 txout = di->txout; 2233 2234 /* 2235 * Walk the chain of packet buffers 2236 * allocating and initializing transmit descriptor entries. 2237 */ 2238 for (p = p0; p; p = next) { 2239 uint nsegs, j; 2240 hnddma_seg_map_t *map; 2241 2242 data = PKTDATA(di->osh, p); 2243 len = PKTLEN(di->osh, p); 2244#ifdef BCM_DMAPAD 2245 len += PKTDMAPAD(di->osh, p); 2246#endif 2247 next = PKTNEXT(di->osh, p); 2248 2249 /* return nonzero if out of tx descriptors */ 2250 if (NEXTTXD(txout) == di->txin) 2251 goto outoftxd; 2252 2253 if (len == 0) 2254 continue; 2255 2256 if (DMASGLIST_ENAB) 2257 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t)); 2258 2259 /* get physical address of buffer start */ 2260#ifdef BCM_SECURE_DMA 2261 pa = SECURE_DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout], 2262 &di->sec_cma_info_tx, 0); 2263#else 2264 pa = DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]); 2265#endif 2266 if (DMASGLIST_ENAB) { 2267 map = &di->txp_dmah[txout]; 2268 2269 /* See if all the segments can be accounted for */ 2270 if (map->nsegs > (uint)(di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1)) 2271 goto outoftxd; 2272 2273 nsegs = map->nsegs; 2274 } else 2275 nsegs = 1; 2276 2277 for (j = 1; j <= nsegs; j++) { 2278 flags = 0; 2279 if (p == p0 && j == 1) 2280 flags |= CTRL_SOF; 2281 2282 /* With a DMA segment list, Descriptor table is filled 2283 * using the segment list instead of looping over 2284 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when 2285 * end of segment list is reached. 2286 */ 2287 if ((!DMASGLIST_ENAB && next == NULL) || 2288 (DMASGLIST_ENAB && j == nsegs)) 2289 flags |= (CTRL_IOC | CTRL_EOF); 2290 if (txout == (di->ntxd - 1)) 2291 flags |= CTRL_EOT; 2292 2293 if (DMASGLIST_ENAB) { 2294 len = map->segs[j - 1].length; 2295 pa = map->segs[j - 1].addr; 2296 } 2297 ASSERT(PHYSADDRHI(pa) == 0); 2298 2299 dma32_dd_upd(di, di->txd32, pa, txout, &flags, len); 2300 ASSERT(di->txp[txout] == NULL); 2301 2302 txout = NEXTTXD(txout); 2303 } 2304 2305 /* See above. No need to loop over individual buffers */ 2306 if (DMASGLIST_ENAB) 2307 break; 2308 } 2309 2310 /* if last txd eof not set, fix it */ 2311 if (!(flags & CTRL_EOF)) 2312 W_SM(&di->txd32[PREVTXD(txout)].ctrl, BUS_SWAP32(flags | CTRL_IOC | CTRL_EOF)); 2313 2314 /* save the packet */ 2315 di->txp[PREVTXD(txout)] = p0; 2316 2317 /* bump the tx descriptor index */ 2318 di->txout = txout; 2319 2320 /* kick the chip */ 2321 if (commit) 2322 W_REG(di->osh, &di->d32txregs->ptr, I2B(txout, dma32dd_t)); 2323 2324 /* tx flow control */ 2325 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2326 2327 return (0); 2328 2329outoftxd: 2330 DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name)); 2331 PKTFREE(di->osh, p0, TRUE); 2332 di->hnddma.txavail = 0; 2333 di->hnddma.txnobuf++; 2334 di->hnddma.txnodesc++; 2335 return (-1); 2336} 2337 2338/** 2339 * Reclaim next completed txd (txds if using chained buffers) in the range 2340 * specified and return associated packet. 2341 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be 2342 * transmitted as noted by the hardware "CurrDescr" pointer. 2343 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be 2344 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer. 2345 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and 2346 * return associated packet regardless of the value of hardware pointers. 2347 */ 2348static void * 2349dma32_getnexttxp(dma_info_t *di, txd_range_t range) 2350{ 2351 uint16 start, end, i; 2352 uint16 active_desc; 2353 void *txp; 2354 2355 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, 2356 (range == HNDDMA_RANGE_ALL) ? "all" : 2357 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 2358 2359 if (di->ntxd == 0) 2360 return (NULL); 2361 2362 txp = NULL; 2363 2364 start = di->txin; 2365 if (range == HNDDMA_RANGE_ALL) 2366 end = di->txout; 2367 else { 2368 dma32regs_t *dregs = di->d32txregs; 2369 2370 if (di->txin == di->xs0cd) { 2371 end = (uint16)B2I(R_REG(di->osh, &dregs->status) & XS_CD_MASK, dma32dd_t); 2372 di->xs0cd = end; 2373 } else 2374 end = di->xs0cd; 2375 2376 2377 if (range == HNDDMA_RANGE_TRANSFERED) { 2378 active_desc = (uint16)((R_REG(di->osh, &dregs->status) & XS_AD_MASK) >> 2379 XS_AD_SHIFT); 2380 active_desc = (uint16)B2I(active_desc, dma32dd_t); 2381 if (end != active_desc) 2382 end = PREVTXD(active_desc); 2383 } 2384 } 2385 2386 if ((start == 0) && (end > di->txout)) 2387 goto bogus; 2388 2389 for (i = start; i != end && !txp; i = NEXTTXD(i)) { 2390 dmaaddr_t pa; 2391 hnddma_seg_map_t *map = NULL; 2392 uint size, j, nsegs; 2393 2394 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->txd32[i].addr)) - di->dataoffsetlow)); 2395 PHYSADDRHISET(pa, 0); 2396 2397 if (DMASGLIST_ENAB) { 2398 map = &di->txp_dmah[i]; 2399 size = map->origsize; 2400 nsegs = map->nsegs; 2401 } else { 2402 size = (BUS_SWAP32(R_SM(&di->txd32[i].ctrl)) & CTRL_BC_MASK); 2403 nsegs = 1; 2404 } 2405 2406 for (j = nsegs; j > 0; j--) { 2407 W_SM(&di->txd32[i].addr, 0xdeadbeef); 2408 2409 txp = di->txp[i]; 2410 di->txp[i] = NULL; 2411 if (j > 1) 2412 i = NEXTTXD(i); 2413 } 2414#ifdef BCM_SECURE_DMA 2415 SECURE_DMA_UNMAP(di->osh, pa, size, DMA_TX, NULL, NULL, &di->sec_cma_info_tx, 0); 2416#else 2417 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map); 2418#endif 2419 } 2420 2421 di->txin = i; 2422 2423 /* tx flow control */ 2424 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2425 2426 return (txp); 2427 2428bogus: 2429 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", 2430 start, end, di->txout, forceall)); 2431 return (NULL); 2432} 2433 2434static void * 2435dma32_getnextrxp(dma_info_t *di, bool forceall) 2436{ 2437 uint16 i, curr; 2438 void *rxp; 2439 dmaaddr_t pa; 2440 /* if forcing, dma engine must be disabled */ 2441 ASSERT(!forceall || !dma32_rxenabled(di)); 2442 2443 i = di->rxin; 2444 2445 /* return if no packets posted */ 2446 if (i == di->rxout) 2447 return (NULL); 2448 2449 if (di->rxin == di->rs0cd) { 2450 curr = (uint16)B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t); 2451 di->rs0cd = curr; 2452 } else 2453 curr = di->rs0cd; 2454 2455 /* ignore curr if forceall */ 2456 if (!forceall && (i == curr)) 2457 return (NULL); 2458 2459 /* get the packet pointer that corresponds to the rx descriptor */ 2460 rxp = di->rxp[i]; 2461 ASSERT(rxp); 2462 di->rxp[i] = NULL; 2463 2464 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->rxd32[i].addr)) - di->dataoffsetlow)); 2465 PHYSADDRHISET(pa, 0); 2466 2467 /* clear this packet from the descriptor ring */ 2468#ifdef BCM_SECURE_DMA 2469 SECURE_DMA_UNMAP(di->osh, pa, di->rxbufsize, DMA_RX, NULL, NULL, &di->sec_cma_info_rx, 0); 2470#else 2471 DMA_UNMAP(di->osh, pa, 2472 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]); 2473#endif 2474 2475 W_SM(&di->rxd32[i].addr, 0xdeadbeef); 2476 2477 di->rxin = NEXTRXD(i); 2478 2479 return (rxp); 2480} 2481 2482/** 2483 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin). 2484 */ 2485static void 2486dma32_txrotate(dma_info_t *di) 2487{ 2488 uint16 ad; 2489 uint nactive; 2490 uint rot; 2491 uint16 old, new; 2492 uint32 w; 2493 uint16 first, last; 2494 2495 ASSERT(dma32_txsuspendedidle(di)); 2496 2497 nactive = _dma_txactive(di); 2498 ad = B2I(((R_REG(di->osh, &di->d32txregs->status) & XS_AD_MASK) >> XS_AD_SHIFT), dma32dd_t); 2499 rot = TXD(ad - di->txin); 2500 2501 ASSERT(rot < di->ntxd); 2502 2503 /* full-ring case is a lot harder - don't worry about this */ 2504 if (rot >= (di->ntxd - nactive)) { 2505 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name)); 2506 return; 2507 } 2508 2509 first = di->txin; 2510 last = PREVTXD(di->txout); 2511 2512 /* move entries starting at last and moving backwards to first */ 2513 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) { 2514 new = TXD(old + rot); 2515 2516 /* 2517 * Move the tx dma descriptor. 2518 * EOT is set only in the last entry in the ring. 2519 */ 2520 w = BUS_SWAP32(R_SM(&di->txd32[old].ctrl)) & ~CTRL_EOT; 2521 if (new == (di->ntxd - 1)) 2522 w |= CTRL_EOT; 2523 W_SM(&di->txd32[new].ctrl, BUS_SWAP32(w)); 2524 W_SM(&di->txd32[new].addr, R_SM(&di->txd32[old].addr)); 2525 2526 /* zap the old tx dma descriptor address field */ 2527 W_SM(&di->txd32[old].addr, BUS_SWAP32(0xdeadbeef)); 2528 2529 /* move the corresponding txp[] entry */ 2530 ASSERT(di->txp[new] == NULL); 2531 di->txp[new] = di->txp[old]; 2532 2533 /* Move the segment map as well */ 2534 if (DMASGLIST_ENAB) { 2535 bcopy(&di->txp_dmah[old], &di->txp_dmah[new], sizeof(hnddma_seg_map_t)); 2536 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t)); 2537 } 2538 2539 di->txp[old] = NULL; 2540 } 2541 2542 /* update txin and txout */ 2543 di->txin = ad; 2544 di->txout = TXD(di->txout + rot); 2545 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2546 2547 /* kick the chip */ 2548 W_REG(di->osh, &di->d32txregs->ptr, I2B(di->txout, dma32dd_t)); 2549} 2550 2551/* 64-bit DMA functions */ 2552 2553static void 2554dma64_txinit(dma_info_t *di) 2555{ 2556 uint32 control; 2557 2558 DMA_TRACE(("%s: dma_txinit\n", di->name)); 2559 2560 if (di->ntxd == 0) 2561 return; 2562 2563 di->txin = di->txout = di->xs0cd = di->xs0cd_snapshot = 0; 2564 di->hnddma.txavail = di->ntxd - 1; 2565 2566 /* clear tx descriptor ring */ 2567 BZERO_SM((void *)(uintptr)di->txd64, (di->ntxd * sizeof(dma64dd_t))); 2568 2569 /* These bits 20:18 (burstLen) of control register can be written but will take 2570 * effect only if these bits are valid. So this will not affect previous versions 2571 * of the DMA. They will continue to have those bits set to 0. 2572 */ 2573 control = R_REG(di->osh, &di->d64txregs->control); 2574 control = (control & ~D64_XC_BL_MASK) | (di->txburstlen << D64_XC_BL_SHIFT); 2575 control = (control & ~D64_XC_MR_MASK) | (di->txmultioutstdrd << D64_XC_MR_SHIFT); 2576 control = (control & ~D64_XC_PC_MASK) | (di->txprefetchctl << D64_XC_PC_SHIFT); 2577 control = (control & ~D64_XC_PT_MASK) | (di->txprefetchthresh << D64_XC_PT_SHIFT); 2578 W_REG(di->osh, &di->d64txregs->control, control); 2579 2580 control = D64_XC_XE; 2581 /* DMA engine with out alignment requirement requires table to be inited 2582 * before enabling the engine 2583 */ 2584 if (!di->aligndesc_4k) 2585 _dma_ddtable_init(di, DMA_TX, di->txdpa); 2586 2587 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0) 2588 control |= D64_XC_PD; 2589 OR_REG(di->osh, &di->d64txregs->control, control); 2590 2591 /* DMA engine with alignment requirement requires table to be inited 2592 * before enabling the engine 2593 */ 2594 if (di->aligndesc_4k) 2595 _dma_ddtable_init(di, DMA_TX, di->txdpa); 2596} 2597 2598static bool 2599dma64_txenabled(dma_info_t *di) 2600{ 2601 uint32 xc; 2602 2603 /* If the chip is dead, it is not enabled :-) */ 2604 xc = R_REG(di->osh, &di->d64txregs->control); 2605 return ((xc != 0xffffffff) && (xc & D64_XC_XE)); 2606} 2607 2608static void 2609dma64_txsuspend(dma_info_t *di) 2610{ 2611 DMA_TRACE(("%s: dma_txsuspend\n", di->name)); 2612 2613 if (di->ntxd == 0) 2614 return; 2615 2616 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE); 2617} 2618 2619static void 2620dma64_txresume(dma_info_t *di) 2621{ 2622 DMA_TRACE(("%s: dma_txresume\n", di->name)); 2623 2624 if (di->ntxd == 0) 2625 return; 2626 2627 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_SE); 2628} 2629 2630static bool 2631dma64_txsuspended(dma_info_t *di) 2632{ 2633 return (di->ntxd == 0) || 2634 ((R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE) == D64_XC_SE); 2635} 2636 2637static void 2638dma64_txflush(dma_info_t *di) 2639{ 2640 DMA_TRACE(("%s: dma_txflush\n", di->name)); 2641 2642 if (di->ntxd == 0) 2643 return; 2644 2645 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE | D64_XC_FL); 2646} 2647 2648static void 2649dma64_txflush_clear(dma_info_t *di) 2650{ 2651 uint32 status; 2652 2653 DMA_TRACE(("%s: dma_txflush_clear\n", di->name)); 2654 2655 if (di->ntxd == 0) 2656 return; 2657 2658 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) != 2659 D64_XS0_XS_DISABLED) && 2660 (status != D64_XS0_XS_IDLE) && 2661 (status != D64_XS0_XS_STOPPED), 2662 10000); 2663 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_FL); 2664} 2665 2666static void BCMFASTPATH 2667dma64_txreclaim(dma_info_t *di, txd_range_t range) 2668{ 2669 void *p; 2670 2671 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, 2672 (range == HNDDMA_RANGE_ALL) ? "all" : 2673 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 2674 2675 if (di->txin == di->txout) 2676 return; 2677 2678 while ((p = dma64_getnexttxp(di, range))) { 2679 /* For unframed data, we don't have any packets to free */ 2680 if (!(di->hnddma.dmactrlflags & DMA_CTRL_UNFRAMED)) 2681 PKTFREE(di->osh, p, TRUE); 2682 } 2683} 2684 2685static bool 2686dma64_txstopped(dma_info_t *di) 2687{ 2688 return ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_STOPPED); 2689} 2690 2691static bool 2692dma64_rxstopped(dma_info_t *di) 2693{ 2694 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK) == D64_RS0_RS_STOPPED); 2695} 2696 2697static bool 2698dma64_alloc(dma_info_t *di, uint direction) 2699{ 2700 uint32 size; 2701 uint ddlen; 2702 void *va; 2703 uint alloced = 0; 2704 uint32 align; 2705 uint16 align_bits; 2706 2707 ddlen = sizeof(dma64dd_t); 2708 2709 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen); 2710 align_bits = di->dmadesc_align; 2711 align = (1 << align_bits); 2712 2713 if (direction == DMA_TX) { 2714 if ((va = dma_ringalloc(di->osh, 2715 (di->d64_xs0_cd_mask == 0x1fff) ? D64RINGBOUNDARY : D64RINGBOUNDARY_LARGE, 2716 size, &align_bits, &alloced, 2717 &di->txdpaorig, &di->tx_dmah)) == NULL) { 2718 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", 2719 di->name)); 2720 return FALSE; 2721 } 2722 align = (1 << align_bits); 2723 2724 /* adjust the pa by rounding up to the alignment */ 2725 PHYSADDRLOSET(di->txdpa, ROUNDUP(PHYSADDRLO(di->txdpaorig), align)); 2726 PHYSADDRHISET(di->txdpa, PHYSADDRHI(di->txdpaorig)); 2727 2728 /* Make sure that alignment didn't overflow */ 2729 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig)); 2730 2731 /* find the alignment offset that was used */ 2732 di->txdalign = (uint)(PHYSADDRLO(di->txdpa) - PHYSADDRLO(di->txdpaorig)); 2733 2734 /* adjust the va by the same offset */ 2735 di->txd64 = (dma64dd_t *)((uintptr)va + di->txdalign); 2736 2737 di->txdalloc = alloced; 2738 ASSERT(ISALIGNED(PHYSADDRLO(di->txdpa), align)); 2739 } else { 2740 if ((va = dma_ringalloc(di->osh, 2741 (di->d64_rs0_cd_mask == 0x1fff) ? D64RINGBOUNDARY : D64RINGBOUNDARY_LARGE, 2742 size, &align_bits, &alloced, 2743 &di->rxdpaorig, &di->rx_dmah)) == NULL) { 2744 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", 2745 di->name)); 2746 return FALSE; 2747 } 2748 align = (1 << align_bits); 2749 2750 /* adjust the pa by rounding up to the alignment */ 2751 PHYSADDRLOSET(di->rxdpa, ROUNDUP(PHYSADDRLO(di->rxdpaorig), align)); 2752 PHYSADDRHISET(di->rxdpa, PHYSADDRHI(di->rxdpaorig)); 2753 2754 /* Make sure that alignment didn't overflow */ 2755 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig)); 2756 2757 /* find the alignment offset that was used */ 2758 di->rxdalign = (uint)(PHYSADDRLO(di->rxdpa) - PHYSADDRLO(di->rxdpaorig)); 2759 2760 /* adjust the va by the same offset */ 2761 di->rxd64 = (dma64dd_t *)((uintptr)va + di->rxdalign); 2762 2763 di->rxdalloc = alloced; 2764 ASSERT(ISALIGNED(PHYSADDRLO(di->rxdpa), align)); 2765 } 2766 2767 return TRUE; 2768} 2769 2770static bool 2771dma64_txreset(dma_info_t *di) 2772{ 2773 uint32 status; 2774 2775 if (di->ntxd == 0) 2776 return TRUE; 2777 2778 /* suspend tx DMA first */ 2779 W_REG(di->osh, &di->d64txregs->control, D64_XC_SE); 2780 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) != 2781 D64_XS0_XS_DISABLED) && 2782 (status != D64_XS0_XS_IDLE) && 2783 (status != D64_XS0_XS_STOPPED), 2784 10000); 2785 2786 W_REG(di->osh, &di->d64txregs->control, 0); 2787 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) != 2788 D64_XS0_XS_DISABLED), 2789 10000); 2790 2791 /* We should be disabled at this point */ 2792 if (status != D64_XS0_XS_DISABLED) { 2793 DMA_ERROR(("%s: status != D64_XS0_XS_DISABLED 0x%x\n", __FUNCTION__, status)); 2794 ASSERT(status == D64_XS0_XS_DISABLED); 2795 OSL_DELAY(300); 2796 } 2797 2798 return (status == D64_XS0_XS_DISABLED); 2799} 2800 2801static bool 2802dma64_rxidle(dma_info_t *di) 2803{ 2804 DMA_TRACE(("%s: dma_rxidle\n", di->name)); 2805 2806 if (di->nrxd == 0) 2807 return TRUE; 2808 2809 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) == 2810 (R_REG(di->osh, &di->d64rxregs->ptr) & D64_RS0_CD_MASK)); 2811} 2812 2813static bool 2814dma64_rxreset(dma_info_t *di) 2815{ 2816 uint32 status; 2817 2818 if (di->nrxd == 0) 2819 return TRUE; 2820 2821 W_REG(di->osh, &di->d64rxregs->control, 0); 2822 SPINWAIT(((status = (R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK)) != 2823 D64_RS0_RS_DISABLED), 10000); 2824 2825 return (status == D64_RS0_RS_DISABLED); 2826} 2827 2828static bool 2829dma64_rxenabled(dma_info_t *di) 2830{ 2831 uint32 rc; 2832 2833 rc = R_REG(di->osh, &di->d64rxregs->control); 2834 return ((rc != 0xffffffff) && (rc & D64_RC_RE)); 2835} 2836 2837static bool 2838dma64_txsuspendedidle(dma_info_t *di) 2839{ 2840 2841 if (di->ntxd == 0) 2842 return TRUE; 2843 2844 if (!(R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE)) 2845 return 0; 2846 2847 if ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_IDLE) 2848 return 1; 2849 2850 return 0; 2851} 2852 2853/** 2854 * Useful when sending unframed data. This allows us to get a progress report from the DMA. 2855 * We return a pointer to the beginning of the data buffer of the current descriptor. 2856 * If DMA is idle, we return NULL. 2857 */ 2858static void* 2859dma64_getpos(dma_info_t *di, bool direction) 2860{ 2861 void *va; 2862 bool idle; 2863 uint16 cur_idx; 2864 2865 if (direction == DMA_TX) { 2866 cur_idx = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 2867 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 2868 idle = !NTXDACTIVE(di->txin, di->txout); 2869 va = di->txp[cur_idx]; 2870 } else { 2871 cur_idx = B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 2872 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 2873 idle = !NRXDACTIVE(di->rxin, di->rxout); 2874 va = di->rxp[cur_idx]; 2875 } 2876 2877 /* If DMA is IDLE, return NULL */ 2878 if (idle) { 2879 DMA_TRACE(("%s: DMA idle, return NULL\n", __FUNCTION__)); 2880 va = NULL; 2881 } 2882 2883 return va; 2884} 2885 2886/** 2887 * TX of unframed data 2888 * 2889 * Adds a DMA ring descriptor for the data pointed to by "buf". 2890 * This is for DMA of a buffer of data and is unlike other hnddma TX functions 2891 * that take a pointer to a "packet" 2892 * Each call to this is results in a single descriptor being added for "len" bytes of 2893 * data starting at "buf", it doesn't handle chained buffers. 2894 */ 2895static int 2896dma64_txunframed(dma_info_t *di, void *buf, uint len, bool commit) 2897{ 2898 uint16 txout; 2899 uint32 flags = 0; 2900 dmaaddr_t pa; /* phys addr */ 2901 2902 txout = di->txout; 2903 2904 /* return nonzero if out of tx descriptors */ 2905 if (NEXTTXD(txout) == di->txin) 2906 goto outoftxd; 2907 2908 if (len == 0) 2909 return 0; 2910#ifdef BCM_SECURE_DMA 2911 pa = SECURE_DMA_MAP(di->osh, buf, len, DMA_TX, NULL, NULL, &di->sec_cma_info_tx, 0); 2912#else 2913 pa = DMA_MAP(di->osh, buf, len, DMA_TX, NULL, &di->txp_dmah[txout]); 2914#endif 2915 2916 flags = (D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF); 2917 2918 if (txout == (di->ntxd - 1)) 2919 flags |= D64_CTRL1_EOT; 2920 2921 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len); 2922 ASSERT(di->txp[txout] == NULL); 2923 2924 /* save the buffer pointer - used by dma_getpos */ 2925 di->txp[txout] = buf; 2926 2927 txout = NEXTTXD(txout); 2928 /* bump the tx descriptor index */ 2929 di->txout = txout; 2930 2931 /* kick the chip */ 2932 if (commit) { 2933 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(txout, dma64dd_t)); 2934 } 2935 2936 /* tx flow control */ 2937 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2938 2939 return (0); 2940 2941outoftxd: 2942 DMA_ERROR(("%s: %s: out of txds !!!\n", di->name, __FUNCTION__)); 2943 di->hnddma.txavail = 0; 2944 di->hnddma.txnobuf++; 2945 return (-1); 2946} 2947 2948 2949/** 2950 * !! tx entry routine 2951 * WARNING: call must check the return value for error. 2952 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems 2953 */ 2954static int BCMFASTPATH 2955dma64_txfast(dma_info_t *di, void *p0, bool commit) 2956{ 2957 void *p, *next; 2958 uchar *data; 2959 uint len; 2960 uint16 txout; 2961 uint32 flags = 0; 2962 dmaaddr_t pa; 2963 bool war; 2964 2965 DMA_TRACE(("%s: dma_txfast\n", di->name)); 2966 2967 txout = di->txout; 2968 war = (di->hnddma.dmactrlflags & DMA_CTRL_DMA_AVOIDANCE_WAR) ? TRUE : FALSE; 2969 2970 /* 2971 * Walk the chain of packet buffers 2972 * allocating and initializing transmit descriptor entries. 2973 */ 2974 for (p = p0; p; p = next) { 2975 uint nsegs, j, segsadd; 2976 hnddma_seg_map_t *map = NULL; 2977 2978 data = PKTDATA(di->osh, p); 2979 len = PKTLEN(di->osh, p); 2980#ifdef BCM_DMAPAD 2981 len += PKTDMAPAD(di->osh, p); 2982#endif /* BCM_DMAPAD */ 2983 next = PKTNEXT(di->osh, p); 2984 2985 /* return nonzero if out of tx descriptors */ 2986 if (NEXTTXD(txout) == di->txin) 2987 goto outoftxd; 2988 2989 if (len == 0) 2990 continue; 2991 2992 /* get physical address of buffer start */ 2993 if (DMASGLIST_ENAB) 2994 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t)); 2995 2996#ifdef BCM_SECURE_DMA 2997 2998 if (DMASGLIST_ENAB) { 2999 pa = SECURE_DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout], 3000 &di->sec_cma_info_tx, 0); 3001 } 3002 else { 3003 pa = SECURE_DMA_MAP(di->osh, data, len, DMA_TX, NULL, NULL, 3004 &di->sec_cma_info_tx, 0); 3005 } 3006#else 3007 pa = DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]); 3008#endif 3009 3010 if (DMASGLIST_ENAB) { 3011 map = &di->txp_dmah[txout]; 3012 3013 /* See if all the segments can be accounted for */ 3014 if (map->nsegs > (uint)(di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1)) 3015 goto outoftxd; 3016 3017 nsegs = map->nsegs; 3018 } else 3019 nsegs = 1; 3020 3021 segsadd = 0; 3022 for (j = 1; j <= nsegs; j++) { 3023 flags = 0; 3024 if (p == p0 && j == 1) 3025 flags |= D64_CTRL1_SOF; 3026 3027 /* With a DMA segment list, Descriptor table is filled 3028 * using the segment list instead of looping over 3029 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when 3030 * end of segment list is reached. 3031 */ 3032 if ((!DMASGLIST_ENAB && next == NULL) || 3033 (DMASGLIST_ENAB && j == nsegs)) 3034 flags |= (D64_CTRL1_IOC | D64_CTRL1_EOF); 3035 if (txout == (di->ntxd - 1)) 3036 flags |= D64_CTRL1_EOT; 3037 3038 if (DMASGLIST_ENAB) { 3039 len = map->segs[j - 1].length; 3040 pa = map->segs[j - 1].addr; 3041 if (len > 128 && war) { 3042 uint remain, new_len, align64; 3043 /* check for 64B aligned of pa */ 3044 align64 = (uint)(PHYSADDRLO(pa) & 0x3f); 3045 align64 = (64 - align64) & 0x3f; 3046 new_len = len - align64; 3047 remain = new_len % 128; 3048 if (remain > 0 && remain <= 4) { 3049 uint32 buf_addr_lo; 3050 uint32 tmp_flags = 3051 flags & (~(D64_CTRL1_EOF | D64_CTRL1_IOC)); 3052 flags &= ~(D64_CTRL1_SOF | D64_CTRL1_EOT); 3053 remain += 64; 3054 dma64_dd_upd(di, di->txd64, pa, txout, 3055 &tmp_flags, len-remain); 3056 ASSERT(di->txp[txout] == NULL); 3057 txout = NEXTTXD(txout); 3058 /* return nonzero if out of tx descriptors */ 3059 if (txout == di->txin) { 3060 DMA_ERROR(("%s: dma_txfast: Out-of-DMA" 3061 " descriptors (txin %d txout %d" 3062 " nsegs %d)\n", __FUNCTION__, 3063 di->txin, di->txout, nsegs)); 3064 goto outoftxd; 3065 } 3066 if (txout == (di->ntxd - 1)) 3067 flags |= D64_CTRL1_EOT; 3068 buf_addr_lo = PHYSADDRLO(pa); 3069 PHYSADDRLOSET(pa, (PHYSADDRLO(pa) + (len-remain))); 3070 if (PHYSADDRLO(pa) < buf_addr_lo) { 3071 PHYSADDRHISET(pa, (PHYSADDRHI(pa) + 1)); 3072 } 3073 len = remain; 3074 segsadd++; 3075 di->dma_avoidance_cnt++; 3076 } 3077 } 3078 } 3079 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len); 3080 ASSERT(di->txp[txout] == NULL); 3081 3082 txout = NEXTTXD(txout); 3083 /* return nonzero if out of tx descriptors */ 3084 if (txout == di->txin) { 3085 DMA_ERROR(("%s: dma_txfast: Out-of-DMA descriptors" 3086 " (txin %d txout %d nsegs %d)\n", __FUNCTION__, 3087 di->txin, di->txout, nsegs)); 3088 goto outoftxd; 3089 } 3090 } 3091 if (segsadd && DMASGLIST_ENAB) 3092 map->nsegs += segsadd; 3093 3094 /* See above. No need to loop over individual buffers */ 3095 if (DMASGLIST_ENAB) 3096 break; 3097 } 3098 3099 /* if last txd eof not set, fix it */ 3100 if (!(flags & D64_CTRL1_EOF)) 3101 W_SM(&di->txd64[PREVTXD(txout)].ctrl1, 3102 BUS_SWAP32(flags | D64_CTRL1_IOC | D64_CTRL1_EOF)); 3103 3104 /* save the packet */ 3105 di->txp[PREVTXD(txout)] = p0; 3106 3107 /* bump the tx descriptor index */ 3108 di->txout = txout; 3109 3110 /* kick the chip */ 3111 if (commit) 3112 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(txout, dma64dd_t)); 3113 3114 /* tx flow control */ 3115 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 3116 3117 return (0); 3118 3119outoftxd: 3120 DMA_ERROR(("%s: dma_txfast: out of txds !!!\n", di->name)); 3121 PKTFREE(di->osh, p0, TRUE); 3122 di->hnddma.txavail = 0; 3123 di->hnddma.txnobuf++; 3124 return (-1); 3125} 3126 3127/** 3128 * Reclaim next completed txd (txds if using chained buffers) in the range 3129 * specified and return associated packet. 3130 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be 3131 * transmitted as noted by the hardware "CurrDescr" pointer. 3132 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be 3133 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer. 3134 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and 3135 * return associated packet regardless of the value of hardware pointers. 3136 */ 3137static void * BCMFASTPATH 3138dma64_getnexttxp(dma_info_t *di, txd_range_t range) 3139{ 3140 uint16 start, end, i; 3141 uint16 active_desc; 3142 void *txp; 3143 3144 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, 3145 (range == HNDDMA_RANGE_ALL) ? "all" : 3146 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 3147 3148 if (di->ntxd == 0) 3149 return (NULL); 3150 3151 txp = NULL; 3152 3153 start = di->txin; 3154 if (range == HNDDMA_RANGE_ALL) 3155 end = di->txout; 3156 else { 3157 dma64regs_t *dregs = di->d64txregs; 3158 3159 if (di->txin == di->xs0cd) { 3160 end = (uint16)(B2I(((R_REG(di->osh, &dregs->status0) & D64_XS0_CD_MASK) - 3161 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t)); 3162 di->xs0cd = end; 3163 } else 3164 end = di->xs0cd; 3165 3166 if (range == HNDDMA_RANGE_TRANSFERED) { 3167 active_desc = (uint16)(R_REG(di->osh, &dregs->status1) & D64_XS1_AD_MASK); 3168 active_desc = (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK; 3169 active_desc = B2I(active_desc, dma64dd_t); 3170 if (end != active_desc) 3171 end = PREVTXD(active_desc); 3172 } 3173 } 3174 3175 3176 if ((start == 0) && (end > di->txout)) 3177 goto bogus; 3178 3179 for (i = start; i != end && !txp; i = NEXTTXD(i)) { 3180 hnddma_seg_map_t *map = NULL; 3181 uint size, j, nsegs; 3182 3183#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) || \ 3184 defined(BCM_SECURE_DMA) 3185 dmaaddr_t pa; 3186 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->txd64[i].addrlow)) - di->dataoffsetlow)); 3187 PHYSADDRHISET(pa, (BUS_SWAP32(R_SM(&di->txd64[i].addrhigh)) - di->dataoffsethigh)); 3188#endif 3189 3190 if (DMASGLIST_ENAB) { 3191 map = &di->txp_dmah[i]; 3192 size = map->origsize; 3193 nsegs = map->nsegs; 3194 if (nsegs > (uint)NTXDACTIVE(i, end)) { 3195 di->xs0cd = i; 3196 break; 3197 } 3198 } else { 3199#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) || \ 3200 defined(BCM_SECURE_DMA) 3201 size = (BUS_SWAP32(R_SM(&di->txd64[i].ctrl2)) & D64_CTRL2_BC_MASK); 3202#endif 3203 nsegs = 1; 3204 } 3205 3206 for (j = nsegs; j > 0; j--) { 3207#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3208 W_SM(&di->txd64[i].addrlow, 0xdeadbeef); 3209 W_SM(&di->txd64[i].addrhigh, 0xdeadbeef); 3210#endif 3211 3212 txp = di->txp[i]; 3213 di->txp[i] = NULL; 3214 if (j > 1) 3215 i = NEXTTXD(i); 3216 } 3217 3218#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3219 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map); 3220#endif 3221 3222#ifdef BCM_SECURE_DMA 3223 SECURE_DMA_UNMAP(di->osh, pa, size, DMA_TX, NULL, NULL, &di->sec_cma_info_tx, 0); 3224#endif 3225 } 3226 3227 di->txin = i; 3228 3229 /* tx flow control */ 3230 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 3231 3232 return (txp); 3233 3234bogus: 3235 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", 3236 start, end, di->txout, forceall)); 3237 return (NULL); 3238} 3239 3240static void * BCMFASTPATH 3241dma64_getnextrxp(dma_info_t *di, bool forceall) 3242{ 3243 uint16 i, curr; 3244 void *rxp; 3245#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) || \ 3246 defined(BCM_SECURE_DMA) 3247 dmaaddr_t pa; 3248#endif 3249 3250 /* if forcing, dma engine must be disabled */ 3251 ASSERT(!forceall || !dma64_rxenabled(di)); 3252 3253 i = di->rxin; 3254 3255 /* return if no packets posted */ 3256 if (i == di->rxout) 3257 return (NULL); 3258 3259 if (di->rxin == di->rs0cd) { 3260 curr = (uint16)B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 3261 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 3262 di->rs0cd = curr; 3263 } else 3264 curr = di->rs0cd; 3265 3266 /* ignore curr if forceall */ 3267 if (!forceall && (i == curr)) 3268 return (NULL); 3269 3270 /* get the packet pointer that corresponds to the rx descriptor */ 3271 rxp = di->rxp[i]; 3272 ASSERT(rxp); 3273 di->rxp[i] = NULL; 3274 3275#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) || \ 3276 defined(BCM_SECURE_DMA) 3277 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->rxd64[i].addrlow)) - di->dataoffsetlow)); 3278 PHYSADDRHISET(pa, (BUS_SWAP32(R_SM(&di->rxd64[i].addrhigh)) - di->dataoffsethigh)); 3279 3280 /* clear this packet from the descriptor ring */ 3281#ifdef BCM_SECURE_DMA 3282 SECURE_DMA_UNMAP(di->osh, pa, di->rxbufsize, DMA_RX, NULL, NULL, &di->sec_cma_info_rx, 0); 3283#else 3284 DMA_UNMAP(di->osh, pa, 3285 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]); 3286#endif 3287 3288 W_SM(&di->rxd64[i].addrlow, 0xdeadbeef); 3289 W_SM(&di->rxd64[i].addrhigh, 0xdeadbeef); 3290#endif /* ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) */ 3291 3292 di->rxin = NEXTRXD(i); 3293 3294 return (rxp); 3295} 3296 3297static bool 3298_dma64_addrext(osl_t *osh, dma64regs_t *dma64regs) 3299{ 3300 uint32 w; 3301 OR_REG(osh, &dma64regs->control, D64_XC_AE); 3302 w = R_REG(osh, &dma64regs->control); 3303 AND_REG(osh, &dma64regs->control, ~D64_XC_AE); 3304 return ((w & D64_XC_AE) == D64_XC_AE); 3305} 3306 3307/** 3308 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin). 3309 */ 3310static void 3311dma64_txrotate(dma_info_t *di) 3312{ 3313 uint16 ad; 3314 uint nactive; 3315 uint rot; 3316 uint16 old, new; 3317 uint32 w; 3318 uint16 first, last; 3319 3320 ASSERT(dma64_txsuspendedidle(di)); 3321 3322 nactive = _dma_txactive(di); 3323 ad = B2I((((R_REG(di->osh, &di->d64txregs->status1) & D64_XS1_AD_MASK) 3324 - di->xmtptrbase) & D64_XS1_AD_MASK), dma64dd_t); 3325 rot = TXD(ad - di->txin); 3326 3327 ASSERT(rot < di->ntxd); 3328 3329 /* full-ring case is a lot harder - don't worry about this */ 3330 if (rot >= (di->ntxd - nactive)) { 3331 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name)); 3332 return; 3333 } 3334 3335 first = di->txin; 3336 last = PREVTXD(di->txout); 3337 3338 /* move entries starting at last and moving backwards to first */ 3339 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) { 3340 new = TXD(old + rot); 3341 3342 /* 3343 * Move the tx dma descriptor. 3344 * EOT is set only in the last entry in the ring. 3345 */ 3346 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl1)) & ~D64_CTRL1_EOT; 3347 if (new == (di->ntxd - 1)) 3348 w |= D64_CTRL1_EOT; 3349 W_SM(&di->txd64[new].ctrl1, BUS_SWAP32(w)); 3350 3351 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl2)); 3352 W_SM(&di->txd64[new].ctrl2, BUS_SWAP32(w)); 3353 3354 W_SM(&di->txd64[new].addrlow, R_SM(&di->txd64[old].addrlow)); 3355 W_SM(&di->txd64[new].addrhigh, R_SM(&di->txd64[old].addrhigh)); 3356 3357 /* zap the old tx dma descriptor address field */ 3358 W_SM(&di->txd64[old].addrlow, BUS_SWAP32(0xdeadbeef)); 3359 W_SM(&di->txd64[old].addrhigh, BUS_SWAP32(0xdeadbeef)); 3360 3361 /* move the corresponding txp[] entry */ 3362 ASSERT(di->txp[new] == NULL); 3363 di->txp[new] = di->txp[old]; 3364 3365 /* Move the map */ 3366 if (DMASGLIST_ENAB) { 3367 bcopy(&di->txp_dmah[old], &di->txp_dmah[new], sizeof(hnddma_seg_map_t)); 3368 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t)); 3369 } 3370 3371 di->txp[old] = NULL; 3372 } 3373 3374 /* update txin and txout */ 3375 di->txin = ad; 3376 di->txout = TXD(di->txout + rot); 3377 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 3378 3379 /* kick the chip */ 3380 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(di->txout, dma64dd_t)); 3381} 3382 3383uint 3384BCMATTACHFN(dma_addrwidth)(si_t *sih, void *dmaregs) 3385{ 3386 dma32regs_t *dma32regs; 3387 osl_t *osh; 3388 3389 osh = si_osh(sih); 3390 3391 /* Perform 64-bit checks only if we want to advertise 64-bit (> 32bit) capability) */ 3392 /* DMA engine is 64-bit capable */ 3393 if ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64) { 3394 /* backplane are 64-bit capable */ 3395 if (si_backplane64(sih)) 3396 /* If bus is System Backplane or PCIE then we can access 64-bits */ 3397 if ((BUSTYPE(sih->bustype) == SI_BUS) || 3398 ((BUSTYPE(sih->bustype) == PCI_BUS) && 3399 ((sih->buscoretype == PCIE_CORE_ID) || 3400 (sih->buscoretype == PCIE2_CORE_ID)))) 3401 return (DMADDRWIDTH_64); 3402 3403 /* DMA64 is always 32-bit capable, AE is always TRUE */ 3404 ASSERT(_dma64_addrext(osh, (dma64regs_t *)dmaregs)); 3405 3406 return (DMADDRWIDTH_32); 3407 } 3408 3409 /* Start checking for 32-bit / 30-bit addressing */ 3410 dma32regs = (dma32regs_t *)dmaregs; 3411 3412 /* For System Backplane, PCIE bus or addrext feature, 32-bits ok */ 3413 if ((BUSTYPE(sih->bustype) == SI_BUS) || 3414 ((BUSTYPE(sih->bustype) == PCI_BUS) && 3415 ((sih->buscoretype == PCIE_CORE_ID) || 3416 (sih->buscoretype == PCIE2_CORE_ID))) || 3417 (_dma32_addrext(osh, dma32regs))) 3418 return (DMADDRWIDTH_32); 3419 3420 /* Fallthru */ 3421 return (DMADDRWIDTH_30); 3422} 3423 3424static int 3425_dma_pktpool_set(dma_info_t *di, pktpool_t *pool) 3426{ 3427 ASSERT(di); 3428 ASSERT(di->pktpool == NULL); 3429 di->pktpool = pool; 3430 return 0; 3431} 3432 3433static bool 3434_dma_rxtx_error(dma_info_t *di, bool istx) 3435{ 3436 uint32 status1 = 0; 3437 uint16 curr; 3438 3439 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 3440 3441 if (istx) { 3442 3443 status1 = R_REG(di->osh, &di->d64txregs->status1); 3444 3445 if ((status1 & D64_XS1_XE_MASK) != D64_XS1_XE_NOERR) 3446 return TRUE; 3447 else if ((si_coreid(di->sih) == GMAC_CORE_ID && si_corerev(di->sih) >= 4) || 3448 (si_coreid(di->sih) == D11_CORE_ID)) { //cathy add D11_CORE_ID 3449 curr = (uint16)(B2I(((R_REG(di->osh, &di->d64txregs->status0) & 3450 D64_XS0_CD_MASK) - di->xmtptrbase) & 3451 D64_XS0_CD_MASK, dma64dd_t)); 3452 3453 if (NTXDACTIVE(di->txin, di->txout) != 0 && 3454 curr == di->xs0cd_snapshot) { 3455 3456 /* suspicious */ 3457 return TRUE; 3458 } 3459 di->xs0cd_snapshot = di->xs0cd = curr; 3460 3461 return FALSE; 3462 } 3463 else 3464 return FALSE; 3465 } 3466 else { 3467 3468 status1 = R_REG(di->osh, &di->d64rxregs->status1); 3469 3470 if ((status1 & D64_RS1_RE_MASK) != D64_RS1_RE_NOERR) 3471 return TRUE; 3472 else 3473 return FALSE; 3474 } 3475 3476 } else if (DMA32_ENAB(di)) { 3477 return FALSE; 3478 3479 } else { 3480 ASSERT(0); 3481 return FALSE; 3482 } 3483 3484} 3485 3486void 3487_dma_burstlen_set(dma_info_t *di, uint8 rxburstlen, uint8 txburstlen) 3488{ 3489 di->rxburstlen = rxburstlen; 3490 di->txburstlen = txburstlen; 3491} 3492 3493void 3494_dma_param_set(dma_info_t *di, uint16 paramid, uint16 paramval) 3495{ 3496 switch (paramid) { 3497 case HNDDMA_PID_TX_MULTI_OUTSTD_RD: 3498 di->txmultioutstdrd = (uint8)paramval; 3499 break; 3500 3501 case HNDDMA_PID_TX_PREFETCH_CTL: 3502 di->txprefetchctl = (uint8)paramval; 3503 break; 3504 3505 case HNDDMA_PID_TX_PREFETCH_THRESH: 3506 di->txprefetchthresh = (uint8)paramval; 3507 break; 3508 3509 case HNDDMA_PID_TX_BURSTLEN: 3510 di->txburstlen = (uint8)paramval; 3511 break; 3512 3513 case HNDDMA_PID_RX_PREFETCH_CTL: 3514 di->rxprefetchctl = (uint8)paramval; 3515 break; 3516 3517 case HNDDMA_PID_RX_PREFETCH_THRESH: 3518 di->rxprefetchthresh = (uint8)paramval; 3519 break; 3520 3521 case HNDDMA_PID_RX_BURSTLEN: 3522 di->rxburstlen = (uint8)paramval; 3523 break; 3524 3525 default: 3526 break; 3527 } 3528} 3529 3530static bool 3531_dma_glom_enable(dma_info_t *di, uint32 val) 3532{ 3533 dma64regs_t *dregs = di->d64rxregs; 3534 bool ret = TRUE; 3535 if (val) { 3536 OR_REG(di->osh, &dregs->control, D64_RC_GE); 3537 if (!(R_REG(di->osh, &dregs->control) & D64_RC_GE)) 3538 ret = FALSE; 3539 } else { 3540 AND_REG(di->osh, &dregs->control, ~D64_RC_GE); 3541 } 3542 return ret; 3543} 3544