1/* 2 * Generic Broadcom Home Networking Division (HND) DMA module. 3 * This supports the following chips: BCM42xx, 44xx, 47xx . 4 * 5 * Copyright (C) 2013, Broadcom Corporation. All Rights Reserved. 6 * 7 * Permission to use, copy, modify, and/or distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 * 19 * $Id: hnddma.c 401759 2013-05-13 16:08:08Z $ 20 */ 21 22#include <bcm_cfg.h> 23#include <typedefs.h> 24#include <bcmdefs.h> 25#include <bcmdevs.h> 26#include <osl.h> 27#include <bcmendian.h> 28#include <hndsoc.h> 29#include <bcmutils.h> 30#include <siutils.h> 31 32#include <sbhnddma.h> 33#include <hnddma.h> 34 35/* debug/trace */ 36#define DMA_ERROR(args) 37#define DMA_TRACE(args) 38 39#define DMA_NONE(args) 40 41 42#define d32txregs dregs.d32_u.txregs_32 43#define d32rxregs dregs.d32_u.rxregs_32 44#define txd32 dregs.d32_u.txd_32 45#define rxd32 dregs.d32_u.rxd_32 46 47#define d64txregs dregs.d64_u.txregs_64 48#define d64rxregs dregs.d64_u.rxregs_64 49#define txd64 dregs.d64_u.txd_64 50#define rxd64 dregs.d64_u.rxd_64 51/* Foxconn added start pling 12/04/2007 */ 52#if (defined BCM4716) || (defined BCM5325E) 53int dma_qos_enable = 0; 54#endif 55/* Foxconn added end pling 12/04/2007 */ 56 57/* default dma message level (if input msg_level pointer is null in dma_attach()) */ 58static uint dma_msg_level = 59 0; 60 61#define MAXNAMEL 8 /* 8 char names */ 62 63#define DI_INFO(dmah) ((dma_info_t *)dmah) 64 65/** dma engine software state */ 66typedef struct dma_info { 67 struct hnddma_pub hnddma; /* exported structure, don't use hnddma_t, 68 * which could be const 69 */ 70 uint *msg_level; /* message level pointer */ 71 char name[MAXNAMEL]; /* callers name for diag msgs */ 72 73 void *osh; /* os handle */ 74 si_t *sih; /* sb handle */ 75 76 bool dma64; /* this dma engine is operating in 64-bit mode */ 77 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */ 78 79 union { 80 struct { 81 dma32regs_t *txregs_32; /* 32-bit dma tx engine registers */ 82 dma32regs_t *rxregs_32; /* 32-bit dma rx engine registers */ 83 dma32dd_t *txd_32; /* pointer to dma32 tx descriptor ring */ 84 dma32dd_t *rxd_32; /* pointer to dma32 rx descriptor ring */ 85 } d32_u; 86 struct { 87 dma64regs_t *txregs_64; /* 64-bit dma tx engine registers */ 88 dma64regs_t *rxregs_64; /* 64-bit dma rx engine registers */ 89 dma64dd_t *txd_64; /* pointer to dma64 tx descriptor ring */ 90 dma64dd_t *rxd_64; /* pointer to dma64 rx descriptor ring */ 91 } d64_u; 92 } dregs; 93 94 uint16 dmadesc_align; /* alignment requirement for dma descriptors */ 95 96 uint16 ntxd; /* # tx descriptors tunable */ 97 uint16 txin; /* index of next descriptor to reclaim */ 98 uint16 txout; /* index of next descriptor to post */ 99 void **txp; /* pointer to parallel array of pointers to packets */ 100 osldma_t *tx_dmah; /* DMA TX descriptor ring handle */ 101 hnddma_seg_map_t *txp_dmah; /* DMA MAP meta-data handle */ 102 dmaaddr_t txdpa; /* Aligned physical address of descriptor ring */ 103 dmaaddr_t txdpaorig; /* Original physical address of descriptor ring */ 104 uint16 txdalign; /* #bytes added to alloc'd mem to align txd */ 105 uint32 txdalloc; /* #bytes allocated for the ring */ 106 uint32 xmtptrbase; /* When using unaligned descriptors, the ptr register 107 * is not just an index, it needs all 13 bits to be 108 * an offset from the addr register. 109 */ 110 111 uint16 nrxd; /* # rx descriptors tunable */ 112 uint16 rxin; /* index of next descriptor to reclaim */ 113 uint16 rxout; /* index of next descriptor to post */ 114 void **rxp; /* pointer to parallel array of pointers to packets */ 115 osldma_t *rx_dmah; /* DMA RX descriptor ring handle */ 116 hnddma_seg_map_t *rxp_dmah; /* DMA MAP meta-data handle */ 117 dmaaddr_t rxdpa; /* Aligned physical address of descriptor ring */ 118 dmaaddr_t rxdpaorig; /* Original physical address of descriptor ring */ 119 uint16 rxdalign; /* #bytes added to alloc'd mem to align rxd */ 120 uint32 rxdalloc; /* #bytes allocated for the ring */ 121 uint32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */ 122 123 /* tunables */ 124 uint16 rxbufsize; /* rx buffer size in bytes, 125 * not including the extra headroom 126 */ 127 uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper stack 128 * e.g. some rx pkt buffers will be bridged to tx side 129 * without byte copying. The extra headroom needs to be 130 * large enough to fit txheader needs. 131 * Some dongle driver may not need it. 132 */ 133 uint nrxpost; /* # rx buffers to keep posted */ 134 uint rxoffset; /* rxcontrol offset */ 135 uint ddoffsetlow; /* add to get dma address of descriptor ring, low 32 bits */ 136 uint ddoffsethigh; /* high 32 bits */ 137 uint dataoffsetlow; /* add to get dma address of data buffer, low 32 bits */ 138 uint dataoffsethigh; /* high 32 bits */ 139 bool aligndesc_4k; /* descriptor base need to be aligned or not */ 140 uint8 rxburstlen; /* burstlen field for rx (for cores supporting burstlen) */ 141 uint8 txburstlen; /* burstlen field for tx (for cores supporting burstlen) */ 142 uint8 txmultioutstdrd; /* tx multiple outstanding reads */ 143 uint8 txprefetchctl; /* prefetch control for tx */ 144 uint8 txprefetchthresh; /* prefetch threshold for tx */ 145 uint8 rxprefetchctl; /* prefetch control for rx */ 146 uint8 rxprefetchthresh; /* prefetch threshold for rx */ 147 pktpool_t *pktpool; /* pktpool */ 148 uint dma_avoidance_cnt; 149 150 uint32 d64_xs0_cd_mask; /* tx current descriptor pointer mask */ 151 uint32 d64_xs1_ad_mask; /* tx active descriptor mask */ 152 uint32 d64_rs0_cd_mask; /* rx current descriptor pointer mask */ 153 uint16 rs0cd; /* cached value of rcvstatus0 currdescr */ 154 uint16 xs0cd; /* cached value of xmtstatus0 currdescr */ 155} dma_info_t; 156 157/* 158 * If BCMDMA32 is defined, hnddma will support both 32-bit and 64-bit DMA engines. 159 * Otherwise it will support only 64-bit. 160 * 161 * DMA32_ENAB indicates whether hnddma is compiled with support for 32-bit DMA engines. 162 * DMA64_ENAB indicates whether hnddma is compiled with support for 64-bit DMA engines. 163 * 164 * DMA64_MODE indicates whether the current DMA engine is running as 64-bit. 165 */ 166#ifdef BCMDMA32 167#define DMA32_ENAB(di) 1 168#define DMA64_ENAB(di) 1 169#define DMA64_MODE(di) ((di)->dma64) 170#else /* !BCMDMA32 */ 171#define DMA32_ENAB(di) 0 172#define DMA64_ENAB(di) 1 173#define DMA64_MODE(di) 1 174#endif /* !BCMDMA32 */ 175 176/* DMA Scatter-gather list is supported. Note this is limited to TX direction only */ 177#ifdef BCMDMASGLISTOSL 178#define DMASGLIST_ENAB TRUE 179#else 180#define DMASGLIST_ENAB FALSE 181#endif /* BCMDMASGLISTOSL */ 182 183/* descriptor bumping macros */ 184#define XXD(x, n) ((x) & ((n) - 1)) /* faster than %, but n must be power of 2 */ 185#define TXD(x) XXD((x), di->ntxd) 186#define RXD(x) XXD((x), di->nrxd) 187#define NEXTTXD(i) TXD((i) + 1) 188#define PREVTXD(i) TXD((i) - 1) 189#define NEXTRXD(i) RXD((i) + 1) 190#define PREVRXD(i) RXD((i) - 1) 191 192#define NTXDACTIVE(h, t) TXD((t) - (h)) 193#define NRXDACTIVE(h, t) RXD((t) - (h)) 194 195/* macros to convert between byte offsets and indexes */ 196#define B2I(bytes, type) ((uint16)((bytes) / sizeof(type))) 197#define I2B(index, type) ((index) * sizeof(type)) 198 199#define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */ 200#define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */ 201 202#define PCI64ADDR_HIGH 0x80000000 /* address[63] */ 203#define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */ 204 205/* Common prototypes */ 206static bool _dma_isaddrext(dma_info_t *di); 207static bool _dma_descriptor_align(dma_info_t *di); 208static bool _dma_alloc(dma_info_t *di, uint direction); 209static void _dma_detach(dma_info_t *di); 210static void _dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa); 211static void _dma_rxinit(dma_info_t *di); 212static void *_dma_rx(dma_info_t *di); 213static bool _dma_rxfill(dma_info_t *di); 214static void _dma_rxreclaim(dma_info_t *di); 215static void _dma_rxenable(dma_info_t *di); 216static void *_dma_getnextrxp(dma_info_t *di, bool forceall); 217static void _dma_rx_param_get(dma_info_t *di, uint16 *rxoffset, uint16 *rxbufsize); 218 219static void _dma_txblock(dma_info_t *di); 220static void _dma_txunblock(dma_info_t *di); 221static uint _dma_txactive(dma_info_t *di); 222static uint _dma_rxactive(dma_info_t *di); 223static uint _dma_activerxbuf(dma_info_t *di); 224static uint _dma_txpending(dma_info_t *di); 225static uint _dma_txcommitted(dma_info_t *di); 226 227static void *_dma_peeknexttxp(dma_info_t *di); 228static int _dma_peekntxp(dma_info_t *di, int *len, void *txps[], txd_range_t range); 229static void *_dma_peeknextrxp(dma_info_t *di); 230static uintptr _dma_getvar(dma_info_t *di, const char *name); 231static void _dma_counterreset(dma_info_t *di); 232static void _dma_fifoloopbackenable(dma_info_t *di); 233static uint _dma_ctrlflags(dma_info_t *di, uint mask, uint flags); 234static uint8 dma_align_sizetobits(uint size); 235static void *dma_ringalloc(osl_t *osh, uint32 boundary, uint size, uint16 *alignbits, uint* alloced, 236 dmaaddr_t *descpa, osldma_t **dmah); 237static int _dma_pktpool_set(dma_info_t *di, pktpool_t *pool); 238static bool _dma_rxtx_error(dma_info_t *di, bool istx); 239static void _dma_burstlen_set(dma_info_t *di, uint8 rxburstlen, uint8 txburstlen); 240static uint _dma_avoidancecnt(dma_info_t *di); 241static void _dma_param_set(dma_info_t *di, uint16 paramid, uint16 paramval); 242static bool _dma_glom_enable(dma_info_t *di, uint32 val); 243 244 245/* Prototypes for 32-bit routines */ 246static bool dma32_alloc(dma_info_t *di, uint direction); 247static bool dma32_txreset(dma_info_t *di); 248static bool dma32_rxreset(dma_info_t *di); 249static bool dma32_txsuspendedidle(dma_info_t *di); 250static int dma32_txfast(dma_info_t *di, void *p0, bool commit); 251static void *dma32_getnexttxp(dma_info_t *di, txd_range_t range); 252static void *dma32_getnextrxp(dma_info_t *di, bool forceall); 253static void dma32_txrotate(dma_info_t *di); 254static bool dma32_rxidle(dma_info_t *di); 255static void dma32_txinit(dma_info_t *di); 256static bool dma32_txenabled(dma_info_t *di); 257static void dma32_txsuspend(dma_info_t *di); 258static void dma32_txresume(dma_info_t *di); 259static bool dma32_txsuspended(dma_info_t *di); 260static void dma32_txflush(dma_info_t *di); 261static void dma32_txflush_clear(dma_info_t *di); 262static void dma32_txreclaim(dma_info_t *di, txd_range_t range); 263static bool dma32_txstopped(dma_info_t *di); 264static bool dma32_rxstopped(dma_info_t *di); 265static bool dma32_rxenabled(dma_info_t *di); 266#if defined(BCMDBG_DUMP) 267static void dma32_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma32dd_t *ring, uint start, 268 uint end, uint max_num); 269static void dma32_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 270static void dma32_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 271static void dma32_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 272#endif 273 274static bool _dma32_addrext(osl_t *osh, dma32regs_t *dma32regs); 275 276/* Prototypes for 64-bit routines */ 277static bool dma64_alloc(dma_info_t *di, uint direction); 278static bool dma64_txreset(dma_info_t *di); 279static bool dma64_rxreset(dma_info_t *di); 280static bool dma64_txsuspendedidle(dma_info_t *di); 281static int dma64_txfast(dma_info_t *di, void *p0, bool commit); 282static int dma64_txunframed(dma_info_t *di, void *p0, uint len, bool commit); 283static void *dma64_getpos(dma_info_t *di, bool direction); 284static void *dma64_getnexttxp(dma_info_t *di, txd_range_t range); 285static void *dma64_getnextrxp(dma_info_t *di, bool forceall); 286static void dma64_txrotate(dma_info_t *di); 287 288static bool dma64_rxidle(dma_info_t *di); 289static void dma64_txinit(dma_info_t *di); 290static bool dma64_txenabled(dma_info_t *di); 291static void dma64_txsuspend(dma_info_t *di); 292static void dma64_txresume(dma_info_t *di); 293static bool dma64_txsuspended(dma_info_t *di); 294static void dma64_txflush(dma_info_t *di); 295static void dma64_txflush_clear(dma_info_t *di); 296static void dma64_txreclaim(dma_info_t *di, txd_range_t range); 297static bool dma64_txstopped(dma_info_t *di); 298static bool dma64_rxstopped(dma_info_t *di); 299static bool dma64_rxenabled(dma_info_t *di); 300static bool _dma64_addrext(osl_t *osh, dma64regs_t *dma64regs); 301 302 303STATIC INLINE uint32 parity32(uint32 data); 304 305#if defined(BCMDBG_DUMP) 306static void dma64_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma64dd_t *ring, uint start, 307 uint end, uint max_num); 308static void dma64_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 309static void dma64_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 310static void dma64_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring); 311#endif 312 313 314const di_fcn_t dma64proc = { 315 (di_detach_t)_dma_detach, 316 (di_txinit_t)dma64_txinit, 317 (di_txreset_t)dma64_txreset, 318 (di_txenabled_t)dma64_txenabled, 319 (di_txsuspend_t)dma64_txsuspend, 320 (di_txresume_t)dma64_txresume, 321 (di_txsuspended_t)dma64_txsuspended, 322 (di_txsuspendedidle_t)dma64_txsuspendedidle, 323 (di_txflush_t)dma64_txflush, 324 (di_txflush_clear_t)dma64_txflush_clear, 325 (di_txfast_t)dma64_txfast, 326 (di_txunframed_t)dma64_txunframed, 327 (di_getpos_t)dma64_getpos, 328 (di_txstopped_t)dma64_txstopped, 329 (di_txreclaim_t)dma64_txreclaim, 330 (di_getnexttxp_t)dma64_getnexttxp, 331 (di_peeknexttxp_t)_dma_peeknexttxp, 332 (di_peekntxp_t)_dma_peekntxp, 333 (di_txblock_t)_dma_txblock, 334 (di_txunblock_t)_dma_txunblock, 335 (di_txactive_t)_dma_txactive, 336 (di_txrotate_t)dma64_txrotate, 337 338 (di_rxinit_t)_dma_rxinit, 339 (di_rxreset_t)dma64_rxreset, 340 (di_rxidle_t)dma64_rxidle, 341 (di_rxstopped_t)dma64_rxstopped, 342 (di_rxenable_t)_dma_rxenable, 343 (di_rxenabled_t)dma64_rxenabled, 344 (di_rx_t)_dma_rx, 345 (di_rxfill_t)_dma_rxfill, 346 (di_rxreclaim_t)_dma_rxreclaim, 347 (di_getnextrxp_t)_dma_getnextrxp, 348 (di_peeknextrxp_t)_dma_peeknextrxp, 349 (di_rxparam_get_t)_dma_rx_param_get, 350 351 (di_fifoloopbackenable_t)_dma_fifoloopbackenable, 352 (di_getvar_t)_dma_getvar, 353 (di_counterreset_t)_dma_counterreset, 354 (di_ctrlflags_t)_dma_ctrlflags, 355 356#if defined(BCMDBG_DUMP) 357 (di_dump_t)dma64_dump, 358 (di_dumptx_t)dma64_dumptx, 359 (di_dumprx_t)dma64_dumprx, 360#else 361 NULL, 362 NULL, 363 NULL, 364#endif 365 (di_rxactive_t)_dma_rxactive, 366 (di_txpending_t)_dma_txpending, 367 (di_txcommitted_t)_dma_txcommitted, 368 (di_pktpool_set_t)_dma_pktpool_set, 369 (di_rxtxerror_t)_dma_rxtx_error, 370 (di_burstlen_set_t)_dma_burstlen_set, 371 (di_avoidancecnt_t)_dma_avoidancecnt, 372 (di_param_set_t)_dma_param_set, 373 (dma_glom_enable_t)_dma_glom_enable, 374 (dma_active_rxbuf_t)_dma_activerxbuf, 375 40 376}; 377 378static const di_fcn_t dma32proc = { 379 (di_detach_t)_dma_detach, 380 (di_txinit_t)dma32_txinit, 381 (di_txreset_t)dma32_txreset, 382 (di_txenabled_t)dma32_txenabled, 383 (di_txsuspend_t)dma32_txsuspend, 384 (di_txresume_t)dma32_txresume, 385 (di_txsuspended_t)dma32_txsuspended, 386 (di_txsuspendedidle_t)dma32_txsuspendedidle, 387 (di_txflush_t)dma32_txflush, 388 (di_txflush_clear_t)dma32_txflush_clear, 389 (di_txfast_t)dma32_txfast, 390 NULL, 391 NULL, 392 (di_txstopped_t)dma32_txstopped, 393 (di_txreclaim_t)dma32_txreclaim, 394 (di_getnexttxp_t)dma32_getnexttxp, 395 (di_peeknexttxp_t)_dma_peeknexttxp, 396 (di_peekntxp_t)_dma_peekntxp, 397 (di_txblock_t)_dma_txblock, 398 (di_txunblock_t)_dma_txunblock, 399 (di_txactive_t)_dma_txactive, 400 (di_txrotate_t)dma32_txrotate, 401 402 (di_rxinit_t)_dma_rxinit, 403 (di_rxreset_t)dma32_rxreset, 404 (di_rxidle_t)dma32_rxidle, 405 (di_rxstopped_t)dma32_rxstopped, 406 (di_rxenable_t)_dma_rxenable, 407 (di_rxenabled_t)dma32_rxenabled, 408 (di_rx_t)_dma_rx, 409 (di_rxfill_t)_dma_rxfill, 410 (di_rxreclaim_t)_dma_rxreclaim, 411 (di_getnextrxp_t)_dma_getnextrxp, 412 (di_peeknextrxp_t)_dma_peeknextrxp, 413 (di_rxparam_get_t)_dma_rx_param_get, 414 415 (di_fifoloopbackenable_t)_dma_fifoloopbackenable, 416 (di_getvar_t)_dma_getvar, 417 (di_counterreset_t)_dma_counterreset, 418 (di_ctrlflags_t)_dma_ctrlflags, 419 420#if defined(BCMDBG_DUMP) 421 (di_dump_t)dma32_dump, 422 (di_dumptx_t)dma32_dumptx, 423 (di_dumprx_t)dma32_dumprx, 424#else 425 NULL, 426 NULL, 427 NULL, 428#endif 429 (di_rxactive_t)_dma_rxactive, 430 (di_txpending_t)_dma_txpending, 431 (di_txcommitted_t)_dma_txcommitted, 432 (di_pktpool_set_t)_dma_pktpool_set, 433 (di_rxtxerror_t)_dma_rxtx_error, 434 (di_burstlen_set_t)_dma_burstlen_set, 435 (di_avoidancecnt_t)_dma_avoidancecnt, 436 (di_param_set_t)_dma_param_set, 437 NULL, 438 NULL, 439 40 440}; 441 442hnddma_t * 443dma_attach(osl_t *osh, const char *name, si_t *sih, 444 volatile void *dmaregstx, volatile void *dmaregsrx, 445 uint ntxd, uint nrxd, uint rxbufsize, int rxextheadroom, uint nrxpost, uint rxoffset, 446 uint *msg_level) 447{ 448 dma_info_t *di; 449 uint size; 450 uint32 mask; 451 452 /* allocate private info structure */ 453 if ((di = MALLOC(osh, sizeof (dma_info_t))) == NULL) { 454 return (NULL); 455 } 456 457 bzero(di, sizeof(dma_info_t)); 458 459 di->msg_level = msg_level ? msg_level : &dma_msg_level; 460 461 /* old chips w/o sb is no longer supported */ 462 ASSERT(sih != NULL); 463 464 if (DMA64_ENAB(di)) 465 di->dma64 = ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64); 466 else 467 di->dma64 = 0; 468 469 /* check arguments */ 470 ASSERT(ISPOWEROF2(ntxd)); 471 ASSERT(ISPOWEROF2(nrxd)); 472 473 if (nrxd == 0) 474 ASSERT(dmaregsrx == NULL); 475 if (ntxd == 0) 476 ASSERT(dmaregstx == NULL); 477 478 /* init dma reg pointer */ 479 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 480 di->d64txregs = (dma64regs_t *)dmaregstx; 481 di->d64rxregs = (dma64regs_t *)dmaregsrx; 482 di->hnddma.di_fn = (const di_fcn_t *)&dma64proc; 483 } else if (DMA32_ENAB(di)) { 484 ASSERT(ntxd <= D32MAXDD); 485 ASSERT(nrxd <= D32MAXDD); 486 di->d32txregs = (dma32regs_t *)dmaregstx; 487 di->d32rxregs = (dma32regs_t *)dmaregsrx; 488 di->hnddma.di_fn = (const di_fcn_t *)&dma32proc; 489 } else { 490 DMA_ERROR(("%s: driver doesn't support 32-bit DMA\n", __FUNCTION__)); 491 ASSERT(0); 492 goto fail; 493 } 494 495 /* Default flags (which can be changed by the driver calling dma_ctrlflags 496 * before enable): For backwards compatibility both Rx Overflow Continue 497 * and Parity are DISABLED. 498 * supports it. 499 */ 500 di->hnddma.di_fn->ctrlflags(&di->hnddma, DMA_CTRL_ROC | DMA_CTRL_PEN, 0); 501 502 DMA_TRACE(("%s: %s: %s osh %p flags 0x%x ntxd %d nrxd %d rxbufsize %d " 503 "rxextheadroom %d nrxpost %d rxoffset %d dmaregstx %p dmaregsrx %p\n", 504 name, __FUNCTION__, (DMA64_MODE(di) ? "DMA64" : "DMA32"), 505 osh, di->hnddma.dmactrlflags, ntxd, nrxd, 506 rxbufsize, rxextheadroom, nrxpost, rxoffset, dmaregstx, dmaregsrx)); 507 508 /* make a private copy of our callers name */ 509 strncpy(di->name, name, MAXNAMEL); 510 di->name[MAXNAMEL-1] = '\0'; 511 512 di->osh = osh; 513 di->sih = sih; 514 515 /* save tunables */ 516 di->ntxd = (uint16)ntxd; 517 di->nrxd = (uint16)nrxd; 518 519 /* the actual dma size doesn't include the extra headroom */ 520 di->rxextrahdrroom = (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom; 521 if (rxbufsize > BCMEXTRAHDROOM) 522 di->rxbufsize = (uint16)(rxbufsize - di->rxextrahdrroom); 523 else 524 di->rxbufsize = (uint16)rxbufsize; 525 526 di->nrxpost = (uint16)nrxpost; 527 di->rxoffset = (uint8)rxoffset; 528 529 /* Get the default values (POR) of the burstlen. This can be overridden by the modules 530 * if this has to be different. Otherwise this value will be used to program the control 531 * register after the reset or during the init. 532 */ 533 if (dmaregsrx) { 534 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 535 /* detect the dma descriptor address mask, 536 * should be 0x1fff before 4360B0, 0xffff start from 4360B0 537 */ 538 W_REG(di->osh, &di->d64rxregs->addrlow, 0xffffffff); 539 mask = R_REG(di->osh, &di->d64rxregs->addrlow); 540 541 if (mask & 0xfff) 542 mask = R_REG(di->osh, &di->d64rxregs->ptr) | 0xf; 543 else 544 mask = 0x1fff; 545 546 DMA_TRACE(("%s: dma_rx_mask: %08x\n", di->name, mask)); 547 di->d64_rs0_cd_mask = mask; 548 549 if (mask == 0x1fff) 550 ASSERT(nrxd <= D64MAXDD); 551 else 552 ASSERT(nrxd <= D64MAXDD_LARGE); 553 554 di->rxburstlen = (R_REG(di->osh, 555 &di->d64rxregs->control) & D64_RC_BL_MASK) >> D64_RC_BL_SHIFT; 556 di->rxprefetchctl = (R_REG(di->osh, 557 &di->d64rxregs->control) & D64_RC_PC_MASK) >> 558 D64_RC_PC_SHIFT; 559 di->rxprefetchthresh = (R_REG(di->osh, 560 &di->d64rxregs->control) & D64_RC_PT_MASK) >> 561 D64_RC_PT_SHIFT; 562 } else if (DMA32_ENAB(di)) { 563 di->rxburstlen = (R_REG(di->osh, 564 &di->d32rxregs->control) & RC_BL_MASK) >> RC_BL_SHIFT; 565 di->rxprefetchctl = (R_REG(di->osh, 566 &di->d32rxregs->control) & RC_PC_MASK) >> RC_PC_SHIFT; 567 di->rxprefetchthresh = (R_REG(di->osh, 568 &di->d32rxregs->control) & RC_PT_MASK) >> RC_PT_SHIFT; 569 } 570 } 571 if (dmaregstx) { 572 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 573 574 /* detect the dma descriptor address mask, 575 * should be 0x1fff before 4360B0, 0xffff start from 4360B0 576 */ 577 W_REG(di->osh, &di->d64txregs->addrlow, 0xffffffff); 578 mask = R_REG(di->osh, &di->d64txregs->addrlow); 579 580 if (mask & 0xfff) 581 mask = R_REG(di->osh, &di->d64txregs->ptr) | 0xf; 582 else 583 mask = 0x1fff; 584 585 DMA_TRACE(("%s: dma_tx_mask: %08x\n", di->name, mask)); 586 di->d64_xs0_cd_mask = mask; 587 di->d64_xs1_ad_mask = mask; 588 589 if (mask == 0x1fff) 590 ASSERT(ntxd <= D64MAXDD); 591 else 592 ASSERT(ntxd <= D64MAXDD_LARGE); 593 594 di->txburstlen = (R_REG(di->osh, 595 &di->d64txregs->control) & D64_XC_BL_MASK) >> D64_XC_BL_SHIFT; 596 di->txmultioutstdrd = (R_REG(di->osh, 597 &di->d64txregs->control) & D64_XC_MR_MASK) >> D64_XC_MR_SHIFT; 598 di->txprefetchctl = (R_REG(di->osh, 599 &di->d64txregs->control) & D64_XC_PC_MASK) >> D64_XC_PC_SHIFT; 600 di->txprefetchthresh = (R_REG(di->osh, 601 &di->d64txregs->control) & D64_XC_PT_MASK) >> D64_XC_PT_SHIFT; 602 } else if (DMA32_ENAB(di)) { 603 di->txburstlen = (R_REG(di->osh, 604 &di->d32txregs->control) & XC_BL_MASK) >> XC_BL_SHIFT; 605 di->txmultioutstdrd = (R_REG(di->osh, 606 &di->d32txregs->control) & XC_MR_MASK) >> XC_MR_SHIFT; 607 di->txprefetchctl = (R_REG(di->osh, 608 &di->d32txregs->control) & XC_PC_MASK) >> XC_PC_SHIFT; 609 di->txprefetchthresh = (R_REG(di->osh, 610 &di->d32txregs->control) & XC_PT_MASK) >> XC_PT_SHIFT; 611 } 612 } 613 614 /* 615 * figure out the DMA physical address offset for dd and data 616 * PCI/PCIE: they map silicon backplace address to zero based memory, need offset 617 * Other bus: use zero 618 * SI_BUS BIGENDIAN kludge: use sdram swapped region for data buffer, not descriptor 619 */ 620 di->ddoffsetlow = 0; 621 di->dataoffsetlow = 0; 622 /* for pci bus, add offset */ 623 if (sih->bustype == PCI_BUS) { 624 if ((sih->buscoretype == PCIE_CORE_ID || 625 sih->buscoretype == PCIE2_CORE_ID) && 626 DMA64_MODE(di)) { 627 /* pcie with DMA64 */ 628 di->ddoffsetlow = 0; 629 di->ddoffsethigh = SI_PCIE_DMA_H32; 630 } else { 631 /* pci(DMA32/DMA64) or pcie with DMA32 */ 632 if ((CHIPID(sih->chip) == BCM4322_CHIP_ID) || 633 (CHIPID(sih->chip) == BCM4342_CHIP_ID) || 634 (CHIPID(sih->chip) == BCM43221_CHIP_ID) || 635 (CHIPID(sih->chip) == BCM43231_CHIP_ID) || 636 (CHIPID(sih->chip) == BCM43111_CHIP_ID) || 637 (CHIPID(sih->chip) == BCM43112_CHIP_ID) || 638 (CHIPID(sih->chip) == BCM43222_CHIP_ID)) 639 di->ddoffsetlow = SI_PCI_DMA2; 640 else 641 di->ddoffsetlow = SI_PCI_DMA; 642 643 di->ddoffsethigh = 0; 644 } 645 di->dataoffsetlow = di->ddoffsetlow; 646 di->dataoffsethigh = di->ddoffsethigh; 647 } 648 649#if defined(__mips__) && defined(IL_BIGENDIAN) 650 di->dataoffsetlow = di->dataoffsetlow + SI_SDRAM_SWAPPED; 651#endif /* defined(__mips__) && defined(IL_BIGENDIAN) */ 652 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */ 653 if ((si_coreid(sih) == SDIOD_CORE_ID) && ((si_corerev(sih) > 0) && (si_corerev(sih) <= 2))) 654 di->addrext = 0; 655 else if ((si_coreid(sih) == I2S_CORE_ID) && 656 ((si_corerev(sih) == 0) || (si_corerev(sih) == 1))) 657 di->addrext = 0; 658 else 659 di->addrext = _dma_isaddrext(di); 660 661 /* does the descriptors need to be aligned and if yes, on 4K/8K or not */ 662 di->aligndesc_4k = _dma_descriptor_align(di); 663 if (di->aligndesc_4k) { 664 if (DMA64_MODE(di)) { 665 di->dmadesc_align = D64RINGALIGN_BITS; 666 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) { 667 /* for smaller dd table, HW relax the alignment requirement */ 668 di->dmadesc_align = D64RINGALIGN_BITS - 1; 669 } 670 } else 671 di->dmadesc_align = D32RINGALIGN_BITS; 672 } else { 673 /* The start address of descriptor table should be algined to cache line size, 674 * or other structure may share a cache line with it, which can lead to memory 675 * overlapping due to cache write-back operation. In the case of MIPS 74k, the 676 * cache line size is 32 bytes. 677 */ 678#ifdef __mips__ 679 di->dmadesc_align = 5; /* 32 byte alignment */ 680#else 681 di->dmadesc_align = 4; /* 16 byte alignment */ 682#endif 683 } 684 685 DMA_NONE(("DMA descriptor align_needed %d, align %d\n", 686 di->aligndesc_4k, di->dmadesc_align)); 687 688 /* allocate tx packet pointer vector */ 689 if (ntxd) { 690 size = ntxd * sizeof(void *); 691 if ((di->txp = MALLOC(osh, size)) == NULL) { 692 DMA_ERROR(("%s: %s: out of tx memory, malloced %d bytes\n", 693 di->name, __FUNCTION__, MALLOCED(osh))); 694 goto fail; 695 } 696 bzero(di->txp, size); 697 } 698 699 /* allocate rx packet pointer vector */ 700 if (nrxd) { 701 size = nrxd * sizeof(void *); 702 if ((di->rxp = MALLOC(osh, size)) == NULL) { 703 DMA_ERROR(("%s: %s: out of rx memory, malloced %d bytes\n", 704 di->name, __FUNCTION__, MALLOCED(osh))); 705 goto fail; 706 } 707 bzero(di->rxp, size); 708 } 709 710 /* allocate transmit descriptor ring, only need ntxd descriptors but it must be aligned */ 711 if (ntxd) { 712 if (!_dma_alloc(di, DMA_TX)) 713 goto fail; 714 } 715 716 /* allocate receive descriptor ring, only need nrxd descriptors but it must be aligned */ 717 if (nrxd) { 718 if (!_dma_alloc(di, DMA_RX)) 719 goto fail; 720 } 721 722 if ((di->ddoffsetlow != 0) && !di->addrext) { 723 if (PHYSADDRLO(di->txdpa) > SI_PCI_DMA_SZ) { 724 DMA_ERROR(("%s: %s: txdpa 0x%x: addrext not supported\n", 725 di->name, __FUNCTION__, (uint32)PHYSADDRLO(di->txdpa))); 726 goto fail; 727 } 728 if (PHYSADDRLO(di->rxdpa) > SI_PCI_DMA_SZ) { 729 DMA_ERROR(("%s: %s: rxdpa 0x%x: addrext not supported\n", 730 di->name, __FUNCTION__, (uint32)PHYSADDRLO(di->rxdpa))); 731 goto fail; 732 } 733 } 734 735 DMA_TRACE(("ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh " 736 "0x%x addrext %d\n", di->ddoffsetlow, di->ddoffsethigh, di->dataoffsetlow, 737 di->dataoffsethigh, di->addrext)); 738 739 /* allocate DMA mapping vectors */ 740 if (DMASGLIST_ENAB) { 741 if (ntxd) { 742 size = ntxd * sizeof(hnddma_seg_map_t); 743 if ((di->txp_dmah = (hnddma_seg_map_t *)MALLOC(osh, size)) == NULL) 744 goto fail; 745 bzero(di->txp_dmah, size); 746 } 747 748 if (nrxd) { 749 size = nrxd * sizeof(hnddma_seg_map_t); 750 if ((di->rxp_dmah = (hnddma_seg_map_t *)MALLOC(osh, size)) == NULL) 751 goto fail; 752 bzero(di->rxp_dmah, size); 753 } 754 } 755 756 return ((hnddma_t *)di); 757 758fail: 759 _dma_detach(di); 760 return (NULL); 761} 762 763/** init the tx or rx descriptor */ 764static INLINE void 765dma32_dd_upd(dma_info_t *di, dma32dd_t *ddring, dmaaddr_t pa, uint outidx, uint32 *flags, 766 uint32 bufcount) 767{ 768 /* dma32 uses 32-bit control to fit both flags and bufcounter */ 769 *flags = *flags | (bufcount & CTRL_BC_MASK); 770 771 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 772 W_SM(&ddring[outidx].addr, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 773 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags)); 774 } else { 775 /* address extension */ 776 uint32 ae; 777 ASSERT(di->addrext); 778 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 779 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 780 781 *flags |= (ae << CTRL_AE_SHIFT); 782 W_SM(&ddring[outidx].addr, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 783 W_SM(&ddring[outidx].ctrl, BUS_SWAP32(*flags)); 784 } 785} 786 787/** Check for odd number of 1's */ 788STATIC INLINE uint32 parity32(uint32 data) 789{ 790 data ^= data >> 16; 791 data ^= data >> 8; 792 data ^= data >> 4; 793 data ^= data >> 2; 794 data ^= data >> 1; 795 796 return (data & 1); 797} 798 799#define DMA64_DD_PARITY(dd) parity32((dd)->addrlow ^ (dd)->addrhigh ^ (dd)->ctrl1 ^ (dd)->ctrl2) 800 801static INLINE void 802dma64_dd_upd(dma_info_t *di, dma64dd_t *ddring, dmaaddr_t pa, uint outidx, uint32 *flags, 803 uint32 bufcount) 804{ 805 uint32 ctrl2 = bufcount & D64_CTRL2_BC_MASK; 806 807 /* PCI bus with big(>1G) physical address, use address extension */ 808#if defined(__mips__) && defined(IL_BIGENDIAN) 809 if ((di->dataoffsetlow == SI_SDRAM_SWAPPED) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 810#else 811 if ((di->dataoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 812#endif /* defined(__mips__) && defined(IL_BIGENDIAN) */ 813 ASSERT((PHYSADDRHI(pa) & PCI64ADDR_HIGH) == 0); 814 815 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 816 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(PHYSADDRHI(pa) + di->dataoffsethigh)); 817 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags)); 818 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2)); 819 } else { 820 /* address extension for 32-bit PCI */ 821 uint32 ae; 822 ASSERT(di->addrext); 823 824 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 825 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 826 ASSERT(PHYSADDRHI(pa) == 0); 827 828 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE; 829 W_SM(&ddring[outidx].addrlow, BUS_SWAP32(PHYSADDRLO(pa) + di->dataoffsetlow)); 830 W_SM(&ddring[outidx].addrhigh, BUS_SWAP32(0 + di->dataoffsethigh)); 831 W_SM(&ddring[outidx].ctrl1, BUS_SWAP32(*flags)); 832 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2)); 833 } 834 if (di->hnddma.dmactrlflags & DMA_CTRL_PEN) { 835 if (DMA64_DD_PARITY(&ddring[outidx])) { 836 W_SM(&ddring[outidx].ctrl2, BUS_SWAP32(ctrl2 | D64_CTRL2_PARITY)); 837 } 838 } 839 840#if defined(BCM47XX_CA9) && !defined(__NetBSD__) 841 DMA_MAP(di->osh, (void *)(((uint)(&ddring[outidx])) & ~0x1f), 32, DMA_TX, NULL, NULL); 842#endif /* BCM47XX_CA9 && !__NetBSD__ */ 843} 844 845static bool 846_dma32_addrext(osl_t *osh, dma32regs_t *dma32regs) 847{ 848 uint32 w; 849 850 OR_REG(osh, &dma32regs->control, XC_AE); 851 w = R_REG(osh, &dma32regs->control); 852 AND_REG(osh, &dma32regs->control, ~XC_AE); 853 return ((w & XC_AE) == XC_AE); 854} 855 856static bool 857_dma_alloc(dma_info_t *di, uint direction) 858{ 859 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 860 return dma64_alloc(di, direction); 861 } else if (DMA32_ENAB(di)) { 862 return dma32_alloc(di, direction); 863 } else 864 ASSERT(0); 865} 866 867/** !! may be called with core in reset */ 868static void 869_dma_detach(dma_info_t *di) 870{ 871 872 DMA_TRACE(("%s: dma_detach\n", di->name)); 873 874 /* shouldn't be here if descriptors are unreclaimed */ 875 ASSERT(di->txin == di->txout); 876 ASSERT(di->rxin == di->rxout); 877 878 /* free dma descriptor rings */ 879 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 880 if (di->txd64) 881 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->txd64 - di->txdalign), 882 di->txdalloc, (di->txdpaorig), di->tx_dmah); 883 if (di->rxd64) 884 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->rxd64 - di->rxdalign), 885 di->rxdalloc, (di->rxdpaorig), di->rx_dmah); 886 } else if (DMA32_ENAB(di)) { 887 if (di->txd32) 888 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->txd32 - di->txdalign), 889 di->txdalloc, (di->txdpaorig), di->tx_dmah); 890 if (di->rxd32) 891 DMA_FREE_CONSISTENT(di->osh, ((int8 *)(uintptr)di->rxd32 - di->rxdalign), 892 di->rxdalloc, (di->rxdpaorig), di->rx_dmah); 893 } else 894 ASSERT(0); 895 896 /* free packet pointer vectors */ 897 if (di->txp) 898 MFREE(di->osh, (void *)di->txp, (di->ntxd * sizeof(void *))); 899 if (di->rxp) 900 MFREE(di->osh, (void *)di->rxp, (di->nrxd * sizeof(void *))); 901 902 /* free tx packet DMA handles */ 903 if (di->txp_dmah) 904 MFREE(di->osh, (void *)di->txp_dmah, di->ntxd * sizeof(hnddma_seg_map_t)); 905 906 /* free rx packet DMA handles */ 907 if (di->rxp_dmah) 908 MFREE(di->osh, (void *)di->rxp_dmah, di->nrxd * sizeof(hnddma_seg_map_t)); 909 910 /* free our private info structure */ 911 MFREE(di->osh, (void *)di, sizeof(dma_info_t)); 912 913} 914 915static bool 916_dma_descriptor_align(dma_info_t *di) 917{ 918 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 919 uint32 addrl; 920 921 /* Check to see if the descriptors need to be aligned on 4K/8K or not */ 922 if (di->d64txregs != NULL) { 923 W_REG(di->osh, &di->d64txregs->addrlow, 0xff0); 924 addrl = R_REG(di->osh, &di->d64txregs->addrlow); 925 if (addrl != 0) 926 return FALSE; 927 } else if (di->d64rxregs != NULL) { 928 W_REG(di->osh, &di->d64rxregs->addrlow, 0xff0); 929 addrl = R_REG(di->osh, &di->d64rxregs->addrlow); 930 if (addrl != 0) 931 return FALSE; 932 } 933 } 934 return TRUE; 935} 936 937/** return TRUE if this dma engine supports DmaExtendedAddrChanges, otherwise FALSE */ 938static bool 939_dma_isaddrext(dma_info_t *di) 940{ 941 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 942 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */ 943 944 /* not all tx or rx channel are available */ 945 if (di->d64txregs != NULL) { 946 if (!_dma64_addrext(di->osh, di->d64txregs)) { 947 DMA_ERROR(("%s: _dma_isaddrext: DMA64 tx doesn't have AE set\n", 948 di->name)); 949 ASSERT(0); 950 } 951 return TRUE; 952 } else if (di->d64rxregs != NULL) { 953 if (!_dma64_addrext(di->osh, di->d64rxregs)) { 954 DMA_ERROR(("%s: _dma_isaddrext: DMA64 rx doesn't have AE set\n", 955 di->name)); 956 ASSERT(0); 957 } 958 return TRUE; 959 } 960 return FALSE; 961 } else if (DMA32_ENAB(di)) { 962 if (di->d32txregs) 963 return (_dma32_addrext(di->osh, di->d32txregs)); 964 else if (di->d32rxregs) 965 return (_dma32_addrext(di->osh, di->d32rxregs)); 966 } else 967 ASSERT(0); 968 969 return FALSE; 970} 971 972/** initialize descriptor table base address */ 973static void 974_dma_ddtable_init(dma_info_t *di, uint direction, dmaaddr_t pa) 975{ 976 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 977 if (!di->aligndesc_4k) { 978 if (direction == DMA_TX) 979 di->xmtptrbase = PHYSADDRLO(pa); 980 else 981 di->rcvptrbase = PHYSADDRLO(pa); 982 } 983 984 if ((di->ddoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 985 if (direction == DMA_TX) { 986 W_REG(di->osh, &di->d64txregs->addrlow, (PHYSADDRLO(pa) + 987 di->ddoffsetlow)); 988 W_REG(di->osh, &di->d64txregs->addrhigh, (PHYSADDRHI(pa) + 989 di->ddoffsethigh)); 990 } else { 991 W_REG(di->osh, &di->d64rxregs->addrlow, (PHYSADDRLO(pa) + 992 di->ddoffsetlow)); 993 W_REG(di->osh, &di->d64rxregs->addrhigh, (PHYSADDRHI(pa) + 994 di->ddoffsethigh)); 995 } 996 } else { 997 /* DMA64 32bits address extension */ 998 uint32 ae; 999 ASSERT(di->addrext); 1000 ASSERT(PHYSADDRHI(pa) == 0); 1001 1002 /* shift the high bit(s) from pa to ae */ 1003 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 1004 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 1005 1006 if (direction == DMA_TX) { 1007 W_REG(di->osh, &di->d64txregs->addrlow, (PHYSADDRLO(pa) + 1008 di->ddoffsetlow)); 1009 W_REG(di->osh, &di->d64txregs->addrhigh, di->ddoffsethigh); 1010 SET_REG(di->osh, &di->d64txregs->control, D64_XC_AE, 1011 (ae << D64_XC_AE_SHIFT)); 1012 } else { 1013 W_REG(di->osh, &di->d64rxregs->addrlow, (PHYSADDRLO(pa) + 1014 di->ddoffsetlow)); 1015 W_REG(di->osh, &di->d64rxregs->addrhigh, di->ddoffsethigh); 1016 SET_REG(di->osh, &di->d64rxregs->control, D64_RC_AE, 1017 (ae << D64_RC_AE_SHIFT)); 1018 } 1019 } 1020 1021 } else if (DMA32_ENAB(di)) { 1022 ASSERT(PHYSADDRHI(pa) == 0); 1023 if ((di->ddoffsetlow == 0) || !(PHYSADDRLO(pa) & PCI32ADDR_HIGH)) { 1024 if (direction == DMA_TX) 1025 W_REG(di->osh, &di->d32txregs->addr, (PHYSADDRLO(pa) + 1026 di->ddoffsetlow)); 1027 else 1028 W_REG(di->osh, &di->d32rxregs->addr, (PHYSADDRLO(pa) + 1029 di->ddoffsetlow)); 1030 } else { 1031 /* dma32 address extension */ 1032 uint32 ae; 1033 ASSERT(di->addrext); 1034 1035 /* shift the high bit(s) from pa to ae */ 1036 ae = (PHYSADDRLO(pa) & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 1037 PHYSADDRLO(pa) &= ~PCI32ADDR_HIGH; 1038 1039 if (direction == DMA_TX) { 1040 W_REG(di->osh, &di->d32txregs->addr, (PHYSADDRLO(pa) + 1041 di->ddoffsetlow)); 1042 SET_REG(di->osh, &di->d32txregs->control, XC_AE, ae <<XC_AE_SHIFT); 1043 } else { 1044 W_REG(di->osh, &di->d32rxregs->addr, (PHYSADDRLO(pa) + 1045 di->ddoffsetlow)); 1046 SET_REG(di->osh, &di->d32rxregs->control, RC_AE, ae <<RC_AE_SHIFT); 1047 } 1048 } 1049 } else 1050 ASSERT(0); 1051} 1052 1053static void 1054_dma_fifoloopbackenable(dma_info_t *di) 1055{ 1056 DMA_TRACE(("%s: dma_fifoloopbackenable\n", di->name)); 1057 1058 if (DMA64_ENAB(di) && DMA64_MODE(di)) 1059 OR_REG(di->osh, &di->d64txregs->control, D64_XC_LE); 1060 else if (DMA32_ENAB(di)) 1061 OR_REG(di->osh, &di->d32txregs->control, XC_LE); 1062 else 1063 ASSERT(0); 1064} 1065 1066static void 1067_dma_rxinit(dma_info_t *di) 1068{ 1069 DMA_TRACE(("%s: dma_rxinit\n", di->name)); 1070 1071 if (di->nrxd == 0) 1072 return; 1073 1074 /* During the reset procedure, the active rxd may not be zero if pktpool is 1075 * enabled, we need to reclaim active rxd to avoid rxd being leaked. 1076 */ 1077 if ((POOL_ENAB(di->pktpool)) && (NRXDACTIVE(di->rxin, di->rxout))) { 1078 _dma_rxreclaim(di); 1079 } 1080 1081 ASSERT(di->rxin == di->rxout); 1082 di->rxin = di->rxout = di->rs0cd = 0; 1083 1084 /* clear rx descriptor ring */ 1085 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1086 BZERO_SM((void *)(uintptr)di->rxd64, (di->nrxd * sizeof(dma64dd_t))); 1087 1088 /* DMA engine with out alignment requirement requires table to be inited 1089 * before enabling the engine 1090 */ 1091 if (!di->aligndesc_4k) 1092 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 1093 1094 _dma_rxenable(di); 1095 1096 if (di->aligndesc_4k) 1097 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 1098 } else if (DMA32_ENAB(di)) { 1099 BZERO_SM((void *)(uintptr)di->rxd32, (di->nrxd * sizeof(dma32dd_t))); 1100 _dma_rxenable(di); 1101 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 1102 } else 1103 ASSERT(0); 1104} 1105 1106static void 1107_dma_rxenable(dma_info_t *di) 1108{ 1109 uint dmactrlflags = di->hnddma.dmactrlflags; 1110 1111 DMA_TRACE(("%s: dma_rxenable\n", di->name)); 1112 1113 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1114 uint32 control = (R_REG(di->osh, &di->d64rxregs->control) & D64_RC_AE) | D64_RC_RE; 1115 1116 if ((dmactrlflags & DMA_CTRL_PEN) == 0) 1117 control |= D64_RC_PD; 1118 1119 if (dmactrlflags & DMA_CTRL_ROC) 1120 control |= D64_RC_OC; 1121 1122 /* These bits 20:18 (burstLen) of control register can be written but will take 1123 * effect only if these bits are valid. So this will not affect previous versions 1124 * of the DMA. They will continue to have those bits set to 0. 1125 */ 1126 control &= ~D64_RC_BL_MASK; 1127 control |= (di->rxburstlen << D64_RC_BL_SHIFT); 1128 1129 control &= ~D64_RC_PC_MASK; 1130 control |= (di->rxprefetchctl << D64_RC_PC_SHIFT); 1131 1132 control &= ~D64_RC_PT_MASK; 1133 control |= (di->rxprefetchthresh << D64_RC_PT_SHIFT); 1134 1135 W_REG(di->osh, &di->d64rxregs->control, 1136 ((di->rxoffset << D64_RC_RO_SHIFT) | control)); 1137 } else if (DMA32_ENAB(di)) { 1138 uint32 control = (R_REG(di->osh, &di->d32rxregs->control) & RC_AE) | RC_RE; 1139 1140 if ((dmactrlflags & DMA_CTRL_PEN) == 0) 1141 control |= RC_PD; 1142 1143 if (dmactrlflags & DMA_CTRL_ROC) 1144 control |= RC_OC; 1145 1146 /* These bits 20:18 (burstLen) of control register can be written but will take 1147 * effect only if these bits are valid. So this will not affect previous versions 1148 * of the DMA. They will continue to have those bits set to 0. 1149 */ 1150 control &= ~RC_BL_MASK; 1151 control |= (di->rxburstlen << RC_BL_SHIFT); 1152 1153 control &= ~RC_PC_MASK; 1154 control |= (di->rxprefetchctl << RC_PC_SHIFT); 1155 1156 control &= ~RC_PT_MASK; 1157 control |= (di->rxprefetchthresh << RC_PT_SHIFT); 1158 1159 W_REG(di->osh, &di->d32rxregs->control, 1160 ((di->rxoffset << RC_RO_SHIFT) | control)); 1161 } else 1162 ASSERT(0); 1163} 1164 1165static void 1166_dma_rx_param_get(dma_info_t *di, uint16 *rxoffset, uint16 *rxbufsize) 1167{ 1168 /* the normal values fit into 16 bits */ 1169 *rxoffset = (uint16)di->rxoffset; 1170 *rxbufsize = (uint16)di->rxbufsize; 1171} 1172 1173/** 1174 * !! rx entry routine 1175 * returns a pointer to the next frame received, or NULL if there are no more 1176 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is supported 1177 * with pkts chain 1178 * otherwise, it's treated as giant pkt and will be tossed. 1179 * The DMA scattering starts with normal DMA header, followed by first buffer data. 1180 * After it reaches the max size of buffer, the data continues in next DMA descriptor 1181 * buffer WITHOUT DMA header 1182 */ 1183static void * BCMFASTPATH 1184_dma_rx(dma_info_t *di) 1185{ 1186 void *p, *head, *tail; 1187 uint len; 1188 uint pkt_len; 1189 int resid = 0; 1190#if defined(BCM4335) || defined(BCM4345) || defined(BCM4350) || defined(BCM43602) 1191 dma64regs_t *dregs = di->d64rxregs; 1192#endif 1193 1194next_frame: 1195 head = _dma_getnextrxp(di, FALSE); 1196 if (head == NULL) 1197 return (NULL); 1198 1199#if (!defined(__mips__) && !defined(BCM47XX_CA9)) 1200#if defined(BCM4335) || defined(BCM4345) || defined(BCM4350) || defined(BCM43602) 1201 if ((R_REG(osh, &dregs->control) & D64_RC_GE)) { 1202 /* In case of glommed pkt get length from hwheader */ 1203 len = ltoh16(*((uint16 *)(PKTDATA(di->osh, head)) + di->rxoffset/2 + 2)) + 4; 1204 1205 *(uint16 *)(PKTDATA(di->osh, head)) = len; 1206 } else { 1207 len = ltoh16(*(uint16 *)(PKTDATA(di->osh, head))); 1208 } 1209#else 1210 len = ltoh16(*(uint16 *)(PKTDATA(di->osh, head))); 1211#endif 1212#else 1213 { 1214 int read_count = 0; 1215#if defined(__mips__) || defined(__NetBSD__) 1216 for (read_count = 200; 1217 (!(len = ltoh16(*(uint16 *)OSL_UNCACHED(PKTDATA(di->osh, head)))) && 1218 read_count); read_count--) { 1219 if (CHIPID(di->sih->chip) == BCM5356_CHIP_ID) 1220 break; 1221 OSL_DELAY(1); 1222 } 1223#else 1224 for (read_count = 200; read_count; read_count--) { 1225 len = ltoh16(*(uint16 *)PKTDATA(di->osh, head)); 1226 if (len != 0) 1227 break; 1228 DMA_MAP(di->osh, PKTDATA(di->osh, head), 32, DMA_RX, NULL, NULL); 1229 OSL_DELAY(1); 1230 } 1231#endif /* __mips__ */ 1232 1233 if (!len) { 1234 DMA_ERROR(("%s: dma_rx: frame length (%d)\n", di->name, len)); 1235 PKTFREE(di->osh, head, FALSE); 1236 goto next_frame; 1237 } 1238 1239 } 1240#endif /* defined(__mips__) */ 1241 DMA_TRACE(("%s: dma_rx len %d\n", di->name, len)); 1242 1243 /* set actual length */ 1244 pkt_len = MIN((di->rxoffset + len), di->rxbufsize); 1245 PKTSETLEN(di->osh, head, pkt_len); 1246 resid = len - (di->rxbufsize - di->rxoffset); 1247 1248 /* check for single or multi-buffer rx */ 1249 if (resid > 0) { 1250 tail = head; 1251 while ((resid > 0) && (p = _dma_getnextrxp(di, FALSE))) { 1252 PKTSETNEXT(di->osh, tail, p); 1253 pkt_len = MIN(resid, (int)di->rxbufsize); 1254 PKTSETLEN(di->osh, p, pkt_len); 1255 1256 tail = p; 1257 resid -= di->rxbufsize; 1258 } 1259 1260 1261 if ((di->hnddma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) { 1262 DMA_ERROR(("%s: dma_rx: bad frame length (%d)\n", di->name, len)); 1263 PKTFREE(di->osh, head, FALSE); 1264 di->hnddma.rxgiants++; 1265 goto next_frame; 1266 } 1267 } 1268 1269 return (head); 1270} 1271 1272/** 1273 * post receive buffers 1274 * return FALSE is refill failed completely and ring is empty 1275 * this will stall the rx dma and user might want to call rxfill again asap 1276 * This unlikely happens on memory-rich NIC, but often on memory-constrained dongle 1277 */ 1278static bool BCMFASTPATH 1279_dma_rxfill(dma_info_t *di) 1280{ 1281 void *p; 1282 uint16 rxin, rxout; 1283 uint32 flags = 0; 1284 uint n; 1285 uint i; 1286 dmaaddr_t pa; 1287 uint extra_offset = 0, extra_pad; 1288 bool ring_empty; 1289 uint alignment_req = (di->hnddma.dmactrlflags & DMA_CTRL_USB_BOUNDRY4KB_WAR) ? 1290 16 : 1; /* MUST BE POWER of 2 */ 1291 1292 ring_empty = FALSE; 1293 1294 /* 1295 * Determine how many receive buffers we're lacking 1296 * from the full complement, allocate, initialize, 1297 * and post them, then update the chip rx lastdscr. 1298 */ 1299 1300 rxin = di->rxin; 1301 rxout = di->rxout; 1302 1303 n = di->nrxpost - NRXDACTIVE(rxin, rxout); 1304 1305 if (di->rxbufsize > BCMEXTRAHDROOM) 1306 extra_offset = di->rxextrahdrroom; 1307 1308 DMA_TRACE(("%s: dma_rxfill: post %d\n", di->name, n)); 1309 1310 for (i = 0; i < n; i++) { 1311 /* the di->rxbufsize doesn't include the extra headroom, we need to add it to the 1312 size to be allocated 1313 */ 1314 if (POOL_ENAB(di->pktpool)) { 1315 ASSERT(di->pktpool); 1316 p = pktpool_get(di->pktpool); 1317#ifdef BCMDBG_POOL 1318 if (p) 1319 PKTPOOLSETSTATE(p, POOL_RXFILL); 1320#endif /* BCMDBG_POOL */ 1321 } 1322 else { 1323 p = PKTGET(di->osh, (di->rxbufsize + extra_offset + alignment_req - 1), 1324 FALSE); 1325 } 1326 if (p == NULL) { 1327 DMA_TRACE(("%s: dma_rxfill: out of rxbufs\n", di->name)); 1328 if (i == 0) { 1329 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1330 if (dma64_rxidle(di)) { 1331 DMA_TRACE(("%s: rxfill64: ring is empty !\n", 1332 di->name)); 1333 ring_empty = TRUE; 1334 } 1335 } else if (DMA32_ENAB(di)) { 1336 if (dma32_rxidle(di)) { 1337 DMA_TRACE(("%s: rxfill32: ring is empty !\n", 1338 di->name)); 1339 ring_empty = TRUE; 1340 } 1341 } else 1342 ASSERT(0); 1343 } 1344 di->hnddma.rxnobuf++; 1345 break; 1346 } 1347 /* reserve an extra headroom, if applicable */ 1348 if (di->hnddma.dmactrlflags & DMA_CTRL_USB_BOUNDRY4KB_WAR) { 1349 extra_pad = ((alignment_req - (uint)(((unsigned long)PKTDATA(di->osh, p) - 1350 (unsigned long)(uchar *)0))) & (alignment_req - 1)); 1351 } else 1352 extra_pad = 0; 1353 1354 if (extra_offset + extra_pad) 1355 PKTPULL(di->osh, p, extra_offset + extra_pad); 1356 1357#ifdef CTFMAP 1358 /* mark as ctf buffer for fast mapping */ 1359 if (CTF_ENAB(kcih)) { 1360 ASSERT((((uint32)PKTDATA(di->osh, p)) & 31) == 0); 1361 PKTSETCTF(di->osh, p); 1362 } 1363#endif /* CTFMAP */ 1364 1365 /* Do a cached write instead of uncached write since DMA_MAP 1366 * will flush the cache. 1367 */ 1368 *(uint32 *)(PKTDATA(di->osh, p)) = 0; 1369#if defined(linux) && defined(BCM47XX_CA9) 1370 DMA_MAP(di->osh, (void *)((uint)PKTDATA(di->osh, p) & ~0x1f), 1371 32, DMA_TX, NULL, NULL); 1372#endif 1373 1374 if (DMASGLIST_ENAB) 1375 bzero(&di->rxp_dmah[rxout], sizeof(hnddma_seg_map_t)); 1376 1377 pa = DMA_MAP(di->osh, PKTDATA(di->osh, p), 1378 di->rxbufsize, DMA_RX, p, 1379 &di->rxp_dmah[rxout]); 1380 1381 ASSERT(ISALIGNED(PHYSADDRLO(pa), 4)); 1382 1383#ifdef __mips__ 1384 /* Do a un-cached write now that DMA_MAP has invalidated the cache 1385 */ 1386 *(uint32 *)OSL_UNCACHED((PKTDATA(di->osh, p))) = 0; 1387#endif /* __mips__ */ 1388 1389 /* save the free packet pointer */ 1390 ASSERT(di->rxp[rxout] == NULL); 1391 di->rxp[rxout] = p; 1392 1393 /* reset flags for each descriptor */ 1394 flags = 0; 1395 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1396 if (rxout == (di->nrxd - 1)) 1397 flags = D64_CTRL1_EOT; 1398 1399 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags, di->rxbufsize); 1400 } else if (DMA32_ENAB(di)) { 1401 if (rxout == (di->nrxd - 1)) 1402 flags = CTRL_EOT; 1403 1404 ASSERT(PHYSADDRHI(pa) == 0); 1405 dma32_dd_upd(di, di->rxd32, pa, rxout, &flags, di->rxbufsize); 1406 } else 1407 ASSERT(0); 1408 rxout = NEXTRXD(rxout); 1409 } 1410 1411 di->rxout = rxout; 1412 1413 /* update the chip lastdscr pointer */ 1414 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1415 W_REG(di->osh, &di->d64rxregs->ptr, di->rcvptrbase + I2B(rxout, dma64dd_t)); 1416 } else if (DMA32_ENAB(di)) { 1417 W_REG(di->osh, &di->d32rxregs->ptr, I2B(rxout, dma32dd_t)); 1418 } else 1419 ASSERT(0); 1420 1421 return ring_empty; 1422} 1423 1424/** like getnexttxp but no reclaim */ 1425static void * 1426_dma_peeknexttxp(dma_info_t *di) 1427{ 1428 uint16 end, i; 1429 1430 if (di->ntxd == 0) 1431 return (NULL); 1432 1433 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1434 end = (uint16)B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 1435 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 1436 di->xs0cd = end; 1437 } else if (DMA32_ENAB(di)) { 1438 end = (uint16)B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t); 1439 di->xs0cd = end; 1440 } else 1441 ASSERT(0); 1442 1443 for (i = di->txin; i != end; i = NEXTTXD(i)) 1444 if (di->txp[i]) 1445 return (di->txp[i]); 1446 1447 return (NULL); 1448} 1449 1450int 1451_dma_peekntxp(dma_info_t *di, int *len, void *txps[], txd_range_t range) 1452{ 1453 uint16 start, end, i; 1454 uint act; 1455 void *txp = NULL; 1456 int k, len_max; 1457 1458 DMA_TRACE(("%s: dma_peekntxp\n", di->name)); 1459 1460 ASSERT(len); 1461 ASSERT(txps); 1462 ASSERT(di); 1463 if (di->ntxd == 0) { 1464 *len = 0; 1465 return BCME_ERROR; 1466 } 1467 1468 len_max = *len; 1469 *len = 0; 1470 1471 start = di->txin; 1472 1473 if (range == HNDDMA_RANGE_ALL) 1474 end = di->txout; 1475 else { 1476 if (DMA64_ENAB(di)) { 1477 end = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 1478 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 1479 1480 act = (uint)(R_REG(di->osh, &di->d64txregs->status1) & D64_XS1_AD_MASK); 1481 act = (act - di->xmtptrbase) & D64_XS0_CD_MASK; 1482 act = (uint)B2I(act, dma64dd_t); 1483 } else { 1484 end = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t); 1485 1486 act = (uint)((R_REG(di->osh, &di->d32txregs->status) & XS_AD_MASK) >> 1487 XS_AD_SHIFT); 1488 act = (uint)B2I(act, dma32dd_t); 1489 } 1490 1491 di->xs0cd = end; 1492 if (end != act) 1493 end = PREVTXD(act); 1494 } 1495 1496 if ((start == 0) && (end > di->txout)) 1497 return BCME_ERROR; 1498 1499 k = 0; 1500 for (i = start; i != end; i = NEXTTXD(i)) { 1501 txp = di->txp[i]; 1502 if (txp != NULL) { 1503 if (k < len_max) 1504 txps[k++] = txp; 1505 else 1506 break; 1507 } 1508 } 1509 *len = k; 1510 1511 return BCME_OK; 1512} 1513 1514/** like getnextrxp but not take off the ring */ 1515static void * 1516_dma_peeknextrxp(dma_info_t *di) 1517{ 1518 uint16 end, i; 1519 1520 if (di->nrxd == 0) 1521 return (NULL); 1522 1523 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1524 end = (uint16)B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 1525 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 1526 di->rs0cd = end; 1527 } else if (DMA32_ENAB(di)) { 1528 end = (uint16)B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t); 1529 di->rs0cd = end; 1530 } else 1531 ASSERT(0); 1532 1533 for (i = di->rxin; i != end; i = NEXTRXD(i)) 1534 if (di->rxp[i]) 1535 return (di->rxp[i]); 1536 1537 return (NULL); 1538} 1539 1540static void 1541_dma_rxreclaim(dma_info_t *di) 1542{ 1543 void *p; 1544 bool origcb = TRUE; 1545 1546#ifndef EFI 1547 /* "unused local" warning suppression for OSLs that 1548 * define PKTFREE() without using the di->osh arg 1549 */ 1550 di = di; 1551#endif /* EFI */ 1552 1553 DMA_TRACE(("%s: dma_rxreclaim\n", di->name)); 1554 1555 if (POOL_ENAB(di->pktpool) && 1556 ((origcb = pktpool_emptycb_disabled(di->pktpool)) == FALSE)) 1557 pktpool_emptycb_disable(di->pktpool, TRUE); 1558 1559 while ((p = _dma_getnextrxp(di, TRUE))) 1560 PKTFREE(di->osh, p, FALSE); 1561 1562 if (origcb == FALSE) 1563 pktpool_emptycb_disable(di->pktpool, FALSE); 1564} 1565 1566static void * BCMFASTPATH 1567_dma_getnextrxp(dma_info_t *di, bool forceall) 1568{ 1569 if (di->nrxd == 0) 1570 return (NULL); 1571 1572 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1573 return dma64_getnextrxp(di, forceall); 1574 } else if (DMA32_ENAB(di)) { 1575 return dma32_getnextrxp(di, forceall); 1576 } else 1577 ASSERT(0); 1578} 1579 1580static void 1581_dma_txblock(dma_info_t *di) 1582{ 1583 di->hnddma.txavail = 0; 1584} 1585 1586static void 1587_dma_txunblock(dma_info_t *di) 1588{ 1589 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 1590} 1591 1592static uint 1593_dma_txactive(dma_info_t *di) 1594{ 1595 return NTXDACTIVE(di->txin, di->txout); 1596} 1597 1598static uint 1599_dma_txpending(dma_info_t *di) 1600{ 1601 uint16 curr; 1602 1603 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1604 curr = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 1605 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 1606 di->xs0cd = curr; 1607 } else if (DMA32_ENAB(di)) { 1608 curr = B2I(R_REG(di->osh, &di->d32txregs->status) & XS_CD_MASK, dma32dd_t); 1609 di->xs0cd = curr; 1610 } else 1611 ASSERT(0); 1612 1613 return NTXDACTIVE(curr, di->txout); 1614} 1615 1616static uint 1617_dma_txcommitted(dma_info_t *di) 1618{ 1619 uint16 ptr; 1620 uint txin = di->txin; 1621 1622 if (txin == di->txout) 1623 return 0; 1624 1625 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1626 ptr = B2I(R_REG(di->osh, &di->d64txregs->ptr), dma64dd_t); 1627 } else if (DMA32_ENAB(di)) { 1628 ptr = B2I(R_REG(di->osh, &di->d32txregs->ptr), dma32dd_t); 1629 } else 1630 ASSERT(0); 1631 1632 return NTXDACTIVE(di->txin, ptr); 1633} 1634 1635static uint 1636_dma_rxactive(dma_info_t *di) 1637{ 1638 return NRXDACTIVE(di->rxin, di->rxout); 1639} 1640 1641static uint 1642_dma_activerxbuf(dma_info_t *di) 1643{ 1644 uint16 curr, ptr; 1645 curr = B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 1646 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 1647 ptr = B2I(((R_REG(di->osh, &di->d64rxregs->ptr) & D64_RS0_CD_MASK) - 1648 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 1649 return NRXDACTIVE(curr, ptr); 1650} 1651 1652 1653static void 1654_dma_counterreset(dma_info_t *di) 1655{ 1656 /* reset all software counter */ 1657 di->hnddma.rxgiants = 0; 1658 di->hnddma.rxnobuf = 0; 1659 di->hnddma.txnobuf = 0; 1660} 1661 1662static uint 1663_dma_ctrlflags(dma_info_t *di, uint mask, uint flags) 1664{ 1665 uint dmactrlflags; 1666 1667 if (!di) { 1668 DMA_ERROR(("_dma_ctrlflags: NULL dma handle\n")); 1669 return (0); 1670 } 1671 1672 dmactrlflags = di->hnddma.dmactrlflags; 1673 ASSERT((flags & ~mask) == 0); 1674 1675 dmactrlflags &= ~mask; 1676 dmactrlflags |= flags; 1677 1678 /* If trying to enable parity, check if parity is actually supported */ 1679 if (dmactrlflags & DMA_CTRL_PEN) { 1680 uint32 control; 1681 1682 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 1683 control = R_REG(di->osh, &di->d64txregs->control); 1684 W_REG(di->osh, &di->d64txregs->control, control | D64_XC_PD); 1685 if (R_REG(di->osh, &di->d64txregs->control) & D64_XC_PD) { 1686 /* We *can* disable it so it is supported, 1687 * restore control register 1688 */ 1689 W_REG(di->osh, &di->d64txregs->control, control); 1690 } else { 1691 /* Not supported, don't allow it to be enabled */ 1692 dmactrlflags &= ~DMA_CTRL_PEN; 1693 } 1694 } else if (DMA32_ENAB(di)) { 1695 control = R_REG(di->osh, &di->d32txregs->control); 1696 W_REG(di->osh, &di->d32txregs->control, control | XC_PD); 1697 if (R_REG(di->osh, &di->d32txregs->control) & XC_PD) { 1698 W_REG(di->osh, &di->d32txregs->control, control); 1699 } else { 1700 /* Not supported, don't allow it to be enabled */ 1701 dmactrlflags &= ~DMA_CTRL_PEN; 1702 } 1703 } else 1704 ASSERT(0); 1705 } 1706 1707 di->hnddma.dmactrlflags = dmactrlflags; 1708 1709 return (dmactrlflags); 1710} 1711 1712/** get the address of the var in order to change later */ 1713static uintptr 1714_dma_getvar(dma_info_t *di, const char *name) 1715{ 1716 if (!strcmp(name, "&txavail")) 1717 return ((uintptr) &(di->hnddma.txavail)); 1718 else { 1719 ASSERT(0); 1720 } 1721 return (0); 1722} 1723 1724static uint 1725_dma_avoidancecnt(dma_info_t *di) 1726{ 1727 return (di->dma_avoidance_cnt); 1728} 1729 1730void 1731dma_txpioloopback(osl_t *osh, dma32regs_t *regs) 1732{ 1733 OR_REG(osh, ®s->control, XC_LE); 1734} 1735 1736static 1737uint8 dma_align_sizetobits(uint size) 1738{ 1739 uint8 bitpos = 0; 1740 ASSERT(size); 1741 ASSERT(!(size & (size-1))); 1742 while (size >>= 1) { 1743 bitpos ++; 1744 } 1745 return (bitpos); 1746} 1747 1748/** 1749 * This function ensures that the DMA descriptor ring will not get allocated 1750 * across Page boundary. If the allocation is done across the page boundary 1751 * at the first time, then it is freed and the allocation is done at 1752 * descriptor ring size aligned location. This will ensure that the ring will 1753 * not cross page boundary 1754 */ 1755static void * 1756dma_ringalloc(osl_t *osh, uint32 boundary, uint size, uint16 *alignbits, uint* alloced, 1757 dmaaddr_t *descpa, osldma_t **dmah) 1758{ 1759 void * va; 1760 uint32 desc_strtaddr; 1761 uint32 alignbytes = 1 << *alignbits; 1762 1763 if ((va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced, descpa, dmah)) == NULL) 1764 return NULL; 1765 1766 desc_strtaddr = (uint32)ROUNDUP((uint)PHYSADDRLO(*descpa), alignbytes); 1767 if (((desc_strtaddr + size - 1) & boundary) != 1768 (desc_strtaddr & boundary)) { 1769 *alignbits = dma_align_sizetobits(size); 1770 DMA_FREE_CONSISTENT(osh, va, 1771 size, *descpa, *dmah); 1772 va = DMA_ALLOC_CONSISTENT(osh, size, *alignbits, alloced, descpa, dmah); 1773 } 1774 return va; 1775} 1776 1777#if defined(BCMDBG_DUMP) 1778static void 1779dma32_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma32dd_t *ring, uint start, uint end, 1780 uint max_num) 1781{ 1782 uint i; 1783 1784 for (i = start; i != end; i = XXD((i + 1), max_num)) { 1785 /* in the format of high->low 8 bytes */ 1786 bcm_bprintf(b, "ring index %d: 0x%x %x\n", 1787 i, R_SM(&ring[i].addr), R_SM(&ring[i].ctrl)); 1788 } 1789} 1790 1791static void 1792dma32_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1793{ 1794 if (di->ntxd == 0) 1795 return; 1796 1797 bcm_bprintf(b, "DMA32: txd32 %p txdpa 0x%lx txp %p txin %d txout %d " 1798 "txavail %d txnodesc %d\n", di->txd32, PHYSADDRLO(di->txdpa), di->txp, di->txin, 1799 di->txout, di->hnddma.txavail, di->hnddma.txnodesc); 1800 1801 bcm_bprintf(b, "xmtcontrol 0x%x xmtaddr 0x%x xmtptr 0x%x xmtstatus 0x%x\n", 1802 R_REG(di->osh, &di->d32txregs->control), 1803 R_REG(di->osh, &di->d32txregs->addr), 1804 R_REG(di->osh, &di->d32txregs->ptr), 1805 R_REG(di->osh, &di->d32txregs->status)); 1806 1807 if (dumpring && di->txd32) 1808 dma32_dumpring(di, b, di->txd32, di->txin, di->txout, di->ntxd); 1809} 1810 1811static void 1812dma32_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1813{ 1814 if (di->nrxd == 0) 1815 return; 1816 1817 bcm_bprintf(b, "DMA32: rxd32 %p rxdpa 0x%lx rxp %p rxin %d rxout %d\n", 1818 di->rxd32, PHYSADDRLO(di->rxdpa), di->rxp, di->rxin, di->rxout); 1819 1820 bcm_bprintf(b, "rcvcontrol 0x%x rcvaddr 0x%x rcvptr 0x%x rcvstatus 0x%x\n", 1821 R_REG(di->osh, &di->d32rxregs->control), 1822 R_REG(di->osh, &di->d32rxregs->addr), 1823 R_REG(di->osh, &di->d32rxregs->ptr), 1824 R_REG(di->osh, &di->d32rxregs->status)); 1825 if (di->rxd32 && dumpring) 1826 dma32_dumpring(di, b, di->rxd32, di->rxin, di->rxout, di->nrxd); 1827} 1828 1829static void 1830dma32_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1831{ 1832 dma32_dumptx(di, b, dumpring); 1833 dma32_dumprx(di, b, dumpring); 1834} 1835 1836static void 1837dma64_dumpring(dma_info_t *di, struct bcmstrbuf *b, dma64dd_t *ring, uint start, uint end, 1838 uint max_num) 1839{ 1840 uint i; 1841 1842 for (i = start; i != end; i = XXD((i + 1), max_num)) { 1843 /* in the format of high->low 16 bytes */ 1844 bcm_bprintf(b, "ring index %d: 0x%x %x %x %x\n", 1845 i, R_SM(&ring[i].addrhigh), R_SM(&ring[i].addrlow), 1846 R_SM(&ring[i].ctrl2), R_SM(&ring[i].ctrl1)); 1847 } 1848} 1849 1850static void 1851dma64_dumptx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1852{ 1853 if (di->ntxd == 0) 1854 return; 1855 1856 bcm_bprintf(b, "DMA64: txd64 %p txdpa 0x%lx txdpahi 0x%lx txp %p txin %d txout %d " 1857 "txavail %d txnodesc %d\n", di->txd64, PHYSADDRLO(di->txdpa), 1858 PHYSADDRHI(di->txdpaorig), di->txp, di->txin, di->txout, di->hnddma.txavail, 1859 di->hnddma.txnodesc); 1860 1861 bcm_bprintf(b, "xmtcontrol 0x%x xmtaddrlow 0x%x xmtaddrhigh 0x%x " 1862 "xmtptr 0x%x xmtstatus0 0x%x xmtstatus1 0x%x\n", 1863 R_REG(di->osh, &di->d64txregs->control), 1864 R_REG(di->osh, &di->d64txregs->addrlow), 1865 R_REG(di->osh, &di->d64txregs->addrhigh), 1866 R_REG(di->osh, &di->d64txregs->ptr), 1867 R_REG(di->osh, &di->d64txregs->status0), 1868 R_REG(di->osh, &di->d64txregs->status1)); 1869 1870 bcm_bprintf(b, "DMA64: DMA avoidance applied %d\n", di->dma_avoidance_cnt); 1871 1872 if (dumpring && di->txd64) { 1873 dma64_dumpring(di, b, di->txd64, di->txin, di->txout, di->ntxd); 1874 } 1875} 1876 1877static void 1878dma64_dumprx(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1879{ 1880 if (di->nrxd == 0) 1881 return; 1882 1883 bcm_bprintf(b, "DMA64: rxd64 %p rxdpa 0x%lx rxdpahi 0x%lx rxp %p rxin %d rxout %d\n", 1884 di->rxd64, PHYSADDRLO(di->rxdpa), PHYSADDRHI(di->rxdpaorig), di->rxp, 1885 di->rxin, di->rxout); 1886 1887 bcm_bprintf(b, "rcvcontrol 0x%x rcvaddrlow 0x%x rcvaddrhigh 0x%x rcvptr " 1888 "0x%x rcvstatus0 0x%x rcvstatus1 0x%x\n", 1889 R_REG(di->osh, &di->d64rxregs->control), 1890 R_REG(di->osh, &di->d64rxregs->addrlow), 1891 R_REG(di->osh, &di->d64rxregs->addrhigh), 1892 R_REG(di->osh, &di->d64rxregs->ptr), 1893 R_REG(di->osh, &di->d64rxregs->status0), 1894 R_REG(di->osh, &di->d64rxregs->status1)); 1895 if (di->rxd64 && dumpring) { 1896 dma64_dumpring(di, b, di->rxd64, di->rxin, di->rxout, di->nrxd); 1897 } 1898} 1899 1900static void 1901dma64_dump(dma_info_t *di, struct bcmstrbuf *b, bool dumpring) 1902{ 1903 dma64_dumptx(di, b, dumpring); 1904 dma64_dumprx(di, b, dumpring); 1905} 1906 1907#endif 1908 1909 1910/* 32-bit DMA functions */ 1911 1912static void 1913dma32_txinit(dma_info_t *di) 1914{ 1915 uint32 control = XC_XE; 1916 1917 DMA_TRACE(("%s: dma_txinit\n", di->name)); 1918 1919 if (di->ntxd == 0) 1920 return; 1921 1922 di->txin = di->txout = di->xs0cd = 0; 1923 di->hnddma.txavail = di->ntxd - 1; 1924 1925 /* clear tx descriptor ring */ 1926 BZERO_SM(DISCARD_QUAL(di->txd32, void), (di->ntxd * sizeof(dma32dd_t))); 1927 1928 /* These bits 20:18 (burstLen) of control register can be written but will take 1929 * effect only if these bits are valid. So this will not affect previous versions 1930 * of the DMA. They will continue to have those bits set to 0. 1931 */ 1932 control |= (di->txburstlen << XC_BL_SHIFT); 1933 control |= (di->txmultioutstdrd << XC_MR_SHIFT); 1934 control |= (di->txprefetchctl << XC_PC_SHIFT); 1935 control |= (di->txprefetchthresh << XC_PT_SHIFT); 1936 1937 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0) 1938 control |= XC_PD; 1939 W_REG(di->osh, &di->d32txregs->control, control); 1940 _dma_ddtable_init(di, DMA_TX, di->txdpa); 1941} 1942 1943static bool 1944dma32_txenabled(dma_info_t *di) 1945{ 1946 uint32 xc; 1947 1948 /* If the chip is dead, it is not enabled :-) */ 1949 xc = R_REG(di->osh, &di->d32txregs->control); 1950 return ((xc != 0xffffffff) && (xc & XC_XE)); 1951} 1952 1953static void 1954dma32_txsuspend(dma_info_t *di) 1955{ 1956 DMA_TRACE(("%s: dma_txsuspend\n", di->name)); 1957 1958 if (di->ntxd == 0) 1959 return; 1960 1961 OR_REG(di->osh, &di->d32txregs->control, XC_SE); 1962} 1963 1964static void 1965dma32_txresume(dma_info_t *di) 1966{ 1967 DMA_TRACE(("%s: dma_txresume\n", di->name)); 1968 1969 if (di->ntxd == 0) 1970 return; 1971 1972 AND_REG(di->osh, &di->d32txregs->control, ~XC_SE); 1973} 1974 1975static bool 1976dma32_txsuspended(dma_info_t *di) 1977{ 1978 return (di->ntxd == 0) || ((R_REG(di->osh, &di->d32txregs->control) & XC_SE) == XC_SE); 1979} 1980 1981static void 1982dma32_txflush(dma_info_t *di) 1983{ 1984 DMA_TRACE(("%s: dma_txflush\n", di->name)); 1985 1986 if (di->ntxd == 0) 1987 return; 1988 1989 OR_REG(di->osh, &di->d32txregs->control, XC_SE | XC_FL); 1990} 1991 1992static void 1993dma32_txflush_clear(dma_info_t *di) 1994{ 1995 uint32 status; 1996 1997 DMA_TRACE(("%s: dma_txflush_clear\n", di->name)); 1998 1999 if (di->ntxd == 0) 2000 return; 2001 2002 SPINWAIT(((status = (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK)) 2003 != XS_XS_DISABLED) && 2004 (status != XS_XS_IDLE) && 2005 (status != XS_XS_STOPPED), 2006 (10000)); 2007 AND_REG(di->osh, &di->d32txregs->control, ~XC_FL); 2008} 2009 2010static void 2011dma32_txreclaim(dma_info_t *di, txd_range_t range) 2012{ 2013 void *p; 2014 2015 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, 2016 (range == HNDDMA_RANGE_ALL) ? "all" : 2017 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 2018 2019 if (di->txin == di->txout) 2020 return; 2021 2022 while ((p = dma32_getnexttxp(di, range))) 2023 PKTFREE(di->osh, p, TRUE); 2024} 2025 2026static bool 2027dma32_txstopped(dma_info_t *di) 2028{ 2029 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_STOPPED); 2030} 2031 2032static bool 2033dma32_rxstopped(dma_info_t *di) 2034{ 2035 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_RS_MASK) == RS_RS_STOPPED); 2036} 2037 2038static bool 2039dma32_alloc(dma_info_t *di, uint direction) 2040{ 2041 uint size; 2042 uint ddlen; 2043 void *va; 2044 uint alloced; 2045 uint16 align; 2046 uint16 align_bits; 2047 2048 ddlen = sizeof(dma32dd_t); 2049 2050 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen); 2051 2052 alloced = 0; 2053 align_bits = di->dmadesc_align; 2054 align = (1 << align_bits); 2055 2056 if (direction == DMA_TX) { 2057 if ((va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits, &alloced, 2058 &di->txdpaorig, &di->tx_dmah)) == NULL) { 2059 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", 2060 di->name)); 2061 return FALSE; 2062 } 2063 2064 PHYSADDRHISET(di->txdpa, 0); 2065 ASSERT(PHYSADDRHI(di->txdpaorig) == 0); 2066 di->txd32 = (dma32dd_t *)ROUNDUP((uintptr)va, align); 2067 di->txdalign = (uint)((int8 *)(uintptr)di->txd32 - (int8 *)va); 2068 2069 PHYSADDRLOSET(di->txdpa, PHYSADDRLO(di->txdpaorig) + di->txdalign); 2070 /* Make sure that alignment didn't overflow */ 2071 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig)); 2072 2073 di->txdalloc = alloced; 2074 ASSERT(ISALIGNED(di->txd32, align)); 2075 } else { 2076 if ((va = dma_ringalloc(di->osh, D32RINGALIGN, size, &align_bits, &alloced, 2077 &di->rxdpaorig, &di->rx_dmah)) == NULL) { 2078 DMA_ERROR(("%s: dma_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", 2079 di->name)); 2080 return FALSE; 2081 } 2082 2083 PHYSADDRHISET(di->rxdpa, 0); 2084 ASSERT(PHYSADDRHI(di->rxdpaorig) == 0); 2085 di->rxd32 = (dma32dd_t *)ROUNDUP((uintptr)va, align); 2086 di->rxdalign = (uint)((int8 *)(uintptr)di->rxd32 - (int8 *)va); 2087 2088 PHYSADDRLOSET(di->rxdpa, PHYSADDRLO(di->rxdpaorig) + di->rxdalign); 2089 /* Make sure that alignment didn't overflow */ 2090 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig)); 2091 di->rxdalloc = alloced; 2092 ASSERT(ISALIGNED(di->rxd32, align)); 2093 } 2094 2095 return TRUE; 2096} 2097 2098static bool 2099dma32_txreset(dma_info_t *di) 2100{ 2101 uint32 status; 2102 2103 if (di->ntxd == 0) 2104 return TRUE; 2105 2106 /* suspend tx DMA first */ 2107 W_REG(di->osh, &di->d32txregs->control, XC_SE); 2108 SPINWAIT(((status = (R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK)) 2109 != XS_XS_DISABLED) && 2110 (status != XS_XS_IDLE) && 2111 (status != XS_XS_STOPPED), 2112 (10000)); 2113 2114 W_REG(di->osh, &di->d32txregs->control, 0); 2115 SPINWAIT(((status = (R_REG(di->osh, 2116 &di->d32txregs->status) & XS_XS_MASK)) != XS_XS_DISABLED), 2117 10000); 2118 2119 /* We should be disabled at this point */ 2120 if (status != XS_XS_DISABLED) { 2121 DMA_ERROR(("%s: status != D64_XS0_XS_DISABLED 0x%x\n", __FUNCTION__, status)); 2122 ASSERT(status == XS_XS_DISABLED); 2123 OSL_DELAY(300); 2124 } 2125 2126 return (status == XS_XS_DISABLED); 2127} 2128 2129static bool 2130dma32_rxidle(dma_info_t *di) 2131{ 2132 DMA_TRACE(("%s: dma_rxidle\n", di->name)); 2133 2134 if (di->nrxd == 0) 2135 return TRUE; 2136 2137 return ((R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK) == 2138 R_REG(di->osh, &di->d32rxregs->ptr)); 2139} 2140 2141static bool 2142dma32_rxreset(dma_info_t *di) 2143{ 2144 uint32 status; 2145 2146 if (di->nrxd == 0) 2147 return TRUE; 2148 2149 W_REG(di->osh, &di->d32rxregs->control, 0); 2150 SPINWAIT(((status = (R_REG(di->osh, 2151 &di->d32rxregs->status) & RS_RS_MASK)) != RS_RS_DISABLED), 2152 10000); 2153 2154 return (status == RS_RS_DISABLED); 2155} 2156 2157static bool 2158dma32_rxenabled(dma_info_t *di) 2159{ 2160 uint32 rc; 2161 2162 rc = R_REG(di->osh, &di->d32rxregs->control); 2163 return ((rc != 0xffffffff) && (rc & RC_RE)); 2164} 2165 2166static bool 2167dma32_txsuspendedidle(dma_info_t *di) 2168{ 2169 if (di->ntxd == 0) 2170 return TRUE; 2171 2172 if (!(R_REG(di->osh, &di->d32txregs->control) & XC_SE)) 2173 return 0; 2174 2175 if ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) != XS_XS_IDLE) 2176 return 0; 2177 2178 OSL_DELAY(2); 2179 return ((R_REG(di->osh, &di->d32txregs->status) & XS_XS_MASK) == XS_XS_IDLE); 2180} 2181 2182/** 2183 * !! tx entry routine 2184 * supports full 32bit dma engine buffer addressing so 2185 * dma buffers can cross 4 Kbyte page boundaries. 2186 * 2187 * WARNING: call must check the return value for error. 2188 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems 2189 */ 2190static int 2191dma32_txfast(dma_info_t *di, void *p0, bool commit) 2192{ 2193 void *p, *next; 2194 uchar *data; 2195 uint len; 2196 uint16 txout; 2197 uint32 flags = 0; 2198 dmaaddr_t pa; 2199 2200 DMA_TRACE(("%s: dma_txfast\n", di->name)); 2201 2202 txout = di->txout; 2203 2204 /* 2205 * Walk the chain of packet buffers 2206 * allocating and initializing transmit descriptor entries. 2207 */ 2208 for (p = p0; p; p = next) { 2209 uint nsegs, j; 2210 hnddma_seg_map_t *map; 2211 2212 data = PKTDATA(di->osh, p); 2213 len = PKTLEN(di->osh, p); 2214#ifdef BCM_DMAPAD 2215 len += PKTDMAPAD(di->osh, p); 2216#endif 2217 next = PKTNEXT(di->osh, p); 2218 2219 /* return nonzero if out of tx descriptors */ 2220 if (NEXTTXD(txout) == di->txin) 2221 goto outoftxd; 2222 2223 if (len == 0) 2224 continue; 2225 2226 if (DMASGLIST_ENAB) 2227 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t)); 2228 2229 /* get physical address of buffer start */ 2230 pa = DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]); 2231 2232 if (DMASGLIST_ENAB) { 2233 map = &di->txp_dmah[txout]; 2234 2235 /* See if all the segments can be accounted for */ 2236 if (map->nsegs > (uint)(di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1)) 2237 goto outoftxd; 2238 2239 nsegs = map->nsegs; 2240 } else 2241 nsegs = 1; 2242 2243 for (j = 1; j <= nsegs; j++) { 2244 flags = 0; 2245 if (p == p0 && j == 1) 2246 flags |= CTRL_SOF; 2247 2248 /* With a DMA segment list, Descriptor table is filled 2249 * using the segment list instead of looping over 2250 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when 2251 * end of segment list is reached. 2252 */ 2253 if ((!DMASGLIST_ENAB && next == NULL) || 2254 (DMASGLIST_ENAB && j == nsegs)) 2255 flags |= (CTRL_IOC | CTRL_EOF); 2256 if (txout == (di->ntxd - 1)) 2257 flags |= CTRL_EOT; 2258 2259 if (DMASGLIST_ENAB) { 2260 len = map->segs[j - 1].length; 2261 pa = map->segs[j - 1].addr; 2262 } 2263 ASSERT(PHYSADDRHI(pa) == 0); 2264 2265 dma32_dd_upd(di, di->txd32, pa, txout, &flags, len); 2266 ASSERT(di->txp[txout] == NULL); 2267 2268 txout = NEXTTXD(txout); 2269 } 2270 2271 /* See above. No need to loop over individual buffers */ 2272 if (DMASGLIST_ENAB) 2273 break; 2274 } 2275 2276 /* if last txd eof not set, fix it */ 2277 if (!(flags & CTRL_EOF)) 2278 W_SM(&di->txd32[PREVTXD(txout)].ctrl, BUS_SWAP32(flags | CTRL_IOC | CTRL_EOF)); 2279 2280 /* save the packet */ 2281 di->txp[PREVTXD(txout)] = p0; 2282 2283 /* bump the tx descriptor index */ 2284 di->txout = txout; 2285 2286 /* kick the chip */ 2287 if (commit) 2288 W_REG(di->osh, &di->d32txregs->ptr, I2B(txout, dma32dd_t)); 2289 2290 /* tx flow control */ 2291 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2292 2293 return (0); 2294 2295outoftxd: 2296 DMA_ERROR(("%s: dma_txfast: out of txds\n", di->name)); 2297 PKTFREE(di->osh, p0, TRUE); 2298 di->hnddma.txavail = 0; 2299 di->hnddma.txnobuf++; 2300 di->hnddma.txnodesc++; 2301 return (-1); 2302} 2303 2304/** 2305 * Reclaim next completed txd (txds if using chained buffers) in the range 2306 * specified and return associated packet. 2307 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be 2308 * transmitted as noted by the hardware "CurrDescr" pointer. 2309 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be 2310 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer. 2311 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and 2312 * return associated packet regardless of the value of hardware pointers. 2313 */ 2314static void * 2315dma32_getnexttxp(dma_info_t *di, txd_range_t range) 2316{ 2317 uint16 start, end, i; 2318 uint16 active_desc; 2319 void *txp; 2320 2321 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, 2322 (range == HNDDMA_RANGE_ALL) ? "all" : 2323 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 2324 2325 if (di->ntxd == 0) 2326 return (NULL); 2327 2328 txp = NULL; 2329 2330 start = di->txin; 2331 if (range == HNDDMA_RANGE_ALL) 2332 end = di->txout; 2333 else { 2334 dma32regs_t *dregs = di->d32txregs; 2335 2336 if (di->txin == di->xs0cd) { 2337 end = (uint16)B2I(R_REG(di->osh, &dregs->status) & XS_CD_MASK, dma32dd_t); 2338 di->xs0cd = end; 2339 } else 2340 end = di->xs0cd; 2341 2342 2343 if (range == HNDDMA_RANGE_TRANSFERED) { 2344 active_desc = (uint16)((R_REG(di->osh, &dregs->status) & XS_AD_MASK) >> 2345 XS_AD_SHIFT); 2346 active_desc = (uint16)B2I(active_desc, dma32dd_t); 2347 if (end != active_desc) 2348 end = PREVTXD(active_desc); 2349 } 2350 } 2351 2352 if ((start == 0) && (end > di->txout)) 2353 goto bogus; 2354 2355 for (i = start; i != end && !txp; i = NEXTTXD(i)) { 2356 dmaaddr_t pa; 2357 hnddma_seg_map_t *map = NULL; 2358 uint size, j, nsegs; 2359 2360 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->txd32[i].addr)) - di->dataoffsetlow)); 2361 PHYSADDRHISET(pa, 0); 2362 2363 if (DMASGLIST_ENAB) { 2364 map = &di->txp_dmah[i]; 2365 size = map->origsize; 2366 nsegs = map->nsegs; 2367 } else { 2368 size = (BUS_SWAP32(R_SM(&di->txd32[i].ctrl)) & CTRL_BC_MASK); 2369 nsegs = 1; 2370 } 2371 2372 for (j = nsegs; j > 0; j--) { 2373 W_SM(&di->txd32[i].addr, 0xdeadbeef); 2374 2375 txp = di->txp[i]; 2376 di->txp[i] = NULL; 2377 if (j > 1) 2378 i = NEXTTXD(i); 2379 } 2380 2381 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map); 2382 } 2383 2384 di->txin = i; 2385 2386 /* tx flow control */ 2387 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2388 2389 return (txp); 2390 2391bogus: 2392 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", 2393 start, end, di->txout, forceall)); 2394 return (NULL); 2395} 2396 2397static void * 2398dma32_getnextrxp(dma_info_t *di, bool forceall) 2399{ 2400 uint16 i, curr; 2401 void *rxp; 2402 dmaaddr_t pa; 2403 /* if forcing, dma engine must be disabled */ 2404 ASSERT(!forceall || !dma32_rxenabled(di)); 2405 2406 i = di->rxin; 2407 2408 /* return if no packets posted */ 2409 if (i == di->rxout) 2410 return (NULL); 2411 2412 if (di->rxin == di->rs0cd) { 2413 curr = (uint16)B2I(R_REG(di->osh, &di->d32rxregs->status) & RS_CD_MASK, dma32dd_t); 2414 di->rs0cd = curr; 2415 } else 2416 curr = di->rs0cd; 2417 2418 /* ignore curr if forceall */ 2419 if (!forceall && (i == curr)) 2420 return (NULL); 2421 2422 /* get the packet pointer that corresponds to the rx descriptor */ 2423 rxp = di->rxp[i]; 2424 ASSERT(rxp); 2425 di->rxp[i] = NULL; 2426 2427 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->rxd32[i].addr)) - di->dataoffsetlow)); 2428 PHYSADDRHISET(pa, 0); 2429 2430 /* clear this packet from the descriptor ring */ 2431 DMA_UNMAP(di->osh, pa, 2432 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]); 2433 2434 W_SM(&di->rxd32[i].addr, 0xdeadbeef); 2435 2436 di->rxin = NEXTRXD(i); 2437 2438 return (rxp); 2439} 2440 2441/** 2442 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin). 2443 */ 2444static void 2445dma32_txrotate(dma_info_t *di) 2446{ 2447 uint16 ad; 2448 uint nactive; 2449 uint rot; 2450 uint16 old, new; 2451 uint32 w; 2452 uint16 first, last; 2453 2454 ASSERT(dma32_txsuspendedidle(di)); 2455 2456 nactive = _dma_txactive(di); 2457 ad = B2I(((R_REG(di->osh, &di->d32txregs->status) & XS_AD_MASK) >> XS_AD_SHIFT), dma32dd_t); 2458 rot = TXD(ad - di->txin); 2459 2460 ASSERT(rot < di->ntxd); 2461 2462 /* full-ring case is a lot harder - don't worry about this */ 2463 if (rot >= (di->ntxd - nactive)) { 2464 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name)); 2465 return; 2466 } 2467 2468 first = di->txin; 2469 last = PREVTXD(di->txout); 2470 2471 /* move entries starting at last and moving backwards to first */ 2472 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) { 2473 new = TXD(old + rot); 2474 2475 /* 2476 * Move the tx dma descriptor. 2477 * EOT is set only in the last entry in the ring. 2478 */ 2479 w = BUS_SWAP32(R_SM(&di->txd32[old].ctrl)) & ~CTRL_EOT; 2480 if (new == (di->ntxd - 1)) 2481 w |= CTRL_EOT; 2482 W_SM(&di->txd32[new].ctrl, BUS_SWAP32(w)); 2483 W_SM(&di->txd32[new].addr, R_SM(&di->txd32[old].addr)); 2484 2485 /* zap the old tx dma descriptor address field */ 2486 W_SM(&di->txd32[old].addr, BUS_SWAP32(0xdeadbeef)); 2487 2488 /* move the corresponding txp[] entry */ 2489 ASSERT(di->txp[new] == NULL); 2490 di->txp[new] = di->txp[old]; 2491 2492 /* Move the segment map as well */ 2493 if (DMASGLIST_ENAB) { 2494 bcopy(&di->txp_dmah[old], &di->txp_dmah[new], sizeof(hnddma_seg_map_t)); 2495 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t)); 2496 } 2497 2498 di->txp[old] = NULL; 2499 } 2500 2501 /* update txin and txout */ 2502 di->txin = ad; 2503 di->txout = TXD(di->txout + rot); 2504 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2505 2506 /* kick the chip */ 2507 W_REG(di->osh, &di->d32txregs->ptr, I2B(di->txout, dma32dd_t)); 2508} 2509 2510/* 64-bit DMA functions */ 2511 2512static void 2513dma64_txinit(dma_info_t *di) 2514{ 2515 uint32 control; 2516 2517 DMA_TRACE(("%s: dma_txinit\n", di->name)); 2518 2519 if (di->ntxd == 0) 2520 return; 2521 2522 di->txin = di->txout = di->xs0cd = 0; 2523 di->hnddma.txavail = di->ntxd - 1; 2524 2525 /* clear tx descriptor ring */ 2526 BZERO_SM((void *)(uintptr)di->txd64, (di->ntxd * sizeof(dma64dd_t))); 2527 2528 /* These bits 20:18 (burstLen) of control register can be written but will take 2529 * effect only if these bits are valid. So this will not affect previous versions 2530 * of the DMA. They will continue to have those bits set to 0. 2531 */ 2532 control = R_REG(di->osh, &di->d64txregs->control); 2533 control = (control & ~D64_XC_BL_MASK) | (di->txburstlen << D64_XC_BL_SHIFT); 2534 control = (control & ~D64_XC_MR_MASK) | (di->txmultioutstdrd << D64_XC_MR_SHIFT); 2535 control = (control & ~D64_XC_PC_MASK) | (di->txprefetchctl << D64_XC_PC_SHIFT); 2536 control = (control & ~D64_XC_PT_MASK) | (di->txprefetchthresh << D64_XC_PT_SHIFT); 2537 W_REG(di->osh, &di->d64txregs->control, control); 2538 2539 control = D64_XC_XE; 2540 /* DMA engine with out alignment requirement requires table to be inited 2541 * before enabling the engine 2542 */ 2543 if (!di->aligndesc_4k) 2544 _dma_ddtable_init(di, DMA_TX, di->txdpa); 2545 2546 if ((di->hnddma.dmactrlflags & DMA_CTRL_PEN) == 0) 2547 control |= D64_XC_PD; 2548 OR_REG(di->osh, &di->d64txregs->control, control); 2549 2550 /* DMA engine with alignment requirement requires table to be inited 2551 * before enabling the engine 2552 */ 2553 if (di->aligndesc_4k) 2554 _dma_ddtable_init(di, DMA_TX, di->txdpa); 2555} 2556 2557static bool 2558dma64_txenabled(dma_info_t *di) 2559{ 2560 uint32 xc; 2561 2562 /* If the chip is dead, it is not enabled :-) */ 2563 xc = R_REG(di->osh, &di->d64txregs->control); 2564 return ((xc != 0xffffffff) && (xc & D64_XC_XE)); 2565} 2566 2567static void 2568dma64_txsuspend(dma_info_t *di) 2569{ 2570 DMA_TRACE(("%s: dma_txsuspend\n", di->name)); 2571 2572 if (di->ntxd == 0) 2573 return; 2574 2575 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE); 2576} 2577 2578static void 2579dma64_txresume(dma_info_t *di) 2580{ 2581 DMA_TRACE(("%s: dma_txresume\n", di->name)); 2582 2583 if (di->ntxd == 0) 2584 return; 2585 2586 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_SE); 2587} 2588 2589static bool 2590dma64_txsuspended(dma_info_t *di) 2591{ 2592 return (di->ntxd == 0) || 2593 ((R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE) == D64_XC_SE); 2594} 2595 2596static void 2597dma64_txflush(dma_info_t *di) 2598{ 2599 DMA_TRACE(("%s: dma_txflush\n", di->name)); 2600 2601 if (di->ntxd == 0) 2602 return; 2603 2604 OR_REG(di->osh, &di->d64txregs->control, D64_XC_SE | D64_XC_FL); 2605} 2606 2607static void 2608dma64_txflush_clear(dma_info_t *di) 2609{ 2610 uint32 status; 2611 2612 DMA_TRACE(("%s: dma_txflush_clear\n", di->name)); 2613 2614 if (di->ntxd == 0) 2615 return; 2616 2617 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) != 2618 D64_XS0_XS_DISABLED) && 2619 (status != D64_XS0_XS_IDLE) && 2620 (status != D64_XS0_XS_STOPPED), 2621 10000); 2622 AND_REG(di->osh, &di->d64txregs->control, ~D64_XC_FL); 2623} 2624 2625static void BCMFASTPATH 2626dma64_txreclaim(dma_info_t *di, txd_range_t range) 2627{ 2628 void *p; 2629 2630 DMA_TRACE(("%s: dma_txreclaim %s\n", di->name, 2631 (range == HNDDMA_RANGE_ALL) ? "all" : 2632 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 2633 2634 if (di->txin == di->txout) 2635 return; 2636 2637 while ((p = dma64_getnexttxp(di, range))) { 2638 /* For unframed data, we don't have any packets to free */ 2639 if (!(di->hnddma.dmactrlflags & DMA_CTRL_UNFRAMED)) 2640 PKTFREE(di->osh, p, TRUE); 2641 } 2642} 2643 2644static bool 2645dma64_txstopped(dma_info_t *di) 2646{ 2647 return ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_STOPPED); 2648} 2649 2650static bool 2651dma64_rxstopped(dma_info_t *di) 2652{ 2653 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK) == D64_RS0_RS_STOPPED); 2654} 2655 2656static bool 2657dma64_alloc(dma_info_t *di, uint direction) 2658{ 2659 uint32 size; 2660 uint ddlen; 2661 void *va; 2662 uint alloced = 0; 2663 uint32 align; 2664 uint16 align_bits; 2665 2666 ddlen = sizeof(dma64dd_t); 2667 2668 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen); 2669 align_bits = di->dmadesc_align; 2670 align = (1 << align_bits); 2671 2672 if (direction == DMA_TX) { 2673 if ((va = dma_ringalloc(di->osh, 2674 (di->d64_xs0_cd_mask == 0x1fff) ? D64RINGBOUNDARY : D64RINGBOUNDARY_LARGE, 2675 size, &align_bits, &alloced, 2676 &di->txdpaorig, &di->tx_dmah)) == NULL) { 2677 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(ntxd) failed\n", 2678 di->name)); 2679 return FALSE; 2680 } 2681 align = (1 << align_bits); 2682 2683 /* adjust the pa by rounding up to the alignment */ 2684 PHYSADDRLOSET(di->txdpa, ROUNDUP(PHYSADDRLO(di->txdpaorig), align)); 2685 PHYSADDRHISET(di->txdpa, PHYSADDRHI(di->txdpaorig)); 2686 2687 /* Make sure that alignment didn't overflow */ 2688 ASSERT(PHYSADDRLO(di->txdpa) >= PHYSADDRLO(di->txdpaorig)); 2689 2690 /* find the alignment offset that was used */ 2691 di->txdalign = (uint)(PHYSADDRLO(di->txdpa) - PHYSADDRLO(di->txdpaorig)); 2692 2693 /* adjust the va by the same offset */ 2694 di->txd64 = (dma64dd_t *)((uintptr)va + di->txdalign); 2695 2696 di->txdalloc = alloced; 2697 ASSERT(ISALIGNED(PHYSADDRLO(di->txdpa), align)); 2698 } else { 2699 if ((va = dma_ringalloc(di->osh, 2700 (di->d64_rs0_cd_mask == 0x1fff) ? D64RINGBOUNDARY : D64RINGBOUNDARY_LARGE, 2701 size, &align_bits, &alloced, 2702 &di->rxdpaorig, &di->rx_dmah)) == NULL) { 2703 DMA_ERROR(("%s: dma64_alloc: DMA_ALLOC_CONSISTENT(nrxd) failed\n", 2704 di->name)); 2705 return FALSE; 2706 } 2707 align = (1 << align_bits); 2708 2709 /* adjust the pa by rounding up to the alignment */ 2710 PHYSADDRLOSET(di->rxdpa, ROUNDUP(PHYSADDRLO(di->rxdpaorig), align)); 2711 PHYSADDRHISET(di->rxdpa, PHYSADDRHI(di->rxdpaorig)); 2712 2713 /* Make sure that alignment didn't overflow */ 2714 ASSERT(PHYSADDRLO(di->rxdpa) >= PHYSADDRLO(di->rxdpaorig)); 2715 2716 /* find the alignment offset that was used */ 2717 di->rxdalign = (uint)(PHYSADDRLO(di->rxdpa) - PHYSADDRLO(di->rxdpaorig)); 2718 2719 /* adjust the va by the same offset */ 2720 di->rxd64 = (dma64dd_t *)((uintptr)va + di->rxdalign); 2721 2722 di->rxdalloc = alloced; 2723 ASSERT(ISALIGNED(PHYSADDRLO(di->rxdpa), align)); 2724 } 2725 2726 return TRUE; 2727} 2728 2729static bool 2730dma64_txreset(dma_info_t *di) 2731{ 2732 uint32 status; 2733 2734 if (di->ntxd == 0) 2735 return TRUE; 2736 2737 /* suspend tx DMA first */ 2738 W_REG(di->osh, &di->d64txregs->control, D64_XC_SE); 2739 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) != 2740 D64_XS0_XS_DISABLED) && 2741 (status != D64_XS0_XS_IDLE) && 2742 (status != D64_XS0_XS_STOPPED), 2743 10000); 2744 2745 W_REG(di->osh, &di->d64txregs->control, 0); 2746 SPINWAIT(((status = (R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK)) != 2747 D64_XS0_XS_DISABLED), 2748 10000); 2749 2750 /* We should be disabled at this point */ 2751 if (status != D64_XS0_XS_DISABLED) { 2752 DMA_ERROR(("%s: status != D64_XS0_XS_DISABLED 0x%x\n", __FUNCTION__, status)); 2753 ASSERT(status == D64_XS0_XS_DISABLED); 2754 OSL_DELAY(300); 2755 } 2756 2757 return (status == D64_XS0_XS_DISABLED); 2758} 2759 2760static bool 2761dma64_rxidle(dma_info_t *di) 2762{ 2763 DMA_TRACE(("%s: dma_rxidle\n", di->name)); 2764 2765 if (di->nrxd == 0) 2766 return TRUE; 2767 2768 return ((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) == 2769 (R_REG(di->osh, &di->d64rxregs->ptr) & D64_RS0_CD_MASK)); 2770} 2771 2772static bool 2773dma64_rxreset(dma_info_t *di) 2774{ 2775 uint32 status; 2776 2777 if (di->nrxd == 0) 2778 return TRUE; 2779 2780 W_REG(di->osh, &di->d64rxregs->control, 0); 2781 SPINWAIT(((status = (R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_RS_MASK)) != 2782 D64_RS0_RS_DISABLED), 10000); 2783 2784 return (status == D64_RS0_RS_DISABLED); 2785} 2786 2787static bool 2788dma64_rxenabled(dma_info_t *di) 2789{ 2790 uint32 rc; 2791 2792 rc = R_REG(di->osh, &di->d64rxregs->control); 2793 return ((rc != 0xffffffff) && (rc & D64_RC_RE)); 2794} 2795 2796static bool 2797dma64_txsuspendedidle(dma_info_t *di) 2798{ 2799 2800 if (di->ntxd == 0) 2801 return TRUE; 2802 2803 if (!(R_REG(di->osh, &di->d64txregs->control) & D64_XC_SE)) 2804 return 0; 2805 2806 if ((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_XS_MASK) == D64_XS0_XS_IDLE) 2807 return 1; 2808 2809 return 0; 2810} 2811 2812/** 2813 * Useful when sending unframed data. This allows us to get a progress report from the DMA. 2814 * We return a pointer to the beginning of the data buffer of the current descriptor. 2815 * If DMA is idle, we return NULL. 2816 */ 2817static void* 2818dma64_getpos(dma_info_t *di, bool direction) 2819{ 2820 void *va; 2821 bool idle; 2822 uint16 cur_idx; 2823 2824 if (direction == DMA_TX) { 2825 cur_idx = B2I(((R_REG(di->osh, &di->d64txregs->status0) & D64_XS0_CD_MASK) - 2826 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t); 2827 idle = !NTXDACTIVE(di->txin, di->txout); 2828 va = di->txp[cur_idx]; 2829 } else { 2830 cur_idx = B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 2831 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 2832 idle = !NRXDACTIVE(di->rxin, di->rxout); 2833 va = di->rxp[cur_idx]; 2834 } 2835 2836 /* If DMA is IDLE, return NULL */ 2837 if (idle) { 2838 DMA_TRACE(("%s: DMA idle, return NULL\n", __FUNCTION__)); 2839 va = NULL; 2840 } 2841 2842 return va; 2843} 2844 2845/** 2846 * TX of unframed data 2847 * 2848 * Adds a DMA ring descriptor for the data pointed to by "buf". 2849 * This is for DMA of a buffer of data and is unlike other hnddma TX functions 2850 * that take a pointer to a "packet" 2851 * Each call to this is results in a single descriptor being added for "len" bytes of 2852 * data starting at "buf", it doesn't handle chained buffers. 2853 */ 2854static int 2855dma64_txunframed(dma_info_t *di, void *buf, uint len, bool commit) 2856{ 2857 uint16 txout; 2858 uint32 flags = 0; 2859 dmaaddr_t pa; /* phys addr */ 2860 2861 txout = di->txout; 2862 2863 /* return nonzero if out of tx descriptors */ 2864 if (NEXTTXD(txout) == di->txin) 2865 goto outoftxd; 2866 2867 if (len == 0) 2868 return 0; 2869 2870 pa = DMA_MAP(di->osh, buf, len, DMA_TX, NULL, &di->txp_dmah[txout]); 2871 2872 flags = (D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF); 2873 2874 if (txout == (di->ntxd - 1)) 2875 flags |= D64_CTRL1_EOT; 2876 2877 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len); 2878 ASSERT(di->txp[txout] == NULL); 2879 2880 /* save the buffer pointer - used by dma_getpos */ 2881 di->txp[txout] = buf; 2882 2883 txout = NEXTTXD(txout); 2884 /* bump the tx descriptor index */ 2885 di->txout = txout; 2886 2887 /* kick the chip */ 2888 if (commit) { 2889 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(txout, dma64dd_t)); 2890 } 2891 2892 /* tx flow control */ 2893 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 2894 2895 return (0); 2896 2897outoftxd: 2898 DMA_ERROR(("%s: %s: out of txds !!!\n", di->name, __FUNCTION__)); 2899 di->hnddma.txavail = 0; 2900 di->hnddma.txnobuf++; 2901 return (-1); 2902} 2903 2904 2905/** 2906 * !! tx entry routine 2907 * WARNING: call must check the return value for error. 2908 * the error(toss frames) could be fatal and cause many subsequent hard to debug problems 2909 */ 2910static int BCMFASTPATH 2911dma64_txfast(dma_info_t *di, void *p0, bool commit) 2912{ 2913 void *p, *next; 2914 uchar *data; 2915 uint len; 2916 uint16 txout; 2917 uint32 flags = 0; 2918 dmaaddr_t pa; 2919 bool war; 2920 2921 DMA_TRACE(("%s: dma_txfast\n", di->name)); 2922 2923 txout = di->txout; 2924 war = (di->hnddma.dmactrlflags & DMA_CTRL_DMA_AVOIDANCE_WAR) ? TRUE : FALSE; 2925 2926 /* 2927 * Walk the chain of packet buffers 2928 * allocating and initializing transmit descriptor entries. 2929 */ 2930 for (p = p0; p; p = next) { 2931 uint nsegs, j, segsadd; 2932 hnddma_seg_map_t *map = NULL; 2933 2934 data = PKTDATA(di->osh, p); 2935 len = PKTLEN(di->osh, p); 2936#ifdef BCM_DMAPAD 2937 len += PKTDMAPAD(di->osh, p); 2938#endif /* BCM_DMAPAD */ 2939 next = PKTNEXT(di->osh, p); 2940 2941 /* return nonzero if out of tx descriptors */ 2942 if (NEXTTXD(txout) == di->txin) 2943 goto outoftxd; 2944 2945 if (len == 0) 2946 continue; 2947 2948 /* get physical address of buffer start */ 2949 if (DMASGLIST_ENAB) 2950 bzero(&di->txp_dmah[txout], sizeof(hnddma_seg_map_t)); 2951 2952 pa = DMA_MAP(di->osh, data, len, DMA_TX, p, &di->txp_dmah[txout]); 2953 2954 if (DMASGLIST_ENAB) { 2955 map = &di->txp_dmah[txout]; 2956 2957 /* See if all the segments can be accounted for */ 2958 if (map->nsegs > (uint)(di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1)) 2959 goto outoftxd; 2960 2961 nsegs = map->nsegs; 2962 } else 2963 nsegs = 1; 2964 2965 segsadd = 0; 2966 for (j = 1; j <= nsegs; j++) { 2967 flags = 0; 2968 if (p == p0 && j == 1) 2969 flags |= D64_CTRL1_SOF; 2970 2971 /* With a DMA segment list, Descriptor table is filled 2972 * using the segment list instead of looping over 2973 * buffers in multi-chain DMA. Therefore, EOF for SGLIST is when 2974 * end of segment list is reached. 2975 */ 2976 if ((!DMASGLIST_ENAB && next == NULL) || 2977 (DMASGLIST_ENAB && j == nsegs)) 2978 flags |= (D64_CTRL1_IOC | D64_CTRL1_EOF); 2979 if (txout == (di->ntxd - 1)) 2980 flags |= D64_CTRL1_EOT; 2981 2982 if (DMASGLIST_ENAB) { 2983 len = map->segs[j - 1].length; 2984 pa = map->segs[j - 1].addr; 2985 if (len > 128 && war) { 2986 uint remain, new_len, align64; 2987 /* check for 64B aligned of pa */ 2988 align64 = (uint)(PHYSADDRLO(pa) & 0x3f); 2989 align64 = (64 - align64) & 0x3f; 2990 new_len = len - align64; 2991 remain = new_len % 128; 2992 if (remain > 0 && remain <= 4) { 2993 uint32 buf_addr_lo; 2994 uint32 tmp_flags = 2995 flags & (~(D64_CTRL1_EOF | D64_CTRL1_IOC)); 2996 flags &= ~(D64_CTRL1_SOF | D64_CTRL1_EOT); 2997 remain += 64; 2998 dma64_dd_upd(di, di->txd64, pa, txout, 2999 &tmp_flags, len-remain); 3000 ASSERT(di->txp[txout] == NULL); 3001 txout = NEXTTXD(txout); 3002 /* return nonzero if out of tx descriptors */ 3003 if (txout == di->txin) { 3004 DMA_ERROR(("%s: dma_txfast: Out-of-DMA" 3005 " descriptors (txin %d txout %d" 3006 " nsegs %d)\n", __FUNCTION__, 3007 di->txin, di->txout, nsegs)); 3008 goto outoftxd; 3009 } 3010 if (txout == (di->ntxd - 1)) 3011 flags |= D64_CTRL1_EOT; 3012 buf_addr_lo = PHYSADDRLO(pa); 3013 PHYSADDRLOSET(pa, (PHYSADDRLO(pa) + (len-remain))); 3014 if (PHYSADDRLO(pa) < buf_addr_lo) { 3015 PHYSADDRHISET(pa, (PHYSADDRHI(pa) + 1)); 3016 } 3017 len = remain; 3018 segsadd++; 3019 di->dma_avoidance_cnt++; 3020 } 3021 } 3022 } 3023 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len); 3024 ASSERT(di->txp[txout] == NULL); 3025 3026 txout = NEXTTXD(txout); 3027 /* return nonzero if out of tx descriptors */ 3028 if (txout == di->txin) { 3029 DMA_ERROR(("%s: dma_txfast: Out-of-DMA descriptors" 3030 " (txin %d txout %d nsegs %d)\n", __FUNCTION__, 3031 di->txin, di->txout, nsegs)); 3032 goto outoftxd; 3033 } 3034 } 3035 if (segsadd && DMASGLIST_ENAB) 3036 map->nsegs += segsadd; 3037 3038 /* See above. No need to loop over individual buffers */ 3039 if (DMASGLIST_ENAB) 3040 break; 3041 } 3042 3043 /* if last txd eof not set, fix it */ 3044 if (!(flags & D64_CTRL1_EOF)) 3045 W_SM(&di->txd64[PREVTXD(txout)].ctrl1, 3046 BUS_SWAP32(flags | D64_CTRL1_IOC | D64_CTRL1_EOF)); 3047 3048 /* save the packet */ 3049 di->txp[PREVTXD(txout)] = p0; 3050 3051 /* bump the tx descriptor index */ 3052 di->txout = txout; 3053 3054 /* kick the chip */ 3055 if (commit) 3056 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(txout, dma64dd_t)); 3057 3058 /* tx flow control */ 3059 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 3060 3061 return (0); 3062 3063outoftxd: 3064 DMA_ERROR(("%s: dma_txfast: out of txds !!!\n", di->name)); 3065 PKTFREE(di->osh, p0, TRUE); 3066 di->hnddma.txavail = 0; 3067 di->hnddma.txnobuf++; 3068 return (-1); 3069} 3070 3071/** 3072 * Reclaim next completed txd (txds if using chained buffers) in the range 3073 * specified and return associated packet. 3074 * If range is HNDDMA_RANGE_TRANSMITTED, reclaim descriptors that have be 3075 * transmitted as noted by the hardware "CurrDescr" pointer. 3076 * If range is HNDDMA_RANGE_TRANSFERED, reclaim descriptors that have be 3077 * transfered by the DMA as noted by the hardware "ActiveDescr" pointer. 3078 * If range is HNDDMA_RANGE_ALL, reclaim all txd(s) posted to the ring and 3079 * return associated packet regardless of the value of hardware pointers. 3080 */ 3081static void * BCMFASTPATH 3082dma64_getnexttxp(dma_info_t *di, txd_range_t range) 3083{ 3084 uint16 start, end, i; 3085 uint16 active_desc; 3086 void *txp; 3087 3088 DMA_TRACE(("%s: dma_getnexttxp %s\n", di->name, 3089 (range == HNDDMA_RANGE_ALL) ? "all" : 3090 ((range == HNDDMA_RANGE_TRANSMITTED) ? "transmitted" : "transfered"))); 3091 3092 if (di->ntxd == 0) 3093 return (NULL); 3094 3095 txp = NULL; 3096 3097 start = di->txin; 3098 if (range == HNDDMA_RANGE_ALL) 3099 end = di->txout; 3100 else { 3101 dma64regs_t *dregs = di->d64txregs; 3102 3103 if (di->txin == di->xs0cd) { 3104 end = (uint16)(B2I(((R_REG(di->osh, &dregs->status0) & D64_XS0_CD_MASK) - 3105 di->xmtptrbase) & D64_XS0_CD_MASK, dma64dd_t)); 3106 di->xs0cd = end; 3107 } else 3108 end = di->xs0cd; 3109 3110 if (range == HNDDMA_RANGE_TRANSFERED) { 3111 active_desc = (uint16)(R_REG(di->osh, &dregs->status1) & D64_XS1_AD_MASK); 3112 active_desc = (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK; 3113 active_desc = B2I(active_desc, dma64dd_t); 3114 if (end != active_desc) 3115 end = PREVTXD(active_desc); 3116 } 3117 } 3118 3119 3120 if ((start == 0) && (end > di->txout)) 3121 goto bogus; 3122 3123 for (i = start; i != end && !txp; i = NEXTTXD(i)) { 3124 hnddma_seg_map_t *map = NULL; 3125 uint size, j, nsegs; 3126 3127#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3128 dmaaddr_t pa; 3129 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->txd64[i].addrlow)) - di->dataoffsetlow)); 3130 PHYSADDRHISET(pa, (BUS_SWAP32(R_SM(&di->txd64[i].addrhigh)) - di->dataoffsethigh)); 3131#endif 3132 3133 if (DMASGLIST_ENAB) { 3134 map = &di->txp_dmah[i]; 3135 size = map->origsize; 3136 nsegs = map->nsegs; 3137 if (nsegs > (uint)NTXDACTIVE(i, end)) { 3138 di->xs0cd = i; 3139 break; 3140 } 3141 } else { 3142#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3143 size = (BUS_SWAP32(R_SM(&di->txd64[i].ctrl2)) & D64_CTRL2_BC_MASK); 3144#endif 3145 nsegs = 1; 3146 } 3147 3148 for (j = nsegs; j > 0; j--) { 3149#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3150 W_SM(&di->txd64[i].addrlow, 0xdeadbeef); 3151 W_SM(&di->txd64[i].addrhigh, 0xdeadbeef); 3152#endif 3153 3154 txp = di->txp[i]; 3155 di->txp[i] = NULL; 3156 if (j > 1) 3157 i = NEXTTXD(i); 3158 } 3159 3160#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3161 DMA_UNMAP(di->osh, pa, size, DMA_TX, txp, map); 3162#endif 3163 } 3164 3165 di->txin = i; 3166 3167 /* tx flow control */ 3168 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 3169 3170 return (txp); 3171 3172bogus: 3173 DMA_NONE(("dma_getnexttxp: bogus curr: start %d end %d txout %d force %d\n", 3174 start, end, di->txout, forceall)); 3175 return (NULL); 3176} 3177 3178static void * BCMFASTPATH 3179dma64_getnextrxp(dma_info_t *di, bool forceall) 3180{ 3181 uint16 i, curr; 3182 void *rxp; 3183#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3184 dmaaddr_t pa; 3185#endif 3186 3187 /* if forcing, dma engine must be disabled */ 3188 ASSERT(!forceall || !dma64_rxenabled(di)); 3189 3190 i = di->rxin; 3191 3192 /* return if no packets posted */ 3193 if (i == di->rxout) 3194 return (NULL); 3195 3196 if (di->rxin == di->rs0cd) { 3197 curr = (uint16)B2I(((R_REG(di->osh, &di->d64rxregs->status0) & D64_RS0_CD_MASK) - 3198 di->rcvptrbase) & D64_RS0_CD_MASK, dma64dd_t); 3199 di->rs0cd = curr; 3200 } else 3201 curr = di->rs0cd; 3202 3203 /* ignore curr if forceall */ 3204 if (!forceall && (i == curr)) 3205 return (NULL); 3206 3207 /* get the packet pointer that corresponds to the rx descriptor */ 3208 rxp = di->rxp[i]; 3209 ASSERT(rxp); 3210 di->rxp[i] = NULL; 3211 3212#if ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) 3213 PHYSADDRLOSET(pa, (BUS_SWAP32(R_SM(&di->rxd64[i].addrlow)) - di->dataoffsetlow)); 3214 PHYSADDRHISET(pa, (BUS_SWAP32(R_SM(&di->rxd64[i].addrhigh)) - di->dataoffsethigh)); 3215 3216 /* clear this packet from the descriptor ring */ 3217 DMA_UNMAP(di->osh, pa, 3218 di->rxbufsize, DMA_RX, rxp, &di->rxp_dmah[i]); 3219 3220 W_SM(&di->rxd64[i].addrlow, 0xdeadbeef); 3221 W_SM(&di->rxd64[i].addrhigh, 0xdeadbeef); 3222#endif /* ((!defined(__mips__) && !defined(BCM47XX_CA9)) || defined(__NetBSD__)) */ 3223 3224 di->rxin = NEXTRXD(i); 3225 3226 return (rxp); 3227} 3228 3229static bool 3230_dma64_addrext(osl_t *osh, dma64regs_t *dma64regs) 3231{ 3232 uint32 w; 3233 OR_REG(osh, &dma64regs->control, D64_XC_AE); 3234 w = R_REG(osh, &dma64regs->control); 3235 AND_REG(osh, &dma64regs->control, ~D64_XC_AE); 3236 return ((w & D64_XC_AE) == D64_XC_AE); 3237} 3238 3239/** 3240 * Rotate all active tx dma ring entries "forward" by (ActiveDescriptor - txin). 3241 */ 3242static void 3243dma64_txrotate(dma_info_t *di) 3244{ 3245 uint16 ad; 3246 uint nactive; 3247 uint rot; 3248 uint16 old, new; 3249 uint32 w; 3250 uint16 first, last; 3251 3252 ASSERT(dma64_txsuspendedidle(di)); 3253 3254 nactive = _dma_txactive(di); 3255 ad = B2I((((R_REG(di->osh, &di->d64txregs->status1) & D64_XS1_AD_MASK) 3256 - di->xmtptrbase) & D64_XS1_AD_MASK), dma64dd_t); 3257 rot = TXD(ad - di->txin); 3258 3259 ASSERT(rot < di->ntxd); 3260 3261 /* full-ring case is a lot harder - don't worry about this */ 3262 if (rot >= (di->ntxd - nactive)) { 3263 DMA_ERROR(("%s: dma_txrotate: ring full - punt\n", di->name)); 3264 return; 3265 } 3266 3267 first = di->txin; 3268 last = PREVTXD(di->txout); 3269 3270 /* move entries starting at last and moving backwards to first */ 3271 for (old = last; old != PREVTXD(first); old = PREVTXD(old)) { 3272 new = TXD(old + rot); 3273 3274 /* 3275 * Move the tx dma descriptor. 3276 * EOT is set only in the last entry in the ring. 3277 */ 3278 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl1)) & ~D64_CTRL1_EOT; 3279 if (new == (di->ntxd - 1)) 3280 w |= D64_CTRL1_EOT; 3281 W_SM(&di->txd64[new].ctrl1, BUS_SWAP32(w)); 3282 3283 w = BUS_SWAP32(R_SM(&di->txd64[old].ctrl2)); 3284 W_SM(&di->txd64[new].ctrl2, BUS_SWAP32(w)); 3285 3286 W_SM(&di->txd64[new].addrlow, R_SM(&di->txd64[old].addrlow)); 3287 W_SM(&di->txd64[new].addrhigh, R_SM(&di->txd64[old].addrhigh)); 3288 3289 /* zap the old tx dma descriptor address field */ 3290 W_SM(&di->txd64[old].addrlow, BUS_SWAP32(0xdeadbeef)); 3291 W_SM(&di->txd64[old].addrhigh, BUS_SWAP32(0xdeadbeef)); 3292 3293 /* move the corresponding txp[] entry */ 3294 ASSERT(di->txp[new] == NULL); 3295 di->txp[new] = di->txp[old]; 3296 3297 /* Move the map */ 3298 if (DMASGLIST_ENAB) { 3299 bcopy(&di->txp_dmah[old], &di->txp_dmah[new], sizeof(hnddma_seg_map_t)); 3300 bzero(&di->txp_dmah[old], sizeof(hnddma_seg_map_t)); 3301 } 3302 3303 di->txp[old] = NULL; 3304 } 3305 3306 /* update txin and txout */ 3307 di->txin = ad; 3308 di->txout = TXD(di->txout + rot); 3309 di->hnddma.txavail = di->ntxd - NTXDACTIVE(di->txin, di->txout) - 1; 3310 3311 /* kick the chip */ 3312 W_REG(di->osh, &di->d64txregs->ptr, di->xmtptrbase + I2B(di->txout, dma64dd_t)); 3313} 3314 3315uint 3316BCMATTACHFN(dma_addrwidth)(si_t *sih, void *dmaregs) 3317{ 3318 dma32regs_t *dma32regs; 3319 osl_t *osh; 3320 3321 osh = si_osh(sih); 3322 3323 /* Perform 64-bit checks only if we want to advertise 64-bit (> 32bit) capability) */ 3324 /* DMA engine is 64-bit capable */ 3325 if ((si_core_sflags(sih, 0, 0) & SISF_DMA64) == SISF_DMA64) { 3326 /* backplane are 64-bit capable */ 3327 if (si_backplane64(sih)) 3328 /* If bus is System Backplane or PCIE then we can access 64-bits */ 3329 if ((BUSTYPE(sih->bustype) == SI_BUS) || 3330 ((BUSTYPE(sih->bustype) == PCI_BUS) && 3331 ((sih->buscoretype == PCIE_CORE_ID) || 3332 (sih->buscoretype == PCIE2_CORE_ID)))) 3333 return (DMADDRWIDTH_64); 3334 3335 /* DMA64 is always 32-bit capable, AE is always TRUE */ 3336 ASSERT(_dma64_addrext(osh, (dma64regs_t *)dmaregs)); 3337 3338 return (DMADDRWIDTH_32); 3339 } 3340 3341 /* Start checking for 32-bit / 30-bit addressing */ 3342 dma32regs = (dma32regs_t *)dmaregs; 3343 3344 /* For System Backplane, PCIE bus or addrext feature, 32-bits ok */ 3345 if ((BUSTYPE(sih->bustype) == SI_BUS) || 3346 ((BUSTYPE(sih->bustype) == PCI_BUS) && 3347 ((sih->buscoretype == PCIE_CORE_ID) || 3348 (sih->buscoretype == PCIE2_CORE_ID))) || 3349 (_dma32_addrext(osh, dma32regs))) 3350 return (DMADDRWIDTH_32); 3351 3352 /* Fallthru */ 3353 return (DMADDRWIDTH_30); 3354} 3355 3356static int 3357_dma_pktpool_set(dma_info_t *di, pktpool_t *pool) 3358{ 3359 ASSERT(di); 3360 ASSERT(di->pktpool == NULL); 3361 di->pktpool = pool; 3362 return 0; 3363} 3364 3365static bool 3366_dma_rxtx_error(dma_info_t *di, bool istx) 3367{ 3368 uint32 status1 = 0; 3369 3370 if (DMA64_ENAB(di) && DMA64_MODE(di)) { 3371 3372 if (istx) { 3373 3374 status1 = R_REG(di->osh, &di->d64txregs->status1); 3375 3376 if ((status1 & D64_XS1_XE_MASK) != D64_XS1_XE_NOERR) 3377 return TRUE; 3378 else 3379 return FALSE; 3380 } 3381 else { 3382 3383 status1 = R_REG(di->osh, &di->d64rxregs->status1); 3384 3385 if ((status1 & D64_RS1_RE_MASK) != D64_RS1_RE_NOERR) 3386 return TRUE; 3387 else 3388 return FALSE; 3389 } 3390 3391 } else if (DMA32_ENAB(di)) { 3392 return FALSE; 3393 3394 } else { 3395 ASSERT(0); 3396 return FALSE; 3397 } 3398 3399} 3400 3401void 3402_dma_burstlen_set(dma_info_t *di, uint8 rxburstlen, uint8 txburstlen) 3403{ 3404 di->rxburstlen = rxburstlen; 3405 di->txburstlen = txburstlen; 3406} 3407 3408void 3409_dma_param_set(dma_info_t *di, uint16 paramid, uint16 paramval) 3410{ 3411 switch (paramid) { 3412 case HNDDMA_PID_TX_MULTI_OUTSTD_RD: 3413 di->txmultioutstdrd = (uint8)paramval; 3414 break; 3415 3416 case HNDDMA_PID_TX_PREFETCH_CTL: 3417 di->txprefetchctl = (uint8)paramval; 3418 break; 3419 3420 case HNDDMA_PID_TX_PREFETCH_THRESH: 3421 di->txprefetchthresh = (uint8)paramval; 3422 break; 3423 3424 case HNDDMA_PID_TX_BURSTLEN: 3425 di->txburstlen = (uint8)paramval; 3426 break; 3427 3428 case HNDDMA_PID_RX_PREFETCH_CTL: 3429 di->rxprefetchctl = (uint8)paramval; 3430 break; 3431 3432 case HNDDMA_PID_RX_PREFETCH_THRESH: 3433 di->rxprefetchthresh = (uint8)paramval; 3434 break; 3435 3436 case HNDDMA_PID_RX_BURSTLEN: 3437 di->rxburstlen = (uint8)paramval; 3438 break; 3439 3440 default: 3441 break; 3442 } 3443} 3444 3445static bool 3446_dma_glom_enable(dma_info_t *di, uint32 val) 3447{ 3448 dma64regs_t *dregs = di->d64rxregs; 3449 bool ret = TRUE; 3450 if (val) { 3451 OR_REG(di->osh, &dregs->control, D64_RC_GE); 3452 if (!(R_REG(di->osh, &dregs->control) & D64_RC_GE)) 3453 ret = FALSE; 3454 } else { 3455 AND_REG(di->osh, &dregs->control, ~D64_RC_GE); 3456 } 3457 return ret; 3458} 3459