1/* 2 * Copyright (c) 2010 Broadcom Corporation 3 * 4 * Permission to use, copy, modify, and/or distribute this software for any 5 * purpose with or without fee is hereby granted, provided that the above 6 * copyright notice and this permission notice appear in all copies. 7 * 8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 11 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 13 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 14 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 */ 16 17#include <linux/slab.h> 18#include <linux/delay.h> 19#include <linux/pci.h> 20#include <net/cfg80211.h> 21#include <net/mac80211.h> 22 23#include <brcmu_utils.h> 24#include <aiutils.h> 25#include "types.h" 26#include "main.h" 27#include "dma.h" 28#include "soc.h" 29#include "scb.h" 30#include "ampdu.h" 31#include "debug.h" 32#include "brcms_trace_events.h" 33 34/* 35 * dma register field offset calculation 36 */ 37#define DMA64REGOFFS(field) offsetof(struct dma64regs, field) 38#define DMA64TXREGOFFS(di, field) (di->d64txregbase + DMA64REGOFFS(field)) 39#define DMA64RXREGOFFS(di, field) (di->d64rxregbase + DMA64REGOFFS(field)) 40 41/* 42 * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within 43 * a contiguous 8kB physical address. 44 */ 45#define D64RINGALIGN_BITS 13 46#define D64MAXRINGSZ (1 << D64RINGALIGN_BITS) 47#define D64RINGALIGN (1 << D64RINGALIGN_BITS) 48 49#define D64MAXDD (D64MAXRINGSZ / sizeof(struct dma64desc)) 50 51/* transmit channel control */ 52#define D64_XC_XE 0x00000001 /* transmit enable */ 53#define D64_XC_SE 0x00000002 /* transmit suspend request */ 54#define D64_XC_LE 0x00000004 /* loopback enable */ 55#define D64_XC_FL 0x00000010 /* flush request */ 56#define D64_XC_PD 0x00000800 /* parity check disable */ 57#define D64_XC_AE 0x00030000 /* address extension bits */ 58#define D64_XC_AE_SHIFT 16 59 60/* transmit descriptor table pointer */ 61#define D64_XP_LD_MASK 0x00000fff /* last valid descriptor */ 62 63/* transmit channel status */ 64#define D64_XS0_CD_MASK 0x00001fff /* current descriptor pointer */ 65#define D64_XS0_XS_MASK 0xf0000000 /* transmit state */ 66#define D64_XS0_XS_SHIFT 28 67#define D64_XS0_XS_DISABLED 0x00000000 /* disabled */ 68#define D64_XS0_XS_ACTIVE 0x10000000 /* active */ 69#define D64_XS0_XS_IDLE 0x20000000 /* idle wait */ 70#define D64_XS0_XS_STOPPED 0x30000000 /* stopped */ 71#define D64_XS0_XS_SUSP 0x40000000 /* suspend pending */ 72 73#define D64_XS1_AD_MASK 0x00001fff /* active descriptor */ 74#define D64_XS1_XE_MASK 0xf0000000 /* transmit errors */ 75#define D64_XS1_XE_SHIFT 28 76#define D64_XS1_XE_NOERR 0x00000000 /* no error */ 77#define D64_XS1_XE_DPE 0x10000000 /* descriptor protocol error */ 78#define D64_XS1_XE_DFU 0x20000000 /* data fifo underrun */ 79#define D64_XS1_XE_DTE 0x30000000 /* data transfer error */ 80#define D64_XS1_XE_DESRE 0x40000000 /* descriptor read error */ 81#define D64_XS1_XE_COREE 0x50000000 /* core error */ 82 83/* receive channel control */ 84/* receive enable */ 85#define D64_RC_RE 0x00000001 86/* receive frame offset */ 87#define D64_RC_RO_MASK 0x000000fe 88#define D64_RC_RO_SHIFT 1 89/* direct fifo receive (pio) mode */ 90#define D64_RC_FM 0x00000100 91/* separate rx header descriptor enable */ 92#define D64_RC_SH 0x00000200 93/* overflow continue */ 94#define D64_RC_OC 0x00000400 95/* parity check disable */ 96#define D64_RC_PD 0x00000800 97/* address extension bits */ 98#define D64_RC_AE 0x00030000 99#define D64_RC_AE_SHIFT 16 100 101/* flags for dma controller */ 102/* partity enable */ 103#define DMA_CTRL_PEN (1 << 0) 104/* rx overflow continue */ 105#define DMA_CTRL_ROC (1 << 1) 106/* allow rx scatter to multiple descriptors */ 107#define DMA_CTRL_RXMULTI (1 << 2) 108/* Unframed Rx/Tx data */ 109#define DMA_CTRL_UNFRAMED (1 << 3) 110 111/* receive descriptor table pointer */ 112#define D64_RP_LD_MASK 0x00000fff /* last valid descriptor */ 113 114/* receive channel status */ 115#define D64_RS0_CD_MASK 0x00001fff /* current descriptor pointer */ 116#define D64_RS0_RS_MASK 0xf0000000 /* receive state */ 117#define D64_RS0_RS_SHIFT 28 118#define D64_RS0_RS_DISABLED 0x00000000 /* disabled */ 119#define D64_RS0_RS_ACTIVE 0x10000000 /* active */ 120#define D64_RS0_RS_IDLE 0x20000000 /* idle wait */ 121#define D64_RS0_RS_STOPPED 0x30000000 /* stopped */ 122#define D64_RS0_RS_SUSP 0x40000000 /* suspend pending */ 123 124#define D64_RS1_AD_MASK 0x0001ffff /* active descriptor */ 125#define D64_RS1_RE_MASK 0xf0000000 /* receive errors */ 126#define D64_RS1_RE_SHIFT 28 127#define D64_RS1_RE_NOERR 0x00000000 /* no error */ 128#define D64_RS1_RE_DPO 0x10000000 /* descriptor protocol error */ 129#define D64_RS1_RE_DFU 0x20000000 /* data fifo overflow */ 130#define D64_RS1_RE_DTE 0x30000000 /* data transfer error */ 131#define D64_RS1_RE_DESRE 0x40000000 /* descriptor read error */ 132#define D64_RS1_RE_COREE 0x50000000 /* core error */ 133 134/* fifoaddr */ 135#define D64_FA_OFF_MASK 0xffff /* offset */ 136#define D64_FA_SEL_MASK 0xf0000 /* select */ 137#define D64_FA_SEL_SHIFT 16 138#define D64_FA_SEL_XDD 0x00000 /* transmit dma data */ 139#define D64_FA_SEL_XDP 0x10000 /* transmit dma pointers */ 140#define D64_FA_SEL_RDD 0x40000 /* receive dma data */ 141#define D64_FA_SEL_RDP 0x50000 /* receive dma pointers */ 142#define D64_FA_SEL_XFD 0x80000 /* transmit fifo data */ 143#define D64_FA_SEL_XFP 0x90000 /* transmit fifo pointers */ 144#define D64_FA_SEL_RFD 0xc0000 /* receive fifo data */ 145#define D64_FA_SEL_RFP 0xd0000 /* receive fifo pointers */ 146#define D64_FA_SEL_RSD 0xe0000 /* receive frame status data */ 147#define D64_FA_SEL_RSP 0xf0000 /* receive frame status pointers */ 148 149/* descriptor control flags 1 */ 150#define D64_CTRL_COREFLAGS 0x0ff00000 /* core specific flags */ 151#define D64_CTRL1_EOT ((u32)1 << 28) /* end of descriptor table */ 152#define D64_CTRL1_IOC ((u32)1 << 29) /* interrupt on completion */ 153#define D64_CTRL1_EOF ((u32)1 << 30) /* end of frame */ 154#define D64_CTRL1_SOF ((u32)1 << 31) /* start of frame */ 155 156/* descriptor control flags 2 */ 157/* buffer byte count. real data len must <= 16KB */ 158#define D64_CTRL2_BC_MASK 0x00007fff 159/* address extension bits */ 160#define D64_CTRL2_AE 0x00030000 161#define D64_CTRL2_AE_SHIFT 16 162/* parity bit */ 163#define D64_CTRL2_PARITY 0x00040000 164 165/* control flags in the range [27:20] are core-specific and not defined here */ 166#define D64_CTRL_CORE_MASK 0x0ff00000 167 168#define D64_RX_FRM_STS_LEN 0x0000ffff /* frame length mask */ 169#define D64_RX_FRM_STS_OVFL 0x00800000 /* RxOverFlow */ 170#define D64_RX_FRM_STS_DSCRCNT 0x0f000000 /* no. of descriptors used - 1 */ 171#define D64_RX_FRM_STS_DATATYPE 0xf0000000 /* core-dependent data type */ 172 173/* 174 * packet headroom necessary to accommodate the largest header 175 * in the system, (i.e TXOFF). By doing, we avoid the need to 176 * allocate an extra buffer for the header when bridging to WL. 177 * There is a compile time check in wlc.c which ensure that this 178 * value is at least as big as TXOFF. This value is used in 179 * dma_rxfill(). 180 */ 181 182#define BCMEXTRAHDROOM 172 183 184#define MAXNAMEL 8 /* 8 char names */ 185 186/* macros to convert between byte offsets and indexes */ 187#define B2I(bytes, type) ((bytes) / sizeof(type)) 188#define I2B(index, type) ((index) * sizeof(type)) 189 190#define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */ 191#define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */ 192 193#define PCI64ADDR_HIGH 0x80000000 /* address[63] */ 194#define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */ 195 196/* 197 * DMA Descriptor 198 * Descriptors are only read by the hardware, never written back. 199 */ 200struct dma64desc { 201 __le32 ctrl1; /* misc control bits & bufcount */ 202 __le32 ctrl2; /* buffer count and address extension */ 203 __le32 addrlow; /* memory address of the date buffer, bits 31:0 */ 204 __le32 addrhigh; /* memory address of the date buffer, bits 63:32 */ 205}; 206 207/* dma engine software state */ 208struct dma_info { 209 struct dma_pub dma; /* exported structure */ 210 char name[MAXNAMEL]; /* callers name for diag msgs */ 211 212 struct bcma_device *core; 213 struct device *dmadev; 214 215 /* session information for AMPDU */ 216 struct brcms_ampdu_session ampdu_session; 217 218 bool dma64; /* this dma engine is operating in 64-bit mode */ 219 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */ 220 221 /* 64-bit dma tx engine registers */ 222 uint d64txregbase; 223 /* 64-bit dma rx engine registers */ 224 uint d64rxregbase; 225 /* pointer to dma64 tx descriptor ring */ 226 struct dma64desc *txd64; 227 /* pointer to dma64 rx descriptor ring */ 228 struct dma64desc *rxd64; 229 230 u16 dmadesc_align; /* alignment requirement for dma descriptors */ 231 232 u16 ntxd; /* # tx descriptors tunable */ 233 u16 txin; /* index of next descriptor to reclaim */ 234 u16 txout; /* index of next descriptor to post */ 235 /* pointer to parallel array of pointers to packets */ 236 struct sk_buff **txp; 237 /* Aligned physical address of descriptor ring */ 238 dma_addr_t txdpa; 239 /* Original physical address of descriptor ring */ 240 dma_addr_t txdpaorig; 241 u16 txdalign; /* #bytes added to alloc'd mem to align txd */ 242 u32 txdalloc; /* #bytes allocated for the ring */ 243 u32 xmtptrbase; /* When using unaligned descriptors, the ptr register 244 * is not just an index, it needs all 13 bits to be 245 * an offset from the addr register. 246 */ 247 248 u16 nrxd; /* # rx descriptors tunable */ 249 u16 rxin; /* index of next descriptor to reclaim */ 250 u16 rxout; /* index of next descriptor to post */ 251 /* pointer to parallel array of pointers to packets */ 252 struct sk_buff **rxp; 253 /* Aligned physical address of descriptor ring */ 254 dma_addr_t rxdpa; 255 /* Original physical address of descriptor ring */ 256 dma_addr_t rxdpaorig; 257 u16 rxdalign; /* #bytes added to alloc'd mem to align rxd */ 258 u32 rxdalloc; /* #bytes allocated for the ring */ 259 u32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */ 260 261 /* tunables */ 262 unsigned int rxbufsize; /* rx buffer size in bytes, not including 263 * the extra headroom 264 */ 265 uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper 266 * stack, e.g. some rx pkt buffers will be 267 * bridged to tx side without byte copying. 268 * The extra headroom needs to be large enough 269 * to fit txheader needs. Some dongle driver may 270 * not need it. 271 */ 272 uint nrxpost; /* # rx buffers to keep posted */ 273 unsigned int rxoffset; /* rxcontrol offset */ 274 /* add to get dma address of descriptor ring, low 32 bits */ 275 uint ddoffsetlow; 276 /* high 32 bits */ 277 uint ddoffsethigh; 278 /* add to get dma address of data buffer, low 32 bits */ 279 uint dataoffsetlow; 280 /* high 32 bits */ 281 uint dataoffsethigh; 282 /* descriptor base need to be aligned or not */ 283 bool aligndesc_4k; 284}; 285 286/* Check for odd number of 1's */ 287static u32 parity32(__le32 data) 288{ 289 /* no swap needed for counting 1's */ 290 u32 par_data = *(u32 *)&data; 291 292 par_data ^= par_data >> 16; 293 par_data ^= par_data >> 8; 294 par_data ^= par_data >> 4; 295 par_data ^= par_data >> 2; 296 par_data ^= par_data >> 1; 297 298 return par_data & 1; 299} 300 301static bool dma64_dd_parity(struct dma64desc *dd) 302{ 303 return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2); 304} 305 306/* descriptor bumping functions */ 307 308static uint xxd(uint x, uint n) 309{ 310 return x & (n - 1); /* faster than %, but n must be power of 2 */ 311} 312 313static uint txd(struct dma_info *di, uint x) 314{ 315 return xxd(x, di->ntxd); 316} 317 318static uint rxd(struct dma_info *di, uint x) 319{ 320 return xxd(x, di->nrxd); 321} 322 323static uint nexttxd(struct dma_info *di, uint i) 324{ 325 return txd(di, i + 1); 326} 327 328static uint prevtxd(struct dma_info *di, uint i) 329{ 330 return txd(di, i - 1); 331} 332 333static uint nextrxd(struct dma_info *di, uint i) 334{ 335 return rxd(di, i + 1); 336} 337 338static uint ntxdactive(struct dma_info *di, uint h, uint t) 339{ 340 return txd(di, t-h); 341} 342 343static uint nrxdactive(struct dma_info *di, uint h, uint t) 344{ 345 return rxd(di, t-h); 346} 347 348static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags) 349{ 350 uint dmactrlflags; 351 352 if (di == NULL) 353 return 0; 354 355 dmactrlflags = di->dma.dmactrlflags; 356 dmactrlflags &= ~mask; 357 dmactrlflags |= flags; 358 359 /* If trying to enable parity, check if parity is actually supported */ 360 if (dmactrlflags & DMA_CTRL_PEN) { 361 u32 control; 362 363 control = bcma_read32(di->core, DMA64TXREGOFFS(di, control)); 364 bcma_write32(di->core, DMA64TXREGOFFS(di, control), 365 control | D64_XC_PD); 366 if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) & 367 D64_XC_PD) 368 /* We *can* disable it so it is supported, 369 * restore control register 370 */ 371 bcma_write32(di->core, DMA64TXREGOFFS(di, control), 372 control); 373 else 374 /* Not supported, don't allow it to be enabled */ 375 dmactrlflags &= ~DMA_CTRL_PEN; 376 } 377 378 di->dma.dmactrlflags = dmactrlflags; 379 380 return dmactrlflags; 381} 382 383static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset) 384{ 385 u32 w; 386 bcma_set32(di->core, ctrl_offset, D64_XC_AE); 387 w = bcma_read32(di->core, ctrl_offset); 388 bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE); 389 return (w & D64_XC_AE) == D64_XC_AE; 390} 391 392/* 393 * return true if this dma engine supports DmaExtendedAddrChanges, 394 * otherwise false 395 */ 396static bool _dma_isaddrext(struct dma_info *di) 397{ 398 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */ 399 400 /* not all tx or rx channel are available */ 401 if (di->d64txregbase != 0) { 402 if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control))) 403 brcms_dbg_dma(di->core, 404 "%s: DMA64 tx doesn't have AE set\n", 405 di->name); 406 return true; 407 } else if (di->d64rxregbase != 0) { 408 if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control))) 409 brcms_dbg_dma(di->core, 410 "%s: DMA64 rx doesn't have AE set\n", 411 di->name); 412 return true; 413 } 414 415 return false; 416} 417 418static bool _dma_descriptor_align(struct dma_info *di) 419{ 420 u32 addrl; 421 422 /* Check to see if the descriptors need to be aligned on 4K/8K or not */ 423 if (di->d64txregbase != 0) { 424 bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0); 425 addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow)); 426 if (addrl != 0) 427 return false; 428 } else if (di->d64rxregbase != 0) { 429 bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0); 430 addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow)); 431 if (addrl != 0) 432 return false; 433 } 434 return true; 435} 436 437/* 438 * Descriptor table must start at the DMA hardware dictated alignment, so 439 * allocated memory must be large enough to support this requirement. 440 */ 441static void *dma_alloc_consistent(struct dma_info *di, uint size, 442 u16 align_bits, uint *alloced, 443 dma_addr_t *pap) 444{ 445 if (align_bits) { 446 u16 align = (1 << align_bits); 447 if (!IS_ALIGNED(PAGE_SIZE, align)) 448 size += align; 449 *alloced = size; 450 } 451 return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC); 452} 453 454static 455u8 dma_align_sizetobits(uint size) 456{ 457 u8 bitpos = 0; 458 while (size >>= 1) 459 bitpos++; 460 return bitpos; 461} 462 463/* This function ensures that the DMA descriptor ring will not get allocated 464 * across Page boundary. If the allocation is done across the page boundary 465 * at the first time, then it is freed and the allocation is done at 466 * descriptor ring size aligned location. This will ensure that the ring will 467 * not cross page boundary 468 */ 469static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size, 470 u16 *alignbits, uint *alloced, 471 dma_addr_t *descpa) 472{ 473 void *va; 474 u32 desc_strtaddr; 475 u32 alignbytes = 1 << *alignbits; 476 477 va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa); 478 479 if (NULL == va) 480 return NULL; 481 482 desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes); 483 if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr 484 & boundary)) { 485 *alignbits = dma_align_sizetobits(size); 486 dma_free_coherent(di->dmadev, size, va, *descpa); 487 va = dma_alloc_consistent(di, size, *alignbits, 488 alloced, descpa); 489 } 490 return va; 491} 492 493static bool dma64_alloc(struct dma_info *di, uint direction) 494{ 495 u16 size; 496 uint ddlen; 497 void *va; 498 uint alloced = 0; 499 u16 align; 500 u16 align_bits; 501 502 ddlen = sizeof(struct dma64desc); 503 504 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen); 505 align_bits = di->dmadesc_align; 506 align = (1 << align_bits); 507 508 if (direction == DMA_TX) { 509 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits, 510 &alloced, &di->txdpaorig); 511 if (va == NULL) { 512 brcms_dbg_dma(di->core, 513 "%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n", 514 di->name); 515 return false; 516 } 517 align = (1 << align_bits); 518 di->txd64 = (struct dma64desc *) 519 roundup((unsigned long)va, align); 520 di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va); 521 di->txdpa = di->txdpaorig + di->txdalign; 522 di->txdalloc = alloced; 523 } else { 524 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits, 525 &alloced, &di->rxdpaorig); 526 if (va == NULL) { 527 brcms_dbg_dma(di->core, 528 "%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n", 529 di->name); 530 return false; 531 } 532 align = (1 << align_bits); 533 di->rxd64 = (struct dma64desc *) 534 roundup((unsigned long)va, align); 535 di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va); 536 di->rxdpa = di->rxdpaorig + di->rxdalign; 537 di->rxdalloc = alloced; 538 } 539 540 return true; 541} 542 543static bool _dma_alloc(struct dma_info *di, uint direction) 544{ 545 return dma64_alloc(di, direction); 546} 547 548struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc, 549 uint txregbase, uint rxregbase, uint ntxd, uint nrxd, 550 uint rxbufsize, int rxextheadroom, 551 uint nrxpost, uint rxoffset) 552{ 553 struct si_pub *sih = wlc->hw->sih; 554 struct bcma_device *core = wlc->hw->d11core; 555 struct dma_info *di; 556 u8 rev = core->id.rev; 557 uint size; 558 struct si_info *sii = container_of(sih, struct si_info, pub); 559 560 /* allocate private info structure */ 561 di = kzalloc(sizeof(struct dma_info), GFP_ATOMIC); 562 if (di == NULL) 563 return NULL; 564 565 di->dma64 = 566 ((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64); 567 568 /* init dma reg info */ 569 di->core = core; 570 di->d64txregbase = txregbase; 571 di->d64rxregbase = rxregbase; 572 573 /* 574 * Default flags (which can be changed by the driver calling 575 * dma_ctrlflags before enable): For backwards compatibility 576 * both Rx Overflow Continue and Parity are DISABLED. 577 */ 578 _dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0); 579 580 brcms_dbg_dma(di->core, "%s: %s flags 0x%x ntxd %d nrxd %d " 581 "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d " 582 "txregbase %u rxregbase %u\n", name, "DMA64", 583 di->dma.dmactrlflags, ntxd, nrxd, rxbufsize, 584 rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase); 585 586 /* make a private copy of our callers name */ 587 strscpy(di->name, name, sizeof(di->name)); 588 589 di->dmadev = core->dma_dev; 590 591 /* save tunables */ 592 di->ntxd = (u16) ntxd; 593 di->nrxd = (u16) nrxd; 594 595 /* the actual dma size doesn't include the extra headroom */ 596 di->rxextrahdrroom = 597 (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom; 598 if (rxbufsize > BCMEXTRAHDROOM) 599 di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom); 600 else 601 di->rxbufsize = (u16) rxbufsize; 602 603 di->nrxpost = (u16) nrxpost; 604 di->rxoffset = (u8) rxoffset; 605 606 /* 607 * figure out the DMA physical address offset for dd and data 608 * PCI/PCIE: they map silicon backplace address to zero 609 * based memory, need offset 610 * Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram 611 * swapped region for data buffer, not descriptor 612 */ 613 di->ddoffsetlow = 0; 614 di->dataoffsetlow = 0; 615 /* for pci bus, add offset */ 616 if (sii->icbus->hosttype == BCMA_HOSTTYPE_PCI) { 617 /* add offset for pcie with DMA64 bus */ 618 di->ddoffsetlow = 0; 619 di->ddoffsethigh = SI_PCIE_DMA_H32; 620 } 621 di->dataoffsetlow = di->ddoffsetlow; 622 di->dataoffsethigh = di->ddoffsethigh; 623 624 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */ 625 if ((core->id.id == BCMA_CORE_SDIO_DEV) 626 && ((rev > 0) && (rev <= 2))) 627 di->addrext = false; 628 else if ((core->id.id == BCMA_CORE_I2S) && 629 ((rev == 0) || (rev == 1))) 630 di->addrext = false; 631 else 632 di->addrext = _dma_isaddrext(di); 633 634 /* does the descriptor need to be aligned and if yes, on 4K/8K or not */ 635 di->aligndesc_4k = _dma_descriptor_align(di); 636 if (di->aligndesc_4k) { 637 di->dmadesc_align = D64RINGALIGN_BITS; 638 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2)) 639 /* for smaller dd table, HW relax alignment reqmnt */ 640 di->dmadesc_align = D64RINGALIGN_BITS - 1; 641 } else { 642 di->dmadesc_align = 4; /* 16 byte alignment */ 643 } 644 645 brcms_dbg_dma(di->core, "DMA descriptor align_needed %d, align %d\n", 646 di->aligndesc_4k, di->dmadesc_align); 647 648 /* allocate tx packet pointer vector */ 649 if (ntxd) { 650 size = ntxd * sizeof(void *); 651 di->txp = kzalloc(size, GFP_ATOMIC); 652 if (di->txp == NULL) 653 goto fail; 654 } 655 656 /* allocate rx packet pointer vector */ 657 if (nrxd) { 658 size = nrxd * sizeof(void *); 659 di->rxp = kzalloc(size, GFP_ATOMIC); 660 if (di->rxp == NULL) 661 goto fail; 662 } 663 664 /* 665 * allocate transmit descriptor ring, only need ntxd descriptors 666 * but it must be aligned 667 */ 668 if (ntxd) { 669 if (!_dma_alloc(di, DMA_TX)) 670 goto fail; 671 } 672 673 /* 674 * allocate receive descriptor ring, only need nrxd descriptors 675 * but it must be aligned 676 */ 677 if (nrxd) { 678 if (!_dma_alloc(di, DMA_RX)) 679 goto fail; 680 } 681 682 if ((di->ddoffsetlow != 0) && !di->addrext) { 683 if (di->txdpa > SI_PCI_DMA_SZ) { 684 brcms_dbg_dma(di->core, 685 "%s: txdpa 0x%x: addrext not supported\n", 686 di->name, (u32)di->txdpa); 687 goto fail; 688 } 689 if (di->rxdpa > SI_PCI_DMA_SZ) { 690 brcms_dbg_dma(di->core, 691 "%s: rxdpa 0x%x: addrext not supported\n", 692 di->name, (u32)di->rxdpa); 693 goto fail; 694 } 695 } 696 697 /* Initialize AMPDU session */ 698 brcms_c_ampdu_reset_session(&di->ampdu_session, wlc); 699 700 brcms_dbg_dma(di->core, 701 "ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n", 702 di->ddoffsetlow, di->ddoffsethigh, 703 di->dataoffsetlow, di->dataoffsethigh, 704 di->addrext); 705 706 return (struct dma_pub *) di; 707 708 fail: 709 dma_detach((struct dma_pub *)di); 710 return NULL; 711} 712 713static inline void 714dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring, 715 dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount) 716{ 717 u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK; 718 719 /* PCI bus with big(>1G) physical address, use address extension */ 720 if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) { 721 ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow); 722 ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh); 723 ddring[outidx].ctrl1 = cpu_to_le32(*flags); 724 ddring[outidx].ctrl2 = cpu_to_le32(ctrl2); 725 } else { 726 /* address extension for 32-bit PCI */ 727 u32 ae; 728 729 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 730 pa &= ~PCI32ADDR_HIGH; 731 732 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE; 733 ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow); 734 ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh); 735 ddring[outidx].ctrl1 = cpu_to_le32(*flags); 736 ddring[outidx].ctrl2 = cpu_to_le32(ctrl2); 737 } 738 if (di->dma.dmactrlflags & DMA_CTRL_PEN) { 739 if (dma64_dd_parity(&ddring[outidx])) 740 ddring[outidx].ctrl2 = 741 cpu_to_le32(ctrl2 | D64_CTRL2_PARITY); 742 } 743} 744 745/* !! may be called with core in reset */ 746void dma_detach(struct dma_pub *pub) 747{ 748 struct dma_info *di = container_of(pub, struct dma_info, dma); 749 750 brcms_dbg_dma(di->core, "%s:\n", di->name); 751 752 /* free dma descriptor rings */ 753 if (di->txd64) 754 dma_free_coherent(di->dmadev, di->txdalloc, 755 ((s8 *)di->txd64 - di->txdalign), 756 (di->txdpaorig)); 757 if (di->rxd64) 758 dma_free_coherent(di->dmadev, di->rxdalloc, 759 ((s8 *)di->rxd64 - di->rxdalign), 760 (di->rxdpaorig)); 761 762 /* free packet pointer vectors */ 763 kfree(di->txp); 764 kfree(di->rxp); 765 766 /* free our private info structure */ 767 kfree(di); 768 769} 770 771/* initialize descriptor table base address */ 772static void 773_dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa) 774{ 775 if (!di->aligndesc_4k) { 776 if (direction == DMA_TX) 777 di->xmtptrbase = pa; 778 else 779 di->rcvptrbase = pa; 780 } 781 782 if ((di->ddoffsetlow == 0) 783 || !(pa & PCI32ADDR_HIGH)) { 784 if (direction == DMA_TX) { 785 bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 786 pa + di->ddoffsetlow); 787 bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh), 788 di->ddoffsethigh); 789 } else { 790 bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 791 pa + di->ddoffsetlow); 792 bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh), 793 di->ddoffsethigh); 794 } 795 } else { 796 /* DMA64 32bits address extension */ 797 u32 ae; 798 799 /* shift the high bit(s) from pa to ae */ 800 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT; 801 pa &= ~PCI32ADDR_HIGH; 802 803 if (direction == DMA_TX) { 804 bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 805 pa + di->ddoffsetlow); 806 bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh), 807 di->ddoffsethigh); 808 bcma_maskset32(di->core, DMA64TXREGOFFS(di, control), 809 D64_XC_AE, (ae << D64_XC_AE_SHIFT)); 810 } else { 811 bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 812 pa + di->ddoffsetlow); 813 bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh), 814 di->ddoffsethigh); 815 bcma_maskset32(di->core, DMA64RXREGOFFS(di, control), 816 D64_RC_AE, (ae << D64_RC_AE_SHIFT)); 817 } 818 } 819} 820 821static void _dma_rxenable(struct dma_info *di) 822{ 823 uint dmactrlflags = di->dma.dmactrlflags; 824 u32 control; 825 826 brcms_dbg_dma(di->core, "%s:\n", di->name); 827 828 control = D64_RC_RE | (bcma_read32(di->core, 829 DMA64RXREGOFFS(di, control)) & 830 D64_RC_AE); 831 832 if ((dmactrlflags & DMA_CTRL_PEN) == 0) 833 control |= D64_RC_PD; 834 835 if (dmactrlflags & DMA_CTRL_ROC) 836 control |= D64_RC_OC; 837 838 bcma_write32(di->core, DMA64RXREGOFFS(di, control), 839 ((di->rxoffset << D64_RC_RO_SHIFT) | control)); 840} 841 842void dma_rxinit(struct dma_pub *pub) 843{ 844 struct dma_info *di = container_of(pub, struct dma_info, dma); 845 846 brcms_dbg_dma(di->core, "%s:\n", di->name); 847 848 if (di->nrxd == 0) 849 return; 850 851 di->rxin = di->rxout = 0; 852 853 /* clear rx descriptor ring */ 854 memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc)); 855 856 /* DMA engine with out alignment requirement requires table to be inited 857 * before enabling the engine 858 */ 859 if (!di->aligndesc_4k) 860 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 861 862 _dma_rxenable(di); 863 864 if (di->aligndesc_4k) 865 _dma_ddtable_init(di, DMA_RX, di->rxdpa); 866} 867 868static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall) 869{ 870 uint i, curr; 871 struct sk_buff *rxp; 872 dma_addr_t pa; 873 874 i = di->rxin; 875 876 /* return if no packets posted */ 877 if (i == di->rxout) 878 return NULL; 879 880 curr = 881 B2I(((bcma_read32(di->core, 882 DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) - 883 di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc); 884 885 /* ignore curr if forceall */ 886 if (!forceall && (i == curr)) 887 return NULL; 888 889 /* get the packet pointer that corresponds to the rx descriptor */ 890 rxp = di->rxp[i]; 891 di->rxp[i] = NULL; 892 893 pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow; 894 895 /* clear this packet from the descriptor ring */ 896 dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE); 897 898 di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef); 899 di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef); 900 901 di->rxin = nextrxd(di, i); 902 903 return rxp; 904} 905 906static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall) 907{ 908 if (di->nrxd == 0) 909 return NULL; 910 911 return dma64_getnextrxp(di, forceall); 912} 913 914/* 915 * !! rx entry routine 916 * returns the number packages in the next frame, or 0 if there are no more 917 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is 918 * supported with pkts chain 919 * otherwise, it's treated as giant pkt and will be tossed. 920 * The DMA scattering starts with normal DMA header, followed by first 921 * buffer data. After it reaches the max size of buffer, the data continues 922 * in next DMA descriptor buffer WITHOUT DMA header 923 */ 924int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list) 925{ 926 struct dma_info *di = container_of(pub, struct dma_info, dma); 927 struct sk_buff_head dma_frames; 928 struct sk_buff *p, *next; 929 uint len; 930 uint pkt_len; 931 int resid = 0; 932 int pktcnt = 1; 933 934 skb_queue_head_init(&dma_frames); 935 next_frame: 936 p = _dma_getnextrxp(di, false); 937 if (p == NULL) 938 return 0; 939 940 len = le16_to_cpu(*(__le16 *) (p->data)); 941 brcms_dbg_dma(di->core, "%s: dma_rx len %d\n", di->name, len); 942 dma_spin_for_len(len, p); 943 944 /* set actual length */ 945 pkt_len = min((di->rxoffset + len), di->rxbufsize); 946 __skb_trim(p, pkt_len); 947 skb_queue_tail(&dma_frames, p); 948 resid = len - (di->rxbufsize - di->rxoffset); 949 950 /* check for single or multi-buffer rx */ 951 if (resid > 0) { 952 while ((resid > 0) && (p = _dma_getnextrxp(di, false))) { 953 pkt_len = min_t(uint, resid, di->rxbufsize); 954 __skb_trim(p, pkt_len); 955 skb_queue_tail(&dma_frames, p); 956 resid -= di->rxbufsize; 957 pktcnt++; 958 } 959 960#ifdef DEBUG 961 if (resid > 0) { 962 uint cur; 963 cur = 964 B2I(((bcma_read32(di->core, 965 DMA64RXREGOFFS(di, status0)) & 966 D64_RS0_CD_MASK) - di->rcvptrbase) & 967 D64_RS0_CD_MASK, struct dma64desc); 968 brcms_dbg_dma(di->core, 969 "rxin %d rxout %d, hw_curr %d\n", 970 di->rxin, di->rxout, cur); 971 } 972#endif /* DEBUG */ 973 974 if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) { 975 brcms_dbg_dma(di->core, "%s: bad frame length (%d)\n", 976 di->name, len); 977 skb_queue_walk_safe(&dma_frames, p, next) { 978 skb_unlink(p, &dma_frames); 979 brcmu_pkt_buf_free_skb(p); 980 } 981 di->dma.rxgiants++; 982 pktcnt = 1; 983 goto next_frame; 984 } 985 } 986 987 skb_queue_splice_tail(&dma_frames, skb_list); 988 return pktcnt; 989} 990 991static bool dma64_rxidle(struct dma_info *di) 992{ 993 brcms_dbg_dma(di->core, "%s:\n", di->name); 994 995 if (di->nrxd == 0) 996 return true; 997 998 return ((bcma_read32(di->core, 999 DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) == 1000 (bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) & 1001 D64_RS0_CD_MASK)); 1002} 1003 1004static bool dma64_txidle(struct dma_info *di) 1005{ 1006 if (di->ntxd == 0) 1007 return true; 1008 1009 return ((bcma_read32(di->core, 1010 DMA64TXREGOFFS(di, status0)) & D64_XS0_CD_MASK) == 1011 (bcma_read32(di->core, DMA64TXREGOFFS(di, ptr)) & 1012 D64_XS0_CD_MASK)); 1013} 1014 1015/* 1016 * post receive buffers 1017 * Return false if refill failed completely or dma mapping failed. The ring 1018 * is empty, which will stall the rx dma and user might want to call rxfill 1019 * again asap. This is unlikely to happen on a memory-rich NIC, but often on 1020 * memory-constrained dongle. 1021 */ 1022bool dma_rxfill(struct dma_pub *pub) 1023{ 1024 struct dma_info *di = container_of(pub, struct dma_info, dma); 1025 struct sk_buff *p; 1026 u16 rxin, rxout; 1027 u32 flags = 0; 1028 uint n; 1029 uint i; 1030 dma_addr_t pa; 1031 uint extra_offset = 0; 1032 bool ring_empty; 1033 1034 ring_empty = false; 1035 1036 /* 1037 * Determine how many receive buffers we're lacking 1038 * from the full complement, allocate, initialize, 1039 * and post them, then update the chip rx lastdscr. 1040 */ 1041 1042 rxin = di->rxin; 1043 rxout = di->rxout; 1044 1045 n = di->nrxpost - nrxdactive(di, rxin, rxout); 1046 1047 brcms_dbg_dma(di->core, "%s: post %d\n", di->name, n); 1048 1049 if (di->rxbufsize > BCMEXTRAHDROOM) 1050 extra_offset = di->rxextrahdrroom; 1051 1052 for (i = 0; i < n; i++) { 1053 /* 1054 * the di->rxbufsize doesn't include the extra headroom, 1055 * we need to add it to the size to be allocated 1056 */ 1057 p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset); 1058 1059 if (p == NULL) { 1060 brcms_dbg_dma(di->core, "%s: out of rxbufs\n", 1061 di->name); 1062 if (i == 0 && dma64_rxidle(di)) { 1063 brcms_dbg_dma(di->core, "%s: ring is empty !\n", 1064 di->name); 1065 ring_empty = true; 1066 } 1067 di->dma.rxnobuf++; 1068 break; 1069 } 1070 /* reserve an extra headroom, if applicable */ 1071 if (extra_offset) 1072 skb_pull(p, extra_offset); 1073 1074 /* Do a cached write instead of uncached write since DMA_MAP 1075 * will flush the cache. 1076 */ 1077 *(u32 *) (p->data) = 0; 1078 1079 pa = dma_map_single(di->dmadev, p->data, di->rxbufsize, 1080 DMA_FROM_DEVICE); 1081 if (dma_mapping_error(di->dmadev, pa)) { 1082 brcmu_pkt_buf_free_skb(p); 1083 return false; 1084 } 1085 1086 /* save the free packet pointer */ 1087 di->rxp[rxout] = p; 1088 1089 /* reset flags for each descriptor */ 1090 flags = 0; 1091 if (rxout == (di->nrxd - 1)) 1092 flags = D64_CTRL1_EOT; 1093 1094 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags, 1095 di->rxbufsize); 1096 rxout = nextrxd(di, rxout); 1097 } 1098 1099 di->rxout = rxout; 1100 1101 /* update the chip lastdscr pointer */ 1102 bcma_write32(di->core, DMA64RXREGOFFS(di, ptr), 1103 di->rcvptrbase + I2B(rxout, struct dma64desc)); 1104 1105 return ring_empty; 1106} 1107 1108void dma_rxreclaim(struct dma_pub *pub) 1109{ 1110 struct dma_info *di = container_of(pub, struct dma_info, dma); 1111 struct sk_buff *p; 1112 1113 brcms_dbg_dma(di->core, "%s:\n", di->name); 1114 1115 while ((p = _dma_getnextrxp(di, true))) 1116 brcmu_pkt_buf_free_skb(p); 1117} 1118 1119void dma_counterreset(struct dma_pub *pub) 1120{ 1121 /* reset all software counters */ 1122 pub->rxgiants = 0; 1123 pub->rxnobuf = 0; 1124 pub->txnobuf = 0; 1125} 1126 1127/* get the address of the var in order to change later */ 1128unsigned long dma_getvar(struct dma_pub *pub, const char *name) 1129{ 1130 struct dma_info *di = container_of(pub, struct dma_info, dma); 1131 1132 if (!strcmp(name, "&txavail")) 1133 return (unsigned long)&(di->dma.txavail); 1134 return 0; 1135} 1136 1137/* 64-bit DMA functions */ 1138 1139void dma_txinit(struct dma_pub *pub) 1140{ 1141 struct dma_info *di = container_of(pub, struct dma_info, dma); 1142 u32 control = D64_XC_XE; 1143 1144 brcms_dbg_dma(di->core, "%s:\n", di->name); 1145 1146 if (di->ntxd == 0) 1147 return; 1148 1149 di->txin = di->txout = 0; 1150 di->dma.txavail = di->ntxd - 1; 1151 1152 /* clear tx descriptor ring */ 1153 memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc))); 1154 1155 /* DMA engine with out alignment requirement requires table to be inited 1156 * before enabling the engine 1157 */ 1158 if (!di->aligndesc_4k) 1159 _dma_ddtable_init(di, DMA_TX, di->txdpa); 1160 1161 if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0) 1162 control |= D64_XC_PD; 1163 bcma_set32(di->core, DMA64TXREGOFFS(di, control), control); 1164 1165 /* DMA engine with alignment requirement requires table to be inited 1166 * before enabling the engine 1167 */ 1168 if (di->aligndesc_4k) 1169 _dma_ddtable_init(di, DMA_TX, di->txdpa); 1170} 1171 1172void dma_txsuspend(struct dma_pub *pub) 1173{ 1174 struct dma_info *di = container_of(pub, struct dma_info, dma); 1175 1176 brcms_dbg_dma(di->core, "%s:\n", di->name); 1177 1178 if (di->ntxd == 0) 1179 return; 1180 1181 bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE); 1182} 1183 1184void dma_txresume(struct dma_pub *pub) 1185{ 1186 struct dma_info *di = container_of(pub, struct dma_info, dma); 1187 1188 brcms_dbg_dma(di->core, "%s:\n", di->name); 1189 1190 if (di->ntxd == 0) 1191 return; 1192 1193 bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE); 1194} 1195 1196bool dma_txsuspended(struct dma_pub *pub) 1197{ 1198 struct dma_info *di = container_of(pub, struct dma_info, dma); 1199 1200 return (di->ntxd == 0) || 1201 ((bcma_read32(di->core, 1202 DMA64TXREGOFFS(di, control)) & D64_XC_SE) == 1203 D64_XC_SE); 1204} 1205 1206void dma_txreclaim(struct dma_pub *pub, enum txd_range range) 1207{ 1208 struct dma_info *di = container_of(pub, struct dma_info, dma); 1209 struct sk_buff *p; 1210 1211 brcms_dbg_dma(di->core, "%s: %s\n", 1212 di->name, 1213 range == DMA_RANGE_ALL ? "all" : 1214 range == DMA_RANGE_TRANSMITTED ? "transmitted" : 1215 "transferred"); 1216 1217 if (di->txin == di->txout) 1218 return; 1219 1220 while ((p = dma_getnexttxp(pub, range))) { 1221 /* For unframed data, we don't have any packets to free */ 1222 if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED)) 1223 brcmu_pkt_buf_free_skb(p); 1224 } 1225} 1226 1227bool dma_txreset(struct dma_pub *pub) 1228{ 1229 struct dma_info *di = container_of(pub, struct dma_info, dma); 1230 u32 status; 1231 1232 if (di->ntxd == 0) 1233 return true; 1234 1235 /* suspend tx DMA first */ 1236 bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE); 1237 SPINWAIT(((status = 1238 (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) & 1239 D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) && 1240 (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED), 1241 10000); 1242 1243 bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0); 1244 SPINWAIT(((status = 1245 (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) & 1246 D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000); 1247 1248 /* wait for the last transaction to complete */ 1249 udelay(300); 1250 1251 return status == D64_XS0_XS_DISABLED; 1252} 1253 1254bool dma_rxreset(struct dma_pub *pub) 1255{ 1256 struct dma_info *di = container_of(pub, struct dma_info, dma); 1257 u32 status; 1258 1259 if (di->nrxd == 0) 1260 return true; 1261 1262 bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0); 1263 SPINWAIT(((status = 1264 (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) & 1265 D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000); 1266 1267 return status == D64_RS0_RS_DISABLED; 1268} 1269 1270static void dma_txenq(struct dma_info *di, struct sk_buff *p) 1271{ 1272 unsigned char *data; 1273 uint len; 1274 u16 txout; 1275 u32 flags = 0; 1276 dma_addr_t pa; 1277 1278 txout = di->txout; 1279 1280 if (WARN_ON(nexttxd(di, txout) == di->txin)) 1281 return; 1282 1283 /* 1284 * obtain and initialize transmit descriptor entry. 1285 */ 1286 data = p->data; 1287 len = p->len; 1288 1289 /* get physical address of buffer start */ 1290 pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE); 1291 /* if mapping failed, free skb */ 1292 if (dma_mapping_error(di->dmadev, pa)) { 1293 brcmu_pkt_buf_free_skb(p); 1294 return; 1295 } 1296 /* With a DMA segment list, Descriptor table is filled 1297 * using the segment list instead of looping over 1298 * buffers in multi-chain DMA. Therefore, EOF for SGLIST 1299 * is when end of segment list is reached. 1300 */ 1301 flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF; 1302 if (txout == (di->ntxd - 1)) 1303 flags |= D64_CTRL1_EOT; 1304 1305 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len); 1306 1307 txout = nexttxd(di, txout); 1308 1309 /* save the packet */ 1310 di->txp[prevtxd(di, txout)] = p; 1311 1312 /* bump the tx descriptor index */ 1313 di->txout = txout; 1314} 1315 1316static void ampdu_finalize(struct dma_info *di) 1317{ 1318 struct brcms_ampdu_session *session = &di->ampdu_session; 1319 struct sk_buff *p; 1320 1321 trace_brcms_ampdu_session(&session->wlc->hw->d11core->dev, 1322 session->max_ampdu_len, 1323 session->max_ampdu_frames, 1324 session->ampdu_len, 1325 skb_queue_len(&session->skb_list), 1326 session->dma_len); 1327 1328 if (WARN_ON(skb_queue_empty(&session->skb_list))) 1329 return; 1330 1331 brcms_c_ampdu_finalize(session); 1332 1333 while (!skb_queue_empty(&session->skb_list)) { 1334 p = skb_dequeue(&session->skb_list); 1335 dma_txenq(di, p); 1336 } 1337 1338 bcma_write32(di->core, DMA64TXREGOFFS(di, ptr), 1339 di->xmtptrbase + I2B(di->txout, struct dma64desc)); 1340 brcms_c_ampdu_reset_session(session, session->wlc); 1341} 1342 1343static void prep_ampdu_frame(struct dma_info *di, struct sk_buff *p) 1344{ 1345 struct brcms_ampdu_session *session = &di->ampdu_session; 1346 int ret; 1347 1348 ret = brcms_c_ampdu_add_frame(session, p); 1349 if (ret == -ENOSPC) { 1350 /* 1351 * AMPDU cannot accomodate this frame. Close out the in- 1352 * progress AMPDU session and start a new one. 1353 */ 1354 ampdu_finalize(di); 1355 ret = brcms_c_ampdu_add_frame(session, p); 1356 } 1357 1358 WARN_ON(ret); 1359} 1360 1361/* Update count of available tx descriptors based on current DMA state */ 1362static void dma_update_txavail(struct dma_info *di) 1363{ 1364 /* 1365 * Available space is number of descriptors less the number of 1366 * active descriptors and the number of queued AMPDU frames. 1367 */ 1368 di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) - 1369 skb_queue_len(&di->ampdu_session.skb_list) - 1; 1370} 1371 1372/* 1373 * !! tx entry routine 1374 * WARNING: call must check the return value for error. 1375 * the error(toss frames) could be fatal and cause many subsequent hard 1376 * to debug problems 1377 */ 1378int dma_txfast(struct brcms_c_info *wlc, struct dma_pub *pub, 1379 struct sk_buff *p) 1380{ 1381 struct dma_info *di = container_of(pub, struct dma_info, dma); 1382 struct brcms_ampdu_session *session = &di->ampdu_session; 1383 struct ieee80211_tx_info *tx_info; 1384 bool is_ampdu; 1385 1386 /* no use to transmit a zero length packet */ 1387 if (p->len == 0) 1388 return 0; 1389 1390 /* return nonzero if out of tx descriptors */ 1391 if (di->dma.txavail == 0 || nexttxd(di, di->txout) == di->txin) 1392 goto outoftxd; 1393 1394 tx_info = IEEE80211_SKB_CB(p); 1395 is_ampdu = tx_info->flags & IEEE80211_TX_CTL_AMPDU; 1396 if (is_ampdu) 1397 prep_ampdu_frame(di, p); 1398 else 1399 dma_txenq(di, p); 1400 1401 /* tx flow control */ 1402 dma_update_txavail(di); 1403 1404 /* kick the chip */ 1405 if (is_ampdu) { 1406 /* 1407 * Start sending data if we've got a full AMPDU, there's 1408 * no more space in the DMA ring, or the ring isn't 1409 * currently transmitting. 1410 */ 1411 if (skb_queue_len(&session->skb_list) == session->max_ampdu_frames || 1412 di->dma.txavail == 0 || dma64_txidle(di)) 1413 ampdu_finalize(di); 1414 } else { 1415 bcma_write32(di->core, DMA64TXREGOFFS(di, ptr), 1416 di->xmtptrbase + I2B(di->txout, struct dma64desc)); 1417 } 1418 1419 return 0; 1420 1421 outoftxd: 1422 brcms_dbg_dma(di->core, "%s: out of txds !!!\n", di->name); 1423 brcmu_pkt_buf_free_skb(p); 1424 di->dma.txavail = 0; 1425 di->dma.txnobuf++; 1426 return -ENOSPC; 1427} 1428 1429void dma_txflush(struct dma_pub *pub) 1430{ 1431 struct dma_info *di = container_of(pub, struct dma_info, dma); 1432 struct brcms_ampdu_session *session = &di->ampdu_session; 1433 1434 if (!skb_queue_empty(&session->skb_list)) 1435 ampdu_finalize(di); 1436} 1437 1438int dma_txpending(struct dma_pub *pub) 1439{ 1440 struct dma_info *di = container_of(pub, struct dma_info, dma); 1441 return ntxdactive(di, di->txin, di->txout); 1442} 1443 1444/* 1445 * If we have an active AMPDU session and are not transmitting, 1446 * this function will force tx to start. 1447 */ 1448void dma_kick_tx(struct dma_pub *pub) 1449{ 1450 struct dma_info *di = container_of(pub, struct dma_info, dma); 1451 struct brcms_ampdu_session *session = &di->ampdu_session; 1452 1453 if (!skb_queue_empty(&session->skb_list) && dma64_txidle(di)) 1454 ampdu_finalize(di); 1455} 1456 1457/* 1458 * Reclaim next completed txd (txds if using chained buffers) in the range 1459 * specified and return associated packet. 1460 * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be 1461 * transmitted as noted by the hardware "CurrDescr" pointer. 1462 * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be 1463 * transferred by the DMA as noted by the hardware "ActiveDescr" pointer. 1464 * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and 1465 * return associated packet regardless of the value of hardware pointers. 1466 */ 1467struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range) 1468{ 1469 struct dma_info *di = container_of(pub, struct dma_info, dma); 1470 u16 start, end, i; 1471 u16 active_desc; 1472 struct sk_buff *txp; 1473 1474 brcms_dbg_dma(di->core, "%s: %s\n", 1475 di->name, 1476 range == DMA_RANGE_ALL ? "all" : 1477 range == DMA_RANGE_TRANSMITTED ? "transmitted" : 1478 "transferred"); 1479 1480 if (di->ntxd == 0) 1481 return NULL; 1482 1483 txp = NULL; 1484 1485 start = di->txin; 1486 if (range == DMA_RANGE_ALL) 1487 end = di->txout; 1488 else { 1489 end = (u16) (B2I(((bcma_read32(di->core, 1490 DMA64TXREGOFFS(di, status0)) & 1491 D64_XS0_CD_MASK) - di->xmtptrbase) & 1492 D64_XS0_CD_MASK, struct dma64desc)); 1493 1494 if (range == DMA_RANGE_TRANSFERED) { 1495 active_desc = 1496 (u16)(bcma_read32(di->core, 1497 DMA64TXREGOFFS(di, status1)) & 1498 D64_XS1_AD_MASK); 1499 active_desc = 1500 (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK; 1501 active_desc = B2I(active_desc, struct dma64desc); 1502 if (end != active_desc) 1503 end = prevtxd(di, active_desc); 1504 } 1505 } 1506 1507 if ((start == 0) && (end > di->txout)) 1508 goto bogus; 1509 1510 for (i = start; i != end && !txp; i = nexttxd(di, i)) { 1511 dma_addr_t pa; 1512 uint size; 1513 1514 pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow; 1515 1516 size = 1517 (le32_to_cpu(di->txd64[i].ctrl2) & 1518 D64_CTRL2_BC_MASK); 1519 1520 di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef); 1521 di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef); 1522 1523 txp = di->txp[i]; 1524 di->txp[i] = NULL; 1525 1526 dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE); 1527 } 1528 1529 di->txin = i; 1530 1531 /* tx flow control */ 1532 dma_update_txavail(di); 1533 1534 return txp; 1535 1536 bogus: 1537 brcms_dbg_dma(di->core, "bogus curr: start %d end %d txout %d\n", 1538 start, end, di->txout); 1539 return NULL; 1540} 1541 1542/* 1543 * Mac80211 initiated actions sometimes require packets in the DMA queue to be 1544 * modified. The modified portion of the packet is not under control of the DMA 1545 * engine. This function calls a caller-supplied function for each packet in 1546 * the caller specified dma chain. 1547 */ 1548void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc) 1549 (void *pkt, void *arg_a), void *arg_a) 1550{ 1551 struct dma_info *di = container_of(dmah, struct dma_info, dma); 1552 uint i = di->txin; 1553 uint end = di->txout; 1554 struct sk_buff *skb; 1555 struct ieee80211_tx_info *tx_info; 1556 1557 while (i != end) { 1558 skb = di->txp[i]; 1559 if (skb != NULL) { 1560 tx_info = (struct ieee80211_tx_info *)skb->cb; 1561 (callback_fnc)(tx_info, arg_a); 1562 } 1563 i = nexttxd(di, i); 1564 } 1565} 1566