1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge 4 * 5 * Copyright (c) 2011-2014 Integrated Device Technology, Inc. 6 * Alexandre Bounine <alexandre.bounine@idt.com> 7 */ 8 9#include <linux/io.h> 10#include <linux/errno.h> 11#include <linux/init.h> 12#include <linux/ioport.h> 13#include <linux/kernel.h> 14#include <linux/module.h> 15#include <linux/pci.h> 16#include <linux/rio.h> 17#include <linux/rio_drv.h> 18#include <linux/dma-mapping.h> 19#include <linux/interrupt.h> 20#include <linux/kfifo.h> 21#include <linux/sched.h> 22#include <linux/delay.h> 23#include "../../dma/dmaengine.h" 24 25#include "tsi721.h" 26 27#ifdef CONFIG_PCI_MSI 28static irqreturn_t tsi721_bdma_msix(int irq, void *ptr); 29#endif 30static int tsi721_submit_sg(struct tsi721_tx_desc *desc); 31 32static unsigned int dma_desc_per_channel = 128; 33module_param(dma_desc_per_channel, uint, S_IRUGO); 34MODULE_PARM_DESC(dma_desc_per_channel, 35 "Number of DMA descriptors per channel (default: 128)"); 36 37static unsigned int dma_txqueue_sz = 16; 38module_param(dma_txqueue_sz, uint, S_IRUGO); 39MODULE_PARM_DESC(dma_txqueue_sz, 40 "DMA Transactions Queue Size (default: 16)"); 41 42static u8 dma_sel = 0x7f; 43module_param(dma_sel, byte, S_IRUGO); 44MODULE_PARM_DESC(dma_sel, 45 "DMA Channel Selection Mask (default: 0x7f = all)"); 46 47static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan) 48{ 49 return container_of(chan, struct tsi721_bdma_chan, dchan); 50} 51 52static inline struct tsi721_device *to_tsi721(struct dma_device *ddev) 53{ 54 return container_of(ddev, struct rio_mport, dma)->priv; 55} 56 57static inline 58struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd) 59{ 60 return container_of(txd, struct tsi721_tx_desc, txd); 61} 62 63static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num) 64{ 65 struct tsi721_dma_desc *bd_ptr; 66 struct device *dev = bdma_chan->dchan.device->dev; 67 u64 *sts_ptr; 68 dma_addr_t bd_phys; 69 dma_addr_t sts_phys; 70 int sts_size; 71#ifdef CONFIG_PCI_MSI 72 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); 73#endif 74 75 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); 76 77 /* 78 * Allocate space for DMA descriptors 79 * (add an extra element for link descriptor) 80 */ 81 bd_ptr = dma_alloc_coherent(dev, 82 (bd_num + 1) * sizeof(struct tsi721_dma_desc), 83 &bd_phys, GFP_ATOMIC); 84 if (!bd_ptr) 85 return -ENOMEM; 86 87 bdma_chan->bd_num = bd_num; 88 bdma_chan->bd_phys = bd_phys; 89 bdma_chan->bd_base = bd_ptr; 90 91 tsi_debug(DMA, &bdma_chan->dchan.dev->device, 92 "DMAC%d descriptors @ %p (phys = %pad)", 93 bdma_chan->id, bd_ptr, &bd_phys); 94 95 /* Allocate space for descriptor status FIFO */ 96 sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ? 97 (bd_num + 1) : TSI721_DMA_MINSTSSZ; 98 sts_size = roundup_pow_of_two(sts_size); 99 sts_ptr = dma_alloc_coherent(dev, 100 sts_size * sizeof(struct tsi721_dma_sts), 101 &sts_phys, GFP_ATOMIC); 102 if (!sts_ptr) { 103 /* Free space allocated for DMA descriptors */ 104 dma_free_coherent(dev, 105 (bd_num + 1) * sizeof(struct tsi721_dma_desc), 106 bd_ptr, bd_phys); 107 bdma_chan->bd_base = NULL; 108 return -ENOMEM; 109 } 110 111 bdma_chan->sts_phys = sts_phys; 112 bdma_chan->sts_base = sts_ptr; 113 bdma_chan->sts_size = sts_size; 114 115 tsi_debug(DMA, &bdma_chan->dchan.dev->device, 116 "DMAC%d desc status FIFO @ %p (phys = %pad) size=0x%x", 117 bdma_chan->id, sts_ptr, &sts_phys, sts_size); 118 119 /* Initialize DMA descriptors ring using added link descriptor */ 120 bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29); 121 bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys & 122 TSI721_DMAC_DPTRL_MASK); 123 bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32); 124 125 /* Setup DMA descriptor pointers */ 126 iowrite32(((u64)bd_phys >> 32), 127 bdma_chan->regs + TSI721_DMAC_DPTRH); 128 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK), 129 bdma_chan->regs + TSI721_DMAC_DPTRL); 130 131 /* Setup descriptor status FIFO */ 132 iowrite32(((u64)sts_phys >> 32), 133 bdma_chan->regs + TSI721_DMAC_DSBH); 134 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK), 135 bdma_chan->regs + TSI721_DMAC_DSBL); 136 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size), 137 bdma_chan->regs + TSI721_DMAC_DSSZ); 138 139 /* Clear interrupt bits */ 140 iowrite32(TSI721_DMAC_INT_ALL, 141 bdma_chan->regs + TSI721_DMAC_INT); 142 143 ioread32(bdma_chan->regs + TSI721_DMAC_INT); 144 145#ifdef CONFIG_PCI_MSI 146 /* Request interrupt service if we are in MSI-X mode */ 147 if (priv->flags & TSI721_USING_MSIX) { 148 int rc, idx; 149 150 idx = TSI721_VECT_DMA0_DONE + bdma_chan->id; 151 152 rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, 153 priv->msix[idx].irq_name, (void *)bdma_chan); 154 155 if (rc) { 156 tsi_debug(DMA, &bdma_chan->dchan.dev->device, 157 "Unable to get MSI-X for DMAC%d-DONE", 158 bdma_chan->id); 159 goto err_out; 160 } 161 162 idx = TSI721_VECT_DMA0_INT + bdma_chan->id; 163 164 rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0, 165 priv->msix[idx].irq_name, (void *)bdma_chan); 166 167 if (rc) { 168 tsi_debug(DMA, &bdma_chan->dchan.dev->device, 169 "Unable to get MSI-X for DMAC%d-INT", 170 bdma_chan->id); 171 free_irq( 172 priv->msix[TSI721_VECT_DMA0_DONE + 173 bdma_chan->id].vector, 174 (void *)bdma_chan); 175 } 176 177err_out: 178 if (rc) { 179 /* Free space allocated for DMA descriptors */ 180 dma_free_coherent(dev, 181 (bd_num + 1) * sizeof(struct tsi721_dma_desc), 182 bd_ptr, bd_phys); 183 bdma_chan->bd_base = NULL; 184 185 /* Free space allocated for status descriptors */ 186 dma_free_coherent(dev, 187 sts_size * sizeof(struct tsi721_dma_sts), 188 sts_ptr, sts_phys); 189 bdma_chan->sts_base = NULL; 190 191 return -EIO; 192 } 193 } 194#endif /* CONFIG_PCI_MSI */ 195 196 /* Toggle DMA channel initialization */ 197 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); 198 ioread32(bdma_chan->regs + TSI721_DMAC_CTL); 199 bdma_chan->wr_count = bdma_chan->wr_count_next = 0; 200 bdma_chan->sts_rdptr = 0; 201 udelay(10); 202 203 return 0; 204} 205 206static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan) 207{ 208 u32 ch_stat; 209#ifdef CONFIG_PCI_MSI 210 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); 211#endif 212 213 if (!bdma_chan->bd_base) 214 return 0; 215 216 /* Check if DMA channel still running */ 217 ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS); 218 if (ch_stat & TSI721_DMAC_STS_RUN) 219 return -EFAULT; 220 221 /* Put DMA channel into init state */ 222 iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL); 223 224#ifdef CONFIG_PCI_MSI 225 if (priv->flags & TSI721_USING_MSIX) { 226 free_irq(priv->msix[TSI721_VECT_DMA0_DONE + 227 bdma_chan->id].vector, (void *)bdma_chan); 228 free_irq(priv->msix[TSI721_VECT_DMA0_INT + 229 bdma_chan->id].vector, (void *)bdma_chan); 230 } 231#endif /* CONFIG_PCI_MSI */ 232 233 /* Free space allocated for DMA descriptors */ 234 dma_free_coherent(bdma_chan->dchan.device->dev, 235 (bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc), 236 bdma_chan->bd_base, bdma_chan->bd_phys); 237 bdma_chan->bd_base = NULL; 238 239 /* Free space allocated for status FIFO */ 240 dma_free_coherent(bdma_chan->dchan.device->dev, 241 bdma_chan->sts_size * sizeof(struct tsi721_dma_sts), 242 bdma_chan->sts_base, bdma_chan->sts_phys); 243 bdma_chan->sts_base = NULL; 244 return 0; 245} 246 247static void 248tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable) 249{ 250 if (enable) { 251 /* Clear pending BDMA channel interrupts */ 252 iowrite32(TSI721_DMAC_INT_ALL, 253 bdma_chan->regs + TSI721_DMAC_INT); 254 ioread32(bdma_chan->regs + TSI721_DMAC_INT); 255 /* Enable BDMA channel interrupts */ 256 iowrite32(TSI721_DMAC_INT_ALL, 257 bdma_chan->regs + TSI721_DMAC_INTE); 258 } else { 259 /* Disable BDMA channel interrupts */ 260 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); 261 /* Clear pending BDMA channel interrupts */ 262 iowrite32(TSI721_DMAC_INT_ALL, 263 bdma_chan->regs + TSI721_DMAC_INT); 264 } 265 266} 267 268static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan) 269{ 270 u32 sts; 271 272 sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); 273 return ((sts & TSI721_DMAC_STS_RUN) == 0); 274} 275 276void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan) 277{ 278 /* Disable BDMA channel interrupts */ 279 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE); 280 if (bdma_chan->active) 281 tasklet_hi_schedule(&bdma_chan->tasklet); 282} 283 284#ifdef CONFIG_PCI_MSI 285/** 286 * tsi721_bdma_msix - MSI-X interrupt handler for BDMA channels 287 * @irq: Linux interrupt number 288 * @ptr: Pointer to interrupt-specific data (BDMA channel structure) 289 * 290 * Handles BDMA channel interrupts signaled using MSI-X. 291 * 292 * Returns: %IRQ_HANDLED 293 */ 294static irqreturn_t tsi721_bdma_msix(int irq, void *ptr) 295{ 296 struct tsi721_bdma_chan *bdma_chan = ptr; 297 298 if (bdma_chan->active) 299 tasklet_hi_schedule(&bdma_chan->tasklet); 300 return IRQ_HANDLED; 301} 302#endif /* CONFIG_PCI_MSI */ 303 304/* Must be called with the spinlock held */ 305static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan) 306{ 307 if (!tsi721_dma_is_idle(bdma_chan)) { 308 tsi_err(&bdma_chan->dchan.dev->device, 309 "DMAC%d Attempt to start non-idle channel", 310 bdma_chan->id); 311 return; 312 } 313 314 if (bdma_chan->wr_count == bdma_chan->wr_count_next) { 315 tsi_err(&bdma_chan->dchan.dev->device, 316 "DMAC%d Attempt to start DMA with no BDs ready %d", 317 bdma_chan->id, task_pid_nr(current)); 318 return; 319 } 320 321 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d (wrc=%d) %d", 322 bdma_chan->id, bdma_chan->wr_count_next, 323 task_pid_nr(current)); 324 325 iowrite32(bdma_chan->wr_count_next, 326 bdma_chan->regs + TSI721_DMAC_DWRCNT); 327 ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT); 328 329 bdma_chan->wr_count = bdma_chan->wr_count_next; 330} 331 332static int 333tsi721_desc_fill_init(struct tsi721_tx_desc *desc, 334 struct tsi721_dma_desc *bd_ptr, 335 struct scatterlist *sg, u32 sys_size) 336{ 337 u64 rio_addr; 338 339 if (!bd_ptr) 340 return -EINVAL; 341 342 /* Initialize DMA descriptor */ 343 bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) | 344 (desc->rtype << 19) | desc->destid); 345 bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) | 346 (sys_size << 26)); 347 rio_addr = (desc->rio_addr >> 2) | 348 ((u64)(desc->rio_addr_u & 0x3) << 62); 349 bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff); 350 bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32); 351 bd_ptr->t1.bufptr_lo = cpu_to_le32( 352 (u64)sg_dma_address(sg) & 0xffffffff); 353 bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32); 354 bd_ptr->t1.s_dist = 0; 355 bd_ptr->t1.s_size = 0; 356 357 return 0; 358} 359 360static int 361tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt) 362{ 363 if (!bd_ptr) 364 return -EINVAL; 365 366 /* Update DMA descriptor */ 367 if (interrupt) 368 bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF); 369 bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1); 370 371 return 0; 372} 373 374static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan, 375 struct tsi721_tx_desc *desc) 376{ 377 struct dma_async_tx_descriptor *txd = &desc->txd; 378 dma_async_tx_callback callback = txd->callback; 379 void *param = txd->callback_param; 380 381 list_move(&desc->desc_node, &bdma_chan->free_list); 382 383 if (callback) 384 callback(param); 385} 386 387static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan) 388{ 389 u32 srd_ptr; 390 u64 *sts_ptr; 391 int i, j; 392 393 /* Check and clear descriptor status FIFO entries */ 394 srd_ptr = bdma_chan->sts_rdptr; 395 sts_ptr = bdma_chan->sts_base; 396 j = srd_ptr * 8; 397 while (sts_ptr[j]) { 398 for (i = 0; i < 8 && sts_ptr[j]; i++, j++) 399 sts_ptr[j] = 0; 400 401 ++srd_ptr; 402 srd_ptr %= bdma_chan->sts_size; 403 j = srd_ptr * 8; 404 } 405 406 iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP); 407 bdma_chan->sts_rdptr = srd_ptr; 408} 409 410/* Must be called with the channel spinlock held */ 411static int tsi721_submit_sg(struct tsi721_tx_desc *desc) 412{ 413 struct dma_chan *dchan = desc->txd.chan; 414 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 415 u32 sys_size; 416 u64 rio_addr; 417 dma_addr_t next_addr; 418 u32 bcount; 419 struct scatterlist *sg; 420 unsigned int i; 421 int err = 0; 422 struct tsi721_dma_desc *bd_ptr = NULL; 423 u32 idx, rd_idx; 424 u32 add_count = 0; 425 struct device *ch_dev = &dchan->dev->device; 426 427 if (!tsi721_dma_is_idle(bdma_chan)) { 428 tsi_err(ch_dev, "DMAC%d ERR: Attempt to use non-idle channel", 429 bdma_chan->id); 430 return -EIO; 431 } 432 433 /* 434 * Fill DMA channel's hardware buffer descriptors. 435 * (NOTE: RapidIO destination address is limited to 64 bits for now) 436 */ 437 rio_addr = desc->rio_addr; 438 next_addr = -1; 439 bcount = 0; 440 sys_size = dma_to_mport(dchan->device)->sys_size; 441 442 rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT); 443 rd_idx %= (bdma_chan->bd_num + 1); 444 445 idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1); 446 if (idx == bdma_chan->bd_num) { 447 /* wrap around link descriptor */ 448 idx = 0; 449 add_count++; 450 } 451 452 tsi_debug(DMA, ch_dev, "DMAC%d BD ring status: rdi=%d wri=%d", 453 bdma_chan->id, rd_idx, idx); 454 455 for_each_sg(desc->sg, sg, desc->sg_len, i) { 456 457 tsi_debug(DMAV, ch_dev, "DMAC%d sg%d/%d addr: 0x%llx len: %d", 458 bdma_chan->id, i, desc->sg_len, 459 (unsigned long long)sg_dma_address(sg), sg_dma_len(sg)); 460 461 if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) { 462 tsi_err(ch_dev, "DMAC%d SG entry %d is too large", 463 bdma_chan->id, i); 464 err = -EINVAL; 465 break; 466 } 467 468 /* 469 * If this sg entry forms contiguous block with previous one, 470 * try to merge it into existing DMA descriptor 471 */ 472 if (next_addr == sg_dma_address(sg) && 473 bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) { 474 /* Adjust byte count of the descriptor */ 475 bcount += sg_dma_len(sg); 476 goto entry_done; 477 } else if (next_addr != -1) { 478 /* Finalize descriptor using total byte count value */ 479 tsi721_desc_fill_end(bd_ptr, bcount, 0); 480 tsi_debug(DMAV, ch_dev, "DMAC%d prev desc final len: %d", 481 bdma_chan->id, bcount); 482 } 483 484 desc->rio_addr = rio_addr; 485 486 if (i && idx == rd_idx) { 487 tsi_debug(DMAV, ch_dev, 488 "DMAC%d HW descriptor ring is full @ %d", 489 bdma_chan->id, i); 490 desc->sg = sg; 491 desc->sg_len -= i; 492 break; 493 } 494 495 bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx]; 496 err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size); 497 if (err) { 498 tsi_err(ch_dev, "Failed to build desc: err=%d", err); 499 break; 500 } 501 502 tsi_debug(DMAV, ch_dev, "DMAC%d bd_ptr = %p did=%d raddr=0x%llx", 503 bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr); 504 505 next_addr = sg_dma_address(sg); 506 bcount = sg_dma_len(sg); 507 508 add_count++; 509 if (++idx == bdma_chan->bd_num) { 510 /* wrap around link descriptor */ 511 idx = 0; 512 add_count++; 513 } 514 515entry_done: 516 if (sg_is_last(sg)) { 517 tsi721_desc_fill_end(bd_ptr, bcount, 0); 518 tsi_debug(DMAV, ch_dev, 519 "DMAC%d last desc final len: %d", 520 bdma_chan->id, bcount); 521 desc->sg_len = 0; 522 } else { 523 rio_addr += sg_dma_len(sg); 524 next_addr += sg_dma_len(sg); 525 } 526 } 527 528 if (!err) 529 bdma_chan->wr_count_next += add_count; 530 531 return err; 532} 533 534static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan, 535 struct tsi721_tx_desc *desc) 536{ 537 int err; 538 539 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id); 540 541 if (!tsi721_dma_is_idle(bdma_chan)) 542 return; 543 544 /* 545 * If there is no data transfer in progress, fetch new descriptor from 546 * the pending queue. 547 */ 548 if (!desc && !bdma_chan->active_tx && !list_empty(&bdma_chan->queue)) { 549 desc = list_first_entry(&bdma_chan->queue, 550 struct tsi721_tx_desc, desc_node); 551 list_del_init((&desc->desc_node)); 552 bdma_chan->active_tx = desc; 553 } 554 555 if (desc) { 556 err = tsi721_submit_sg(desc); 557 if (!err) 558 tsi721_start_dma(bdma_chan); 559 else { 560 tsi721_dma_tx_err(bdma_chan, desc); 561 tsi_debug(DMA, &bdma_chan->dchan.dev->device, 562 "DMAC%d ERR: tsi721_submit_sg failed with err=%d", 563 bdma_chan->id, err); 564 } 565 } 566 567 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d Exit", 568 bdma_chan->id); 569} 570 571static void tsi721_dma_tasklet(unsigned long data) 572{ 573 struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data; 574 u32 dmac_int, dmac_sts; 575 576 dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); 577 tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d_INT = 0x%x", 578 bdma_chan->id, dmac_int); 579 /* Clear channel interrupts */ 580 iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT); 581 582 if (dmac_int & TSI721_DMAC_INT_ERR) { 583 int i = 10000; 584 struct tsi721_tx_desc *desc; 585 586 desc = bdma_chan->active_tx; 587 dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); 588 tsi_err(&bdma_chan->dchan.dev->device, 589 "DMAC%d_STS = 0x%x did=%d raddr=0x%llx", 590 bdma_chan->id, dmac_sts, desc->destid, desc->rio_addr); 591 592 /* Re-initialize DMA channel if possible */ 593 594 if ((dmac_sts & TSI721_DMAC_STS_ABORT) == 0) 595 goto err_out; 596 597 tsi721_clr_stat(bdma_chan); 598 599 spin_lock(&bdma_chan->lock); 600 601 /* Put DMA channel into init state */ 602 iowrite32(TSI721_DMAC_CTL_INIT, 603 bdma_chan->regs + TSI721_DMAC_CTL); 604 do { 605 udelay(1); 606 dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS); 607 i--; 608 } while ((dmac_sts & TSI721_DMAC_STS_ABORT) && i); 609 610 if (dmac_sts & TSI721_DMAC_STS_ABORT) { 611 tsi_err(&bdma_chan->dchan.dev->device, 612 "Failed to re-initiate DMAC%d", bdma_chan->id); 613 spin_unlock(&bdma_chan->lock); 614 goto err_out; 615 } 616 617 /* Setup DMA descriptor pointers */ 618 iowrite32(((u64)bdma_chan->bd_phys >> 32), 619 bdma_chan->regs + TSI721_DMAC_DPTRH); 620 iowrite32(((u64)bdma_chan->bd_phys & TSI721_DMAC_DPTRL_MASK), 621 bdma_chan->regs + TSI721_DMAC_DPTRL); 622 623 /* Setup descriptor status FIFO */ 624 iowrite32(((u64)bdma_chan->sts_phys >> 32), 625 bdma_chan->regs + TSI721_DMAC_DSBH); 626 iowrite32(((u64)bdma_chan->sts_phys & TSI721_DMAC_DSBL_MASK), 627 bdma_chan->regs + TSI721_DMAC_DSBL); 628 iowrite32(TSI721_DMAC_DSSZ_SIZE(bdma_chan->sts_size), 629 bdma_chan->regs + TSI721_DMAC_DSSZ); 630 631 /* Clear interrupt bits */ 632 iowrite32(TSI721_DMAC_INT_ALL, 633 bdma_chan->regs + TSI721_DMAC_INT); 634 635 ioread32(bdma_chan->regs + TSI721_DMAC_INT); 636 637 bdma_chan->wr_count = bdma_chan->wr_count_next = 0; 638 bdma_chan->sts_rdptr = 0; 639 udelay(10); 640 641 desc = bdma_chan->active_tx; 642 desc->status = DMA_ERROR; 643 dma_cookie_complete(&desc->txd); 644 list_add(&desc->desc_node, &bdma_chan->free_list); 645 bdma_chan->active_tx = NULL; 646 if (bdma_chan->active) 647 tsi721_advance_work(bdma_chan, NULL); 648 spin_unlock(&bdma_chan->lock); 649 } 650 651 if (dmac_int & TSI721_DMAC_INT_STFULL) { 652 tsi_err(&bdma_chan->dchan.dev->device, 653 "DMAC%d descriptor status FIFO is full", 654 bdma_chan->id); 655 } 656 657 if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) { 658 struct tsi721_tx_desc *desc; 659 660 tsi721_clr_stat(bdma_chan); 661 spin_lock(&bdma_chan->lock); 662 desc = bdma_chan->active_tx; 663 664 if (desc->sg_len == 0) { 665 dma_async_tx_callback callback = NULL; 666 void *param = NULL; 667 668 desc->status = DMA_COMPLETE; 669 dma_cookie_complete(&desc->txd); 670 if (desc->txd.flags & DMA_PREP_INTERRUPT) { 671 callback = desc->txd.callback; 672 param = desc->txd.callback_param; 673 } 674 list_add(&desc->desc_node, &bdma_chan->free_list); 675 bdma_chan->active_tx = NULL; 676 if (bdma_chan->active) 677 tsi721_advance_work(bdma_chan, NULL); 678 spin_unlock(&bdma_chan->lock); 679 if (callback) 680 callback(param); 681 } else { 682 if (bdma_chan->active) 683 tsi721_advance_work(bdma_chan, 684 bdma_chan->active_tx); 685 spin_unlock(&bdma_chan->lock); 686 } 687 } 688err_out: 689 /* Re-Enable BDMA channel interrupts */ 690 iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE); 691} 692 693static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd) 694{ 695 struct tsi721_tx_desc *desc = to_tsi721_desc(txd); 696 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan); 697 dma_cookie_t cookie; 698 699 /* Check if the descriptor is detached from any lists */ 700 if (!list_empty(&desc->desc_node)) { 701 tsi_err(&bdma_chan->dchan.dev->device, 702 "DMAC%d wrong state of descriptor %p", 703 bdma_chan->id, txd); 704 return -EIO; 705 } 706 707 spin_lock_bh(&bdma_chan->lock); 708 709 if (!bdma_chan->active) { 710 spin_unlock_bh(&bdma_chan->lock); 711 return -ENODEV; 712 } 713 714 cookie = dma_cookie_assign(txd); 715 desc->status = DMA_IN_PROGRESS; 716 list_add_tail(&desc->desc_node, &bdma_chan->queue); 717 tsi721_advance_work(bdma_chan, NULL); 718 719 spin_unlock_bh(&bdma_chan->lock); 720 return cookie; 721} 722 723static int tsi721_alloc_chan_resources(struct dma_chan *dchan) 724{ 725 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 726 struct tsi721_tx_desc *desc; 727 int i; 728 729 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); 730 731 if (bdma_chan->bd_base) 732 return dma_txqueue_sz; 733 734 /* Initialize BDMA channel */ 735 if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) { 736 tsi_err(&dchan->dev->device, "Unable to initialize DMAC%d", 737 bdma_chan->id); 738 return -ENODEV; 739 } 740 741 /* Allocate queue of transaction descriptors */ 742 desc = kcalloc(dma_txqueue_sz, sizeof(struct tsi721_tx_desc), 743 GFP_ATOMIC); 744 if (!desc) { 745 tsi721_bdma_ch_free(bdma_chan); 746 return -ENOMEM; 747 } 748 749 bdma_chan->tx_desc = desc; 750 751 for (i = 0; i < dma_txqueue_sz; i++) { 752 dma_async_tx_descriptor_init(&desc[i].txd, dchan); 753 desc[i].txd.tx_submit = tsi721_tx_submit; 754 desc[i].txd.flags = DMA_CTRL_ACK; 755 list_add(&desc[i].desc_node, &bdma_chan->free_list); 756 } 757 758 dma_cookie_init(dchan); 759 760 bdma_chan->active = true; 761 tsi721_bdma_interrupt_enable(bdma_chan, 1); 762 763 return dma_txqueue_sz; 764} 765 766static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan) 767{ 768 struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device); 769 770#ifdef CONFIG_PCI_MSI 771 if (priv->flags & TSI721_USING_MSIX) { 772 synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE + 773 bdma_chan->id].vector); 774 synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT + 775 bdma_chan->id].vector); 776 } else 777#endif 778 synchronize_irq(priv->pdev->irq); 779} 780 781static void tsi721_free_chan_resources(struct dma_chan *dchan) 782{ 783 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 784 785 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); 786 787 if (!bdma_chan->bd_base) 788 return; 789 790 tsi721_bdma_interrupt_enable(bdma_chan, 0); 791 bdma_chan->active = false; 792 tsi721_sync_dma_irq(bdma_chan); 793 tasklet_kill(&bdma_chan->tasklet); 794 INIT_LIST_HEAD(&bdma_chan->free_list); 795 kfree(bdma_chan->tx_desc); 796 tsi721_bdma_ch_free(bdma_chan); 797} 798 799static 800enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, 801 struct dma_tx_state *txstate) 802{ 803 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 804 enum dma_status status; 805 806 spin_lock_bh(&bdma_chan->lock); 807 status = dma_cookie_status(dchan, cookie, txstate); 808 spin_unlock_bh(&bdma_chan->lock); 809 return status; 810} 811 812static void tsi721_issue_pending(struct dma_chan *dchan) 813{ 814 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 815 816 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); 817 818 spin_lock_bh(&bdma_chan->lock); 819 if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) { 820 tsi721_advance_work(bdma_chan, NULL); 821 } 822 spin_unlock_bh(&bdma_chan->lock); 823} 824 825static 826struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan, 827 struct scatterlist *sgl, unsigned int sg_len, 828 enum dma_transfer_direction dir, unsigned long flags, 829 void *tinfo) 830{ 831 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 832 struct tsi721_tx_desc *desc; 833 struct rio_dma_ext *rext = tinfo; 834 enum dma_rtype rtype; 835 struct dma_async_tx_descriptor *txd = NULL; 836 837 if (!sgl || !sg_len) { 838 tsi_err(&dchan->dev->device, "DMAC%d No SG list", 839 bdma_chan->id); 840 return ERR_PTR(-EINVAL); 841 } 842 843 tsi_debug(DMA, &dchan->dev->device, "DMAC%d %s", bdma_chan->id, 844 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); 845 846 if (dir == DMA_DEV_TO_MEM) 847 rtype = NREAD; 848 else if (dir == DMA_MEM_TO_DEV) { 849 switch (rext->wr_type) { 850 case RDW_ALL_NWRITE: 851 rtype = ALL_NWRITE; 852 break; 853 case RDW_ALL_NWRITE_R: 854 rtype = ALL_NWRITE_R; 855 break; 856 case RDW_LAST_NWRITE_R: 857 default: 858 rtype = LAST_NWRITE_R; 859 break; 860 } 861 } else { 862 tsi_err(&dchan->dev->device, 863 "DMAC%d Unsupported DMA direction option", 864 bdma_chan->id); 865 return ERR_PTR(-EINVAL); 866 } 867 868 spin_lock_bh(&bdma_chan->lock); 869 870 if (!list_empty(&bdma_chan->free_list)) { 871 desc = list_first_entry(&bdma_chan->free_list, 872 struct tsi721_tx_desc, desc_node); 873 list_del_init(&desc->desc_node); 874 desc->destid = rext->destid; 875 desc->rio_addr = rext->rio_addr; 876 desc->rio_addr_u = 0; 877 desc->rtype = rtype; 878 desc->sg_len = sg_len; 879 desc->sg = sgl; 880 txd = &desc->txd; 881 txd->flags = flags; 882 } 883 884 spin_unlock_bh(&bdma_chan->lock); 885 886 if (!txd) { 887 tsi_debug(DMA, &dchan->dev->device, 888 "DMAC%d free TXD is not available", bdma_chan->id); 889 return ERR_PTR(-EBUSY); 890 } 891 892 return txd; 893} 894 895static int tsi721_terminate_all(struct dma_chan *dchan) 896{ 897 struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan); 898 struct tsi721_tx_desc *desc, *_d; 899 LIST_HEAD(list); 900 901 tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id); 902 903 spin_lock_bh(&bdma_chan->lock); 904 905 bdma_chan->active = false; 906 907 while (!tsi721_dma_is_idle(bdma_chan)) { 908 909 udelay(5); 910#if (0) 911 /* make sure to stop the transfer */ 912 iowrite32(TSI721_DMAC_CTL_SUSP, 913 bdma_chan->regs + TSI721_DMAC_CTL); 914 915 /* Wait until DMA channel stops */ 916 do { 917 dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT); 918 } while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0); 919#endif 920 } 921 922 if (bdma_chan->active_tx) 923 list_add(&bdma_chan->active_tx->desc_node, &list); 924 list_splice_init(&bdma_chan->queue, &list); 925 926 list_for_each_entry_safe(desc, _d, &list, desc_node) 927 tsi721_dma_tx_err(bdma_chan, desc); 928 929 spin_unlock_bh(&bdma_chan->lock); 930 931 return 0; 932} 933 934static void tsi721_dma_stop(struct tsi721_bdma_chan *bdma_chan) 935{ 936 if (!bdma_chan->active) 937 return; 938 spin_lock_bh(&bdma_chan->lock); 939 if (!tsi721_dma_is_idle(bdma_chan)) { 940 int timeout = 100000; 941 942 /* stop the transfer in progress */ 943 iowrite32(TSI721_DMAC_CTL_SUSP, 944 bdma_chan->regs + TSI721_DMAC_CTL); 945 946 /* Wait until DMA channel stops */ 947 while (!tsi721_dma_is_idle(bdma_chan) && --timeout) 948 udelay(1); 949 } 950 951 spin_unlock_bh(&bdma_chan->lock); 952} 953 954void tsi721_dma_stop_all(struct tsi721_device *priv) 955{ 956 int i; 957 958 for (i = 0; i < TSI721_DMA_MAXCH; i++) { 959 if ((i != TSI721_DMACH_MAINT) && (dma_sel & (1 << i))) 960 tsi721_dma_stop(&priv->bdma[i]); 961 } 962} 963 964int tsi721_register_dma(struct tsi721_device *priv) 965{ 966 int i; 967 int nr_channels = 0; 968 int err; 969 struct rio_mport *mport = &priv->mport; 970 971 INIT_LIST_HEAD(&mport->dma.channels); 972 973 for (i = 0; i < TSI721_DMA_MAXCH; i++) { 974 struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i]; 975 976 if ((i == TSI721_DMACH_MAINT) || (dma_sel & (1 << i)) == 0) 977 continue; 978 979 bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i); 980 981 bdma_chan->dchan.device = &mport->dma; 982 bdma_chan->dchan.cookie = 1; 983 bdma_chan->dchan.chan_id = i; 984 bdma_chan->id = i; 985 bdma_chan->active = false; 986 987 spin_lock_init(&bdma_chan->lock); 988 989 bdma_chan->active_tx = NULL; 990 INIT_LIST_HEAD(&bdma_chan->queue); 991 INIT_LIST_HEAD(&bdma_chan->free_list); 992 993 tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet, 994 (unsigned long)bdma_chan); 995 list_add_tail(&bdma_chan->dchan.device_node, 996 &mport->dma.channels); 997 nr_channels++; 998 } 999 1000 mport->dma.chancnt = nr_channels; 1001 dma_cap_zero(mport->dma.cap_mask); 1002 dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask); 1003 dma_cap_set(DMA_SLAVE, mport->dma.cap_mask); 1004 1005 mport->dma.dev = &priv->pdev->dev; 1006 mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources; 1007 mport->dma.device_free_chan_resources = tsi721_free_chan_resources; 1008 mport->dma.device_tx_status = tsi721_tx_status; 1009 mport->dma.device_issue_pending = tsi721_issue_pending; 1010 mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg; 1011 mport->dma.device_terminate_all = tsi721_terminate_all; 1012 1013 err = dma_async_device_register(&mport->dma); 1014 if (err) 1015 tsi_err(&priv->pdev->dev, "Failed to register DMA device"); 1016 1017 return err; 1018} 1019 1020void tsi721_unregister_dma(struct tsi721_device *priv) 1021{ 1022 struct rio_mport *mport = &priv->mport; 1023 struct dma_chan *chan, *_c; 1024 struct tsi721_bdma_chan *bdma_chan; 1025 1026 tsi721_dma_stop_all(priv); 1027 dma_async_device_unregister(&mport->dma); 1028 1029 list_for_each_entry_safe(chan, _c, &mport->dma.channels, 1030 device_node) { 1031 bdma_chan = to_tsi721_chan(chan); 1032 if (bdma_chan->active) { 1033 tsi721_bdma_interrupt_enable(bdma_chan, 0); 1034 bdma_chan->active = false; 1035 tsi721_sync_dma_irq(bdma_chan); 1036 tasklet_kill(&bdma_chan->tasklet); 1037 INIT_LIST_HEAD(&bdma_chan->free_list); 1038 kfree(bdma_chan->tx_desc); 1039 tsi721_bdma_ch_free(bdma_chan); 1040 } 1041 1042 list_del(&chan->device_node); 1043 } 1044} 1045