1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Freescale MPC85xx, MPC83xx DMA Engine support 4 * 5 * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved. 6 * 7 * Author: 8 * Zhang Wei <wei.zhang@freescale.com>, Jul 2007 9 * Ebony Zhu <ebony.zhu@freescale.com>, May 2007 10 * 11 * Description: 12 * DMA engine driver for Freescale MPC8540 DMA controller, which is 13 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc. 14 * The support for MPC8349 DMA controller is also added. 15 * 16 * This driver instructs the DMA controller to issue the PCI Read Multiple 17 * command for PCI read operations, instead of using the default PCI Read Line 18 * command. Please be aware that this setting may result in read pre-fetching 19 * on some platforms. 20 */ 21 22#include <linux/init.h> 23#include <linux/module.h> 24#include <linux/pci.h> 25#include <linux/slab.h> 26#include <linux/interrupt.h> 27#include <linux/dmaengine.h> 28#include <linux/delay.h> 29#include <linux/dma-mapping.h> 30#include <linux/dmapool.h> 31#include <linux/of.h> 32#include <linux/of_address.h> 33#include <linux/of_irq.h> 34#include <linux/platform_device.h> 35#include <linux/fsldma.h> 36#include "dmaengine.h" 37#include "fsldma.h" 38 39#define chan_dbg(chan, fmt, arg...) \ 40 dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg) 41#define chan_err(chan, fmt, arg...) \ 42 dev_err(chan->dev, "%s: " fmt, chan->name, ##arg) 43 44static const char msg_ld_oom[] = "No free memory for link descriptor"; 45 46/* 47 * Register Helpers 48 */ 49 50static void set_sr(struct fsldma_chan *chan, u32 val) 51{ 52 FSL_DMA_OUT(chan, &chan->regs->sr, val, 32); 53} 54 55static u32 get_sr(struct fsldma_chan *chan) 56{ 57 return FSL_DMA_IN(chan, &chan->regs->sr, 32); 58} 59 60static void set_mr(struct fsldma_chan *chan, u32 val) 61{ 62 FSL_DMA_OUT(chan, &chan->regs->mr, val, 32); 63} 64 65static u32 get_mr(struct fsldma_chan *chan) 66{ 67 return FSL_DMA_IN(chan, &chan->regs->mr, 32); 68} 69 70static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr) 71{ 72 FSL_DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); 73} 74 75static dma_addr_t get_cdar(struct fsldma_chan *chan) 76{ 77 return FSL_DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; 78} 79 80static void set_bcr(struct fsldma_chan *chan, u32 val) 81{ 82 FSL_DMA_OUT(chan, &chan->regs->bcr, val, 32); 83} 84 85static u32 get_bcr(struct fsldma_chan *chan) 86{ 87 return FSL_DMA_IN(chan, &chan->regs->bcr, 32); 88} 89 90/* 91 * Descriptor Helpers 92 */ 93 94static void set_desc_cnt(struct fsldma_chan *chan, 95 struct fsl_dma_ld_hw *hw, u32 count) 96{ 97 hw->count = CPU_TO_DMA(chan, count, 32); 98} 99 100static void set_desc_src(struct fsldma_chan *chan, 101 struct fsl_dma_ld_hw *hw, dma_addr_t src) 102{ 103 u64 snoop_bits; 104 105 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 106 ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; 107 hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64); 108} 109 110static void set_desc_dst(struct fsldma_chan *chan, 111 struct fsl_dma_ld_hw *hw, dma_addr_t dst) 112{ 113 u64 snoop_bits; 114 115 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) 116 ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; 117 hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64); 118} 119 120static void set_desc_next(struct fsldma_chan *chan, 121 struct fsl_dma_ld_hw *hw, dma_addr_t next) 122{ 123 u64 snoop_bits; 124 125 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) 126 ? FSL_DMA_SNEN : 0; 127 hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64); 128} 129 130static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc) 131{ 132 u64 snoop_bits; 133 134 snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) 135 ? FSL_DMA_SNEN : 0; 136 137 desc->hw.next_ln_addr = CPU_TO_DMA(chan, 138 DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL 139 | snoop_bits, 64); 140} 141 142/* 143 * DMA Engine Hardware Control Helpers 144 */ 145 146static void dma_init(struct fsldma_chan *chan) 147{ 148 /* Reset the channel */ 149 set_mr(chan, 0); 150 151 switch (chan->feature & FSL_DMA_IP_MASK) { 152 case FSL_DMA_IP_85XX: 153 /* Set the channel to below modes: 154 * EIE - Error interrupt enable 155 * EOLNIE - End of links interrupt enable 156 * BWC - Bandwidth sharing among channels 157 */ 158 set_mr(chan, FSL_DMA_MR_BWC | FSL_DMA_MR_EIE 159 | FSL_DMA_MR_EOLNIE); 160 break; 161 case FSL_DMA_IP_83XX: 162 /* Set the channel to below modes: 163 * EOTIE - End-of-transfer interrupt enable 164 * PRC_RM - PCI read multiple 165 */ 166 set_mr(chan, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM); 167 break; 168 } 169} 170 171static int dma_is_idle(struct fsldma_chan *chan) 172{ 173 u32 sr = get_sr(chan); 174 return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH); 175} 176 177/* 178 * Start the DMA controller 179 * 180 * Preconditions: 181 * - the CDAR register must point to the start descriptor 182 * - the MRn[CS] bit must be cleared 183 */ 184static void dma_start(struct fsldma_chan *chan) 185{ 186 u32 mode; 187 188 mode = get_mr(chan); 189 190 if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { 191 set_bcr(chan, 0); 192 mode |= FSL_DMA_MR_EMP_EN; 193 } else { 194 mode &= ~FSL_DMA_MR_EMP_EN; 195 } 196 197 if (chan->feature & FSL_DMA_CHAN_START_EXT) { 198 mode |= FSL_DMA_MR_EMS_EN; 199 } else { 200 mode &= ~FSL_DMA_MR_EMS_EN; 201 mode |= FSL_DMA_MR_CS; 202 } 203 204 set_mr(chan, mode); 205} 206 207static void dma_halt(struct fsldma_chan *chan) 208{ 209 u32 mode; 210 int i; 211 212 /* read the mode register */ 213 mode = get_mr(chan); 214 215 /* 216 * The 85xx controller supports channel abort, which will stop 217 * the current transfer. On 83xx, this bit is the transfer error 218 * mask bit, which should not be changed. 219 */ 220 if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { 221 mode |= FSL_DMA_MR_CA; 222 set_mr(chan, mode); 223 224 mode &= ~FSL_DMA_MR_CA; 225 } 226 227 /* stop the DMA controller */ 228 mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN); 229 set_mr(chan, mode); 230 231 /* wait for the DMA controller to become idle */ 232 for (i = 0; i < 100; i++) { 233 if (dma_is_idle(chan)) 234 return; 235 236 udelay(10); 237 } 238 239 if (!dma_is_idle(chan)) 240 chan_err(chan, "DMA halt timeout!\n"); 241} 242 243/** 244 * fsl_chan_set_src_loop_size - Set source address hold transfer size 245 * @chan : Freescale DMA channel 246 * @size : Address loop size, 0 for disable loop 247 * 248 * The set source address hold transfer size. The source 249 * address hold or loop transfer size is when the DMA transfer 250 * data from source address (SA), if the loop size is 4, the DMA will 251 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA, 252 * SA + 1 ... and so on. 253 */ 254static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size) 255{ 256 u32 mode; 257 258 mode = get_mr(chan); 259 260 switch (size) { 261 case 0: 262 mode &= ~FSL_DMA_MR_SAHE; 263 break; 264 case 1: 265 case 2: 266 case 4: 267 case 8: 268 mode &= ~FSL_DMA_MR_SAHTS_MASK; 269 mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14); 270 break; 271 } 272 273 set_mr(chan, mode); 274} 275 276/** 277 * fsl_chan_set_dst_loop_size - Set destination address hold transfer size 278 * @chan : Freescale DMA channel 279 * @size : Address loop size, 0 for disable loop 280 * 281 * The set destination address hold transfer size. The destination 282 * address hold or loop transfer size is when the DMA transfer 283 * data to destination address (TA), if the loop size is 4, the DMA will 284 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA, 285 * TA + 1 ... and so on. 286 */ 287static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size) 288{ 289 u32 mode; 290 291 mode = get_mr(chan); 292 293 switch (size) { 294 case 0: 295 mode &= ~FSL_DMA_MR_DAHE; 296 break; 297 case 1: 298 case 2: 299 case 4: 300 case 8: 301 mode &= ~FSL_DMA_MR_DAHTS_MASK; 302 mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16); 303 break; 304 } 305 306 set_mr(chan, mode); 307} 308 309/** 310 * fsl_chan_set_request_count - Set DMA Request Count for external control 311 * @chan : Freescale DMA channel 312 * @size : Number of bytes to transfer in a single request 313 * 314 * The Freescale DMA channel can be controlled by the external signal DREQ#. 315 * The DMA request count is how many bytes are allowed to transfer before 316 * pausing the channel, after which a new assertion of DREQ# resumes channel 317 * operation. 318 * 319 * A size of 0 disables external pause control. The maximum size is 1024. 320 */ 321static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size) 322{ 323 u32 mode; 324 325 BUG_ON(size > 1024); 326 327 mode = get_mr(chan); 328 mode &= ~FSL_DMA_MR_BWC_MASK; 329 mode |= (__ilog2(size) << 24) & FSL_DMA_MR_BWC_MASK; 330 331 set_mr(chan, mode); 332} 333 334/** 335 * fsl_chan_toggle_ext_pause - Toggle channel external pause status 336 * @chan : Freescale DMA channel 337 * @enable : 0 is disabled, 1 is enabled. 338 * 339 * The Freescale DMA channel can be controlled by the external signal DREQ#. 340 * The DMA Request Count feature should be used in addition to this feature 341 * to set the number of bytes to transfer before pausing the channel. 342 */ 343static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable) 344{ 345 if (enable) 346 chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; 347 else 348 chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; 349} 350 351/** 352 * fsl_chan_toggle_ext_start - Toggle channel external start status 353 * @chan : Freescale DMA channel 354 * @enable : 0 is disabled, 1 is enabled. 355 * 356 * If enable the external start, the channel can be started by an 357 * external DMA start pin. So the dma_start() does not start the 358 * transfer immediately. The DMA channel will wait for the 359 * control pin asserted. 360 */ 361static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable) 362{ 363 if (enable) 364 chan->feature |= FSL_DMA_CHAN_START_EXT; 365 else 366 chan->feature &= ~FSL_DMA_CHAN_START_EXT; 367} 368 369int fsl_dma_external_start(struct dma_chan *dchan, int enable) 370{ 371 struct fsldma_chan *chan; 372 373 if (!dchan) 374 return -EINVAL; 375 376 chan = to_fsl_chan(dchan); 377 378 fsl_chan_toggle_ext_start(chan, enable); 379 return 0; 380} 381EXPORT_SYMBOL_GPL(fsl_dma_external_start); 382 383static void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc) 384{ 385 struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev); 386 387 if (list_empty(&chan->ld_pending)) 388 goto out_splice; 389 390 /* 391 * Add the hardware descriptor to the chain of hardware descriptors 392 * that already exists in memory. 393 * 394 * This will un-set the EOL bit of the existing transaction, and the 395 * last link in this transaction will become the EOL descriptor. 396 */ 397 set_desc_next(chan, &tail->hw, desc->async_tx.phys); 398 399 /* 400 * Add the software descriptor and all children to the list 401 * of pending transactions 402 */ 403out_splice: 404 list_splice_tail_init(&desc->tx_list, &chan->ld_pending); 405} 406 407static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) 408{ 409 struct fsldma_chan *chan = to_fsl_chan(tx->chan); 410 struct fsl_desc_sw *desc = tx_to_fsl_desc(tx); 411 struct fsl_desc_sw *child; 412 dma_cookie_t cookie = -EINVAL; 413 414 spin_lock_bh(&chan->desc_lock); 415 416#ifdef CONFIG_PM 417 if (unlikely(chan->pm_state != RUNNING)) { 418 chan_dbg(chan, "cannot submit due to suspend\n"); 419 spin_unlock_bh(&chan->desc_lock); 420 return -1; 421 } 422#endif 423 424 /* 425 * assign cookies to all of the software descriptors 426 * that make up this transaction 427 */ 428 list_for_each_entry(child, &desc->tx_list, node) { 429 cookie = dma_cookie_assign(&child->async_tx); 430 } 431 432 /* put this transaction onto the tail of the pending queue */ 433 append_ld_queue(chan, desc); 434 435 spin_unlock_bh(&chan->desc_lock); 436 437 return cookie; 438} 439 440/** 441 * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool. 442 * @chan : Freescale DMA channel 443 * @desc: descriptor to be freed 444 */ 445static void fsl_dma_free_descriptor(struct fsldma_chan *chan, 446 struct fsl_desc_sw *desc) 447{ 448 list_del(&desc->node); 449 chan_dbg(chan, "LD %p free\n", desc); 450 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 451} 452 453/** 454 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool. 455 * @chan : Freescale DMA channel 456 * 457 * Return - The descriptor allocated. NULL for failed. 458 */ 459static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan) 460{ 461 struct fsl_desc_sw *desc; 462 dma_addr_t pdesc; 463 464 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc); 465 if (!desc) { 466 chan_dbg(chan, "out of memory for link descriptor\n"); 467 return NULL; 468 } 469 470 INIT_LIST_HEAD(&desc->tx_list); 471 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 472 desc->async_tx.tx_submit = fsl_dma_tx_submit; 473 desc->async_tx.phys = pdesc; 474 475 chan_dbg(chan, "LD %p allocated\n", desc); 476 477 return desc; 478} 479 480/** 481 * fsldma_clean_completed_descriptor - free all descriptors which 482 * has been completed and acked 483 * @chan: Freescale DMA channel 484 * 485 * This function is used on all completed and acked descriptors. 486 * All descriptors should only be freed in this function. 487 */ 488static void fsldma_clean_completed_descriptor(struct fsldma_chan *chan) 489{ 490 struct fsl_desc_sw *desc, *_desc; 491 492 /* Run the callback for each descriptor, in order */ 493 list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node) 494 if (async_tx_test_ack(&desc->async_tx)) 495 fsl_dma_free_descriptor(chan, desc); 496} 497 498/** 499 * fsldma_run_tx_complete_actions - cleanup a single link descriptor 500 * @chan: Freescale DMA channel 501 * @desc: descriptor to cleanup and free 502 * @cookie: Freescale DMA transaction identifier 503 * 504 * This function is used on a descriptor which has been executed by the DMA 505 * controller. It will run any callbacks, submit any dependencies. 506 */ 507static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan, 508 struct fsl_desc_sw *desc, dma_cookie_t cookie) 509{ 510 struct dma_async_tx_descriptor *txd = &desc->async_tx; 511 dma_cookie_t ret = cookie; 512 513 BUG_ON(txd->cookie < 0); 514 515 if (txd->cookie > 0) { 516 ret = txd->cookie; 517 518 dma_descriptor_unmap(txd); 519 /* Run the link descriptor callback function */ 520 dmaengine_desc_get_callback_invoke(txd, NULL); 521 } 522 523 /* Run any dependencies */ 524 dma_run_dependencies(txd); 525 526 return ret; 527} 528 529/** 530 * fsldma_clean_running_descriptor - move the completed descriptor from 531 * ld_running to ld_completed 532 * @chan: Freescale DMA channel 533 * @desc: the descriptor which is completed 534 * 535 * Free the descriptor directly if acked by async_tx api, or move it to 536 * queue ld_completed. 537 */ 538static void fsldma_clean_running_descriptor(struct fsldma_chan *chan, 539 struct fsl_desc_sw *desc) 540{ 541 /* Remove from the list of transactions */ 542 list_del(&desc->node); 543 544 /* 545 * the client is allowed to attach dependent operations 546 * until 'ack' is set 547 */ 548 if (!async_tx_test_ack(&desc->async_tx)) { 549 /* 550 * Move this descriptor to the list of descriptors which is 551 * completed, but still awaiting the 'ack' bit to be set. 552 */ 553 list_add_tail(&desc->node, &chan->ld_completed); 554 return; 555 } 556 557 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 558} 559 560/** 561 * fsl_chan_xfer_ld_queue - transfer any pending transactions 562 * @chan : Freescale DMA channel 563 * 564 * HARDWARE STATE: idle 565 * LOCKING: must hold chan->desc_lock 566 */ 567static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan) 568{ 569 struct fsl_desc_sw *desc; 570 571 /* 572 * If the list of pending descriptors is empty, then we 573 * don't need to do any work at all 574 */ 575 if (list_empty(&chan->ld_pending)) { 576 chan_dbg(chan, "no pending LDs\n"); 577 return; 578 } 579 580 /* 581 * The DMA controller is not idle, which means that the interrupt 582 * handler will start any queued transactions when it runs after 583 * this transaction finishes 584 */ 585 if (!chan->idle) { 586 chan_dbg(chan, "DMA controller still busy\n"); 587 return; 588 } 589 590 /* 591 * If there are some link descriptors which have not been 592 * transferred, we need to start the controller 593 */ 594 595 /* 596 * Move all elements from the queue of pending transactions 597 * onto the list of running transactions 598 */ 599 chan_dbg(chan, "idle, starting controller\n"); 600 desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node); 601 list_splice_tail_init(&chan->ld_pending, &chan->ld_running); 602 603 /* 604 * The 85xx DMA controller doesn't clear the channel start bit 605 * automatically at the end of a transfer. Therefore we must clear 606 * it in software before starting the transfer. 607 */ 608 if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { 609 u32 mode; 610 611 mode = get_mr(chan); 612 mode &= ~FSL_DMA_MR_CS; 613 set_mr(chan, mode); 614 } 615 616 /* 617 * Program the descriptor's address into the DMA controller, 618 * then start the DMA transaction 619 */ 620 set_cdar(chan, desc->async_tx.phys); 621 get_cdar(chan); 622 623 dma_start(chan); 624 chan->idle = false; 625} 626 627/** 628 * fsldma_cleanup_descriptors - cleanup link descriptors which are completed 629 * and move them to ld_completed to free until flag 'ack' is set 630 * @chan: Freescale DMA channel 631 * 632 * This function is used on descriptors which have been executed by the DMA 633 * controller. It will run any callbacks, submit any dependencies, then 634 * free these descriptors if flag 'ack' is set. 635 */ 636static void fsldma_cleanup_descriptors(struct fsldma_chan *chan) 637{ 638 struct fsl_desc_sw *desc, *_desc; 639 dma_cookie_t cookie = 0; 640 dma_addr_t curr_phys = get_cdar(chan); 641 int seen_current = 0; 642 643 fsldma_clean_completed_descriptor(chan); 644 645 /* Run the callback for each descriptor, in order */ 646 list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) { 647 /* 648 * do not advance past the current descriptor loaded into the 649 * hardware channel, subsequent descriptors are either in 650 * process or have not been submitted 651 */ 652 if (seen_current) 653 break; 654 655 /* 656 * stop the search if we reach the current descriptor and the 657 * channel is busy 658 */ 659 if (desc->async_tx.phys == curr_phys) { 660 seen_current = 1; 661 if (!dma_is_idle(chan)) 662 break; 663 } 664 665 cookie = fsldma_run_tx_complete_actions(chan, desc, cookie); 666 667 fsldma_clean_running_descriptor(chan, desc); 668 } 669 670 /* 671 * Start any pending transactions automatically 672 * 673 * In the ideal case, we keep the DMA controller busy while we go 674 * ahead and free the descriptors below. 675 */ 676 fsl_chan_xfer_ld_queue(chan); 677 678 if (cookie > 0) 679 chan->common.completed_cookie = cookie; 680} 681 682/** 683 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. 684 * @chan : Freescale DMA channel 685 * 686 * This function will create a dma pool for descriptor allocation. 687 * 688 * Return - The number of descriptors allocated. 689 */ 690static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan) 691{ 692 struct fsldma_chan *chan = to_fsl_chan(dchan); 693 694 /* Has this channel already been allocated? */ 695 if (chan->desc_pool) 696 return 1; 697 698 /* 699 * We need the descriptor to be aligned to 32bytes 700 * for meeting FSL DMA specification requirement. 701 */ 702 chan->desc_pool = dma_pool_create(chan->name, chan->dev, 703 sizeof(struct fsl_desc_sw), 704 __alignof__(struct fsl_desc_sw), 0); 705 if (!chan->desc_pool) { 706 chan_err(chan, "unable to allocate descriptor pool\n"); 707 return -ENOMEM; 708 } 709 710 /* there is at least one descriptor free to be allocated */ 711 return 1; 712} 713 714/** 715 * fsldma_free_desc_list - Free all descriptors in a queue 716 * @chan: Freescae DMA channel 717 * @list: the list to free 718 * 719 * LOCKING: must hold chan->desc_lock 720 */ 721static void fsldma_free_desc_list(struct fsldma_chan *chan, 722 struct list_head *list) 723{ 724 struct fsl_desc_sw *desc, *_desc; 725 726 list_for_each_entry_safe(desc, _desc, list, node) 727 fsl_dma_free_descriptor(chan, desc); 728} 729 730static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, 731 struct list_head *list) 732{ 733 struct fsl_desc_sw *desc, *_desc; 734 735 list_for_each_entry_safe_reverse(desc, _desc, list, node) 736 fsl_dma_free_descriptor(chan, desc); 737} 738 739/** 740 * fsl_dma_free_chan_resources - Free all resources of the channel. 741 * @chan : Freescale DMA channel 742 */ 743static void fsl_dma_free_chan_resources(struct dma_chan *dchan) 744{ 745 struct fsldma_chan *chan = to_fsl_chan(dchan); 746 747 chan_dbg(chan, "free all channel resources\n"); 748 spin_lock_bh(&chan->desc_lock); 749 fsldma_cleanup_descriptors(chan); 750 fsldma_free_desc_list(chan, &chan->ld_pending); 751 fsldma_free_desc_list(chan, &chan->ld_running); 752 fsldma_free_desc_list(chan, &chan->ld_completed); 753 spin_unlock_bh(&chan->desc_lock); 754 755 dma_pool_destroy(chan->desc_pool); 756 chan->desc_pool = NULL; 757} 758 759static struct dma_async_tx_descriptor * 760fsl_dma_prep_memcpy(struct dma_chan *dchan, 761 dma_addr_t dma_dst, dma_addr_t dma_src, 762 size_t len, unsigned long flags) 763{ 764 struct fsldma_chan *chan; 765 struct fsl_desc_sw *first = NULL, *prev = NULL, *new; 766 size_t copy; 767 768 if (!dchan) 769 return NULL; 770 771 if (!len) 772 return NULL; 773 774 chan = to_fsl_chan(dchan); 775 776 do { 777 778 /* Allocate the link descriptor from DMA pool */ 779 new = fsl_dma_alloc_descriptor(chan); 780 if (!new) { 781 chan_err(chan, "%s\n", msg_ld_oom); 782 goto fail; 783 } 784 785 copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT); 786 787 set_desc_cnt(chan, &new->hw, copy); 788 set_desc_src(chan, &new->hw, dma_src); 789 set_desc_dst(chan, &new->hw, dma_dst); 790 791 if (!first) 792 first = new; 793 else 794 set_desc_next(chan, &prev->hw, new->async_tx.phys); 795 796 new->async_tx.cookie = 0; 797 async_tx_ack(&new->async_tx); 798 799 prev = new; 800 len -= copy; 801 dma_src += copy; 802 dma_dst += copy; 803 804 /* Insert the link descriptor to the LD ring */ 805 list_add_tail(&new->node, &first->tx_list); 806 } while (len); 807 808 new->async_tx.flags = flags; /* client is in control of this ack */ 809 new->async_tx.cookie = -EBUSY; 810 811 /* Set End-of-link to the last link descriptor of new list */ 812 set_ld_eol(chan, new); 813 814 return &first->async_tx; 815 816fail: 817 if (!first) 818 return NULL; 819 820 fsldma_free_desc_list_reverse(chan, &first->tx_list); 821 return NULL; 822} 823 824static int fsl_dma_device_terminate_all(struct dma_chan *dchan) 825{ 826 struct fsldma_chan *chan; 827 828 if (!dchan) 829 return -EINVAL; 830 831 chan = to_fsl_chan(dchan); 832 833 spin_lock_bh(&chan->desc_lock); 834 835 /* Halt the DMA engine */ 836 dma_halt(chan); 837 838 /* Remove and free all of the descriptors in the LD queue */ 839 fsldma_free_desc_list(chan, &chan->ld_pending); 840 fsldma_free_desc_list(chan, &chan->ld_running); 841 fsldma_free_desc_list(chan, &chan->ld_completed); 842 chan->idle = true; 843 844 spin_unlock_bh(&chan->desc_lock); 845 return 0; 846} 847 848static int fsl_dma_device_config(struct dma_chan *dchan, 849 struct dma_slave_config *config) 850{ 851 struct fsldma_chan *chan; 852 int size; 853 854 if (!dchan) 855 return -EINVAL; 856 857 chan = to_fsl_chan(dchan); 858 859 /* make sure the channel supports setting burst size */ 860 if (!chan->set_request_count) 861 return -ENXIO; 862 863 /* we set the controller burst size depending on direction */ 864 if (config->direction == DMA_MEM_TO_DEV) 865 size = config->dst_addr_width * config->dst_maxburst; 866 else 867 size = config->src_addr_width * config->src_maxburst; 868 869 chan->set_request_count(chan, size); 870 return 0; 871} 872 873 874/** 875 * fsl_dma_memcpy_issue_pending - Issue the DMA start command 876 * @chan : Freescale DMA channel 877 */ 878static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan) 879{ 880 struct fsldma_chan *chan = to_fsl_chan(dchan); 881 882 spin_lock_bh(&chan->desc_lock); 883 fsl_chan_xfer_ld_queue(chan); 884 spin_unlock_bh(&chan->desc_lock); 885} 886 887/** 888 * fsl_tx_status - Determine the DMA status 889 * @chan : Freescale DMA channel 890 */ 891static enum dma_status fsl_tx_status(struct dma_chan *dchan, 892 dma_cookie_t cookie, 893 struct dma_tx_state *txstate) 894{ 895 struct fsldma_chan *chan = to_fsl_chan(dchan); 896 enum dma_status ret; 897 898 ret = dma_cookie_status(dchan, cookie, txstate); 899 if (ret == DMA_COMPLETE) 900 return ret; 901 902 spin_lock_bh(&chan->desc_lock); 903 fsldma_cleanup_descriptors(chan); 904 spin_unlock_bh(&chan->desc_lock); 905 906 return dma_cookie_status(dchan, cookie, txstate); 907} 908 909/*----------------------------------------------------------------------------*/ 910/* Interrupt Handling */ 911/*----------------------------------------------------------------------------*/ 912 913static irqreturn_t fsldma_chan_irq(int irq, void *data) 914{ 915 struct fsldma_chan *chan = data; 916 u32 stat; 917 918 /* save and clear the status register */ 919 stat = get_sr(chan); 920 set_sr(chan, stat); 921 chan_dbg(chan, "irq: stat = 0x%x\n", stat); 922 923 /* check that this was really our device */ 924 stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH); 925 if (!stat) 926 return IRQ_NONE; 927 928 if (stat & FSL_DMA_SR_TE) 929 chan_err(chan, "Transfer Error!\n"); 930 931 /* 932 * Programming Error 933 * The DMA_INTERRUPT async_tx is a NULL transfer, which will 934 * trigger a PE interrupt. 935 */ 936 if (stat & FSL_DMA_SR_PE) { 937 chan_dbg(chan, "irq: Programming Error INT\n"); 938 stat &= ~FSL_DMA_SR_PE; 939 if (get_bcr(chan) != 0) 940 chan_err(chan, "Programming Error!\n"); 941 } 942 943 /* 944 * For MPC8349, EOCDI event need to update cookie 945 * and start the next transfer if it exist. 946 */ 947 if (stat & FSL_DMA_SR_EOCDI) { 948 chan_dbg(chan, "irq: End-of-Chain link INT\n"); 949 stat &= ~FSL_DMA_SR_EOCDI; 950 } 951 952 /* 953 * If it current transfer is the end-of-transfer, 954 * we should clear the Channel Start bit for 955 * prepare next transfer. 956 */ 957 if (stat & FSL_DMA_SR_EOLNI) { 958 chan_dbg(chan, "irq: End-of-link INT\n"); 959 stat &= ~FSL_DMA_SR_EOLNI; 960 } 961 962 /* check that the DMA controller is really idle */ 963 if (!dma_is_idle(chan)) 964 chan_err(chan, "irq: controller not idle!\n"); 965 966 /* check that we handled all of the bits */ 967 if (stat) 968 chan_err(chan, "irq: unhandled sr 0x%08x\n", stat); 969 970 /* 971 * Schedule the tasklet to handle all cleanup of the current 972 * transaction. It will start a new transaction if there is 973 * one pending. 974 */ 975 tasklet_schedule(&chan->tasklet); 976 chan_dbg(chan, "irq: Exit\n"); 977 return IRQ_HANDLED; 978} 979 980static void dma_do_tasklet(struct tasklet_struct *t) 981{ 982 struct fsldma_chan *chan = from_tasklet(chan, t, tasklet); 983 984 chan_dbg(chan, "tasklet entry\n"); 985 986 spin_lock(&chan->desc_lock); 987 988 /* the hardware is now idle and ready for more */ 989 chan->idle = true; 990 991 /* Run all cleanup for descriptors which have been completed */ 992 fsldma_cleanup_descriptors(chan); 993 994 spin_unlock(&chan->desc_lock); 995 996 chan_dbg(chan, "tasklet exit\n"); 997} 998 999static irqreturn_t fsldma_ctrl_irq(int irq, void *data) 1000{ 1001 struct fsldma_device *fdev = data; 1002 struct fsldma_chan *chan; 1003 unsigned int handled = 0; 1004 u32 gsr, mask; 1005 int i; 1006 1007 gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs) 1008 : in_le32(fdev->regs); 1009 mask = 0xff000000; 1010 dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr); 1011 1012 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1013 chan = fdev->chan[i]; 1014 if (!chan) 1015 continue; 1016 1017 if (gsr & mask) { 1018 dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id); 1019 fsldma_chan_irq(irq, chan); 1020 handled++; 1021 } 1022 1023 gsr &= ~mask; 1024 mask >>= 8; 1025 } 1026 1027 return IRQ_RETVAL(handled); 1028} 1029 1030static void fsldma_free_irqs(struct fsldma_device *fdev) 1031{ 1032 struct fsldma_chan *chan; 1033 int i; 1034 1035 if (fdev->irq) { 1036 dev_dbg(fdev->dev, "free per-controller IRQ\n"); 1037 free_irq(fdev->irq, fdev); 1038 return; 1039 } 1040 1041 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1042 chan = fdev->chan[i]; 1043 if (chan && chan->irq) { 1044 chan_dbg(chan, "free per-channel IRQ\n"); 1045 free_irq(chan->irq, chan); 1046 } 1047 } 1048} 1049 1050static int fsldma_request_irqs(struct fsldma_device *fdev) 1051{ 1052 struct fsldma_chan *chan; 1053 int ret; 1054 int i; 1055 1056 /* if we have a per-controller IRQ, use that */ 1057 if (fdev->irq) { 1058 dev_dbg(fdev->dev, "request per-controller IRQ\n"); 1059 ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED, 1060 "fsldma-controller", fdev); 1061 return ret; 1062 } 1063 1064 /* no per-controller IRQ, use the per-channel IRQs */ 1065 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1066 chan = fdev->chan[i]; 1067 if (!chan) 1068 continue; 1069 1070 if (!chan->irq) { 1071 chan_err(chan, "interrupts property missing in device tree\n"); 1072 ret = -ENODEV; 1073 goto out_unwind; 1074 } 1075 1076 chan_dbg(chan, "request per-channel IRQ\n"); 1077 ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED, 1078 "fsldma-chan", chan); 1079 if (ret) { 1080 chan_err(chan, "unable to request per-channel IRQ\n"); 1081 goto out_unwind; 1082 } 1083 } 1084 1085 return 0; 1086 1087out_unwind: 1088 for (/* none */; i >= 0; i--) { 1089 chan = fdev->chan[i]; 1090 if (!chan) 1091 continue; 1092 1093 if (!chan->irq) 1094 continue; 1095 1096 free_irq(chan->irq, chan); 1097 } 1098 1099 return ret; 1100} 1101 1102/*----------------------------------------------------------------------------*/ 1103/* OpenFirmware Subsystem */ 1104/*----------------------------------------------------------------------------*/ 1105 1106static int fsl_dma_chan_probe(struct fsldma_device *fdev, 1107 struct device_node *node, u32 feature, const char *compatible) 1108{ 1109 struct fsldma_chan *chan; 1110 struct resource res; 1111 int err; 1112 1113 /* alloc channel */ 1114 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 1115 if (!chan) { 1116 err = -ENOMEM; 1117 goto out_return; 1118 } 1119 1120 /* ioremap registers for use */ 1121 chan->regs = of_iomap(node, 0); 1122 if (!chan->regs) { 1123 dev_err(fdev->dev, "unable to ioremap registers\n"); 1124 err = -ENOMEM; 1125 goto out_free_chan; 1126 } 1127 1128 err = of_address_to_resource(node, 0, &res); 1129 if (err) { 1130 dev_err(fdev->dev, "unable to find 'reg' property\n"); 1131 goto out_iounmap_regs; 1132 } 1133 1134 chan->feature = feature; 1135 if (!fdev->feature) 1136 fdev->feature = chan->feature; 1137 1138 /* 1139 * If the DMA device's feature is different than the feature 1140 * of its channels, report the bug 1141 */ 1142 WARN_ON(fdev->feature != chan->feature); 1143 1144 chan->dev = fdev->dev; 1145 chan->id = (res.start & 0xfff) < 0x300 ? 1146 ((res.start - 0x100) & 0xfff) >> 7 : 1147 ((res.start - 0x200) & 0xfff) >> 7; 1148 if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) { 1149 dev_err(fdev->dev, "too many channels for device\n"); 1150 err = -EINVAL; 1151 goto out_iounmap_regs; 1152 } 1153 1154 fdev->chan[chan->id] = chan; 1155 tasklet_setup(&chan->tasklet, dma_do_tasklet); 1156 snprintf(chan->name, sizeof(chan->name), "chan%d", chan->id); 1157 1158 /* Initialize the channel */ 1159 dma_init(chan); 1160 1161 /* Clear cdar registers */ 1162 set_cdar(chan, 0); 1163 1164 switch (chan->feature & FSL_DMA_IP_MASK) { 1165 case FSL_DMA_IP_85XX: 1166 chan->toggle_ext_pause = fsl_chan_toggle_ext_pause; 1167 fallthrough; 1168 case FSL_DMA_IP_83XX: 1169 chan->toggle_ext_start = fsl_chan_toggle_ext_start; 1170 chan->set_src_loop_size = fsl_chan_set_src_loop_size; 1171 chan->set_dst_loop_size = fsl_chan_set_dst_loop_size; 1172 chan->set_request_count = fsl_chan_set_request_count; 1173 } 1174 1175 spin_lock_init(&chan->desc_lock); 1176 INIT_LIST_HEAD(&chan->ld_pending); 1177 INIT_LIST_HEAD(&chan->ld_running); 1178 INIT_LIST_HEAD(&chan->ld_completed); 1179 chan->idle = true; 1180#ifdef CONFIG_PM 1181 chan->pm_state = RUNNING; 1182#endif 1183 1184 chan->common.device = &fdev->common; 1185 dma_cookie_init(&chan->common); 1186 1187 /* find the IRQ line, if it exists in the device tree */ 1188 chan->irq = irq_of_parse_and_map(node, 0); 1189 1190 /* Add the channel to DMA device channel list */ 1191 list_add_tail(&chan->common.device_node, &fdev->common.channels); 1192 1193 dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible, 1194 chan->irq ? chan->irq : fdev->irq); 1195 1196 return 0; 1197 1198out_iounmap_regs: 1199 iounmap(chan->regs); 1200out_free_chan: 1201 kfree(chan); 1202out_return: 1203 return err; 1204} 1205 1206static void fsl_dma_chan_remove(struct fsldma_chan *chan) 1207{ 1208 irq_dispose_mapping(chan->irq); 1209 list_del(&chan->common.device_node); 1210 iounmap(chan->regs); 1211 kfree(chan); 1212} 1213 1214static int fsldma_of_probe(struct platform_device *op) 1215{ 1216 struct fsldma_device *fdev; 1217 struct device_node *child; 1218 unsigned int i; 1219 int err; 1220 1221 fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); 1222 if (!fdev) { 1223 err = -ENOMEM; 1224 goto out_return; 1225 } 1226 1227 fdev->dev = &op->dev; 1228 INIT_LIST_HEAD(&fdev->common.channels); 1229 1230 /* ioremap the registers for use */ 1231 fdev->regs = of_iomap(op->dev.of_node, 0); 1232 if (!fdev->regs) { 1233 dev_err(&op->dev, "unable to ioremap registers\n"); 1234 err = -ENOMEM; 1235 goto out_free; 1236 } 1237 1238 /* map the channel IRQ if it exists, but don't hookup the handler yet */ 1239 fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0); 1240 1241 dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask); 1242 dma_cap_set(DMA_SLAVE, fdev->common.cap_mask); 1243 fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources; 1244 fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources; 1245 fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy; 1246 fdev->common.device_tx_status = fsl_tx_status; 1247 fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending; 1248 fdev->common.device_config = fsl_dma_device_config; 1249 fdev->common.device_terminate_all = fsl_dma_device_terminate_all; 1250 fdev->common.dev = &op->dev; 1251 1252 fdev->common.src_addr_widths = FSL_DMA_BUSWIDTHS; 1253 fdev->common.dst_addr_widths = FSL_DMA_BUSWIDTHS; 1254 fdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1255 fdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1256 1257 dma_set_mask(&(op->dev), DMA_BIT_MASK(36)); 1258 1259 platform_set_drvdata(op, fdev); 1260 1261 /* 1262 * We cannot use of_platform_bus_probe() because there is no 1263 * of_platform_bus_remove(). Instead, we manually instantiate every DMA 1264 * channel object. 1265 */ 1266 for_each_child_of_node(op->dev.of_node, child) { 1267 if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) { 1268 fsl_dma_chan_probe(fdev, child, 1269 FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN, 1270 "fsl,eloplus-dma-channel"); 1271 } 1272 1273 if (of_device_is_compatible(child, "fsl,elo-dma-channel")) { 1274 fsl_dma_chan_probe(fdev, child, 1275 FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN, 1276 "fsl,elo-dma-channel"); 1277 } 1278 } 1279 1280 /* 1281 * Hookup the IRQ handler(s) 1282 * 1283 * If we have a per-controller interrupt, we prefer that to the 1284 * per-channel interrupts to reduce the number of shared interrupt 1285 * handlers on the same IRQ line 1286 */ 1287 err = fsldma_request_irqs(fdev); 1288 if (err) { 1289 dev_err(fdev->dev, "unable to request IRQs\n"); 1290 goto out_free_fdev; 1291 } 1292 1293 dma_async_device_register(&fdev->common); 1294 return 0; 1295 1296out_free_fdev: 1297 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1298 if (fdev->chan[i]) 1299 fsl_dma_chan_remove(fdev->chan[i]); 1300 } 1301 irq_dispose_mapping(fdev->irq); 1302 iounmap(fdev->regs); 1303out_free: 1304 kfree(fdev); 1305out_return: 1306 return err; 1307} 1308 1309static void fsldma_of_remove(struct platform_device *op) 1310{ 1311 struct fsldma_device *fdev; 1312 unsigned int i; 1313 1314 fdev = platform_get_drvdata(op); 1315 dma_async_device_unregister(&fdev->common); 1316 1317 fsldma_free_irqs(fdev); 1318 1319 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1320 if (fdev->chan[i]) 1321 fsl_dma_chan_remove(fdev->chan[i]); 1322 } 1323 irq_dispose_mapping(fdev->irq); 1324 1325 iounmap(fdev->regs); 1326 kfree(fdev); 1327} 1328 1329#ifdef CONFIG_PM 1330static int fsldma_suspend_late(struct device *dev) 1331{ 1332 struct fsldma_device *fdev = dev_get_drvdata(dev); 1333 struct fsldma_chan *chan; 1334 int i; 1335 1336 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1337 chan = fdev->chan[i]; 1338 if (!chan) 1339 continue; 1340 1341 spin_lock_bh(&chan->desc_lock); 1342 if (unlikely(!chan->idle)) 1343 goto out; 1344 chan->regs_save.mr = get_mr(chan); 1345 chan->pm_state = SUSPENDED; 1346 spin_unlock_bh(&chan->desc_lock); 1347 } 1348 return 0; 1349 1350out: 1351 for (; i >= 0; i--) { 1352 chan = fdev->chan[i]; 1353 if (!chan) 1354 continue; 1355 chan->pm_state = RUNNING; 1356 spin_unlock_bh(&chan->desc_lock); 1357 } 1358 return -EBUSY; 1359} 1360 1361static int fsldma_resume_early(struct device *dev) 1362{ 1363 struct fsldma_device *fdev = dev_get_drvdata(dev); 1364 struct fsldma_chan *chan; 1365 u32 mode; 1366 int i; 1367 1368 for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { 1369 chan = fdev->chan[i]; 1370 if (!chan) 1371 continue; 1372 1373 spin_lock_bh(&chan->desc_lock); 1374 mode = chan->regs_save.mr 1375 & ~FSL_DMA_MR_CS & ~FSL_DMA_MR_CC & ~FSL_DMA_MR_CA; 1376 set_mr(chan, mode); 1377 chan->pm_state = RUNNING; 1378 spin_unlock_bh(&chan->desc_lock); 1379 } 1380 1381 return 0; 1382} 1383 1384static const struct dev_pm_ops fsldma_pm_ops = { 1385 .suspend_late = fsldma_suspend_late, 1386 .resume_early = fsldma_resume_early, 1387}; 1388#endif 1389 1390static const struct of_device_id fsldma_of_ids[] = { 1391 { .compatible = "fsl,elo3-dma", }, 1392 { .compatible = "fsl,eloplus-dma", }, 1393 { .compatible = "fsl,elo-dma", }, 1394 {} 1395}; 1396MODULE_DEVICE_TABLE(of, fsldma_of_ids); 1397 1398static struct platform_driver fsldma_of_driver = { 1399 .driver = { 1400 .name = "fsl-elo-dma", 1401 .of_match_table = fsldma_of_ids, 1402#ifdef CONFIG_PM 1403 .pm = &fsldma_pm_ops, 1404#endif 1405 }, 1406 .probe = fsldma_of_probe, 1407 .remove_new = fsldma_of_remove, 1408}; 1409 1410/*----------------------------------------------------------------------------*/ 1411/* Module Init / Exit */ 1412/*----------------------------------------------------------------------------*/ 1413 1414static __init int fsldma_init(void) 1415{ 1416 pr_info("Freescale Elo series DMA driver\n"); 1417 return platform_driver_register(&fsldma_of_driver); 1418} 1419 1420static void __exit fsldma_exit(void) 1421{ 1422 platform_driver_unregister(&fsldma_of_driver); 1423} 1424 1425subsys_initcall(fsldma_init); 1426module_exit(fsldma_exit); 1427 1428MODULE_DESCRIPTION("Freescale Elo series DMA driver"); 1429MODULE_LICENSE("GPL"); 1430