1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * IMG Multi-threaded DMA Controller (MDC) 4 * 5 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd. 6 * Copyright (C) 2014 Google, Inc. 7 */ 8 9#include <linux/clk.h> 10#include <linux/dma-mapping.h> 11#include <linux/dmaengine.h> 12#include <linux/dmapool.h> 13#include <linux/interrupt.h> 14#include <linux/io.h> 15#include <linux/irq.h> 16#include <linux/kernel.h> 17#include <linux/mfd/syscon.h> 18#include <linux/module.h> 19#include <linux/of.h> 20#include <linux/of_dma.h> 21#include <linux/platform_device.h> 22#include <linux/pm_runtime.h> 23#include <linux/regmap.h> 24#include <linux/slab.h> 25#include <linux/spinlock.h> 26 27#include "dmaengine.h" 28#include "virt-dma.h" 29 30#define MDC_MAX_DMA_CHANNELS 32 31 32#define MDC_GENERAL_CONFIG 0x000 33#define MDC_GENERAL_CONFIG_LIST_IEN BIT(31) 34#define MDC_GENERAL_CONFIG_IEN BIT(29) 35#define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28) 36#define MDC_GENERAL_CONFIG_INC_W BIT(12) 37#define MDC_GENERAL_CONFIG_INC_R BIT(8) 38#define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7) 39#define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4 40#define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7 41#define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3) 42#define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0 43#define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7 44 45#define MDC_READ_PORT_CONFIG 0x004 46#define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28 47#define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf 48#define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24 49#define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf 50#define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16 51#define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf 52#define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4 53#define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff 54#define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1) 55 56#define MDC_READ_ADDRESS 0x008 57 58#define MDC_WRITE_ADDRESS 0x00c 59 60#define MDC_TRANSFER_SIZE 0x010 61#define MDC_TRANSFER_SIZE_MASK 0xffffff 62 63#define MDC_LIST_NODE_ADDRESS 0x014 64 65#define MDC_CMDS_PROCESSED 0x018 66#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16 67#define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f 68#define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8) 69#define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0 70#define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f 71 72#define MDC_CONTROL_AND_STATUS 0x01c 73#define MDC_CONTROL_AND_STATUS_CANCEL BIT(20) 74#define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4) 75#define MDC_CONTROL_AND_STATUS_EN BIT(0) 76 77#define MDC_ACTIVE_TRANSFER_SIZE 0x030 78 79#define MDC_GLOBAL_CONFIG_A 0x900 80#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16 81#define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff 82#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8 83#define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff 84#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0 85#define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff 86 87struct mdc_hw_list_desc { 88 u32 gen_conf; 89 u32 readport_conf; 90 u32 read_addr; 91 u32 write_addr; 92 u32 xfer_size; 93 u32 node_addr; 94 u32 cmds_done; 95 u32 ctrl_status; 96 /* 97 * Not part of the list descriptor, but instead used by the CPU to 98 * traverse the list. 99 */ 100 struct mdc_hw_list_desc *next_desc; 101}; 102 103struct mdc_tx_desc { 104 struct mdc_chan *chan; 105 struct virt_dma_desc vd; 106 dma_addr_t list_phys; 107 struct mdc_hw_list_desc *list; 108 bool cyclic; 109 bool cmd_loaded; 110 unsigned int list_len; 111 unsigned int list_period_len; 112 size_t list_xfer_size; 113 unsigned int list_cmds_done; 114}; 115 116struct mdc_chan { 117 struct mdc_dma *mdma; 118 struct virt_dma_chan vc; 119 struct dma_slave_config config; 120 struct mdc_tx_desc *desc; 121 int irq; 122 unsigned int periph; 123 unsigned int thread; 124 unsigned int chan_nr; 125}; 126 127struct mdc_dma_soc_data { 128 void (*enable_chan)(struct mdc_chan *mchan); 129 void (*disable_chan)(struct mdc_chan *mchan); 130}; 131 132struct mdc_dma { 133 struct dma_device dma_dev; 134 void __iomem *regs; 135 struct clk *clk; 136 struct dma_pool *desc_pool; 137 struct regmap *periph_regs; 138 spinlock_t lock; 139 unsigned int nr_threads; 140 unsigned int nr_channels; 141 unsigned int bus_width; 142 unsigned int max_burst_mult; 143 unsigned int max_xfer_size; 144 const struct mdc_dma_soc_data *soc; 145 struct mdc_chan channels[MDC_MAX_DMA_CHANNELS]; 146}; 147 148static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg) 149{ 150 return readl(mdma->regs + reg); 151} 152 153static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg) 154{ 155 writel(val, mdma->regs + reg); 156} 157 158static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg) 159{ 160 return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg); 161} 162 163static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg) 164{ 165 mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg); 166} 167 168static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c) 169{ 170 return container_of(to_virt_chan(c), struct mdc_chan, vc); 171} 172 173static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t) 174{ 175 struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx); 176 177 return container_of(vdesc, struct mdc_tx_desc, vd); 178} 179 180static inline struct device *mdma2dev(struct mdc_dma *mdma) 181{ 182 return mdma->dma_dev.dev; 183} 184 185static inline unsigned int to_mdc_width(unsigned int bytes) 186{ 187 return ffs(bytes) - 1; 188} 189 190static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc, 191 unsigned int bytes) 192{ 193 ldesc->gen_conf |= to_mdc_width(bytes) << 194 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT; 195} 196 197static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc, 198 unsigned int bytes) 199{ 200 ldesc->gen_conf |= to_mdc_width(bytes) << 201 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT; 202} 203 204static void mdc_list_desc_config(struct mdc_chan *mchan, 205 struct mdc_hw_list_desc *ldesc, 206 enum dma_transfer_direction dir, 207 dma_addr_t src, dma_addr_t dst, size_t len) 208{ 209 struct mdc_dma *mdma = mchan->mdma; 210 unsigned int max_burst, burst_size; 211 212 ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN | 213 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | 214 MDC_GENERAL_CONFIG_PHYSICAL_R; 215 ldesc->readport_conf = 216 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | 217 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | 218 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); 219 ldesc->read_addr = src; 220 ldesc->write_addr = dst; 221 ldesc->xfer_size = len - 1; 222 ldesc->node_addr = 0; 223 ldesc->cmds_done = 0; 224 ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN | 225 MDC_CONTROL_AND_STATUS_EN; 226 ldesc->next_desc = NULL; 227 228 if (IS_ALIGNED(dst, mdma->bus_width) && 229 IS_ALIGNED(src, mdma->bus_width)) 230 max_burst = mdma->bus_width * mdma->max_burst_mult; 231 else 232 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1); 233 234 if (dir == DMA_MEM_TO_DEV) { 235 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R; 236 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; 237 mdc_set_read_width(ldesc, mdma->bus_width); 238 mdc_set_write_width(ldesc, mchan->config.dst_addr_width); 239 burst_size = min(max_burst, mchan->config.dst_maxburst * 240 mchan->config.dst_addr_width); 241 } else if (dir == DMA_DEV_TO_MEM) { 242 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W; 243 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE; 244 mdc_set_read_width(ldesc, mchan->config.src_addr_width); 245 mdc_set_write_width(ldesc, mdma->bus_width); 246 burst_size = min(max_burst, mchan->config.src_maxburst * 247 mchan->config.src_addr_width); 248 } else { 249 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R | 250 MDC_GENERAL_CONFIG_INC_W; 251 mdc_set_read_width(ldesc, mdma->bus_width); 252 mdc_set_write_width(ldesc, mdma->bus_width); 253 burst_size = max_burst; 254 } 255 ldesc->readport_conf |= (burst_size - 1) << 256 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT; 257} 258 259static void mdc_list_desc_free(struct mdc_tx_desc *mdesc) 260{ 261 struct mdc_dma *mdma = mdesc->chan->mdma; 262 struct mdc_hw_list_desc *curr, *next; 263 dma_addr_t curr_phys, next_phys; 264 265 curr = mdesc->list; 266 curr_phys = mdesc->list_phys; 267 while (curr) { 268 next = curr->next_desc; 269 next_phys = curr->node_addr; 270 dma_pool_free(mdma->desc_pool, curr, curr_phys); 271 curr = next; 272 curr_phys = next_phys; 273 } 274} 275 276static void mdc_desc_free(struct virt_dma_desc *vd) 277{ 278 struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx); 279 280 mdc_list_desc_free(mdesc); 281 kfree(mdesc); 282} 283 284static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy( 285 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, 286 unsigned long flags) 287{ 288 struct mdc_chan *mchan = to_mdc_chan(chan); 289 struct mdc_dma *mdma = mchan->mdma; 290 struct mdc_tx_desc *mdesc; 291 struct mdc_hw_list_desc *curr, *prev = NULL; 292 dma_addr_t curr_phys; 293 294 if (!len) 295 return NULL; 296 297 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 298 if (!mdesc) 299 return NULL; 300 mdesc->chan = mchan; 301 mdesc->list_xfer_size = len; 302 303 while (len > 0) { 304 size_t xfer_size; 305 306 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys); 307 if (!curr) 308 goto free_desc; 309 310 if (prev) { 311 prev->node_addr = curr_phys; 312 prev->next_desc = curr; 313 } else { 314 mdesc->list_phys = curr_phys; 315 mdesc->list = curr; 316 } 317 318 xfer_size = min_t(size_t, mdma->max_xfer_size, len); 319 320 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest, 321 xfer_size); 322 323 prev = curr; 324 325 mdesc->list_len++; 326 src += xfer_size; 327 dest += xfer_size; 328 len -= xfer_size; 329 } 330 331 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 332 333free_desc: 334 mdc_desc_free(&mdesc->vd); 335 336 return NULL; 337} 338 339static int mdc_check_slave_width(struct mdc_chan *mchan, 340 enum dma_transfer_direction dir) 341{ 342 enum dma_slave_buswidth width; 343 344 if (dir == DMA_MEM_TO_DEV) 345 width = mchan->config.dst_addr_width; 346 else 347 width = mchan->config.src_addr_width; 348 349 switch (width) { 350 case DMA_SLAVE_BUSWIDTH_1_BYTE: 351 case DMA_SLAVE_BUSWIDTH_2_BYTES: 352 case DMA_SLAVE_BUSWIDTH_4_BYTES: 353 case DMA_SLAVE_BUSWIDTH_8_BYTES: 354 break; 355 default: 356 return -EINVAL; 357 } 358 359 if (width > mchan->mdma->bus_width) 360 return -EINVAL; 361 362 return 0; 363} 364 365static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic( 366 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 367 size_t period_len, enum dma_transfer_direction dir, 368 unsigned long flags) 369{ 370 struct mdc_chan *mchan = to_mdc_chan(chan); 371 struct mdc_dma *mdma = mchan->mdma; 372 struct mdc_tx_desc *mdesc; 373 struct mdc_hw_list_desc *curr, *prev = NULL; 374 dma_addr_t curr_phys; 375 376 if (!buf_len && !period_len) 377 return NULL; 378 379 if (!is_slave_direction(dir)) 380 return NULL; 381 382 if (mdc_check_slave_width(mchan, dir) < 0) 383 return NULL; 384 385 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 386 if (!mdesc) 387 return NULL; 388 mdesc->chan = mchan; 389 mdesc->cyclic = true; 390 mdesc->list_xfer_size = buf_len; 391 mdesc->list_period_len = DIV_ROUND_UP(period_len, 392 mdma->max_xfer_size); 393 394 while (buf_len > 0) { 395 size_t remainder = min(period_len, buf_len); 396 397 while (remainder > 0) { 398 size_t xfer_size; 399 400 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, 401 &curr_phys); 402 if (!curr) 403 goto free_desc; 404 405 if (!prev) { 406 mdesc->list_phys = curr_phys; 407 mdesc->list = curr; 408 } else { 409 prev->node_addr = curr_phys; 410 prev->next_desc = curr; 411 } 412 413 xfer_size = min_t(size_t, mdma->max_xfer_size, 414 remainder); 415 416 if (dir == DMA_MEM_TO_DEV) { 417 mdc_list_desc_config(mchan, curr, dir, 418 buf_addr, 419 mchan->config.dst_addr, 420 xfer_size); 421 } else { 422 mdc_list_desc_config(mchan, curr, dir, 423 mchan->config.src_addr, 424 buf_addr, 425 xfer_size); 426 } 427 428 prev = curr; 429 430 mdesc->list_len++; 431 buf_addr += xfer_size; 432 buf_len -= xfer_size; 433 remainder -= xfer_size; 434 } 435 } 436 prev->node_addr = mdesc->list_phys; 437 438 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 439 440free_desc: 441 mdc_desc_free(&mdesc->vd); 442 443 return NULL; 444} 445 446static struct dma_async_tx_descriptor *mdc_prep_slave_sg( 447 struct dma_chan *chan, struct scatterlist *sgl, 448 unsigned int sg_len, enum dma_transfer_direction dir, 449 unsigned long flags, void *context) 450{ 451 struct mdc_chan *mchan = to_mdc_chan(chan); 452 struct mdc_dma *mdma = mchan->mdma; 453 struct mdc_tx_desc *mdesc; 454 struct scatterlist *sg; 455 struct mdc_hw_list_desc *curr, *prev = NULL; 456 dma_addr_t curr_phys; 457 unsigned int i; 458 459 if (!sgl) 460 return NULL; 461 462 if (!is_slave_direction(dir)) 463 return NULL; 464 465 if (mdc_check_slave_width(mchan, dir) < 0) 466 return NULL; 467 468 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT); 469 if (!mdesc) 470 return NULL; 471 mdesc->chan = mchan; 472 473 for_each_sg(sgl, sg, sg_len, i) { 474 dma_addr_t buf = sg_dma_address(sg); 475 size_t buf_len = sg_dma_len(sg); 476 477 while (buf_len > 0) { 478 size_t xfer_size; 479 480 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, 481 &curr_phys); 482 if (!curr) 483 goto free_desc; 484 485 if (!prev) { 486 mdesc->list_phys = curr_phys; 487 mdesc->list = curr; 488 } else { 489 prev->node_addr = curr_phys; 490 prev->next_desc = curr; 491 } 492 493 xfer_size = min_t(size_t, mdma->max_xfer_size, 494 buf_len); 495 496 if (dir == DMA_MEM_TO_DEV) { 497 mdc_list_desc_config(mchan, curr, dir, buf, 498 mchan->config.dst_addr, 499 xfer_size); 500 } else { 501 mdc_list_desc_config(mchan, curr, dir, 502 mchan->config.src_addr, 503 buf, xfer_size); 504 } 505 506 prev = curr; 507 508 mdesc->list_len++; 509 mdesc->list_xfer_size += xfer_size; 510 buf += xfer_size; 511 buf_len -= xfer_size; 512 } 513 } 514 515 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags); 516 517free_desc: 518 mdc_desc_free(&mdesc->vd); 519 520 return NULL; 521} 522 523static void mdc_issue_desc(struct mdc_chan *mchan) 524{ 525 struct mdc_dma *mdma = mchan->mdma; 526 struct virt_dma_desc *vd; 527 struct mdc_tx_desc *mdesc; 528 u32 val; 529 530 vd = vchan_next_desc(&mchan->vc); 531 if (!vd) 532 return; 533 534 list_del(&vd->node); 535 536 mdesc = to_mdc_desc(&vd->tx); 537 mchan->desc = mdesc; 538 539 dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n", 540 mchan->chan_nr); 541 542 mdma->soc->enable_chan(mchan); 543 544 val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG); 545 val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN | 546 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W | 547 MDC_GENERAL_CONFIG_PHYSICAL_R; 548 mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG); 549 val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) | 550 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) | 551 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT); 552 mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG); 553 mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS); 554 val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS); 555 val |= MDC_CONTROL_AND_STATUS_LIST_EN; 556 mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS); 557} 558 559static void mdc_issue_pending(struct dma_chan *chan) 560{ 561 struct mdc_chan *mchan = to_mdc_chan(chan); 562 unsigned long flags; 563 564 spin_lock_irqsave(&mchan->vc.lock, flags); 565 if (vchan_issue_pending(&mchan->vc) && !mchan->desc) 566 mdc_issue_desc(mchan); 567 spin_unlock_irqrestore(&mchan->vc.lock, flags); 568} 569 570static enum dma_status mdc_tx_status(struct dma_chan *chan, 571 dma_cookie_t cookie, struct dma_tx_state *txstate) 572{ 573 struct mdc_chan *mchan = to_mdc_chan(chan); 574 struct mdc_tx_desc *mdesc; 575 struct virt_dma_desc *vd; 576 unsigned long flags; 577 size_t bytes = 0; 578 int ret; 579 580 ret = dma_cookie_status(chan, cookie, txstate); 581 if (ret == DMA_COMPLETE) 582 return ret; 583 584 if (!txstate) 585 return ret; 586 587 spin_lock_irqsave(&mchan->vc.lock, flags); 588 vd = vchan_find_desc(&mchan->vc, cookie); 589 if (vd) { 590 mdesc = to_mdc_desc(&vd->tx); 591 bytes = mdesc->list_xfer_size; 592 } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) { 593 struct mdc_hw_list_desc *ldesc; 594 u32 val1, val2, done, processed, residue; 595 int i, cmds; 596 597 mdesc = mchan->desc; 598 599 /* 600 * Determine the number of commands that haven't been 601 * processed (handled by the IRQ handler) yet. 602 */ 603 do { 604 val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & 605 ~MDC_CMDS_PROCESSED_INT_ACTIVE; 606 residue = mdc_chan_readl(mchan, 607 MDC_ACTIVE_TRANSFER_SIZE); 608 val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) & 609 ~MDC_CMDS_PROCESSED_INT_ACTIVE; 610 } while (val1 != val2); 611 612 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 613 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 614 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & 615 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; 616 cmds = (done - processed) % 617 (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1); 618 619 /* 620 * If the command loaded event hasn't been processed yet, then 621 * the difference above includes an extra command. 622 */ 623 if (!mdesc->cmd_loaded) 624 cmds--; 625 else 626 cmds += mdesc->list_cmds_done; 627 628 bytes = mdesc->list_xfer_size; 629 ldesc = mdesc->list; 630 for (i = 0; i < cmds; i++) { 631 bytes -= ldesc->xfer_size + 1; 632 ldesc = ldesc->next_desc; 633 } 634 if (ldesc) { 635 if (residue != MDC_TRANSFER_SIZE_MASK) 636 bytes -= ldesc->xfer_size - residue; 637 else 638 bytes -= ldesc->xfer_size + 1; 639 } 640 } 641 spin_unlock_irqrestore(&mchan->vc.lock, flags); 642 643 dma_set_residue(txstate, bytes); 644 645 return ret; 646} 647 648static unsigned int mdc_get_new_events(struct mdc_chan *mchan) 649{ 650 u32 val, processed, done1, done2; 651 unsigned int ret; 652 653 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 654 processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & 655 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; 656 /* 657 * CMDS_DONE may have incremented between reading CMDS_PROCESSED 658 * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we 659 * didn't miss a command completion. 660 */ 661 do { 662 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 663 664 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 665 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 666 667 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK << 668 MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) | 669 MDC_CMDS_PROCESSED_INT_ACTIVE); 670 671 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT; 672 673 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED); 674 675 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); 676 677 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & 678 MDC_CMDS_PROCESSED_CMDS_DONE_MASK; 679 } while (done1 != done2); 680 681 if (done1 >= processed) 682 ret = done1 - processed; 683 else 684 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) - 685 processed) + done1; 686 687 return ret; 688} 689 690static int mdc_terminate_all(struct dma_chan *chan) 691{ 692 struct mdc_chan *mchan = to_mdc_chan(chan); 693 unsigned long flags; 694 LIST_HEAD(head); 695 696 spin_lock_irqsave(&mchan->vc.lock, flags); 697 698 mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL, 699 MDC_CONTROL_AND_STATUS); 700 701 if (mchan->desc) { 702 vchan_terminate_vdesc(&mchan->desc->vd); 703 mchan->desc = NULL; 704 } 705 vchan_get_all_descriptors(&mchan->vc, &head); 706 707 mdc_get_new_events(mchan); 708 709 spin_unlock_irqrestore(&mchan->vc.lock, flags); 710 711 vchan_dma_desc_free_list(&mchan->vc, &head); 712 713 return 0; 714} 715 716static void mdc_synchronize(struct dma_chan *chan) 717{ 718 struct mdc_chan *mchan = to_mdc_chan(chan); 719 720 vchan_synchronize(&mchan->vc); 721} 722 723static int mdc_slave_config(struct dma_chan *chan, 724 struct dma_slave_config *config) 725{ 726 struct mdc_chan *mchan = to_mdc_chan(chan); 727 unsigned long flags; 728 729 spin_lock_irqsave(&mchan->vc.lock, flags); 730 mchan->config = *config; 731 spin_unlock_irqrestore(&mchan->vc.lock, flags); 732 733 return 0; 734} 735 736static int mdc_alloc_chan_resources(struct dma_chan *chan) 737{ 738 struct mdc_chan *mchan = to_mdc_chan(chan); 739 struct device *dev = mdma2dev(mchan->mdma); 740 741 return pm_runtime_get_sync(dev); 742} 743 744static void mdc_free_chan_resources(struct dma_chan *chan) 745{ 746 struct mdc_chan *mchan = to_mdc_chan(chan); 747 struct mdc_dma *mdma = mchan->mdma; 748 struct device *dev = mdma2dev(mdma); 749 750 mdc_terminate_all(chan); 751 mdma->soc->disable_chan(mchan); 752 pm_runtime_put(dev); 753} 754 755static irqreturn_t mdc_chan_irq(int irq, void *dev_id) 756{ 757 struct mdc_chan *mchan = (struct mdc_chan *)dev_id; 758 struct mdc_tx_desc *mdesc; 759 unsigned int i, new_events; 760 761 spin_lock(&mchan->vc.lock); 762 763 dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr); 764 765 new_events = mdc_get_new_events(mchan); 766 767 if (!new_events) 768 goto out; 769 770 mdesc = mchan->desc; 771 if (!mdesc) { 772 dev_warn(mdma2dev(mchan->mdma), 773 "IRQ with no active descriptor on channel %d\n", 774 mchan->chan_nr); 775 goto out; 776 } 777 778 for (i = 0; i < new_events; i++) { 779 /* 780 * The first interrupt in a transfer indicates that the 781 * command list has been loaded, not that a command has 782 * been completed. 783 */ 784 if (!mdesc->cmd_loaded) { 785 mdesc->cmd_loaded = true; 786 continue; 787 } 788 789 mdesc->list_cmds_done++; 790 if (mdesc->cyclic) { 791 mdesc->list_cmds_done %= mdesc->list_len; 792 if (mdesc->list_cmds_done % mdesc->list_period_len == 0) 793 vchan_cyclic_callback(&mdesc->vd); 794 } else if (mdesc->list_cmds_done == mdesc->list_len) { 795 mchan->desc = NULL; 796 vchan_cookie_complete(&mdesc->vd); 797 mdc_issue_desc(mchan); 798 break; 799 } 800 } 801out: 802 spin_unlock(&mchan->vc.lock); 803 804 return IRQ_HANDLED; 805} 806 807static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec, 808 struct of_dma *ofdma) 809{ 810 struct mdc_dma *mdma = ofdma->of_dma_data; 811 struct dma_chan *chan; 812 813 if (dma_spec->args_count != 3) 814 return NULL; 815 816 list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) { 817 struct mdc_chan *mchan = to_mdc_chan(chan); 818 819 if (!(dma_spec->args[1] & BIT(mchan->chan_nr))) 820 continue; 821 if (dma_get_slave_channel(chan)) { 822 mchan->periph = dma_spec->args[0]; 823 mchan->thread = dma_spec->args[2]; 824 return chan; 825 } 826 } 827 828 return NULL; 829} 830 831#define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4)) 832#define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4)) 833#define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f 834 835static void pistachio_mdc_enable_chan(struct mdc_chan *mchan) 836{ 837 struct mdc_dma *mdma = mchan->mdma; 838 839 regmap_update_bits(mdma->periph_regs, 840 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), 841 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << 842 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), 843 mchan->periph << 844 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr)); 845} 846 847static void pistachio_mdc_disable_chan(struct mdc_chan *mchan) 848{ 849 struct mdc_dma *mdma = mchan->mdma; 850 851 regmap_update_bits(mdma->periph_regs, 852 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr), 853 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK << 854 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr), 855 0); 856} 857 858static const struct mdc_dma_soc_data pistachio_mdc_data = { 859 .enable_chan = pistachio_mdc_enable_chan, 860 .disable_chan = pistachio_mdc_disable_chan, 861}; 862 863static const struct of_device_id mdc_dma_of_match[] = { 864 { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, }, 865 { }, 866}; 867MODULE_DEVICE_TABLE(of, mdc_dma_of_match); 868 869static int img_mdc_runtime_suspend(struct device *dev) 870{ 871 struct mdc_dma *mdma = dev_get_drvdata(dev); 872 873 clk_disable_unprepare(mdma->clk); 874 875 return 0; 876} 877 878static int img_mdc_runtime_resume(struct device *dev) 879{ 880 struct mdc_dma *mdma = dev_get_drvdata(dev); 881 882 return clk_prepare_enable(mdma->clk); 883} 884 885static int mdc_dma_probe(struct platform_device *pdev) 886{ 887 struct mdc_dma *mdma; 888 unsigned int i; 889 u32 val; 890 int ret; 891 892 mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL); 893 if (!mdma) 894 return -ENOMEM; 895 platform_set_drvdata(pdev, mdma); 896 897 mdma->soc = of_device_get_match_data(&pdev->dev); 898 899 mdma->regs = devm_platform_ioremap_resource(pdev, 0); 900 if (IS_ERR(mdma->regs)) 901 return PTR_ERR(mdma->regs); 902 903 mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, 904 "img,cr-periph"); 905 if (IS_ERR(mdma->periph_regs)) 906 return PTR_ERR(mdma->periph_regs); 907 908 mdma->clk = devm_clk_get(&pdev->dev, "sys"); 909 if (IS_ERR(mdma->clk)) 910 return PTR_ERR(mdma->clk); 911 912 dma_cap_zero(mdma->dma_dev.cap_mask); 913 dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask); 914 dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask); 915 dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask); 916 dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask); 917 918 val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A); 919 mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) & 920 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK; 921 mdma->nr_threads = 922 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) & 923 MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK); 924 mdma->bus_width = 925 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) & 926 MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8; 927 /* 928 * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes 929 * are supported, this makes it possible for the value reported in 930 * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size 931 * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or 932 * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this 933 * ambiguity, restrict transfer sizes to one bus-width less than the 934 * actual maximum. 935 */ 936 mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width; 937 938 of_property_read_u32(pdev->dev.of_node, "dma-channels", 939 &mdma->nr_channels); 940 ret = of_property_read_u32(pdev->dev.of_node, 941 "img,max-burst-multiplier", 942 &mdma->max_burst_mult); 943 if (ret) 944 return ret; 945 946 mdma->dma_dev.dev = &pdev->dev; 947 mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg; 948 mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic; 949 mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy; 950 mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources; 951 mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources; 952 mdma->dma_dev.device_tx_status = mdc_tx_status; 953 mdma->dma_dev.device_issue_pending = mdc_issue_pending; 954 mdma->dma_dev.device_terminate_all = mdc_terminate_all; 955 mdma->dma_dev.device_synchronize = mdc_synchronize; 956 mdma->dma_dev.device_config = mdc_slave_config; 957 958 mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 959 mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 960 for (i = 1; i <= mdma->bus_width; i <<= 1) { 961 mdma->dma_dev.src_addr_widths |= BIT(i); 962 mdma->dma_dev.dst_addr_widths |= BIT(i); 963 } 964 965 INIT_LIST_HEAD(&mdma->dma_dev.channels); 966 for (i = 0; i < mdma->nr_channels; i++) { 967 struct mdc_chan *mchan = &mdma->channels[i]; 968 969 mchan->mdma = mdma; 970 mchan->chan_nr = i; 971 mchan->irq = platform_get_irq(pdev, i); 972 if (mchan->irq < 0) 973 return mchan->irq; 974 975 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq, 976 IRQ_TYPE_LEVEL_HIGH, 977 dev_name(&pdev->dev), mchan); 978 if (ret < 0) 979 return ret; 980 981 mchan->vc.desc_free = mdc_desc_free; 982 vchan_init(&mchan->vc, &mdma->dma_dev); 983 } 984 985 mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, 986 sizeof(struct mdc_hw_list_desc), 987 4, 0); 988 if (!mdma->desc_pool) 989 return -ENOMEM; 990 991 pm_runtime_enable(&pdev->dev); 992 if (!pm_runtime_enabled(&pdev->dev)) { 993 ret = img_mdc_runtime_resume(&pdev->dev); 994 if (ret) 995 return ret; 996 } 997 998 ret = dma_async_device_register(&mdma->dma_dev); 999 if (ret) 1000 goto suspend; 1001 1002 ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma); 1003 if (ret) 1004 goto unregister; 1005 1006 dev_info(&pdev->dev, "MDC with %u channels and %u threads\n", 1007 mdma->nr_channels, mdma->nr_threads); 1008 1009 return 0; 1010 1011unregister: 1012 dma_async_device_unregister(&mdma->dma_dev); 1013suspend: 1014 if (!pm_runtime_enabled(&pdev->dev)) 1015 img_mdc_runtime_suspend(&pdev->dev); 1016 pm_runtime_disable(&pdev->dev); 1017 return ret; 1018} 1019 1020static void mdc_dma_remove(struct platform_device *pdev) 1021{ 1022 struct mdc_dma *mdma = platform_get_drvdata(pdev); 1023 struct mdc_chan *mchan, *next; 1024 1025 of_dma_controller_free(pdev->dev.of_node); 1026 dma_async_device_unregister(&mdma->dma_dev); 1027 1028 list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels, 1029 vc.chan.device_node) { 1030 list_del(&mchan->vc.chan.device_node); 1031 1032 devm_free_irq(&pdev->dev, mchan->irq, mchan); 1033 1034 tasklet_kill(&mchan->vc.task); 1035 } 1036 1037 pm_runtime_disable(&pdev->dev); 1038 if (!pm_runtime_status_suspended(&pdev->dev)) 1039 img_mdc_runtime_suspend(&pdev->dev); 1040} 1041 1042#ifdef CONFIG_PM_SLEEP 1043static int img_mdc_suspend_late(struct device *dev) 1044{ 1045 struct mdc_dma *mdma = dev_get_drvdata(dev); 1046 int i; 1047 1048 /* Check that all channels are idle */ 1049 for (i = 0; i < mdma->nr_channels; i++) { 1050 struct mdc_chan *mchan = &mdma->channels[i]; 1051 1052 if (unlikely(mchan->desc)) 1053 return -EBUSY; 1054 } 1055 1056 return pm_runtime_force_suspend(dev); 1057} 1058 1059static int img_mdc_resume_early(struct device *dev) 1060{ 1061 return pm_runtime_force_resume(dev); 1062} 1063#endif /* CONFIG_PM_SLEEP */ 1064 1065static const struct dev_pm_ops img_mdc_pm_ops = { 1066 SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend, 1067 img_mdc_runtime_resume, NULL) 1068 SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late, 1069 img_mdc_resume_early) 1070}; 1071 1072static struct platform_driver mdc_dma_driver = { 1073 .driver = { 1074 .name = "img-mdc-dma", 1075 .pm = &img_mdc_pm_ops, 1076 .of_match_table = of_match_ptr(mdc_dma_of_match), 1077 }, 1078 .probe = mdc_dma_probe, 1079 .remove_new = mdc_dma_remove, 1080}; 1081module_platform_driver(mdc_dma_driver); 1082 1083MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver"); 1084MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>"); 1085MODULE_LICENSE("GPL v2"); 1086