Lines Matching refs:chan
142 * @chan: base DMA channel object
160 struct dma_chan chan;
186 #define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan)
214 #define for_each_rcar_dmac_chan(i, dmac, chan) \
215 for (i = 0, chan = &(dmac)->channels[0]; i < (dmac)->n_channels; i++, chan++) \
328 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg)
331 return readw(chan->iomem + reg);
333 return readl(chan->iomem + reg);
336 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data)
339 writew(data, chan->iomem + reg);
341 writel(data, chan->iomem + reg);
345 struct rcar_dmac_chan *chan)
348 rcar_dmac_chan_write(chan, RCAR_GEN4_DMACHCLR, 1);
350 rcar_dmac_write(dmac, RCAR_DMACHCLR, BIT(chan->index));
355 struct rcar_dmac_chan *chan;
359 for_each_rcar_dmac_chan(i, dmac, chan)
360 rcar_dmac_chan_write(chan, RCAR_GEN4_DMACHCLR, 1);
370 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan)
372 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
377 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan)
379 struct rcar_dmac_desc *desc = chan->desc.running;
382 WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan));
384 if (chan->mid_rid >= 0)
385 rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid);
392 dev_dbg(chan->chan.device->dev,
393 "chan%u: queue desc %p: %u@%pad\n",
394 chan->index, desc, desc->nchunks, &desc->hwdescs.dma);
397 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
399 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
401 rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE,
404 rcar_dmac_chan_write(chan, RCAR_DMADPBASE,
407 rcar_dmac_chan_write(chan, RCAR_DMACHCRB,
418 rcar_dmac_chan_write(chan, RCAR_DMADAR,
425 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1));
451 dev_dbg(chan->chan.device->dev,
452 "chan%u: queue chunk %p: %u@%pad -> %pad\n",
453 chan->index, chunk, chunk->size, &chunk->src_addr,
457 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
459 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
462 rcar_dmac_chan_write(chan, RCAR_DMASAR,
464 rcar_dmac_chan_write(chan, RCAR_DMADAR,
466 rcar_dmac_chan_write(chan, RCAR_DMATCR,
472 rcar_dmac_chan_write(chan, RCAR_DMACHCR,
500 struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan);
505 spin_lock_irqsave(&chan->lock, flags);
509 dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n",
510 chan->index, tx->cookie, desc);
512 list_add_tail(&desc->node, &chan->desc.pending);
516 spin_unlock_irqrestore(&chan->lock, flags);
527 * @chan: the DMA channel
530 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
544 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
551 spin_lock_irqsave(&chan->lock, flags);
552 list_splice_tail(&list, &chan->desc.free);
553 list_add_tail(&page->node, &chan->desc.pages);
554 spin_unlock_irqrestore(&chan->lock, flags);
561 * @chan: the DMA channel
571 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan,
576 spin_lock_irqsave(&chan->lock, flags);
577 list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free);
578 list_add(&desc->node, &chan->desc.free);
579 spin_unlock_irqrestore(&chan->lock, flags);
582 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan)
594 spin_lock_irqsave(&chan->lock, flags);
595 list_splice_init(&chan->desc.wait, &list);
596 spin_unlock_irqrestore(&chan->lock, flags);
601 rcar_dmac_desc_put(chan, desc);
609 spin_lock_irqsave(&chan->lock, flags);
610 list_splice(&list, &chan->desc.wait);
611 spin_unlock_irqrestore(&chan->lock, flags);
616 * @chan: the DMA channel
623 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan)
630 rcar_dmac_desc_recycle_acked(chan);
632 spin_lock_irqsave(&chan->lock, flags);
634 while (list_empty(&chan->desc.free)) {
641 spin_unlock_irqrestore(&chan->lock, flags);
642 ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT);
645 spin_lock_irqsave(&chan->lock, flags);
648 desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node);
651 spin_unlock_irqrestore(&chan->lock, flags);
658 * @chan: the DMA channel
661 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
678 spin_lock_irqsave(&chan->lock, flags);
679 list_splice_tail(&list, &chan->desc.chunks_free);
680 list_add_tail(&page->node, &chan->desc.pages);
681 spin_unlock_irqrestore(&chan->lock, flags);
688 * @chan: the DMA channel
696 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan)
702 spin_lock_irqsave(&chan->lock, flags);
704 while (list_empty(&chan->desc.chunks_free)) {
711 spin_unlock_irqrestore(&chan->lock, flags);
712 ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT);
715 spin_lock_irqsave(&chan->lock, flags);
718 chunk = list_first_entry(&chan->desc.chunks_free,
722 spin_unlock_irqrestore(&chan->lock, flags);
727 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan,
742 dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size,
751 desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size,
759 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan,
765 rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc));
784 static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan)
794 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
800 dev_err(chan->chan.device->dev, "CHCR DE check error\n");
803 static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan)
805 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
808 rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE));
811 rcar_dmac_chcr_de_barrier(chan);
814 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
816 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
821 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);
822 rcar_dmac_chcr_de_barrier(chan);
825 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)
831 spin_lock_irqsave(&chan->lock, flags);
834 list_splice_init(&chan->desc.pending, &descs);
835 list_splice_init(&chan->desc.active, &descs);
836 list_splice_init(&chan->desc.done, &descs);
837 list_splice_init(&chan->desc.wait, &descs);
839 chan->desc.running = NULL;
841 spin_unlock_irqrestore(&chan->lock, flags);
845 rcar_dmac_desc_put(chan, desc);
851 struct rcar_dmac_chan *chan;
855 for_each_rcar_dmac_chan(i, dmac, chan) {
857 spin_lock_irq(&chan->lock);
858 rcar_dmac_chan_halt(chan);
859 spin_unlock_irq(&chan->lock);
863 static int rcar_dmac_chan_pause(struct dma_chan *chan)
866 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
879 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan,
896 xfer_size = chan->src.xfer_size;
902 xfer_size = chan->dst.xfer_size;
928 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl,
946 desc = rcar_dmac_desc_get(chan);
956 rcar_dmac_chan_configure_desc(chan, desc);
998 chunk = rcar_dmac_xfer_chunk_get(chan);
1000 rcar_dmac_desc_put(chan, desc);
1014 dev_dbg(chan->chan.device->dev,
1015 "chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n",
1016 chan->index, chunk, desc, i, sg, size, len,
1043 if (rcar_dmac_fill_hwdesc(chan, desc) < 0)
1054 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan)
1056 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1071 return pm_runtime_get_sync(chan->device->dev);
1074 static void rcar_dmac_free_chan_resources(struct dma_chan *chan)
1076 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1077 struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
1118 dma_unmap_resource(chan->device->dev, map->addr,
1123 pm_runtime_put(chan->device->dev);
1127 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
1130 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1146 static int rcar_dmac_map_slave_addr(struct dma_chan *chan,
1149 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1173 dma_unmap_resource(chan->device->dev, map->addr,
1178 map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size,
1181 if (dma_mapping_error(chan->device->dev, map->addr)) {
1182 dev_err(chan->device->dev,
1183 "chan%u: failed to map %zx@%pap", rchan->index,
1188 dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n",
1200 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1204 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1208 dev_warn(chan->device->dev,
1214 if (rcar_dmac_map_slave_addr(chan, dir))
1224 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
1228 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1236 dev_warn(chan->device->dev,
1242 if (rcar_dmac_map_slave_addr(chan, dir))
1247 dev_err(chan->device->dev,
1248 "chan%u: sg length %d exceeds limit %d",
1279 static int rcar_dmac_device_config(struct dma_chan *chan,
1282 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1296 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan)
1298 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1315 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,
1318 struct rcar_dmac_desc *desc = chan->desc.running;
1337 status = dma_cookie_status(&chan->chan, cookie, NULL);
1352 list_for_each_entry(desc, &chan->desc.done, node) {
1356 list_for_each_entry(desc, &chan->desc.pending, node) {
1360 list_for_each_entry(desc, &chan->desc.active, node) {
1382 chcrb = rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1384 tcrb = rcar_dmac_chan_read(chan, RCAR_DMATCRB);
1386 if (chcrb == (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1422 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan,
1426 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1432 status = dma_cookie_status(chan, cookie, txstate);
1450 static void rcar_dmac_issue_pending(struct dma_chan *chan)
1452 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1481 static void rcar_dmac_device_synchronize(struct dma_chan *chan)
1483 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1492 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan)
1494 struct rcar_dmac_desc *desc = chan->desc.running;
1507 stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1509 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage));
1514 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan)
1516 struct rcar_dmac_desc *desc = chan->desc.running;
1560 list_move_tail(&desc->node, &chan->desc.done);
1563 if (!list_empty(&chan->desc.active))
1564 chan->desc.running = list_first_entry(&chan->desc.active,
1568 chan->desc.running = NULL;
1571 if (chan->desc.running)
1572 rcar_dmac_chan_start_xfer(chan);
1580 struct rcar_dmac_chan *chan = dev;
1585 spin_lock(&chan->lock);
1587 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
1589 struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device);
1596 rcar_dmac_chan_clear(dmac, chan);
1597 rcar_dmac_chcr_de_barrier(chan);
1604 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask);
1606 rcar_dmac_chcr_de_barrier(chan);
1609 ret |= rcar_dmac_isr_desc_stage_end(chan);
1612 ret |= rcar_dmac_isr_transfer_end(chan);
1615 spin_unlock(&chan->lock);
1618 dev_err(chan->chan.device->dev, "Channel Address Error\n");
1620 rcar_dmac_chan_reinit(chan);
1629 struct rcar_dmac_chan *chan = dev;
1633 spin_lock_irq(&chan->lock);
1636 if (chan->desc.running && chan->desc.running->cyclic) {
1637 desc = chan->desc.running;
1641 spin_unlock_irq(&chan->lock);
1643 spin_lock_irq(&chan->lock);
1651 while (!list_empty(&chan->desc.done)) {
1652 desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc,
1659 spin_unlock_irq(&chan->lock);
1666 spin_lock_irq(&chan->lock);
1669 list_add_tail(&desc->node, &chan->desc.wait);
1672 spin_unlock_irq(&chan->lock);
1675 rcar_dmac_desc_recycle_acked(chan);
1684 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg)
1686 struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
1696 if (chan->device->device_config != rcar_dmac_device_config)
1706 struct dma_chan *chan;
1716 chan = __dma_request_channel(&mask, rcar_dmac_chan_filter, dma_spec,
1718 if (!chan)
1721 rchan = to_rcar_dmac_chan(chan);
1724 return chan;
1765 struct dma_chan *chan = &rchan->chan;
1795 chan->device = &dmac->engine;
1796 dma_cookie_init(chan);
1798 list_add_tail(&chan->device_node, &dmac->engine.channels);
1854 struct rcar_dmac_chan *chan;
1914 for_each_rcar_dmac_chan(i, dmac, chan) {
1915 chan->index = i;
1916 chan->iomem = chan_base + i * data->chan_offset_stride;
1963 for_each_rcar_dmac_chan(i, dmac, chan) {
1964 ret = rcar_dmac_chan_probe(dmac, chan);