dma/sh/rz-dmac.c

192 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
196 		writel(val, channel->ch_base + offset);
198 		writel(val, channel->ch_cmn_base + offset);
201 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
205 		return readl(channel->ch_base + offset);
207 		return readl(channel->ch_cmn_base + offset);
215 static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
220 	channel->lmdesc.base = lmdesc;
221 	channel->lmdesc.head = lmdesc;
222 	channel->lmdesc.tail = lmdesc;
223 	nxla = channel->lmdesc.base_dma;
224 	while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
232 	lmdesc->nxla = channel->lmdesc.base_dma;
240 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
242 	struct rz_lmdesc *lmdesc = channel->lmdesc.head;
247 		if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
248 			lmdesc = channel->lmdesc.base;
250 	channel->lmdesc.head = lmdesc;
253 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
255 	struct dma_chan *chan = &channel->vc.chan;
262 	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
266 	rz_dmac_lmdesc_recycle(channel);
268 	nxla = channel->lmdesc.base_dma +
269 		(sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
270 					     channel->lmdesc.base));
272 	chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
274 		chctrl = (channel->chctrl | CHCTRL_SETEN);
275 		rz_dmac_ch_writel(channel, nxla, NXLA, 1);
276 		rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
277 		rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
278 		rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
284 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
286 	struct dma_chan *chan = &channel->vc.chan;
290 	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
293 	rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
310 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
312 	struct dma_chan *chan = &channel->vc.chan;
314 	struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
315 	struct rz_dmac_desc *d = channel->desc;
327 	rz_dmac_set_dmars_register(dmac, channel->index, 0);
329 	channel->chcfg = chcfg;
330 	channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
333 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
335 	struct dma_chan *chan = &channel->vc.chan;
337 	struct rz_dmac_desc *d = channel->desc;
342 	channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
345 		channel->chcfg |= CHCFG_SAD;
346 		channel->chcfg &= ~CHCFG_REQD;
348 		channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
351 	lmdesc = channel->lmdesc.tail;
355 			lmdesc->sa = channel->src_per_address;
359 			lmdesc->da = channel->dst_per_address;
366 			lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
369 			lmdesc->chcfg = channel->chcfg;
372 		if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
373 			lmdesc = channel->lmdesc.base;
376 	channel->lmdesc.tail = lmdesc;
378 	rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
379 	channel->chctrl = CHCTRL_SETEN;
418 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
420 	while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
427 		list_add_tail(&desc->node, &channel->ld_free);
428 		channel->descs_allocated++;
431 	if (!channel->descs_allocated)
434 	return channel->descs_allocated;
439 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
441 	struct rz_lmdesc *lmdesc = channel->lmdesc.base;
446 	spin_lock_irqsave(&channel->vc.lock, flags);
451 	rz_dmac_disable_hw(channel);
452 	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
453 	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
455 	if (channel->mid_rid >= 0) {
456 		clear_bit(channel->mid_rid, dmac->modules);
457 		channel->mid_rid = -EINVAL;
460 	spin_unlock_irqrestore(&channel->vc.lock, flags);
462 	list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
464 		channel->descs_allocated--;
467 	INIT_LIST_HEAD(&channel->ld_free);
468 	vchan_free_chan_resources(&channel->vc);
475 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
479 	dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
480 		__func__, channel->index, &src, &dest, len);
482 	if (list_empty(&channel->ld_free))
485 	desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
493 	list_move_tail(channel->ld_free.next, &channel->ld_queue);
494 	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
503 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
509 	if (list_empty(&channel->ld_free))
512 	desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
525 		desc->src = channel->src_per_address;
527 		desc->dest = channel->dst_per_address;
529 	list_move_tail(channel->ld_free.next, &channel->ld_queue);
530 	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
535 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
539 	rz_dmac_disable_hw(channel);
540 	spin_lock_irqsave(&channel->vc.lock, flags);
541 	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
542 	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
543 	spin_unlock_irqrestore(&channel->vc.lock, flags);
544 	vchan_get_all_descriptors(&channel->vc, &head);
545 	vchan_dma_desc_free_list(&channel->vc, &head);
552 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
557 	spin_lock_irqsave(&channel->vc.lock, flags);
559 	if (!list_empty(&channel->ld_queue)) {
560 		desc = list_first_entry(&channel->ld_queue,
562 		channel->desc = desc;
563 		if (vchan_issue_pending(&channel->vc)) {
564 			if (rz_dmac_xfer_desc(channel) < 0)
566 					 channel->index);
568 				list_move_tail(channel->ld_queue.next,
569 					       &channel->ld_active);
573 	spin_unlock_irqrestore(&channel->vc.lock, flags);
601 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
604 	channel->src_per_address = config->src_addr;
605 	channel->dst_per_address = config->dst_addr;
611 	channel->chcfg &= ~CHCFG_FILL_DDS_MASK;
612 	channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val);
618 	channel->chcfg &= ~CHCFG_FILL_SDS_MASK;
619 	channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val);
637 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
643 				100, 100000, false, channel, CHSTAT, 1);
647 	rz_dmac_set_dmars_register(dmac, channel->index, 0);
655 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
657 	struct dma_chan *chan = &channel->vc.chan;
661 	chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
664 			channel->index, chstat);
665 		rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
669 	chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
670 	rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
677 	struct rz_dmac_chan *channel = dev_id;
679 	if (channel) {
680 		rz_dmac_irq_handle_channel(channel);
689 	struct rz_dmac_chan *channel = dev_id;
693 	spin_lock_irqsave(&channel->vc.lock, flags);
695 	if (list_empty(&channel->ld_active)) {
700 	desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
702 	list_move_tail(channel->ld_active.next, &channel->ld_free);
703 	if (!list_empty(&channel->ld_queue)) {
704 		desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
706 		channel->desc = desc;
707 		if (rz_dmac_xfer_desc(channel) == 0)
708 			list_move_tail(channel->ld_queue.next, &channel->ld_active);
711 	spin_unlock_irqrestore(&channel->vc.lock, flags);
718  * OF xlate and channel filter
723 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
728 	channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
730 	channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
733 	return !test_and_set_bit(channel->mid_rid, dmac->modules);
757 			      struct rz_dmac_chan *channel,
766 	channel->index = index;
767 	channel->mid_rid = -EINVAL;
769 	/* Request the channel interrupt. */
771 	channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
772 	if (channel->irq < 0)
773 		return channel->irq;
780 	ret = devm_request_threaded_irq(dmac->dev, channel->irq,
783 					irqname, channel);
786 			channel->irq, ret);
790 	/* Set io base address for each channel */
792 		channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
794 		channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
796 		channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
798 		channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
804 				    &channel->lmdesc.base_dma, GFP_KERNEL);
809 	rz_lmdesc_setup(channel, lmdesc);
811 	/* Initialize register for each channel */
812 	rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
814 	channel->vc.desc_free = rz_dmac_virt_desc_free;
815 	vchan_init(&channel->vc, &dmac->engine);
816 	INIT_LIST_HEAD(&channel->ld_queue);
817 	INIT_LIST_HEAD(&channel->ld_free);
818 	INIT_LIST_HEAD(&channel->ld_active);
955 		struct rz_dmac_chan *channel = &dmac->channels[i];
959 				  channel->lmdesc.base,
960 				  channel->lmdesc.base_dma);
980 		struct rz_dmac_chan *channel = &dmac->channels[i];
984 				  channel->lmdesc.base,
985 				  channel->lmdesc.base_dma);