drivers/dma/imx-sdma.c

392  * @sdmac:		sdma_channel pointer
404 	struct sdma_channel	*sdmac;
699 static int sdma_config_ownership(struct sdma_channel *sdmac,
702 	struct sdma_engine *sdma = sdmac->sdma;
703 	int channel = sdmac->channel;
802 static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
804 	struct sdma_engine *sdma = sdmac->sdma;
805 	int channel = sdmac->channel;
814 	if (sdmac->sw_done) {
822 static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
824 	struct sdma_engine *sdma = sdmac->sdma;
825 	int channel = sdmac->channel;
839 static void sdma_start_desc(struct sdma_channel *sdmac)
841 	struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
843 	struct sdma_engine *sdma = sdmac->sdma;
844 	int channel = sdmac->channel;
847 		sdmac->desc = NULL;
850 	sdmac->desc = desc = to_sdma_desc(&vd->tx);
856 	sdma_enable_channel(sdma, sdmac->channel);
859 static void sdma_update_channel_loop(struct sdma_channel *sdmac)
863 	enum dma_status	old_status = sdmac->status;
869 	while (sdmac->desc) {
870 		struct sdma_desc *desc = sdmac->desc;
879 			sdmac->status = DMA_ERROR;
899 		spin_unlock(&sdmac->vc.lock);
901 		spin_lock(&sdmac->vc.lock);
907 			sdmac->status = old_status;
914 	if (sdmac->desc && !is_sdma_channel_enabled(sdmac->sdma, sdmac->channel)) {
915 		dev_warn(sdmac->sdma->dev, "restart cyclic channel %d\n", sdmac->channel);
916 		sdma_enable_channel(sdmac->sdma, sdmac->channel);
922 	struct sdma_channel *sdmac = (struct sdma_channel *) data;
926 	sdmac->desc->chn_real_count = 0;
931 	for (i = 0; i < sdmac->desc->num_bd; i++) {
932 		bd = &sdmac->desc->bd[i];
936 		sdmac->desc->chn_real_count += bd->mode.count;
940 		sdmac->status = DMA_ERROR;
942 		sdmac->status = DMA_COMPLETE;
957 		struct sdma_channel *sdmac = &sdma->channel[channel];
960 		spin_lock(&sdmac->vc.lock);
961 		desc = sdmac->desc;
963 			if (sdmac->flags & IMX_DMA_SG_LOOP) {
964 				if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
965 					sdma_update_channel_loop(sdmac);
969 				mxc_sdma_handle_channel_normal(sdmac);
971 				sdma_start_desc(sdmac);
975 		spin_unlock(&sdmac->vc.lock);
985 static int sdma_get_pc(struct sdma_channel *sdmac,
988 	struct sdma_engine *sdma = sdmac->sdma;
996 	sdmac->pc_from_device = 0;
997 	sdmac->pc_to_device = 0;
998 	sdmac->device_to_device = 0;
999 	sdmac->pc_to_pc = 0;
1000 	sdmac->is_ram_script = false;
1030 		if (sdmac->sdma->drvdata->ecspi_fixed) {
1034 			sdmac->is_ram_script = true;
1047 		sdmac->is_ram_script = true;
1062 		sdmac->is_ram_script = true;
1090 		sdmac->is_ram_script = true;
1094 		sdmac->is_ram_script = true;
1102 	sdmac->pc_from_device = per_2_emi;
1103 	sdmac->pc_to_device = emi_2_per;
1104 	sdmac->device_to_device = per_2_per;
1105 	sdmac->pc_to_pc = emi_2_emi;
1110 static int sdma_load_context(struct sdma_channel *sdmac)
1112 	struct sdma_engine *sdma = sdmac->sdma;
1113 	int channel = sdmac->channel;
1120 	if (sdmac->direction == DMA_DEV_TO_MEM)
1121 		load_address = sdmac->pc_from_device;
1122 	else if (sdmac->direction == DMA_DEV_TO_DEV)
1123 		load_address = sdmac->device_to_device;
1124 	else if (sdmac->direction == DMA_MEM_TO_MEM)
1125 		load_address = sdmac->pc_to_pc;
1127 		load_address = sdmac->pc_to_device;
1133 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
1134 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
1135 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
1136 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
1137 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
1147 	if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
1148 		context->gReg[4] = sdmac->per_addr;
1149 		context->gReg[6] = sdmac->shp_addr;
1151 		context->gReg[0] = sdmac->event_mask[1];
1152 		context->gReg[1] = sdmac->event_mask[0];
1153 		context->gReg[2] = sdmac->per_addr;
1154 		context->gReg[6] = sdmac->shp_addr;
1155 		context->gReg[7] = sdmac->watermark_level;
1177 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1178 	struct sdma_engine *sdma = sdmac->sdma;
1179 	int channel = sdmac->channel;
1182 	sdmac->status = DMA_ERROR;
1188 	struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
1198 	vchan_dma_desc_free_list(&sdmac->vc, &sdmac->terminated);
1203 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1206 	spin_lock_irqsave(&sdmac->vc.lock, flags);
1210 	if (sdmac->desc) {
1211 		vchan_terminate_vdesc(&sdmac->desc->vd);
1218 		vchan_get_all_descriptors(&sdmac->vc, &sdmac->terminated);
1219 		sdmac->desc = NULL;
1220 		schedule_work(&sdmac->terminate_worker);
1223 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1230 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1232 	vchan_synchronize(&sdmac->vc);
1234 	flush_work(&sdmac->terminate_worker);
1237 static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
1239 	struct sdma_engine *sdma = sdmac->sdma;
1241 	int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
1242 	int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
1244 	set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
1245 	set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
1247 	if (sdmac->event_id0 > 31)
1248 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
1250 	if (sdmac->event_id1 > 31)
1251 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
1259 		sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
1261 		sdmac->watermark_level |= hwml;
1262 		sdmac->watermark_level |= lwml << 16;
1263 		swap(sdmac->event_mask[0], sdmac->event_mask[1]);
1266 	if (sdmac->per_address2 >= sdma->spba_start_addr &&
1267 			sdmac->per_address2 <= sdma->spba_end_addr)
1268 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
1270 	if (sdmac->per_address >= sdma->spba_start_addr &&
1271 			sdmac->per_address <= sdma->spba_end_addr)
1272 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
1274 	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
1281 	if (sdmac->n_fifos_src > 1)
1282 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD;
1283 	if (sdmac->n_fifos_dst > 1)
1284 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD;
1287 static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac)
1293 	if (sdmac->sw_done)
1294 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE;
1296 	if (sdmac->direction == DMA_DEV_TO_MEM) {
1297 		n_fifos = sdmac->n_fifos_src;
1298 		stride_fifos = sdmac->stride_fifos_src;
1300 		n_fifos = sdmac->n_fifos_dst;
1301 		stride_fifos = sdmac->stride_fifos_dst;
1304 	words_per_fifo = sdmac->words_per_fifo;
1306 	sdmac->watermark_level |=
1308 	sdmac->watermark_level |=
1311 		sdmac->watermark_level |=
1317 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1322 	sdmac->event_mask[0] = 0;
1323 	sdmac->event_mask[1] = 0;
1324 	sdmac->shp_addr = 0;
1325 	sdmac->per_addr = 0;
1327 	switch (sdmac->peripheral_type) {
1329 		sdma_config_ownership(sdmac, false, true, true);
1332 		sdma_config_ownership(sdmac, false, true, false);
1335 		sdma_config_ownership(sdmac, true, true, false);
1339 	ret = sdma_get_pc(sdmac, sdmac->peripheral_type);
1343 	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
1344 			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
1346 		if (sdmac->event_id1) {
1347 			if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
1348 			    sdmac->peripheral_type == IMX_DMATYPE_ASRC)
1349 				sdma_set_watermarklevel_for_p2p(sdmac);
1351 			if (sdmac->peripheral_type ==
1353 				sdma_set_watermarklevel_for_sais(sdmac);
1355 			__set_bit(sdmac->event_id0, sdmac->event_mask);
1359 		sdmac->shp_addr = sdmac->per_address;
1360 		sdmac->per_addr = sdmac->per_address2;
1362 		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
1368 static int sdma_set_channel_priority(struct sdma_channel *sdmac,
1371 	struct sdma_engine *sdma = sdmac->sdma;
1372 	int channel = sdmac->channel;
1415 	struct sdma_engine *sdma = desc->sdmac->sdma;
1434 	struct sdma_engine *sdma = desc->sdmac->sdma;
1439 		dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, desc->bd_phys);
1452 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1467 		dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
1474 		ret = sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
1492 	sdmac->peripheral_type = data->peripheral_type;
1493 	sdmac->event_id0 = data->dma_request;
1494 	sdmac->event_id1 = data->dma_request2;
1496 	ret = clk_enable(sdmac->sdma->clk_ipg);
1499 	ret = clk_enable(sdmac->sdma->clk_ahb);
1503 	ret = sdma_set_channel_priority(sdmac, prio);
1510 	clk_disable(sdmac->sdma->clk_ahb);
1512 	clk_disable(sdmac->sdma->clk_ipg);
1518 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1519 	struct sdma_engine *sdma = sdmac->sdma;
1525 	sdma_event_disable(sdmac, sdmac->event_id0);
1526 	if (sdmac->event_id1)
1527 		sdma_event_disable(sdmac, sdmac->event_id1);
1529 	sdmac->event_id0 = 0;
1530 	sdmac->event_id1 = 0;
1532 	sdma_set_channel_priority(sdmac, 0);
1538 static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
1543 	if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) {
1544 		dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n");
1552 	sdmac->status = DMA_IN_PROGRESS;
1553 	sdmac->direction = direction;
1554 	sdmac->flags = 0;
1560 	desc->sdmac = sdmac;
1568 		sdma_config_ownership(sdmac, false, true, false);
1570 	if (sdma_load_context(sdmac))
1587 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1588 	struct sdma_engine *sdma = sdmac->sdma;
1589 	int channel = sdmac->channel;
1601 	desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
1636 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1644 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1645 	struct sdma_engine *sdma = sdmac->sdma;
1647 	int channel = sdmac->channel;
1651 	sdma_config_write(chan, &sdmac->slave_config, direction);
1653 	desc = sdma_transfer_init(sdmac, direction, sg_len);
1677 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1680 		switch (sdmac->word_size) {
1717 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1722 	sdmac->status = DMA_ERROR;
1731 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1732 	struct sdma_engine *sdma = sdmac->sdma;
1734 	int channel = sdmac->channel;
1740 	if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
1743 	sdma_config_write(chan, &sdmac->slave_config, direction);
1745 	desc = sdma_transfer_init(sdmac, direction, num_periods);
1751 	sdmac->flags |= IMX_DMA_SG_LOOP;
1759 	if (sdmac->peripheral_type == IMX_DMATYPE_HDMI)
1760 		return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1770 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1772 		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
1775 			bd->mode.command = sdmac->word_size;
1794 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
1799 	sdmac->status = DMA_ERROR;
1807 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1810 		sdmac->per_address = dmaengine_cfg->src_addr;
1811 		sdmac->watermark_level = dmaengine_cfg->src_maxburst *
1813 		sdmac->word_size = dmaengine_cfg->src_addr_width;
1815 		sdmac->per_address2 = dmaengine_cfg->src_addr;
1816 		sdmac->per_address = dmaengine_cfg->dst_addr;
1817 		sdmac->watermark_level = dmaengine_cfg->src_maxburst &
1819 		sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
1821 		sdmac->word_size = dmaengine_cfg->dst_addr_width;
1822 	} else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
1823 		sdmac->per_address = dmaengine_cfg->dst_addr;
1824 		sdmac->per_address2 = dmaengine_cfg->src_addr;
1825 		sdmac->watermark_level = 0;
1827 		sdmac->per_address = dmaengine_cfg->dst_addr;
1828 		sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
1830 		sdmac->word_size = dmaengine_cfg->dst_addr_width;
1832 	sdmac->direction = direction;
1839 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1840 	struct sdma_engine *sdma = sdmac->sdma;
1842 	memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
1852 		sdmac->n_fifos_src = sdmacfg->n_fifos_src;
1853 		sdmac->n_fifos_dst = sdmacfg->n_fifos_dst;
1854 		sdmac->stride_fifos_src = sdmacfg->stride_fifos_src;
1855 		sdmac->stride_fifos_dst = sdmacfg->stride_fifos_dst;
1856 		sdmac->words_per_fifo = sdmacfg->words_per_fifo;
1857 		sdmac->sw_done = sdmacfg->sw_done;
1861 	if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
1863 	sdma_event_enable(sdmac, sdmac->event_id0);
1865 	if (sdmac->event_id1) {
1866 		if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
1868 		sdma_event_enable(sdmac, sdmac->event_id1);
1878 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1889 	spin_lock_irqsave(&sdmac->vc.lock, flags);
1891 	vd = vchan_find_desc(&sdmac->vc, cookie);
1894 	else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
1895 		desc = sdmac->desc;
1898 		if (sdmac->flags & IMX_DMA_SG_LOOP)
1907 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);
1912 	return sdmac->status;
1917 	struct sdma_channel *sdmac = to_sdma_chan(chan);
1920 	spin_lock_irqsave(&sdmac->vc.lock, flags);
1921 	if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
1922 		sdma_start_desc(sdmac);
1923 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);
2199 	struct sdma_channel *sdmac = to_sdma_chan(chan);
2205 	sdmac->data = *data;
2206 	chan->private = &sdmac->data;
2229 	 * be set to sdmac->event_id1.
2312 		struct sdma_channel *sdmac = &sdma->channel[i];
2314 		sdmac->sdma = sdma;
2316 		sdmac->channel = i;
2317 		sdmac->vc.desc_free = sdma_desc_free;
2318 		INIT_LIST_HEAD(&sdmac->terminated);
2319 		INIT_WORK(&sdmac->terminate_worker,
2327 			vchan_init(&sdmac->vc, &sdma->dma_device);
2430 		struct sdma_channel *sdmac = &sdma->channel[i];
2432 		tasklet_kill(&sdmac->vc.task);
2433 		sdma_free_chan_resources(&sdmac->vc.chan);