drivers/dma/hisi_dma.c

146 	struct hisi_dma_dev *hdma_dev;
345 static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
350 	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
355 static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
360 	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
365 static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
367 	void __iomem *q_base = hdma_dev->queue_base;
369 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
381 static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
383 	void __iomem *q_base = hdma_dev->queue_base;
385 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
403 static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
407 	addr = hdma_dev->queue_base +
412 static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
414 	void __iomem *q_base = hdma_dev->queue_base;
423 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
428 	hisi_dma_pause_dma(hdma_dev, index, true);
429 	hisi_dma_enable_dma(hdma_dev, index, false);
430 	hisi_dma_mask_irq(hdma_dev, index);
432 	addr = hdma_dev->queue_base +
439 		dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
443 	hisi_dma_do_reset(hdma_dev, index);
444 	hisi_dma_reset_qp_point(hdma_dev, index);
445 	hisi_dma_pause_dma(hdma_dev, index, false);
448 		hisi_dma_enable_dma(hdma_dev, index, true);
449 		hisi_dma_unmask_irq(hdma_dev, index);
456 		dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
464 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
469 	memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
470 	memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
509 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
532 	chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
535 	hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR,
560 	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
571 	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
583 static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
585 	size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
586 	size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
587 	struct device *dev = &hdma_dev->pdev->dev;
591 	for (i = 0; i < hdma_dev->chan_num; i++) {
592 		chan = &hdma_dev->chan[i];
607 static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
609 	struct hisi_dma_chan *chan = &hdma_dev->chan[index];
610 	void __iomem *q_base = hdma_dev->queue_base;
611 	u32 hw_depth = hdma_dev->chan_depth - 1;
638 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
692 static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
694 	hisi_dma_init_hw_qp(hdma_dev, qp_index);
695 	hisi_dma_unmask_irq(hdma_dev, qp_index);
696 	hisi_dma_enable_dma(hdma_dev, qp_index, true);
699 static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
701 	hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true);
704 static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
708 	for (i = 0; i < hdma_dev->chan_num; i++) {
709 		hdma_dev->chan[i].qp_num = i;
710 		hdma_dev->chan[i].hdma_dev = hdma_dev;
711 		hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
712 		vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
713 		hisi_dma_enable_qp(hdma_dev, i);
717 static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
721 	for (i = 0; i < hdma_dev->chan_num; i++) {
722 		hisi_dma_disable_qp(hdma_dev, i);
723 		tasklet_kill(&hdma_dev->chan[i].vc.task);
730 	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
739 	q_base = hdma_dev->queue_base;
741 		chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
748 			dev_err(&hdma_dev->pdev->dev, "task error!\n");
757 static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
759 	struct pci_dev *pdev = hdma_dev->pdev;
762 	for (i = 0; i < hdma_dev->chan_num; i++) {
765 				       &hdma_dev->chan[i]);
774 static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
778 	ret = hisi_dma_alloc_qps_mem(hdma_dev);
780 		dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
784 	ret = hisi_dma_request_qps_irq(hdma_dev);
786 		dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
790 	hisi_dma_enable_qps(hdma_dev);
800 static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
803 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
805 			       hdma_dev->base + HISI_DMA_HIP08_MODE);
808 static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev)
813 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {
815 			addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i);
822 static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev)
826 	dma_dev = &hdma_dev->dma_dev;
835 	dma_dev->dev = &hdma_dev->pdev->dev;
842 static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev,
845 	struct device *dev = &hdma_dev->pdev->dev;
851 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
862 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
872 static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev)
883 	dev = &hdma_dev->pdev->dev;
885 	regsets = devm_kcalloc(dev, hdma_dev->chan_num,
890 	regs = hisi_dma_get_ch_regs(hdma_dev, &regs_sz);
894 	for (i = 0; i < hdma_dev->chan_num; i++) {
897 		regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET;
906 					      hdma_dev->dma_dev.dbg_dev_root);
913 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev)
919 	dev = &hdma_dev->pdev->dev;
921 	if (hdma_dev->dma_dev.dbg_dev_root == NULL)
928 	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
935 	regset->base = hdma_dev->base;
939 				hdma_dev->dma_dev.dbg_dev_root, regset);
941 	ret = hisi_dma_create_chan_dir(hdma_dev);
943 		dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n");
946 static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { }
954 	struct hisi_dma_dev *hdma_dev;
983 	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
985 	if (!hdma_dev)
988 	hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
989 	hdma_dev->pdev = pdev;
990 	hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
991 	hdma_dev->chan_num = chan_num;
992 	hdma_dev->reg_layout = reg_layout;
993 	hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev);
995 	pci_set_drvdata(pdev, hdma_dev);
1007 	hisi_dma_init_dma_dev(hdma_dev);
1009 	hisi_dma_set_mode(hdma_dev, RC);
1011 	hisi_dma_init_hw(hdma_dev);
1013 	ret = hisi_dma_enable_hw_channels(hdma_dev);
1020 				       hdma_dev);
1024 	dma_dev = &hdma_dev->dma_dev;
1031 	hisi_dma_create_debugfs(hdma_dev);