Lines Matching refs:ring
114 int (*push_tail)(struct k3_ring *ring, void *elm);
115 int (*push_head)(struct k3_ring *ring, void *elm);
116 int (*pop_tail)(struct k3_ring *ring, void *elm);
117 int (*pop_head)(struct k3_ring *ring, void *elm);
147 * @elm_size: Size of the ring element
192 * @num_rings: number of ring in RA
193 * @rings_inuse: bitfield for ring usage tracking
205 * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
242 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
249 val = readl(&ring->rt->occ);
251 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
252 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
258 (4 << ring->elm_size);
261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
263 return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
282 /* Reverse side of the DMA ring can only be popped by SW */
286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
310 static void k3_ringacc_ring_dump(struct k3_ring *ring)
312 struct device *dev = ring->parent->dev;
314 dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
315 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
316 &ring->ring_mem_dma);
318 ring->elm_size, ring->size, ring->mode, ring->proxy_id);
319 dev_dbg(dev, "dump flags %08X\n", ring->flags);
321 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
322 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
323 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
324 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
325 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
327 if (ring->ring_mem_virt)
329 16, 1, ring->ring_mem_virt, 16 * 8, false);
343 /* Request for any general purpose ring */
373 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
376 dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
401 * DMA rings must be requested by ID, completion ring is the reverse
402 * side of the forward ring
423 dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
463 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
466 struct k3_ringacc *ringacc = ring->parent;
470 ring_cfg.index = ring->ring_id;
472 ring_cfg.count = ring->size;
476 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
477 ret, ring->ring_id);
480 void k3_ringacc_ring_reset(struct k3_ring *ring)
482 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
485 memset(&ring->state, 0, sizeof(ring->state));
487 k3_ringacc_ring_reset_sci(ring);
491 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
495 struct k3_ringacc *ringacc = ring->parent;
499 ring_cfg.index = ring->ring_id;
506 ret, ring->ring_id);
509 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
511 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
514 if (!ring->parent->dma_ring_reset_quirk)
518 occ = k3_ringacc_ring_read_occ(ring);
523 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
524 ring->ring_id, occ);
525 /* TI-SCI ring reset */
526 k3_ringacc_ring_reset_sci(ring);
529 * Setup the ring in ring/doorbell mode (if not already in this
532 if (ring->mode != K3_RINGACC_RING_MODE_RING)
534 ring, K3_RINGACC_RING_MODE_RING);
537 * This will wrap the internal UDMAP ring state occupancy
553 writel(db_ring_cnt_cur, &ring->rt->db);
557 /* Restore the original ring mode (if not ring mode) */
558 if (ring->mode != K3_RINGACC_RING_MODE_RING)
559 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
563 /* Reset the ring */
564 k3_ringacc_ring_reset(ring);
568 static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
571 struct k3_ringacc *ringacc = ring->parent;
575 ring_cfg.index = ring->ring_id;
580 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
581 ret, ring->ring_id);
584 int k3_ringacc_ring_free(struct k3_ring *ring)
588 if (!ring)
591 ringacc = ring->parent;
594 * DMA rings: rings shared memory and configuration, only forward ring
595 * is configured and reverse ring considered as slave.
597 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
600 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
602 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
607 if (--ring->use_count)
610 if (!(ring->flags & K3_RING_FLAG_BUSY))
613 k3_ringacc_ring_free_sci(ring);
615 dma_free_coherent(ring->dma_dev,
616 ring->size * (4 << ring->elm_size),
617 ring->ring_mem_virt, ring->ring_mem_dma);
618 ring->flags = 0;
619 ring->ops = NULL;
620 ring->dma_dev = NULL;
621 ring->asel = 0;
623 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
624 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
625 ring->proxy = NULL;
626 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
630 clear_bit(ring->ring_id, ringacc->rings_inuse);
640 u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
642 if (!ring)
645 return ring->ring_id;
649 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
651 if (!ring)
654 return ring->parent->tisci_dev_id;
658 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
662 if (!ring)
665 irq_num = msi_get_virq(ring->parent->dev, ring->ring_id);
672 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
675 struct k3_ringacc *ringacc = ring->parent;
682 ring_cfg.index = ring->ring_id;
684 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma);
685 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma);
686 ring_cfg.count = ring->size;
687 ring_cfg.mode = ring->mode;
688 ring_cfg.size = ring->elm_size;
689 ring_cfg.asel = ring->asel;
693 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
694 ret, ring->ring_id);
699 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
710 ringacc = ring->parent;
713 * DMA rings: rings shared memory and configuration, only forward ring
714 * is configured and reverse ring considered as slave.
716 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
719 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
722 ring->size = cfg->size;
723 ring->elm_size = cfg->elm_size;
724 ring->mode = cfg->mode;
725 ring->asel = cfg->asel;
726 ring->dma_dev = cfg->dma_dev;
727 if (!ring->dma_dev) {
728 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
729 ring->ring_id);
730 ring->dma_dev = ringacc->dev;
733 memset(&ring->state, 0, sizeof(ring->state));
735 ring->ops = &k3_dmaring_fwd_ops;
737 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
738 ring->size * (4 << ring->elm_size),
739 &ring->ring_mem_dma, GFP_KERNEL);
740 if (!ring->ring_mem_virt) {
741 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
746 ret = k3_ringacc_ring_cfg_sci(ring);
750 ring->flags |= K3_RING_FLAG_BUSY;
752 k3_ringacc_ring_dump(ring);
754 /* DMA rings: configure reverse ring */
755 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
763 reverse_ring->ring_mem_virt = ring->ring_mem_virt;
764 reverse_ring->ring_mem_dma = ring->ring_mem_dma;
771 dma_free_coherent(ring->dma_dev,
772 ring->size * (4 << ring->elm_size),
773 ring->ring_mem_virt,
774 ring->ring_mem_dma);
776 ring->ops = NULL;
777 ring->proxy = NULL;
778 ring->dma_dev = NULL;
779 ring->asel = 0;
783 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
788 if (!ring || !cfg)
791 ringacc = ring->parent;
794 return k3_dmaring_cfg(ring, cfg);
799 !test_bit(ring->ring_id, ringacc->rings_inuse))
803 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
807 4 << ring->elm_size);
812 * In case of shared ring only the first user (master user) can
813 * configure the ring. The sequence should be by the client:
814 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
815 * k3_ringacc_ring_cfg(ring, cfg); # master configuration
819 if (ring->use_count != 1)
822 ring->size = cfg->size;
823 ring->elm_size = cfg->elm_size;
824 ring->mode = cfg->mode;
825 memset(&ring->state, 0, sizeof(ring->state));
827 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
828 ring->proxy = ringacc->proxy_target_base +
829 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
831 switch (ring->mode) {
833 ring->ops = &k3_ring_mode_ring_ops;
834 ring->dma_dev = cfg->dma_dev;
835 if (!ring->dma_dev)
836 ring->dma_dev = ringacc->dev;
839 ring->dma_dev = ringacc->dev;
840 if (ring->proxy)
841 ring->ops = &k3_ring_mode_proxy_ops;
843 ring->ops = &k3_ring_mode_msg_ops;
846 ring->ops = NULL;
851 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
852 ring->size * (4 << ring->elm_size),
853 &ring->ring_mem_dma, GFP_KERNEL);
854 if (!ring->ring_mem_virt) {
855 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
860 ret = k3_ringacc_ring_cfg_sci(ring);
865 ring->flags |= K3_RING_FLAG_BUSY;
866 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
869 k3_ringacc_ring_dump(ring);
874 dma_free_coherent(ring->dma_dev,
875 ring->size * (4 << ring->elm_size),
876 ring->ring_mem_virt,
877 ring->ring_mem_dma);
879 ring->ops = NULL;
880 ring->dma_dev = NULL;
882 ring->proxy = NULL;
887 u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
889 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
892 return ring->size;
896 u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
898 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
901 if (!ring->state.free)
902 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
904 return ring->state.free;
908 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
910 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
913 return k3_ringacc_ring_read_occ(ring);
917 u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
919 return !k3_ringacc_ring_get_free(ring);
934 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
939 val = ring->ring_id;
941 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
942 writel(val, &ring->proxy->control);
946 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
951 ptr = (void __iomem *)&ring->proxy->data;
956 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
960 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
966 ptr += k3_ringacc_ring_get_fifo_pos(ring);
971 dev_dbg(ring->parent->dev,
974 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
975 ring->state.occ--;
979 dev_dbg(ring->parent->dev,
982 memcpy_toio(ptr, elem, (4 << ring->elm_size));
983 ring->state.free--;
989 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
990 ring->state.occ);
994 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
996 return k3_ringacc_ring_access_proxy(ring, elem,
1000 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
1002 return k3_ringacc_ring_access_proxy(ring, elem,
1006 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
1008 return k3_ringacc_ring_access_proxy(ring, elem,
1012 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
1014 return k3_ringacc_ring_access_proxy(ring, elem,
1018 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
1026 ptr = (void __iomem *)&ring->fifos->head_data;
1030 ptr = (void __iomem *)&ring->fifos->tail_data;
1036 ptr += k3_ringacc_ring_get_fifo_pos(ring);
1041 dev_dbg(ring->parent->dev,
1044 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
1045 ring->state.occ--;
1049 dev_dbg(ring->parent->dev,
1052 memcpy_toio(ptr, elem, (4 << ring->elm_size));
1053 ring->state.free--;
1059 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
1060 ring->state.free, ring->state.windex, ring->state.occ,
1061 ring->state.rindex);
1065 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
1067 return k3_ringacc_ring_access_io(ring, elem,
1071 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
1073 return k3_ringacc_ring_access_io(ring, elem,
1077 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
1079 return k3_ringacc_ring_access_io(ring, elem,
1083 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
1085 return k3_ringacc_ring_access_io(ring, elem,
1090 * The element is 48 bits of address + ASEL bits in the ring.
1099 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
1105 * DMA rings: forward ring is always tied DMA channel and HW does not
1110 ring->state.occ = k3_ringacc_ring_read_occ(ring);
1111 if (ring->state.windex >= ring->state.occ)
1112 elem_idx = ring->state.windex - ring->state.occ;
1114 elem_idx = ring->size - (ring->state.occ - ring->state.windex);
1116 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
1117 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1120 ring->state.occ--;
1121 writel(-1, &ring->rt->db);
1123 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
1124 __func__, ring->state.occ, ring->state.windex, elem_idx,
1129 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
1133 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1135 if (ring->state.occ) {
1136 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1139 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1140 ring->state.occ--;
1141 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
1142 } else if (ring->state.tdown_complete) {
1146 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
1147 ring->state.tdown_complete = false;
1150 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
1151 __func__, ring->state.occ, ring->state.rindex, elem_ptr);
1155 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
1159 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
1161 memcpy(elem_ptr, elem, (4 << ring->elm_size));
1162 if (ring->parent->dma_rings) {
1165 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
1168 ring->state.windex = (ring->state.windex + 1) % ring->size;
1169 ring->state.free--;
1170 writel(1, &ring->rt->db);
1172 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
1173 ring->state.free, ring->state.windex);
1178 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
1182 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1184 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1186 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1187 ring->state.occ--;
1188 writel(-1, &ring->rt->db);
1190 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
1191 ring->state.occ, ring->state.rindex, elem_ptr);
1195 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
1199 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1202 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
1203 ring->state.free, ring->state.windex);
1205 if (k3_ringacc_ring_is_full(ring))
1208 if (ring->ops && ring->ops->push_tail)
1209 ret = ring->ops->push_tail(ring, elem);
1215 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
1219 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1222 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
1223 ring->state.free, ring->state.windex);
1225 if (k3_ringacc_ring_is_full(ring))
1228 if (ring->ops && ring->ops->push_head)
1229 ret = ring->ops->push_head(ring, elem);
1235 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
1239 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1242 if (!ring->state.occ)
1243 k3_ringacc_ring_update_occ(ring);
1245 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
1246 ring->state.rindex);
1248 if (!ring->state.occ && !ring->state.tdown_complete)
1251 if (ring->ops && ring->ops->pop_head)
1252 ret = ring->ops->pop_head(ring, elem);
1258 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
1262 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1265 if (!ring->state.occ)
1266 k3_ringacc_ring_update_occ(ring);
1268 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
1269 ring->state.occ, ring->state.rindex);
1271 if (!ring->state.occ)
1274 if (ring->ops && ring->ops->pop_tail)
1275 ret = ring->ops->pop_tail(ring, elem);
1438 dev_info(dev, "dma-ring-reset-quirk: %s\n",
1498 struct k3_ring *ring = &ringacc->rings[i];
1500 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
1501 ring->parent = ringacc;
1502 ring->ring_id = i;
1503 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1505 ring = &ringacc->rings[ringacc->num_rings + i];
1506 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
1508 ring->parent = ringacc;
1509 ring->ring_id = i;
1510 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1511 ring->flags = K3_RING_FLAG_REVERSE;