fsl/qbman/qman.c

35 #define IRQNAME		"QMan portal %d"
36 #define MAX_IRQNAME	16	/* big enough for "QMan portal %d" */
121  * synchronisation for portal accesses and data-dependencies. Use of barrier()s
124  * the portal registers as volatile
136  *   Enum values use 3 letter codes. First letter matches the portal mode,
138  *     ci == cache-inhibited portal register
139  *     ce == cache-enabled portal register
421 static inline int qm_eqcr_init(struct qm_portal *portal,
426 	struct qm_eqcr *eqcr = &portal->eqcr;
430 	eqcr->ring = portal->addr.ce + QM_CL_EQCR;
431 	eqcr->ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
432 	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
433 	pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
435 	eqcr->vbit = (qm_in(portal, QM_REG_EQCR_PI_CINH) & QM_EQCR_SIZE) ?
439 	eqcr->ithresh = qm_in(portal, QM_REG_EQCR_ITR);
444 	cfg = (qm_in(portal, QM_REG_CFG) & 0x00ffffff) |
448 	qm_out(portal, QM_REG_CFG, cfg);
452 static inline void qm_eqcr_finish(struct qm_portal *portal)
454 	struct qm_eqcr *eqcr = &portal->eqcr;
455 	u8 pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
456 	u8 ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
468 								 *portal)
470 	struct qm_eqcr *eqcr = &portal->eqcr;
484 								*portal)
486 	struct qm_eqcr *eqcr = &portal->eqcr;
492 		eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) &
513 static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
515 	struct qm_eqcr *eqcr = &portal->eqcr;
531 static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
533 	qm_cl_touch_ro(portal, QM_CL_EQCR_CI_CENA);
536 static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
538 	struct qm_eqcr *eqcr = &portal->eqcr;
541 	eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) & (QM_EQCR_SIZE - 1);
542 	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
548 static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
550 	struct qm_eqcr *eqcr = &portal->eqcr;
553 	qm_out(portal, QM_REG_EQCR_ITR, ithresh);
556 static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
558 	struct qm_eqcr *eqcr = &portal->eqcr;
563 static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
565 	struct qm_eqcr *eqcr = &portal->eqcr;
595 static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
597 	qm_out(portal, QM_REG_CFG, (qm_in(portal, QM_REG_CFG) & 0xff0fffff) |
601 static inline int qm_dqrr_init(struct qm_portal *portal,
607 	struct qm_dqrr *dqrr = &portal->dqrr;
611 	qm_out(portal, QM_REG_DQRR_SDQCR, 0);
612 	qm_out(portal, QM_REG_DQRR_VDQCR, 0);
613 	qm_out(portal, QM_REG_DQRR_PDQCR, 0);
614 	dqrr->ring = portal->addr.ce + QM_CL_DQRR;
615 	dqrr->pi = qm_in(portal, QM_REG_DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
616 	dqrr->ci = qm_in(portal, QM_REG_DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
619 	dqrr->vbit = (qm_in(portal, QM_REG_DQRR_PI_CINH) & QM_DQRR_SIZE) ?
621 	dqrr->ithresh = qm_in(portal, QM_REG_DQRR_ITR);
630 	cfg = (qm_in(portal, QM_REG_CFG) & 0xff000f00) |
637 	qm_out(portal, QM_REG_CFG, cfg);
638 	qm_dqrr_set_maxfill(portal, max_fill);
642 static inline void qm_dqrr_finish(struct qm_portal *portal)
645 	struct qm_dqrr *dqrr = &portal->dqrr;
654 						struct qm_portal *portal)
656 	struct qm_dqrr *dqrr = &portal->dqrr;
663 static inline u8 qm_dqrr_next(struct qm_portal *portal)
665 	struct qm_dqrr *dqrr = &portal->dqrr;
672 static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
674 	struct qm_dqrr *dqrr = &portal->dqrr;
693 static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
697 	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
703 	qm_out(portal, QM_REG_DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
708 static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u32 bitmask)
710 	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
713 	qm_out(portal, QM_REG_DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
717 static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
719 	qm_out(portal, QM_REG_DQRR_SDQCR, sdqcr);
722 static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
724 	qm_out(portal, QM_REG_DQRR_VDQCR, vdqcr);
727 static inline int qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
733 	qm_out(portal, QM_REG_DQRR_ITR, ithresh);
762 static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,
765 	struct qm_mr *mr = &portal->mr;
768 	mr->ring = portal->addr.ce + QM_CL_MR;
769 	mr->pi = qm_in(portal, QM_REG_MR_PI_CINH) & (QM_MR_SIZE - 1);
770 	mr->ci = qm_in(portal, QM_REG_MR_CI_CINH) & (QM_MR_SIZE - 1);
773 	mr->vbit = (qm_in(portal, QM_REG_MR_PI_CINH) & QM_MR_SIZE)
775 	mr->ithresh = qm_in(portal, QM_REG_MR_ITR);
780 	cfg = (qm_in(portal, QM_REG_CFG) & 0xfffff0ff) |
782 	qm_out(portal, QM_REG_CFG, cfg);
786 static inline void qm_mr_finish(struct qm_portal *portal)
788 	struct qm_mr *mr = &portal->mr;
794 static inline const union qm_mr_entry *qm_mr_current(struct qm_portal *portal)
796 	struct qm_mr *mr = &portal->mr;
803 static inline int qm_mr_next(struct qm_portal *portal)
805 	struct qm_mr *mr = &portal->mr;
812 static inline void qm_mr_pvb_update(struct qm_portal *portal)
814 	struct qm_mr *mr = &portal->mr;
829 static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
831 	struct qm_mr *mr = &portal->mr;
835 	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
838 static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
840 	struct qm_mr *mr = &portal->mr;
844 	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
847 static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
849 	qm_out(portal, QM_REG_MR_ITR, ithresh);
854 static inline int qm_mc_init(struct qm_portal *portal)
857 	struct qm_mc *mc = &portal->mc;
859 	mc->cr = portal->addr.ce + QM_CL_CR;
860 	mc->rr = portal->addr.ce + QM_CL_RR0;
881 static inline void qm_mc_finish(struct qm_portal *portal)
884 	struct qm_mc *mc = &portal->mc;
892 static inline union qm_mc_command *qm_mc_start(struct qm_portal *portal)
894 	struct qm_mc *mc = &portal->mc;
904 static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
906 	struct qm_mc *mc = &portal->mc;
919 static inline union qm_mc_result *qm_mc_result(struct qm_portal *portal)
921 	struct qm_mc *mc = &portal->mc;
942 static inline int qm_mc_result_timeout(struct qm_portal *portal,
948 		*mcr = qm_mc_result(portal);
1032 int qman_dqrr_set_ithresh(struct qman_portal *portal, u8 ithresh)
1036 	if (!portal)
1039 	res = qm_dqrr_set_ithresh(&portal->p, ithresh);
1043 	portal->p.dqrr.ithresh = ithresh;
1049 void qman_dqrr_get_ithresh(struct qman_portal *portal, u8 *ithresh)
1051 	if (portal && ithresh)
1052 		*ithresh = qm_in(&portal->p, QM_REG_DQRR_ITR);
1056 void qman_portal_get_iperiod(struct qman_portal *portal, u32 *iperiod)
1058 	if (portal && iperiod)
1059 		*iperiod = qm_in(&portal->p, QM_REG_ITPR);
1063 int qman_portal_set_iperiod(struct qman_portal *portal, u32 iperiod)
1065 	if (!portal || iperiod > QMAN_ITP_MAX)
1068 	qm_out(&portal->p, QM_REG_ITPR, iperiod);
1098  * to the one whose affine portal it is waiting on.
1198 		 * other things before servicing the portal's MR pump, each of
1223 static int qman_create_portal(struct qman_portal *portal,
1231 	p = &portal->p;
1235 	portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
1237 	portal->use_eqcr_ci_stashing = 0;
1240 	 * prep the low-level portal struct with the mapped addresses from the
1252 			portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
1273 	portal->cgrs = kmalloc_array(2, sizeof(*cgrs), GFP_KERNEL);
1274 	if (!portal->cgrs)
1277 	qman_cgrs_init(&portal->cgrs[1]);
1279 		portal->cgrs[0] = *cgrs;
1282 		qman_cgrs_fill(&portal->cgrs[0]);
1283 	INIT_LIST_HEAD(&portal->cgr_cbs);
1284 	raw_spin_lock_init(&portal->cgr_lock);
1285 	INIT_WORK(&portal->congestion_work, qm_congestion_task);
1286 	INIT_WORK(&portal->mr_work, qm_mr_process_task);
1287 	portal->bits = 0;
1288 	portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
1293 	portal->irq_sources = 0;
1295 	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
1297 	if (request_irq(c->irq, portal_isr, 0, portal->irqname,	portal)) {
1328 	portal->config = c;
1334 	qm_dqrr_sdqcr_set(p, portal->sdqcr);
1340 	free_irq(c->irq, portal);
1342 	kfree(portal->cgrs);
1358 	struct qman_portal *portal;
1361 	portal = &per_cpu(qman_affine_portal, c->cpu);
1362 	err = qman_create_portal(portal, c, cgrs);
1369 	affine_portals[c->cpu] = portal;
1372 	return portal;
1379 	/* Stop dequeues on the portal */
1603  * (this is because the function tracks one end of each FIFO in the portal and
1734 		struct qman_portal *portal = get_affine_portal();
1736 		cpu = portal->config->cpu;
2494 		/* attempt to delete from other portal than creator */
2495 		dev_err(p->config->dev, "CGR not owned by current portal");
2723 			dev_err(dev, "Can't find portal for dedicated channel 0x%x\n",
2756 			 * order for the FQ to drain the portal needs to be set
2766 				/* Pool channel, enable the bit in the portal */
2886 						struct qman_portal *portal)
2888 	return portal->config;