fsl/qbman/qman.c

268 	struct qm_mcc_fq fq;
957 static inline void fq_set(struct qman_fq *fq, u32 mask)
959 	fq->flags |= mask;
962 static inline void fq_clear(struct qman_fq *fq, u32 mask)
964 	fq->flags &= ~mask;
967 static inline int fq_isset(struct qman_fq *fq, u32 mask)
969 	return fq->flags & mask;
972 static inline int fq_isclear(struct qman_fq *fq, u32 mask)
974 	return !(fq->flags & mask);
1114 	pr_debug("Allocated fq lookup table at %p, entry count %u\n",
1121 	struct qman_fq *fq;
1127 	fq = fq_table[idx];
1128 	DPAA_ASSERT(!fq || idx == fq->idx);
1130 	return fq;
1134  * Only returns full-service fq objects, not enqueue-only
1151 static u32 fq_to_tag(struct qman_fq *fq)
1154 	return fq->idx;
1156 	return (u32)fq;
1425 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
1430 		DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
1431 		fq_clear(fq, QMAN_FQ_STATE_ORL);
1434 		DPAA_ASSERT(fq->state == qman_fq_state_parked ||
1435 			    fq->state == qman_fq_state_sched);
1436 		DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
1437 		fq_clear(fq, QMAN_FQ_STATE_CHANGING);
1438 		if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
1439 			fq_set(fq, QMAN_FQ_STATE_NE);
1440 		if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
1441 			fq_set(fq, QMAN_FQ_STATE_ORL);
1442 		fq->state = qman_fq_state_retired;
1445 		DPAA_ASSERT(fq->state == qman_fq_state_sched);
1446 		DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
1447 		fq->state = qman_fq_state_parked;
1491 	struct qman_fq *fq;
1512 				fq = fqid_to_fq(qm_fqid_get(&msg->fq));
1513 				if (WARN_ON(!fq))
1515 				fq_state_change(p, fq, msg, verb);
1516 				if (fq->cb.fqs)
1517 					fq->cb.fqs(p, fq, msg);
1521 				fq = tag_to_fq(be32_to_cpu(msg->fq.context_b));
1522 				fq_state_change(p, fq, msg, verb);
1523 				if (fq->cb.fqs)
1524 					fq->cb.fqs(p, fq, msg);
1535 			fq = tag_to_fq(be32_to_cpu(msg->ern.tag));
1536 			fq->cb.ern(p, fq, msg);
1574 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
1577 	fq_clear(fq, QMAN_FQ_STATE_VDQCR);
1611 	struct qman_fq *fq;
1627 			fq = p->vdqcr_owned;
1635 				fq_clear(fq, QMAN_FQ_STATE_NE);
1642 			res = fq->cb.dqrr(p, fq, dq, sched_napi);
1647 				clear_vdqcr(p, fq);
1650 			fq = tag_to_fq(be32_to_cpu(dq->context_b));
1652 			res = fq->cb.dqrr(p, fq, dq, sched_napi);
1798 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1806 	fq->fqid = fqid;
1807 	fq->flags = flags;
1808 	fq->state = qman_fq_state_oos;
1809 	fq->cgr_groupid = 0;
1817 	fq->idx = fqid * 2;
1819 		fq->idx++;
1821 	WARN_ON(fq_table[fq->idx]);
1822 	fq_table[fq->idx] = fq;
1828 void qman_destroy_fq(struct qman_fq *fq)
1834 	switch (fq->state) {
1837 		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1838 			qman_release_fqid(fq->fqid);
1840 		DPAA_ASSERT(fq_table[fq->idx]);
1841 		fq_table[fq->idx] = NULL;
1850 u32 qman_fq_fqid(struct qman_fq *fq)
1852 	return fq->fqid;
1856 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
1867 	if (fq->state != qman_fq_state_oos &&
1868 	    fq->state != qman_fq_state_parked)
1871 	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
1881 	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
1882 	    (fq->state != qman_fq_state_oos &&
1883 	     fq->state != qman_fq_state_parked)) {
1890 	qm_fqid_set(&mcc->fq, fq->fqid);
1897 	if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1901 		mcc->initfq.fqd.context_b = cpu_to_be32(fq_to_tag(fq));
1915 			phys_fq = dma_map_single(p->config->dev, fq,
1916 						 sizeof(*fq), DMA_TO_DEVICE);
1953 				fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1955 				fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1958 			fq->cgr_groupid = opts->fqd.cgid;
1960 	fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1969 int qman_schedule_fq(struct qman_fq *fq)
1976 	if (fq->state != qman_fq_state_parked)
1979 	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
1984 	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
1985 	    fq->state != qman_fq_state_parked) {
1990 	qm_fqid_set(&mcc->fq, fq->fqid);
2003 	fq->state = qman_fq_state_sched;
2010 int qman_retire_fq(struct qman_fq *fq, u32 *flags)
2018 	if (fq->state != qman_fq_state_parked &&
2019 	    fq->state != qman_fq_state_sched)
2022 	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
2026 	if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
2027 	    fq->state == qman_fq_state_retired ||
2028 	    fq->state == qman_fq_state_oos) {
2033 	qm_fqid_set(&mcc->fq, fq->fqid);
2054 		/* Process 'fq' right away, we'll ignore FQRNI */
2056 			fq_set(fq, QMAN_FQ_STATE_NE);
2058 			fq_set(fq, QMAN_FQ_STATE_ORL);
2060 			*flags = fq->flags;
2061 		fq->state = qman_fq_state_retired;
2062 		if (fq->cb.fqs) {
2067 			 * fq may have been OOS'd and free()'d already). But if
2075 			msg.fq.fqs = mcr->alterfq.fqs;
2076 			qm_fqid_set(&msg.fq, fq->fqid);
2077 			msg.fq.context_b = cpu_to_be32(fq_to_tag(fq));
2078 			fq->cb.fqs(p, fq, &msg);
2082 		fq_set(fq, QMAN_FQ_STATE_CHANGING);
2092 int qman_oos_fq(struct qman_fq *fq)
2099 	if (fq->state != qman_fq_state_retired)
2102 	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
2106 	if (fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS) ||
2107 	    fq->state != qman_fq_state_retired) {
2112 	qm_fqid_set(&mcc->fq, fq->fqid);
2123 	fq->state = qman_fq_state_oos;
2130 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
2138 	qm_fqid_set(&mcc->fq, fq->fqid);
2155 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
2163 	qm_fqid_set(&mcc->fq, fq->fqid);
2226 static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr)
2234 	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2237 	fq_set(fq, QMAN_FQ_STATE_VDQCR);
2238 	p->vdqcr_owned = fq;
2246 static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr)
2251 	ret = set_p_vdqcr(*p, fq, vdqcr);
2256 static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq,
2263 				!set_vdqcr(p, fq, vdqcr));
2265 		wait_event(affine_queue, !set_vdqcr(p, fq, vdqcr));
2269 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr)
2274 	if (fq->state != qman_fq_state_parked &&
2275 	    fq->state != qman_fq_state_retired)
2279 	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2281 	vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
2283 		ret = wait_vdqcr_start(&p, fq, vdqcr, flags);
2285 		ret = set_vdqcr(&p, fq, vdqcr);
2298 				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
2301 				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
2315 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd)
2345 	qm_fqid_set(eq, fq->fqid);
2346 	eq->tag = cpu_to_be32(fq_to_tag(fq));
2692 	qm_fqid_set(&mcc->fq, fqid);
2707 	qm_fqid_set(&mcc->fq, fqid);
2738 		qm_fqid_set(&mcc->fq, fqid);
2835 		qm_fqid_set(&mcc->fq, fqid);
2855 		qm_fqid_set(&mcc->fq, fqid);
2952 	struct qman_fq fq = {
2960 		err = qman_query_fq_np(&fq, &np);
2970 			err = qman_query_fq(&fq, &fqd);
2975 				err = qman_shutdown_fq(fq.fqid);
2985 		fq.fqid++;
3010 	struct qman_fq fq = {
3018 		err = qman_query_fq_np(&fq, &np);
3028 			err = qman_query_fq(&fq, &fqd);
3034 				       cgrid, fq.fqid);
3039 		fq.fqid++;