hw/cxgb4/cq.c

184 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq, u32 srqidx)
188 	pr_debug("wq %p cq %p sw_cidx %u sw_pidx %u\n",
189 		 wq, cq, cq->sw_cidx, cq->sw_pidx);
195 				 CQE_QPID_V(wq->sq.qid));
203 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)
206 	int in_use = wq->rq.in_use - count;
208 	pr_debug("wq %p cq %p rq.in_use %u skip count %u\n",
209 		 wq, cq, wq->rq.in_use, count);
211 		insert_recv_cqe(wq, cq, 0);
217 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,
222 	pr_debug("wq %p cq %p sw_cidx %u sw_pidx %u\n",
223 		 wq, cq, cq->sw_cidx, cq->sw_pidx);
229 				 CQE_QPID_V(wq->sq.qid));
236 static void advance_oldest_read(struct t4_wq *wq);
241 	struct t4_wq *wq = &qhp->wq;
247 	if (wq->sq.flush_cidx == -1)
248 		wq->sq.flush_cidx = wq->sq.cidx;
249 	idx = wq->sq.flush_cidx;
250 	while (idx != wq->sq.pidx) {
251 		swsqe = &wq->sq.sw_sq[idx];
253 		insert_sq_cqe(wq, cq, swsqe);
254 		if (wq->sq.oldest_read == swsqe) {
255 			advance_oldest_read(wq);
258 		if (++idx == wq->sq.size)
261 	wq->sq.flush_cidx += flushed;
262 	if (wq->sq.flush_cidx >= wq->sq.size)
263 		wq->sq.flush_cidx -= wq->sq.size;
267 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
272 	if (wq->sq.flush_cidx == -1)
273 		wq->sq.flush_cidx = wq->sq.cidx;
274 	cidx = wq->sq.flush_cidx;
276 	while (cidx != wq->sq.pidx) {
277 		swsqe = &wq->sq.sw_sq[cidx];
279 			if (++cidx == wq->sq.size)
292 			if (++cidx == wq->sq.size)
294 			wq->sq.flush_cidx = cidx;
300 static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
303 	read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
304 	read_cqe->len = htonl(wq->sq.oldest_read->read_len);
312 static void advance_oldest_read(struct t4_wq *wq)
315 	u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;
317 	if (rptr == wq->sq.size)
319 	while (rptr != wq->sq.pidx) {
320 		wq->sq.oldest_read = &wq->sq.sw_sq[rptr];
322 		if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
324 		if (++rptr == wq->sq.size)
327 	wq->sq.oldest_read = NULL;
362 			if (qhp->wq.flushed == 1)
386 			if (!qhp->wq.sq.oldest_read->signaled) {
387 				advance_oldest_read(&qhp->wq);
395 			create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);
397 			advance_oldest_read(&qhp->wq);
404 			swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
407 			flush_completed_wrs(&qhp->wq, &chp->cq);
422 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)
425 		WARN_ONCE(1, "Unexpected DRAIN CQE qp id %u!\n", wq->sq.qid);
438 	if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))
443 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
454 		    (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))
533  *     supply the wq assicated with the qpid.
544 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
566 	if (wq == NULL) {
572 	* skip hw cqe's if the wq is flushed.
574 	if (wq->flushed && !SW_CQE(hw_cqe)) {
611 				t4_set_wq_in_error(wq, 0);
622 				t4_set_wq_in_error(wq, 0);
630 		if (!wq->sq.oldest_read->signaled) {
631 			advance_oldest_read(wq);
640 		create_read_req_cqe(wq, hw_cqe, &read_cqe);
642 		advance_oldest_read(wq);
645 	if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {
647 		t4_set_wq_in_error(wq, 0);
658 		 * then we complete this with T4_ERR_MSN and mark the wq in
662 			     CQE_WRID_MSN(hw_cqe) != wq->rq.msn)) {
663 			t4_set_wq_in_error(wq, 0);
680 	if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) {
685 		swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
710 		if (idx < wq->sq.cidx)
711 			wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;
713 			wq->sq.in_use -= idx - wq->sq.cidx;
715 		wq->sq.cidx = (uint16_t)idx;
716 		pr_debug("completing sq idx %u\n", wq->sq.cidx);
717 		*cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;
719 			c4iw_log_wr_stats(wq, hw_cqe);
720 		t4_sq_consume(wq);
723 			pr_debug("completing rq idx %u\n", wq->rq.cidx);
724 			*cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;
726 				c4iw_log_wr_stats(wq, hw_cqe);
727 			t4_rq_consume(wq);
731 		wq->rq.msn++;
739 	flush_completed_wrs(wq, cq);
758 	struct t4_wq *wq = qhp ? &qhp->wq : NULL;
764 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit,
765 		      srq ? &srq->wq : NULL);
778 	    srq->wq.in_use < srq->srq_limit)
1186 	insert_recv_cqe(&qhp->wq, &rchp->cq, srqidx);