libcxgb4/src/cq.c

45 static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq)
49 	PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__,
50 	     wq, cq, cq->sw_cidx, cq->sw_pidx);
56 				 V_CQE_QPID(wq->sq.qid));
62 int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)
65 	int in_use = wq->rq.in_use - count;
68 	PDBG("%s wq %p cq %p rq.in_use %u skip count %u\n", __func__,
69 	     wq, cq, wq->rq.in_use, count);
71 		insert_recv_cqe(wq, cq);
77 static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,
82 	PDBG("%s wq %p cq %p sw_cidx %u sw_pidx %u\n", __func__,
83 	     wq, cq, cq->sw_cidx, cq->sw_pidx);
89 			         V_CQE_QPID(wq->sq.qid));
96 static void advance_oldest_read(struct t4_wq *wq);
101 	struct t4_wq *wq = &qhp->wq;
107 	if (wq->sq.flush_cidx == -1)
108 		wq->sq.flush_cidx = wq->sq.cidx;
109 	idx = wq->sq.flush_cidx;
110 	BUG_ON(idx >= wq->sq.size);
111 	while (idx != wq->sq.pidx) {
112 		swsqe = &wq->sq.sw_sq[idx];
115 		insert_sq_cqe(wq, cq, swsqe);
116 		if (wq->sq.oldest_read == swsqe) {
118 			advance_oldest_read(wq);
121 		if (++idx == wq->sq.size)
124 	wq->sq.flush_cidx += flushed;
125 	if (wq->sq.flush_cidx >= wq->sq.size)
126 		wq->sq.flush_cidx -= wq->sq.size;
129 static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
134 	if (wq->sq.flush_cidx == -1)
135 		wq->sq.flush_cidx = wq->sq.cidx;
136 	cidx = wq->sq.flush_cidx;
137 	BUG_ON(cidx >= wq->sq.size);
139 	while (cidx != wq->sq.pidx) {
140 		swsqe = &wq->sq.sw_sq[cidx];
142 			if (++cidx == wq->sq.size)
158 			if (++cidx == wq->sq.size)
160 			wq->sq.flush_cidx = cidx;
166 static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
169 	read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
170 	read_cqe->len = ntohl(wq->sq.oldest_read->read_len);
178 static void advance_oldest_read(struct t4_wq *wq)
181 	u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;
183 	if (rptr == wq->sq.size)
185 	while (rptr != wq->sq.pidx) {
186 		wq->sq.oldest_read = &wq->sq.sw_sq[rptr];
188 		if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
190 		if (++rptr == wq->sq.size)
193 	wq->sq.oldest_read = NULL;
250 			if (!qhp->wq.sq.oldest_read->signaled) {
251 				advance_oldest_read(&qhp->wq);
259 			create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);
261 			advance_oldest_read(&qhp->wq);
270 			BUG_ON(idx >= qhp->wq.sq.size);
271 			swsqe = &qhp->wq.sq.sw_sq[idx];
274 			flush_completed_wrs(&qhp->wq, &chp->cq);
287 static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)
298 	if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))
303 void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
314 		    (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))
337  *     supply the wq assicated with the qpid.
348 static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
371 	if (wq == NULL) {
395 				t4_set_wq_in_error(wq);
408 				t4_set_wq_in_error(wq);
416 		if (!wq->sq.oldest_read->signaled) {
417 			advance_oldest_read(wq);
426 		create_read_req_cqe(wq, hw_cqe, &read_cqe);
428 		advance_oldest_read(wq);
431 	if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {
433 		wq->error = 1;
455 		 * then we complete this with T4_ERR_MSN and mark the wq in
459 		if (t4_rq_empty(wq)) {
460 			t4_set_wq_in_error(wq);
464 		if (unlikely((CQE_WRID_MSN(hw_cqe) != (wq->rq.msn)))) {
465 			t4_set_wq_in_error(wq);
483 	if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) {
489 		BUG_ON(idx >= wq->sq.size);
490 		swsqe = &wq->sq.sw_sq[idx];
506 		BUG_ON(idx >= wq->sq.size);
516 		if (idx < wq->sq.cidx)
517 			wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;
519 			wq->sq.in_use -= idx - wq->sq.cidx;
520 		BUG_ON(wq->sq.in_use <= 0 || wq->sq.in_use >= wq->sq.size);
522 		wq->sq.cidx = (u16)idx;
523 		PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx);
524 		*cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;
525 		t4_sq_consume(wq);
527 		PDBG("%s completing rq idx %u\n", __func__, wq->rq.cidx);
528 		BUG_ON(wq->rq.cidx >= wq->rq.size);
529 		*cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;
530 		BUG_ON(t4_rq_empty(wq));
531 		t4_rq_consume(wq);
539 	flush_completed_wrs(wq, cq);
567 	struct t4_wq *wq;
596 		wq = NULL;
599 		wq = &(qhp->wq);
601 	ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit);
607 	wc->qp_num = qhp->wq.sq.qid;
709 	if (wq)