ulp/iw_cxgb/iw_cxgb_qp.c

246 static int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe,
257         err = iwch_sgl2pbl_map(qhp->rhp, wr->sg_list, wr->num_sge, pbl_addr,
282         qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
283                              qhp->wq.rq_size_log2)].wr_id = wr->wr_id;
284         qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
285                              qhp->wq.rq_size_log2)].pbl_addr = 0;
290 static int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe,
304         pbl_addr = cxio_hal_pblpool_alloc(&qhp->rhp->rdev, T3_STAG0_PBL_SIZE);
311         pbl_offset = (pbl_addr - qhp->rhp->rdev.rnic_info.pbl_base) >> 3;
346         qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
347                              qhp->wq.rq_size_log2)].wr_id = wr->wr_id;
348         qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,
349                              qhp->wq.rq_size_log2)].pbl_addr = pbl_addr;
360 	struct iwch_qp *qhp;
366 	qhp = to_iwch_qp(ibqp);
367 	mtx_lock(&qhp->lock);
368 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {
369 		mtx_unlock(&qhp->lock);
373 	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr,
374 		  qhp->wq.sq_size_log2);
376 		mtx_unlock(&qhp->lock);
385 		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
386 		wqe = (union t3_wr *) (qhp->wq.queue + idx);
394 		sqp = qhp->wq.sq +
395 		      Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);
414 			if (!qhp->wq.oldest_read)
415 				qhp->wq.oldest_read = sqp;
425 		wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;
428 		sqp->sq_wptr = qhp->wq.sq_wptr;
433 			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
437 		     Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2),
441 		++(qhp->wq.wptr);
442 		++(qhp->wq.sq_wptr);
444 	mtx_unlock(&qhp->lock);
445 	ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);
456 	struct iwch_qp *qhp;
461 	qhp = to_iwch_qp(ibqp);
462 	mtx_lock(&qhp->lock);
463 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {
464 		mtx_unlock(&qhp->lock);
468 	num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr,
469 			    qhp->wq.rq_size_log2) - 1;
471 		mtx_unlock(&qhp->lock);
482 		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
483 		wqe = (union t3_wr *) (qhp->wq.queue + idx);
486                                 err = build_rdma_recv(qhp, wqe, wr);
488                                 err = build_zero_stag_recv(qhp, wqe, wr);
495 			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
499 		     idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe);
500 		++(qhp->wq.rq_wptr);
501 		++(qhp->wq.wptr);
505 	mtx_unlock(&qhp->lock);
506 	ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);
519 	struct iwch_qp *qhp;
530 	qhp = to_iwch_qp(qp);
532 	rhp = qhp->rhp;
534 	mtx_lock(&qhp->lock);
535 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {
536 		mtx_unlock(&qhp->lock);
539 	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr,
540 			    qhp->wq.sq_size_log2);
542 		mtx_unlock(&qhp->lock);
545 	idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
548 	wqe = (union t3_wr *) (qhp->wq.queue + idx);
568 		mtx_unlock(&qhp->lock);
571 	wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;
572 	sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);
575 	sqp->sq_wptr = qhp->wq.sq_wptr;
582 		       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0,
584 	++(qhp->wq.wptr);
585 	++(qhp->wq.sq_wptr);
586 	mtx_unlock(&qhp->lock);
588 	ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);
728 int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg)
735 	CTR3(KTR_IW_CXGB, "%s: tid %u, %p", __func__, qhp->ep->hwtid, rsp_msg);
756 	wqe->send.wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(qhp->ep->hwtid));
758 	return t3_offload_tx(qhp->rhp->rdev.adap, m);
762  * Assumes qhp lock is held.
764 static void __flush_qp(struct iwch_qp *qhp, struct iwch_cq *rchp,
770 	CTR4(KTR_IW_CXGB, "%s qhp %p rchp %p schp %p", __FUNCTION__, qhp, rchp, schp);
771 	/* take a ref on the qhp since we must release the lock */
772 	qhp->refcnt++;
773 	mtx_unlock(&qhp->lock);
777 	mtx_lock(&qhp->lock);
779 	cxio_count_rcqes(&rchp->cq, &qhp->wq, &count);
780 	flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count);
781 	mtx_unlock(&qhp->lock);
788 	mtx_lock(&qhp->lock);
790 	cxio_count_scqes(&schp->cq, &qhp->wq, &count);
791 	flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count);
792 	mtx_unlock(&qhp->lock);
798 	mtx_lock(&qhp->lock);
799 	if (--qhp->refcnt == 0)
800 		wakeup(qhp);
803 static void flush_qp(struct iwch_qp *qhp)
807 	rchp = get_chp(qhp->rhp, qhp->attr.rcq);
808 	schp = get_chp(qhp->rhp, qhp->attr.scq);
810 	if (qhp->ibqp.uobject) {
811 		cxio_set_wq_in_error(&qhp->wq);
821        	__flush_qp(qhp, rchp, schp);
828 static int rqes_posted(struct iwch_qp *qhp)
830        union t3_wr *wqe = qhp->wq.queue;
839 static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp,
845 	struct socket *so = qhp->ep->com.so;
850 	init_attr.tid = qhp->ep->hwtid;
851 	init_attr.qpid = qhp->wq.qpid;
852 	init_attr.pdid = qhp->attr.pd;
853 	init_attr.scqid = qhp->attr.scq;
854 	init_attr.rcqid = qhp->attr.rcq;
855 	init_attr.rq_addr = qhp->wq.rq_addr;
856 	init_attr.rq_size = 1 << qhp->wq.rq_size_log2;
858 		qhp->attr.mpa_attr.recv_marker_enabled |
859 		(qhp->attr.mpa_attr.xmit_marker_enabled << 1) |
860 		(qhp->attr.mpa_attr.crc_enabled << 2);
865 	if (!qhp->ibqp.uobject)
867 	init_attr.tcp_emss = qhp->ep->emss;
868 	init_attr.ord = qhp->attr.max_ord;
869 	init_attr.ird = qhp->attr.max_ird;
870 	init_attr.qp_dma_addr = qhp->wq.dma_addr;
871 	init_attr.qp_dma_size = (1UL << qhp->wq.size_log2);
872 	init_attr.rqe_count = rqes_posted(qhp);
873 	init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0;
878 	init_attr.irs = qhp->ep->rcv_seq;
883 	ret = cxio_rdma_init(&rhp->rdev, &init_attr, qhp->ep->com.so);
888 int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp,
894 	struct iwch_qp_attributes newattr = qhp->attr;
901 	CTR6(KTR_IW_CXGB, "%s qhp %p qpid 0x%x ep %p state %d -> %d", __FUNCTION__,
902 	     qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state,
905 	mtx_lock(&qhp->lock);
909 		if (qhp->attr.state != IWCH_QP_STATE_IDLE) {
935 		qhp->attr = newattr;
940 	if (qhp->attr.state == attrs->next_state)
943 	switch (qhp->attr.state) {
955 			qhp->attr.mpa_attr = attrs->mpa_attr;
956 			qhp->attr.llp_stream_handle = attrs->llp_stream_handle;
957 			qhp->ep = qhp->attr.llp_stream_handle;
958 			qhp->attr.state = IWCH_QP_STATE_RTS;
966 			get_ep(&qhp->ep->com);
967 			mtx_unlock(&qhp->lock);
968 			ret = rdma_init(rhp, qhp, mask, attrs);
969 			mtx_lock(&qhp->lock);
974 			qhp->attr.state = IWCH_QP_STATE_ERROR;
975 			flush_qp(qhp);
985 			PANIC_IF(atomic_load_acq_int(&qhp->ep->com.refcount) < 2);
986 			qhp->attr.state = IWCH_QP_STATE_CLOSING;
990 				ep = qhp->ep;
995 			qhp->attr.state = IWCH_QP_STATE_TERMINATE;
996 			if (qhp->ibqp.uobject)
997 				cxio_set_wq_in_error(&qhp->wq);
1002 			qhp->attr.state = IWCH_QP_STATE_ERROR;
1006 				ep = qhp->ep;
1023 				flush_qp(qhp);
1024 				qhp->attr.state = IWCH_QP_STATE_IDLE;
1025 				qhp->attr.llp_stream_handle = NULL;
1026 				put_ep(&qhp->ep->com);
1027 				qhp->ep = NULL;
1028 				wakeup(qhp);
1043 		if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) ||
1044 		    !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) {
1048 		qhp->attr.state = IWCH_QP_STATE_IDLE;
1049 		memset(&qhp->attr, 0, sizeof(qhp->attr));
1060 		       __FUNCTION__, qhp->attr.state);
1067 	CTR3(KTR_IW_CXGB, "%s disassociating ep %p qpid 0x%x", __FUNCTION__, qhp->ep,
1068 	     qhp->wq.qpid);
1071 	qhp->attr.llp_stream_handle = NULL;
1072 	ep = qhp->ep;
1073 	qhp->ep = NULL;
1074 	qhp->attr.state = IWCH_QP_STATE_ERROR;
1076 	wakeup(qhp);
1078 	flush_qp(qhp);
1080 	mtx_unlock(&qhp->lock);
1083 		iwch_post_terminate(qhp, NULL);
1103 	CTR2(KTR_IW_CXGB, "%s exit state %d", __FUNCTION__, qhp->attr.state);