hw/qib/qib_rc.c

42 static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
47 	len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
48 	return rvt_restart_sge(ss, wqe, len);
222 	struct rvt_swqe *wqe;
252 		wqe = rvt_get_swqe_ptr(qp, qp->s_last);
253 		rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
276 	wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
297 			if ((wqe->wr.send_flags & IB_SEND_FENCE) &&
303 			qp->s_psn = wqe->psn;
310 		len = wqe->length;
313 		switch (wqe->wr.opcode) {
317 			if (!rvt_rc_credit_avail(qp, wqe))
324 			if (wqe->wr.opcode == IB_WR_SEND)
329 				ohdr->u.imm_data = wqe->wr.ex.imm_data;
332 			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
345 			if (!rvt_rc_credit_avail(qp, wqe))
349 				cpu_to_be64(wqe->rdma_wr.remote_addr);
351 				cpu_to_be32(wqe->rdma_wr.rkey);
359 			if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE)
365 					wqe->rdma_wr.wr.ex.imm_data;
367 				if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED)
392 				cpu_to_be64(wqe->rdma_wr.remote_addr);
394 				cpu_to_be32(wqe->rdma_wr.rkey);
421 			if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
423 				put_ib_ateth_swap(wqe->atomic_wr.swap,
425 				put_ib_ateth_compare(wqe->atomic_wr.compare_add,
429 				put_ib_ateth_swap(wqe->atomic_wr.compare_add,
433 			put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
436 				wqe->atomic_wr.rkey);
448 		qp->s_sge.sge = wqe->sg_list[0];
449 		qp->s_sge.sg_list = wqe->sg_list + 1;
450 		qp->s_sge.num_sge = wqe->wr.num_sge;
451 		qp->s_sge.total_len = wqe->length;
452 		qp->s_len = wqe->length;
458 		if (wqe->wr.opcode == IB_WR_RDMA_READ)
459 			qp->s_psn = wqe->lpsn + 1;
474 		qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
487 		if (wqe->wr.opcode == IB_WR_SEND)
492 			ohdr->u.imm_data = wqe->wr.ex.imm_data;
495 		if (wqe->wr.send_flags & IB_SEND_SOLICITED)
513 		qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
526 		if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
531 			ohdr->u.imm_data = wqe->wr.ex.imm_data;
533 			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
552 		len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
554 			cpu_to_be64(wqe->rdma_wr.remote_addr + len);
556 			cpu_to_be32(wqe->rdma_wr.rkey);
557 		ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
561 		qp->s_psn = wqe->lpsn + 1;
570 	delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8;
739 	struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
748 	if (qib_cmp24(psn, wqe->psn) <= 0) {
754 	opcode = wqe->wr.opcode;
762 		wqe = rvt_get_swqe_ptr(qp, n);
763 		diff = qib_cmp24(psn, wqe->psn);
775 		opcode = wqe->wr.opcode;
823 	struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
831 			rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
840 	if (wqe->wr.opcode == IB_WR_RDMA_READ)
859 	struct rvt_swqe *wqe;
864 		wqe = rvt_get_swqe_ptr(qp, n);
865 		if (qib_cmp24(psn, wqe->lpsn) <= 0) {
866 			if (wqe->wr.opcode == IB_WR_RDMA_READ)
867 				qp->s_sending_psn = wqe->lpsn + 1;
885 	struct rvt_swqe *wqe;
919 		wqe = rvt_get_swqe_ptr(qp, qp->s_last);
920 		if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&
924 				     wqe,
925 				     ib_qib_wc_opcode[wqe->wr.opcode],
952 					 struct rvt_swqe *wqe,
960 	if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||
963 				     wqe,
964 				     ib_qib_wc_opcode[wqe->wr.opcode],
970 	update_last_psn(qp, wqe->lpsn);
981 		wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
984 			qp->s_psn = wqe->psn;
991 		wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
993 	return wqe;
1012 	struct rvt_swqe *wqe;
1026 	wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1033 	while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) {
1040 		if (wqe->wr.opcode == IB_WR_RDMA_READ &&
1055 		if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
1057 		    ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1058 		      wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
1077 		if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1078 		    wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
1079 			u64 *vaddr = wqe->sg_list[0].vaddr;
1083 		    (wqe->wr.opcode == IB_WR_RDMA_READ ||
1084 		     wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1085 		     wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
1099 		wqe = do_rc_completion(qp, wqe, ibp);
1196 				rvt_send_complete(qp, wqe, status);
1227 	struct rvt_swqe *wqe;
1232 	wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1234 	while (qib_cmp24(psn, wqe->lpsn) > 0) {
1235 		if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1236 		    wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1237 		    wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
1239 		wqe = do_rc_completion(qp, wqe, ibp);
1277 	struct rvt_swqe *wqe;
1341 	wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1357 		wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1358 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1366 						  wqe, psn, pmtu);
1373 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1423 		wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1425 						  wqe, psn, pmtu);
1432 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1468 		rvt_send_complete(qp, wqe, status);