hw/hfi1/verbs.c

21 #include "qp.h"
101 static int pio_wait(struct rvt_qp *qp,
276 	if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
279 	     packet->qp->allowed_ops) ||
286 static u64 hfi1_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc)
319 	if (packet->qp->ibqp.qp_type != IB_QPT_RC ||
320 	    !(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
360 	packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
361 	if (!packet->qp)
363 	spin_lock_irqsave(&packet->qp->r_lock, flags);
369 	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
374 	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
414 	packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
415 	if (!packet->qp)
417 	spin_lock_irqsave(&packet->qp->r_lock, flags);
423 	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
428 	spin_unlock_irqrestore(&packet->qp->r_lock, flags);
449 				  packet->qp->s_pkey_index,
479 			packet->qp = p->qp;
482 			spin_lock_irqsave(&packet->qp->r_lock, flags);
488 			spin_unlock_irqrestore(&packet->qp->r_lock, flags);
506 		packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
507 		if (!packet->qp)
513 		spin_lock_irqsave(&packet->qp->r_lock, flags);
519 		spin_unlock_irqrestore(&packet->qp->r_lock, flags);
559 	struct rvt_qp *qp = NULL;
567 		qp = iowait_to_qp(wait);
568 		priv = qp->priv;
577 	if (qp)
578 		hfi1_qp_wakeup(qp, RVT_S_WAIT_KMEM);
591 	struct rvt_qp *qp = tx->qp;
593 	spin_lock(&qp->s_lock);
595 		rvt_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
596 	} else if (qp->ibqp.qp_type == IB_QPT_RC) {
601 			hfi1_rc_verbs_aborted(qp, hdr);
602 		hfi1_rc_send_complete(qp, hdr);
604 	spin_unlock(&qp->s_lock);
609 void hfi1_wait_kmem(struct rvt_qp *qp)
611 	struct hfi1_qp_priv *priv = qp->priv;
612 	struct ib_qp *ibqp = &qp->ibqp;
619 		qp->s_flags |= RVT_S_WAIT_KMEM;
622 		trace_hfi1_qpsleep(qp, RVT_S_WAIT_KMEM);
623 		rvt_get_qp(qp);
628 		     struct rvt_qp *qp,
634 	spin_lock_irqsave(&qp->s_lock, flags);
635 	if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
639 		hfi1_wait_kmem(qp);
641 		hfi1_qp_unbusy(qp, ps->wait);
644 	spin_unlock_irqrestore(&qp->s_lock, flags);
690  * @qp: the qp
697 static void update_tx_opstats(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
701 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
797 int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
800 	struct hfi1_qp_priv *priv = qp->priv;
825 			u32 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
837 					 qp->srate_mbps,
841 			if (unlikely(hfi1_dbg_should_fault_tx(qp, ps->opcode)))
842 				pbc = hfi1_fault_tx(qp, ps->opcode, pbc);
847 		tx->wqe = qp->s_wqe;
859 	update_tx_opstats(qp, ps, plen);
860 	trace_sdma_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
868 	ret = wait_kmem(dev, qp, ps);
881 static int pio_wait(struct rvt_qp *qp,
886 	struct hfi1_qp_priv *priv = qp->priv;
897 	spin_lock_irqsave(&qp->s_lock, flags);
898 	if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
908 			qp->s_flags |= flag;
914 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_PIO);
915 			rvt_get_qp(qp);
921 		hfi1_qp_unbusy(qp, ps->wait);
924 	spin_unlock_irqrestore(&qp->s_lock, flags);
930 	struct rvt_qp *qp = (struct rvt_qp *)arg;
931 	struct hfi1_qp_priv *priv = qp->priv;
937 int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
940 	struct hfi1_qp_priv *priv = qp->priv;
970 	switch (qp->ibqp.qp_type) {
984 		u8 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
992 		pbc = create_pbc(ppd, pbc, qp->srate_mbps, vl, plen);
993 		if (unlikely(hfi1_dbg_should_fault_tx(qp, ps->opcode)))
994 			pbc = hfi1_fault_tx(qp, ps->opcode, pbc);
1001 	pbuf = sc_buffer_alloc(sc, plen, cb, qp);
1004 			verbs_pio_complete(qp, 0);
1024 			ret = pio_wait(qp, sc, ps, RVT_S_WAIT_PIO);
1056 	update_tx_opstats(qp, ps, plen);
1057 	trace_pio_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
1061 	spin_lock_irqsave(&qp->s_lock, flags);
1062 	if (qp->s_wqe) {
1063 		rvt_send_complete(qp, qp->s_wqe, wc_status);
1064 	} else if (qp->ibqp.qp_type == IB_QPT_RC) {
1066 			hfi1_rc_verbs_aborted(qp, &ps->s_txreq->phdr.hdr);
1067 		hfi1_rc_send_complete(qp, &ps->s_txreq->phdr.hdr);
1069 	spin_unlock_irqrestore(&qp->s_lock, flags);
1135 	 * For the kernel contexts only, if a qp is passed into the function,
1136 	 * the most likely matching pkey has index qp->s_pkey_index
1173 static inline send_routine get_send_routine(struct rvt_qp *qp,
1176 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
1177 	struct hfi1_qp_priv *priv = qp->priv;
1182 	switch (qp->ibqp.qp_type) {
1193 		    priv->s_running_pkt_size <= min(piothreshold, qp->pmtu) &&
1207  * @qp: the QP to send on
1211  * Return non-zero and clear qp->s_flags RVT_S_BUSY otherwise.
1213 int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
1215 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
1216 	struct hfi1_qp_priv *priv = qp->priv;
1253 	sr = get_send_routine(qp, ps);
1255 				priv->s_sc, qp->s_pkey_index);
1270 			spin_lock_irqsave(&qp->s_lock, flags);
1271 			rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
1272 			spin_unlock_irqrestore(&qp->s_lock, flags);
1277 		return pio_wait(qp,
1281 	return sr(qp, ps, 0);
1573 	RCU_INIT_POINTER(ibp->rvp.qp[0], NULL);
1574 	RCU_INIT_POINTER(ibp->rvp.qp[1], NULL);
1917 	struct rvt_qp *qp = packet->qp;
1922 	switch (packet->qp->ibqp.qp_type) {
1924 		rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
1925 		rqpn = qp->remote_qpn;
1929 		rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
1930 		rqpn = qp->remote_qpn;
1945 	lqpn = qp->ibqp.qp_num;