Lines Matching refs:qp
54 * @qp: a pointer to the QP
62 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp,
72 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
78 switch (qp->s_ack_state) {
81 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
93 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC)
94 qp->s_tail_ack_queue = 0;
99 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
100 if (qp->s_flags & RVT_S_ACK_PENDING)
105 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
115 qp->s_tail_ack_queue = qp->r_head_ack_queue;
119 qp->s_rdma_mr = e->rdma_sge.mr;
120 if (qp->s_rdma_mr)
121 rvt_get_mr(qp->s_rdma_mr);
122 qp->s_ack_rdma_sge.sge = e->rdma_sge;
123 qp->s_ack_rdma_sge.num_sge = 1;
124 qp->s_cur_sge = &qp->s_ack_rdma_sge;
127 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
129 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
132 ohdr->u.aeth = rvt_compute_aeth(qp);
134 qp->s_ack_rdma_psn = e->psn;
135 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
138 qp->s_cur_sge = NULL;
140 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
141 ohdr->u.at.aeth = rvt_compute_aeth(qp);
147 bth0 = qp->s_ack_state << 24;
151 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
154 qp->s_cur_sge = &qp->s_ack_rdma_sge;
155 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr;
156 if (qp->s_rdma_mr)
157 rvt_get_mr(qp->s_rdma_mr);
158 len = qp->s_ack_rdma_sge.sge.sge_length;
162 ohdr->u.aeth = rvt_compute_aeth(qp);
164 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
165 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
168 bth0 = qp->s_ack_state << 24;
169 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
180 qp->s_ack_state = OP(SEND_ONLY);
181 qp->s_flags &= ~RVT_S_ACK_PENDING;
182 qp->s_cur_sge = NULL;
183 if (qp->s_nak_state)
185 cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
186 (qp->s_nak_state <<
189 ohdr->u.aeth = rvt_compute_aeth(qp);
193 bth2 = qp->s_ack_psn & QIB_PSN_MASK;
195 qp->s_rdma_ack_cnt++;
196 qp->s_hdrwords = hwords;
197 qp->s_cur_size = len;
198 qib_make_ruc_header(qp, ohdr, bth0, bth2);
202 qp->s_ack_state = OP(ACKNOWLEDGE);
203 qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING);
209 * @qp: a pointer to the QP
216 int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
218 struct qib_qp_priv *priv = qp->priv;
219 struct qib_ibdev *dev = to_idev(qp->ibqp.device);
227 u32 pmtu = qp->pmtu;
233 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
237 if ((qp->s_flags & RVT_S_RESP_PENDING) &&
238 qib_make_rc_ack(dev, qp, ohdr, pmtu))
241 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
242 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
245 if (qp->s_last == READ_ONCE(qp->s_head))
249 qp->s_flags |= RVT_S_WAIT_DMA;
252 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
253 rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
259 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK))
262 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) {
263 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
264 qp->s_flags |= RVT_S_WAIT_PSN;
267 qp->s_sending_psn = qp->s_psn;
268 qp->s_sending_hpsn = qp->s_psn - 1;
276 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
277 switch (qp->s_state) {
279 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK))
289 if (qp->s_cur == qp->s_tail) {
291 if (qp->s_tail == READ_ONCE(qp->s_head))
298 qp->s_num_rd_atomic) {
299 qp->s_flags |= RVT_S_WAIT_FENCE;
303 qp->s_psn = wqe->psn;
311 ss = &qp->s_sge;
312 bth2 = qp->s_psn & QIB_PSN_MASK;
317 if (!rvt_rc_credit_avail(qp, wqe))
320 qp->s_state = OP(SEND_FIRST);
325 qp->s_state = OP(SEND_ONLY);
327 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
335 if (++qp->s_cur == qp->s_size)
336 qp->s_cur = 0;
340 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
341 qp->s_lsn++;
345 if (!rvt_rc_credit_avail(qp, wqe))
355 qp->s_state = OP(RDMA_WRITE_FIRST);
360 qp->s_state = OP(RDMA_WRITE_ONLY);
362 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
371 if (++qp->s_cur == qp->s_size)
372 qp->s_cur = 0;
381 if (qp->s_num_rd_atomic >=
382 qp->s_max_rd_atomic) {
383 qp->s_flags |= RVT_S_WAIT_RDMAR;
386 qp->s_num_rd_atomic++;
387 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
388 qp->s_lsn++;
396 qp->s_state = OP(RDMA_READ_REQUEST);
401 if (++qp->s_cur == qp->s_size)
402 qp->s_cur = 0;
412 if (qp->s_num_rd_atomic >=
413 qp->s_max_rd_atomic) {
414 qp->s_flags |= RVT_S_WAIT_RDMAR;
417 qp->s_num_rd_atomic++;
418 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
419 qp->s_lsn++;
422 qp->s_state = OP(COMPARE_SWAP);
428 qp->s_state = OP(FETCH_ADD);
441 if (++qp->s_cur == qp->s_size)
442 qp->s_cur = 0;
448 qp->s_sge.sge = wqe->sg_list[0];
449 qp->s_sge.sg_list = wqe->sg_list + 1;
450 qp->s_sge.num_sge = wqe->wr.num_sge;
451 qp->s_sge.total_len = wqe->length;
452 qp->s_len = wqe->length;
454 qp->s_tail++;
455 if (qp->s_tail >= qp->s_size)
456 qp->s_tail = 0;
459 qp->s_psn = wqe->lpsn + 1;
461 qp->s_psn++;
466 * qp->s_state is normally set to the opcode of the
474 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
477 qp->s_state = OP(SEND_MIDDLE);
480 bth2 = qp->s_psn++ & QIB_PSN_MASK;
481 ss = &qp->s_sge;
482 len = qp->s_len;
488 qp->s_state = OP(SEND_LAST);
490 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
498 qp->s_cur++;
499 if (qp->s_cur >= qp->s_size)
500 qp->s_cur = 0;
505 * qp->s_state is normally set to the opcode of the
513 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
516 qp->s_state = OP(RDMA_WRITE_MIDDLE);
519 bth2 = qp->s_psn++ & QIB_PSN_MASK;
520 ss = &qp->s_sge;
521 len = qp->s_len;
527 qp->s_state = OP(RDMA_WRITE_LAST);
529 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
537 qp->s_cur++;
538 if (qp->s_cur >= qp->s_size)
539 qp->s_cur = 0;
544 * qp->s_state is normally set to the opcode of the
552 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
558 qp->s_state = OP(RDMA_READ_REQUEST);
560 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK;
561 qp->s_psn = wqe->lpsn + 1;
564 qp->s_cur++;
565 if (qp->s_cur == qp->s_size)
566 qp->s_cur = 0;
569 qp->s_sending_hpsn = bth2;
573 if (qp->s_flags & RVT_S_SEND_ONE) {
574 qp->s_flags &= ~RVT_S_SEND_ONE;
575 qp->s_flags |= RVT_S_WAIT_ACK;
578 qp->s_len -= len;
579 qp->s_hdrwords = hwords;
580 qp->s_cur_sge = ss;
581 qp->s_cur_size = len;
582 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2);
586 qp->s_flags &= ~RVT_S_BUSY;
592 * @qp: a pointer to the QP
598 void qib_send_rc_ack(struct rvt_qp *qp)
600 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
601 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
614 spin_lock_irqsave(&qp->s_lock, flags);
616 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
620 if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt)
628 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) &
631 rdma_ah_read_grh(&qp->remote_ah_attr),
637 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24);
638 if (qp->s_mig_state == IB_MIG_MIGRATED)
640 if (qp->r_nak_state)
641 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
642 (qp->r_nak_state <<
645 ohdr->u.aeth = rvt_compute_aeth(qp);
646 lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 |
647 rdma_ah_get_sl(&qp->remote_ah_attr) << 4;
649 hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr));
652 rdma_ah_get_path_bits(&qp->remote_ah_attr));
654 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
655 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK);
657 spin_unlock_irqrestore(&qp->s_lock, flags);
664 qp->s_srate, lrh0 >> 12);
677 spin_lock_irqsave(&qp->s_lock, flags);
712 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
714 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
715 qp->s_nak_state = qp->r_nak_state;
716 qp->s_ack_psn = qp->r_ack_psn;
719 qib_schedule_send(qp);
722 spin_unlock_irqrestore(&qp->s_lock, flags);
729 * @qp: the QP
736 static void reset_psn(struct rvt_qp *qp, u32 psn)
738 u32 n = qp->s_acked;
739 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
742 qp->s_cur = n;
749 qp->s_state = OP(SEND_LAST);
758 if (++n == qp->s_size)
760 if (n == qp->s_tail)
762 wqe = rvt_get_swqe_ptr(qp, n);
766 qp->s_cur = n;
772 qp->s_state = OP(SEND_LAST);
786 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
791 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
795 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
803 qp->s_state = OP(SEND_LAST);
806 qp->s_psn = psn;
812 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
813 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
814 qp->s_flags |= RVT_S_WAIT_PSN;
821 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
823 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
826 if (qp->s_retry == 0) {
827 if (qp->s_mig_state == IB_MIG_ARMED) {
828 qib_migrate_qp(qp);
829 qp->s_retry = qp->s_retry_cnt;
830 } else if (qp->s_last == qp->s_acked) {
831 rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
832 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
837 qp->s_retry--;
839 ibp = to_iport(qp->ibqp.device, qp->port_num);
843 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
845 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR |
849 qp->s_flags |= RVT_S_SEND_ONE;
850 reset_psn(qp, psn);
854 * Set qp->s_sending_psn to the next PSN after the given one.
857 static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
860 u32 n = qp->s_last;
864 wqe = rvt_get_swqe_ptr(qp, n);
867 qp->s_sending_psn = wqe->lpsn + 1;
869 qp->s_sending_psn = psn + 1;
872 if (++n == qp->s_size)
874 if (n == qp->s_tail)
882 void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
889 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK))
901 WARN_ON(!qp->s_rdma_ack_cnt);
902 qp->s_rdma_ack_cnt--;
907 reset_sending_psn(qp, psn);
913 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail &&
914 !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) &&
915 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
916 rvt_add_retry_timer(qp);
918 while (qp->s_last != qp->s_acked) {
919 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
920 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&
921 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
923 rvt_qp_complete_swqe(qp,
932 if (qp->s_flags & RVT_S_WAIT_PSN &&
933 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
934 qp->s_flags &= ~RVT_S_WAIT_PSN;
935 qp->s_sending_psn = qp->s_psn;
936 qp->s_sending_hpsn = qp->s_psn - 1;
937 qib_schedule_send(qp);
941 static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
943 qp->s_last_psn = psn;
951 static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
960 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||
961 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0)
962 rvt_qp_complete_swqe(qp,
969 qp->s_retry = qp->s_retry_cnt;
970 update_last_psn(qp, wqe->lpsn);
977 if (qp->s_acked == qp->s_cur) {
978 if (++qp->s_cur >= qp->s_size)
979 qp->s_cur = 0;
980 qp->s_acked = qp->s_cur;
981 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
982 if (qp->s_acked != qp->s_tail) {
983 qp->s_state = OP(SEND_LAST);
984 qp->s_psn = wqe->psn;
987 if (++qp->s_acked >= qp->s_size)
988 qp->s_acked = 0;
989 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
990 qp->s_draining = 0;
991 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
998 * @qp: the QP the ACK came in on
1007 static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
1026 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1027 ibp = to_iport(qp->ibqp.device, qp->port_num);
1061 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
1062 qp->r_flags |= RVT_R_RDMAR_SEQ;
1063 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1064 if (list_empty(&qp->rspwait)) {
1065 qp->r_flags |= RVT_R_RSP_SEND;
1066 rvt_get_qp(qp);
1067 list_add_tail(&qp->rspwait,
1082 if (qp->s_num_rd_atomic &&
1086 qp->s_num_rd_atomic--;
1088 if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
1089 !qp->s_num_rd_atomic) {
1090 qp->s_flags &= ~(RVT_S_WAIT_FENCE |
1092 qib_schedule_send(qp);
1093 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) {
1094 qp->s_flags &= ~(RVT_S_WAIT_RDMAR |
1096 qib_schedule_send(qp);
1099 wqe = do_rc_completion(qp, wqe, ibp);
1100 if (qp->s_acked == qp->s_tail)
1107 if (qp->s_acked != qp->s_tail) {
1112 rvt_mod_retry_timer(qp);
1117 if (qib_cmp24(qp->s_psn, psn) <= 0)
1118 reset_psn(qp, psn + 1);
1121 rvt_stop_rc_timers(qp);
1122 if (qib_cmp24(qp->s_psn, psn) <= 0) {
1123 qp->s_state = OP(SEND_LAST);
1124 qp->s_psn = psn + 1;
1127 if (qp->s_flags & RVT_S_WAIT_ACK) {
1128 qp->s_flags &= ~RVT_S_WAIT_ACK;
1129 qib_schedule_send(qp);
1131 rvt_get_credit(qp, aeth);
1132 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1133 qp->s_retry = qp->s_retry_cnt;
1134 update_last_psn(qp, psn);
1139 if (qp->s_acked == qp->s_tail)
1141 if (qp->s_flags & RVT_S_WAIT_RNR)
1143 if (qp->s_rnr_retry == 0) {
1147 if (qp->s_rnr_retry_cnt < 7)
1148 qp->s_rnr_retry--;
1151 update_last_psn(qp, psn - 1);
1153 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
1155 reset_psn(qp, psn);
1157 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK);
1158 rvt_stop_rc_timers(qp);
1159 rvt_add_rnr_timer(qp, aeth);
1163 if (qp->s_acked == qp->s_tail)
1166 update_last_psn(qp, psn - 1);
1177 qib_restart_rc(qp, psn, 0);
1178 qib_schedule_send(qp);
1195 if (qp->s_last == qp->s_acked) {
1196 rvt_send_complete(qp, wqe, status);
1197 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1205 qp->s_retry = qp->s_retry_cnt;
1206 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1216 rvt_stop_rc_timers(qp);
1224 static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn,
1230 rvt_stop_rc_timers(qp);
1232 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1239 wqe = do_rc_completion(qp, wqe, ibp);
1243 qp->r_flags |= RVT_R_RDMAR_SEQ;
1244 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1245 if (list_empty(&qp->rspwait)) {
1246 qp->r_flags |= RVT_R_RSP_SEND;
1247 rvt_get_qp(qp);
1248 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1258 * @qp: the QP for this packet
1272 struct rvt_qp *qp,
1291 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) &&
1292 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) {
1298 if (!(qp->s_flags & RVT_S_BUSY)) {
1309 spin_lock_irqsave(&qp->s_lock, flags);
1310 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
1314 if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0)
1318 diff = qib_cmp24(psn, qp->s_last_psn);
1324 rvt_get_credit(qp, aeth);
1333 if (qp->r_flags & RVT_R_RDMAR_SEQ) {
1334 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0)
1336 qp->r_flags &= ~RVT_R_RDMAR_SEQ;
1339 if (unlikely(qp->s_acked == qp->s_tail))
1341 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1353 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
1357 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1365 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1371 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1378 if (unlikely(pmtu >= qp->s_rdma_read_len))
1383 * 4.096 usec. * (1 << qp->timeout)
1385 rvt_mod_retry_timer(qp);
1386 if (qp->s_flags & RVT_S_WAIT_ACK) {
1387 qp->s_flags &= ~RVT_S_WAIT_ACK;
1388 qib_schedule_send(qp);
1392 qp->s_retry = qp->s_retry_cnt;
1398 qp->s_rdma_read_len -= pmtu;
1399 update_last_psn(qp, psn);
1400 spin_unlock_irqrestore(&qp->s_lock, flags);
1401 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
1407 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
1423 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1424 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1430 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1445 if (unlikely(tlen != qp->s_rdma_read_len))
1448 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
1450 WARN_ON(qp->s_rdma_read_sge.num_sge);
1451 (void) do_rc_ack(qp, aeth, psn,
1461 rdma_seq_err(qp, ibp, psn, rcd);
1467 if (qp->s_last == qp->s_acked) {
1468 rvt_send_complete(qp, wqe, status);
1469 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1472 spin_unlock_irqrestore(&qp->s_lock, flags);
1481 * @qp: the QP for this packet
1495 struct rvt_qp *qp,
1501 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
1513 if (!qp->r_nak_state) {
1515 qp->r_nak_state = IB_NAK_PSN_ERROR;
1517 qp->r_ack_psn = qp->r_psn;
1523 if (list_empty(&qp->rspwait)) {
1524 qp->r_flags |= RVT_R_RSP_NAK;
1525 rvt_get_qp(qp);
1526 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1552 spin_lock_irqsave(&qp->s_lock, flags);
1554 for (i = qp->r_head_ack_queue; ; i = prev) {
1555 if (i == qp->s_tail_ack_queue)
1561 if (prev == qp->r_head_ack_queue) {
1565 e = &qp->s_ack_queue[prev];
1571 if (prev == qp->s_tail_ack_queue &&
1599 qp->pmtu;
1612 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
1624 qp->s_tail_ack_queue = prev;
1637 qp->s_tail_ack_queue = prev;
1652 if (i == qp->r_head_ack_queue) {
1653 spin_unlock_irqrestore(&qp->s_lock, flags);
1654 qp->r_nak_state = 0;
1655 qp->r_ack_psn = qp->r_psn - 1;
1663 if (!(qp->s_flags & RVT_S_RESP_PENDING)) {
1664 spin_unlock_irqrestore(&qp->s_lock, flags);
1665 qp->r_nak_state = 0;
1666 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1;
1673 qp->s_tail_ack_queue = i;
1676 qp->s_ack_state = OP(ACKNOWLEDGE);
1677 qp->s_flags |= RVT_S_RESP_PENDING;
1678 qp->r_nak_state = 0;
1679 qib_schedule_send(qp);
1682 spin_unlock_irqrestore(&qp->s_lock, flags);
1690 static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n)
1697 qp->s_tail_ack_queue = next;
1698 qp->s_ack_state = OP(ACKNOWLEDGE);
1708 * @qp: the QP for this packet
1715 int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
1724 u32 pmtu = qp->pmtu;
1740 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode))
1754 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn,
1760 diff = qib_cmp24(psn, qp->r_psn);
1762 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
1768 switch (qp->r_state) {
1801 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
1802 rvt_comm_est(qp);
1807 ret = rvt_get_rwqe(qp, false);
1812 qp->r_rcv_len = 0;
1820 qp->r_rcv_len += pmtu;
1821 if (unlikely(qp->r_rcv_len > qp->r_len))
1823 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);
1828 ret = rvt_get_rwqe(qp, true);
1837 ret = rvt_get_rwqe(qp, false);
1842 qp->r_rcv_len = 0;
1866 wc.byte_len = tlen + qp->r_rcv_len;
1867 if (unlikely(wc.byte_len > qp->r_len))
1869 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false);
1870 rvt_put_ss(&qp->r_sge);
1871 qp->r_msn++;
1872 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
1874 wc.wr_id = qp->r_wr_id;
1881 wc.qp = &qp->ibqp;
1882 wc.src_qp = qp->remote_qpn;
1883 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr);
1884 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
1891 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr));
1897 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
1902 qp->r_len = be32_to_cpu(reth->length);
1903 qp->r_rcv_len = 0;
1904 qp->r_sge.sg_list = NULL;
1905 if (qp->r_len != 0) {
1911 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
1915 qp->r_sge.num_sge = 1;
1917 qp->r_sge.num_sge = 0;
1918 qp->r_sge.sge.mr = NULL;
1919 qp->r_sge.sge.vaddr = NULL;
1920 qp->r_sge.sge.length = 0;
1921 qp->r_sge.sge.sge_length = 0;
1927 ret = rvt_get_rwqe(qp, true);
1931 rvt_put_ss(&qp->r_sge);
1944 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
1946 next = qp->r_head_ack_queue + 1;
1950 spin_lock_irqsave(&qp->s_lock, flags);
1951 if (unlikely(next == qp->s_tail_ack_queue)) {
1952 if (!qp->s_ack_queue[next].sent)
1954 qib_update_ack_queue(qp, next);
1956 e = &qp->s_ack_queue[qp->r_head_ack_queue];
1969 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
1977 qp->r_psn += rvt_div_mtu(qp, len - 1);
1987 e->lpsn = qp->r_psn;
1993 qp->r_msn++;
1994 qp->r_psn++;
1995 qp->r_state = opcode;
1996 qp->r_nak_state = 0;
1997 qp->r_head_ack_queue = next;
2000 qp->s_flags |= RVT_S_RESP_PENDING;
2001 qib_schedule_send(qp);
2016 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
2018 next = qp->r_head_ack_queue + 1;
2021 spin_lock_irqsave(&qp->s_lock, flags);
2022 if (unlikely(next == qp->s_tail_ack_queue)) {
2023 if (!qp->s_ack_queue[next].sent)
2025 qib_update_ack_queue(qp, next);
2027 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2038 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
2043 maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
2047 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
2050 rvt_put_mr(qp->r_sge.sge.mr);
2051 qp->r_sge.num_sge = 0;
2056 qp->r_msn++;
2057 qp->r_psn++;
2058 qp->r_state = opcode;
2059 qp->r_nak_state = 0;
2060 qp->r_head_ack_queue = next;
2063 qp->s_flags |= RVT_S_RESP_PENDING;
2064 qib_schedule_send(qp);
2073 qp->r_psn++;
2074 qp->r_state = opcode;
2075 qp->r_ack_psn = psn;
2076 qp->r_nak_state = 0;
2083 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer;
2084 qp->r_ack_psn = qp->r_psn;
2086 if (list_empty(&qp->rspwait)) {
2087 qp->r_flags |= RVT_R_RSP_NAK;
2088 rvt_get_qp(qp);
2089 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2094 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2095 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
2096 qp->r_ack_psn = qp->r_psn;
2098 if (list_empty(&qp->rspwait)) {
2099 qp->r_flags |= RVT_R_RSP_NAK;
2100 rvt_get_qp(qp);
2101 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2106 spin_unlock_irqrestore(&qp->s_lock, flags);
2108 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2109 qp->r_nak_state = IB_NAK_INVALID_REQUEST;
2110 qp->r_ack_psn = qp->r_psn;
2112 if (list_empty(&qp->rspwait)) {
2113 qp->r_flags |= RVT_R_RSP_NAK;
2114 rvt_get_qp(qp);
2115 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2120 spin_unlock_irqrestore(&qp->s_lock, flags);
2122 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
2123 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
2124 qp->r_ack_psn = qp->r_psn;
2126 qib_send_rc_ack(qp);
2130 spin_unlock_irqrestore(&qp->s_lock, flags);