Lines Matching refs:ep
88 static void start_ep_timer(struct c4iw_ep *ep);
89 static int stop_ep_timer(struct c4iw_ep *ep);
90 static int set_tcpinfo(struct c4iw_ep *ep);
91 static void process_timeout(struct c4iw_ep *ep);
95 static int send_mpa_req(struct c4iw_ep *ep);
96 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen);
97 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen);
98 static void close_complete_upcall(struct c4iw_ep *ep, int status);
99 static int send_abort(struct c4iw_ep *ep);
100 static void peer_close_upcall(struct c4iw_ep *ep);
101 static void peer_abort_upcall(struct c4iw_ep *ep);
102 static void connect_reply_upcall(struct c4iw_ep *ep, int status);
103 static int connect_request_upcall(struct c4iw_ep *ep);
104 static void established_upcall(struct c4iw_ep *ep);
105 static int process_mpa_reply(struct c4iw_ep *ep);
106 static int process_mpa_request(struct c4iw_ep *ep);
107 static void process_peer_close(struct c4iw_ep *ep);
108 static void process_conn_error(struct c4iw_ep *ep);
109 static void process_close_complete(struct c4iw_ep *ep);
114 static void process_data(struct c4iw_ep *ep);
115 static void process_connected(struct c4iw_ep *ep);
117 static void process_socket_event(struct c4iw_ep *ep);
118 static void release_ep_resources(struct c4iw_ep *ep);
119 static int process_terminate(struct c4iw_ep *ep);
122 static int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events);
132 #define START_EP_TIMER(ep) \
134 CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \
135 __func__, __LINE__, (ep)); \
136 start_ep_timer(ep); \
139 #define STOP_EP_TIMER(ep) \
141 CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \
142 __func__, __LINE__, (ep)); \
143 stop_ep_timer(ep); \
197 static void deref_qp(struct c4iw_ep *ep)
199 c4iw_qp_rem_ref(&ep->com.qp->ibqp);
200 clear_bit(QP_REFERENCED, &ep->com.flags);
201 set_bit(QP_DEREFED, &ep->com.history);
204 static void ref_qp(struct c4iw_ep *ep)
206 set_bit(QP_REFERENCED, &ep->com.flags);
207 set_bit(QP_REFED, &ep->com.history);
208 c4iw_qp_add_ref(&ep->com.qp->ibqp);
387 static void process_timeout(struct c4iw_ep *ep)
392 CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__,
393 ep, ep->hwtid, ep->com.state);
394 set_bit(TIMEDOUT, &ep->com.history);
395 switch (ep->com.state) {
397 connect_reply_upcall(ep, -ETIMEDOUT);
406 if (ep->com.cm_id && ep->com.qp) {
408 c4iw_modify_qp(ep->com.dev, ep->com.qp,
411 close_complete_upcall(ep, -ETIMEDOUT);
416 * These states are expected if the ep timed out at the same
423 CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u"
424 , __func__, ep, ep->hwtid, ep->com.state);
428 c4iw_ep_disconnect(ep, 1, GFP_KERNEL);
429 c4iw_put_ep(&ep->com);
479 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, ep_state %s events 0x%x",
501 set_tcpinfo(struct c4iw_ep *ep)
503 struct socket *so = ep->com.so;
513 log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n",
514 __func__, so, ep);
519 ep->hwtid = toep->tid;
520 ep->snd_seq = tp->snd_nxt;
521 ep->rcv_seq = tp->rcv_nxt;
566 process_peer_close(struct c4iw_ep *ep)
572 CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep,
573 ep->com.so, states[ep->com.state]);
575 switch (ep->com.state) {
579 __func__, ep);
583 __func__, ep);
584 ep->com.state = DEAD;
585 connect_reply_upcall(ep, -ECONNABORTED);
588 STOP_EP_TIMER(ep);
589 close_socket(ep->com.so);
590 deref_cm_id(&ep->com);
602 __func__, ep);
603 ep->com.state = CLOSING;
608 __func__, ep);
609 ep->com.state = CLOSING;
614 __func__, ep);
615 START_EP_TIMER(ep);
616 ep->com.state = CLOSING;
618 c4iw_modify_qp(ep->com.dev, ep->com.qp,
620 peer_close_upcall(ep);
625 __func__, ep);
631 __func__, ep);
632 ep->com.state = MORIBUND;
638 ep);
639 STOP_EP_TIMER(ep);
640 if (ep->com.cm_id && ep->com.qp) {
642 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
645 close_socket(ep->com.so);
646 close_complete_upcall(ep, 0);
647 ep->com.state = DEAD;
654 __func__, ep);
659 panic("%s: ep %p state %d", __func__, ep,
660 ep->com.state);
667 CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep);
668 c4iw_ep_disconnect(ep, 0, M_NOWAIT);
672 CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep);
673 c4iw_put_ep(&ep->com);
675 CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep);
680 process_conn_error(struct c4iw_ep *ep)
686 state = ep->com.state;
687 CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s",
688 __func__, ep, ep->com.so, ep->com.so->so_error,
689 states[ep->com.state]);
694 STOP_EP_TIMER(ep);
695 c4iw_put_ep(&ep->parent_ep->com);
699 STOP_EP_TIMER(ep);
700 connect_reply_upcall(ep, -ECONNRESET);
704 ep->com.rpl_err = ECONNRESET;
705 CTR1(KTR_IW_CXGBE, "waking up ep %p", ep);
713 STOP_EP_TIMER(ep);
717 if (ep->com.cm_id && ep->com.qp) {
720 ret = c4iw_modify_qp(ep->com.qp->rhp,
721 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE,
728 peer_abort_upcall(ep);
736 __func__, ep->com.so->so_error);
740 panic("%s: ep %p state %d", __func__, ep, state);
745 close_socket(ep->com.so);
746 ep->com.state = DEAD;
747 c4iw_put_ep(&ep->com);
749 CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep);
754 process_close_complete(struct c4iw_ep *ep)
759 CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep,
760 ep->com.so, states[ep->com.state]);
763 set_bit(CLOSE_CON_RPL, &ep->com.history);
765 switch (ep->com.state) {
769 __func__, ep);
770 ep->com.state = MORIBUND;
775 ep);
776 STOP_EP_TIMER(ep);
778 if ((ep->com.cm_id) && (ep->com.qp)) {
781 __func__, ep);
783 c4iw_modify_qp(ep->com.dev,
784 ep->com.qp,
789 close_socket(ep->com.so);
790 close_complete_upcall(ep, 0);
791 ep->com.state = DEAD;
796 CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep);
800 CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep);
803 CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state",
804 __func__, ep);
805 panic("%s:pcc6 %p unknown ep state", __func__, ep);
811 CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep);
812 release_ep_resources(ep);
814 CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep);
869 process_data(struct c4iw_ep *ep)
875 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__,
876 ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv));
878 switch (ep->com.state) {
880 disconnect = process_mpa_reply(ep);
883 disconnect = process_mpa_request(ep);
886 c4iw_put_ep(&ep->parent_ep->com);
889 MPASS(ep->com.qp != NULL);
891 ret = c4iw_modify_qp(ep->com.dev, ep->com.qp,
897 log(LOG_ERR, "%s: Unexpected streaming data. ep %p, "
899 __func__, ep, ep->com.state, ep->com.so,
900 ep->com.so->so_state, sbused(&ep->com.so->so_rcv));
904 c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL);
909 process_connected(struct c4iw_ep *ep)
911 struct socket *so = ep->com.so;
914 if (send_mpa_req(ep))
917 connect_reply_upcall(ep, -so->so_error);
923 ep->com.state = DEAD;
924 c4iw_put_ep(&ep->com);
980 * created 'ep' can be attached to the real 'lep'.
985 "ep for sock: %p", __func__, new_so);
986 log(LOG_ERR,"%s: Could not find the real listen ep for "
999 CTR6(KTR_IW_CXGBE, "%s: master_lep %p, real_lep: %p, new ep %p, "
1034 avoid freeing of ep before ep unlock. */
1050 add_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event)
1055 if (ep && ep->com.so) {
1056 ep->com.ep_events |= new_ep_event;
1057 if (!ep->com.entry.tqe_prev) {
1058 c4iw_get_ep(&ep->com);
1059 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry);
1071 struct c4iw_ep *ep = arg;
1074 "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p",
1075 __func__, so, so->so_state, ep, states[ep->com.state],
1076 ep->com.entry.tqe_prev);
1078 MPASS(ep->com.so == so);
1083 if (so->so_error || (ep->com.dev->rdev.flags & T4_FATAL_ERROR))
1084 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET);
1085 add_ep_to_req_list(ep, C4IW_EVENT_SOCKET);
1099 struct c4iw_ep *ep;
1103 ep = so->so_rcv.sb_upcallarg;
1106 CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep);
1107 add_ep_to_req_list(ep, C4IW_EVENT_TERM);
1113 process_socket_event(struct c4iw_ep *ep)
1115 int state = ep->com.state;
1116 struct socket *so = ep->com.so;
1118 if (ep->com.state == DEAD) {
1120 "ep %p ep_state %s", __func__, ep, states[state]);
1125 "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state,
1126 so->so_error, so->so_rcv.sb_state, ep, states[state]);
1129 process_connected(ep);
1134 struct c4iw_listen_ep *lep = (struct c4iw_listen_ep *)ep;
1162 process_conn_error(ep);
1168 process_peer_close(ep);
1179 process_close_complete(ep);
1184 if (sbused(&ep->com.so->so_rcv)) {
1185 process_data(ep);
1194 MPASS((ep->com.state == MPA_REQ_RCVD) || (ep->com.state == MORIBUND));
1196 if ((ep->com.state != MPA_REQ_RCVD) && (ep->com.state != MORIBUND))
1198 "so_state 0x%x, so_err %d, sb_state 0x%x, ep %p, ep_state %s\n",
1200 ep, states[state]);
1276 start_ep_timer(struct c4iw_ep *ep)
1279 if (timer_pending(&ep->timer)) {
1280 CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep);
1281 printk(KERN_ERR "%s timer already started! ep %p\n", __func__,
1282 ep);
1285 clear_bit(TIMEOUT, &ep->com.flags);
1286 c4iw_get_ep(&ep->com);
1287 ep->timer.expires = jiffies + ep_timeout_secs * HZ;
1288 ep->timer.data = (unsigned long)ep;
1289 ep->timer.function = ep_timeout;
1290 add_timer(&ep->timer);
1294 stop_ep_timer(struct c4iw_ep *ep)
1297 del_timer_sync(&ep->timer);
1298 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) {
1299 c4iw_put_ep(&ep->com);
1323 struct c4iw_ep *ep;
1328 ep = container_of(kref, struct c4iw_ep, com.kref);
1330 epc = &ep->com;
1334 if (test_bit(QP_REFERENCED, &ep->com.flags))
1335 deref_qp(ep);
1336 CTR4(KTR_IW_CXGBE, "%s: ep %p, history 0x%lx, flags 0x%lx",
1337 __func__, ep, epc->history, epc->flags);
1338 kfree(ep);
1341 static void release_ep_resources(struct c4iw_ep *ep)
1343 CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep);
1344 set_bit(RELEASE_RESOURCES, &ep->com.flags);
1345 c4iw_put_ep(&ep->com);
1346 CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep);
1350 send_mpa_req(struct c4iw_ep *ep)
1359 if (ep->retry_with_mpa_v1)
1361 mpalen = sizeof(*mpa) + ep->plen;
1368 CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d",
1369 __func__, ep, err);
1378 mpa->private_data_size = htons(ep->plen);
1382 ep->tried_with_mpa_v1 = 1;
1383 ep->retry_with_mpa_v1 = 0;
1389 mpa_v2_params.ird = htons((u16)ep->ird);
1390 mpa_v2_params.ord = htons((u16)ep->ord);
1406 if (ep->plen) {
1410 ep->mpa_pkt + sizeof(*mpa), ep->plen);
1414 if (ep->plen)
1416 ep->mpa_pkt + sizeof(*mpa), ep->plen);
1417 CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep);
1423 CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d",
1424 __func__, ep, err);
1431 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT,
1432 ep->com.thread);
1434 CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d",
1435 __func__, ep, err);
1439 START_EP_TIMER(ep);
1440 ep->com.state = MPA_REQ_SENT;
1441 ep->mpa_attr.initiator = 1;
1442 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err);
1445 connect_reply_upcall(ep, err);
1446 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err);
1450 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen)
1458 CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid,
1459 ep->plen);
1463 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
1466 CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep,
1467 ep->mpa_attr.version, mpalen);
1480 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
1485 mpa_v2_params.ird = htons(((u16)ep->ird) |
1488 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ?
1497 if (ep->plen)
1500 CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep,
1501 mpa_v2_params.ird, mpa_v2_params.ord, ep->plen);
1514 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread);
1516 ep->snd_seq += mpalen;
1517 CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err);
1521 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen)
1529 CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep);
1533 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
1535 CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep,
1536 ep->mpa_attr.version);
1546 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
1548 mpa->revision = ep->mpa_attr.version;
1551 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
1556 mpa_v2_params.ird = htons((u16)ep->ird);
1557 mpa_v2_params.ord = htons((u16)ep->ord);
1558 CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep,
1559 ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord);
1561 if (peer2peer && (ep->mpa_attr.p2p_type !=
1571 __func__, ep, p2p_type, mpa_v2_params.ird,
1579 __func__, ep, p2p_type, mpa_v2_params.ird,
1587 if (ep->plen)
1603 ep->com.state = MPA_REP_SENT;
1604 ep->snd_seq += mpalen;
1605 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT,
1606 ep->com.thread);
1607 CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err);
1613 static void close_complete_upcall(struct c4iw_ep *ep, int status)
1617 CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep);
1622 if (ep->com.cm_id) {
1624 CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep);
1625 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
1626 deref_cm_id(&ep->com);
1627 set_bit(CLOSE_UPCALL, &ep->com.history);
1629 CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep);
1633 send_abort(struct c4iw_ep *ep)
1635 struct socket *so = ep->com.so;
1640 CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so,
1641 states[ep->com.state], ep->hwtid);
1661 set_bit(ABORT_CONN, &ep->com.history);
1667 * around de-refererece 'ep' here instead of doing it in abort_rpl()
1670 release_ep_resources(ep);
1671 ep->com.state = DEAD;
1676 static void peer_close_upcall(struct c4iw_ep *ep)
1680 CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep);
1684 if (ep->com.cm_id) {
1686 CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep);
1687 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
1688 set_bit(DISCONN_UPCALL, &ep->com.history);
1690 CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep);
1693 static void peer_abort_upcall(struct c4iw_ep *ep)
1697 CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep);
1702 if (ep->com.cm_id) {
1704 CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep);
1705 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
1706 deref_cm_id(&ep->com);
1707 set_bit(ABORT_UPCALL, &ep->com.history);
1709 CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep);
1712 static void connect_reply_upcall(struct c4iw_ep *ep, int status)
1716 CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status);
1721 event.local_addr = ep->com.local_addr;
1722 event.remote_addr = ep->com.remote_addr;
1726 if (!ep->tried_with_mpa_v1) {
1728 CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep);
1730 event.ord = ep->ird;
1731 event.ird = ep->ord;
1732 event.private_data_len = ep->plen -
1734 event.private_data = ep->mpa_pkt +
1739 CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep);
1743 event.private_data_len = ep->plen;
1744 event.private_data = ep->mpa_pkt +
1749 if (ep->com.cm_id) {
1751 CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep);
1752 set_bit(CONN_RPL_UPCALL, &ep->com.history);
1753 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
1758 CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status);
1764 CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status);
1765 deref_cm_id(&ep->com);
1768 CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep);
1771 static int connect_request_upcall(struct c4iw_ep *ep)
1776 CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep,
1777 ep->tried_with_mpa_v1);
1781 event.local_addr = ep->com.local_addr;
1782 event.remote_addr = ep->com.remote_addr;
1783 event.provider_data = ep;
1785 if (!ep->tried_with_mpa_v1) {
1787 event.ord = ep->ord;
1788 event.ird = ep->ird;
1789 event.private_data_len = ep->plen -
1791 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) +
1798 event.private_data_len = ep->plen;
1799 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
1802 c4iw_get_ep(&ep->com);
1803 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id,
1806 CTR3(KTR_IW_CXGBE, "%s: ep %p, Failure while notifying event to"
1807 " IWCM, err:%d", __func__, ep, ret);
1808 c4iw_put_ep(&ep->com);
1814 c4iw_put_ep(&ep->parent_ep->com);
1816 set_bit(CONNREQ_UPCALL, &ep->com.history);
1820 static void established_upcall(struct c4iw_ep *ep)
1824 CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep);
1827 event.ird = ep->ord;
1828 event.ord = ep->ird;
1830 if (ep->com.cm_id) {
1832 CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep);
1833 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
1834 set_bit(ESTAB_UPCALL, &ep->com.history);
1836 CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep);
1854 static int process_mpa_reply(struct c4iw_ep *ep)
1869 CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep);
1876 if (STOP_EP_TIMER(ep))
1880 uio.uio_td = ep->com.thread;
1881 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags);
1887 CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep);
1888 START_EP_TIMER(ep);
1892 CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep);
1896 if (ep->com.so->so_rcv.sb_mb) {
1898 CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep);
1900 __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top);
1907 CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep);
1912 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) {
1914 CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep,
1915 ep->mpa_pkt_len + m->m_len);
1923 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len]));
1924 ep->mpa_pkt_len += m->m_len;
1935 if (ep->mpa_pkt_len < sizeof(*mpa)) {
1938 mpa = (struct mpa_message *) ep->mpa_pkt;
1943 CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep,
1953 CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep);
1965 CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep);
1973 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
1975 CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep);
1976 STOP_EP_TIMER(ep);
1981 ep->plen = (u8) plen;
1987 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) {
1989 CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep);
1995 CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep);
2005 ep->com.state = FPDU_MODE;
2006 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
2007 ep->mpa_attr.recv_marker_enabled = markers_enabled;
2008 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
2009 ep->mpa_attr.version = mpa->revision;
2010 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED;
2014 CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep);
2015 ep->mpa_attr.enhanced_rdma_conn =
2018 if (ep->mpa_attr.enhanced_rdma_conn) {
2020 CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep);
2022 (ep->mpa_pkt + sizeof(*mpa));
2033 if (ep->ird < resp_ord) {
2035 ep->com.dev->rdev.adap->params.max_ordird_qp)
2036 ep->ird = resp_ord;
2039 } else if (ep->ird > resp_ord) {
2040 ep->ird = resp_ord;
2042 if (ep->ord > resp_ird) {
2044 ep->ord = resp_ird;
2050 ep->ird = resp_ord;
2051 ep->ord = resp_ird;
2057 CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep);
2061 CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep);
2062 ep->mpa_attr.p2p_type =
2068 CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep);
2069 ep->mpa_attr.p2p_type =
2076 CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep);
2080 CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep);
2084 CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep);
2085 ep->mpa_attr.p2p_type = p2p_type;
2090 if (set_tcpinfo(ep)) {
2092 CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep);
2100 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
2101 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version,
2102 ep->mpa_attr.p2p_type);
2110 if ((ep->mpa_attr.version == 2) && peer2peer &&
2111 (ep->mpa_attr.p2p_type != p2p_type)) {
2113 CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep);
2114 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED;
2119 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq;
2120 attrs.mpa_attr = ep->mpa_attr;
2121 attrs.max_ird = ep->ird;
2122 attrs.max_ord = ep->ord;
2123 attrs.llp_stream_handle = ep;
2131 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1);
2135 CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep);
2145 CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep);
2151 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
2166 CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep);
2173 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
2181 STOP_EP_TIMER(ep);
2185 connect_reply_upcall(ep, err);
2186 CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep);
2203 process_mpa_request(struct c4iw_ep *ep)
2212 enum c4iw_ep_state state = ep->com.state;
2214 CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]);
2219 iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len];
2220 iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len;
2224 uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len;
2227 uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */
2229 rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags);
2236 __func__, ep->com.so));
2237 ep->mpa_pkt_len += uio.uio_offset;
2247 if (ep->mpa_pkt_len < sizeof(*mpa))
2249 mpa = (struct mpa_message *) ep->mpa_pkt;
2273 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen))
2276 ep->plen = (u8) plen;
2281 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
2288 ep->mpa_attr.initiator = 0;
2289 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
2290 ep->mpa_attr.recv_marker_enabled = markers_enabled;
2291 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
2292 ep->mpa_attr.version = mpa->revision;
2294 ep->tried_with_mpa_v1 = 1;
2295 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED;
2298 ep->mpa_attr.enhanced_rdma_conn =
2300 if (ep->mpa_attr.enhanced_rdma_conn) {
2302 (ep->mpa_pkt + sizeof(*mpa));
2303 ep->ird = ntohs(mpa_v2_params->ird) &
2305 ep->ird = min_t(u32, ep->ird,
2306 cur_max_read_depth(ep->com.dev));
2307 ep->ord = ntohs(mpa_v2_params->ord) &
2309 ep->ord = min_t(u32, ep->ord,
2310 cur_max_read_depth(ep->com.dev));
2312 __func__, ep->ird, ep->ord);
2317 ep->mpa_attr.p2p_type =
2321 ep->mpa_attr.p2p_type =
2326 ep->mpa_attr.p2p_type = p2p_type;
2328 if (set_tcpinfo(ep))
2333 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
2334 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
2336 ep->com.state = MPA_REQ_RCVD;
2337 STOP_EP_TIMER(ep);
2340 if (ep->parent_ep->com.state != DEAD)
2341 if (connect_request_upcall(ep))
2346 STOP_EP_TIMER(ep);
2361 struct c4iw_ep *ep = to_ep(cm_id);
2364 mutex_lock(&ep->com.mutex);
2365 CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep);
2367 if ((ep->com.state == DEAD) ||
2368 (ep->com.state != MPA_REQ_RCVD)) {
2370 CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep);
2371 mutex_unlock(&ep->com.mutex);
2372 c4iw_put_ep(&ep->com);
2375 set_bit(ULP_REJECT, &ep->com.history);
2379 CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep);
2384 CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep);
2385 abort = send_mpa_reject(ep, pdata, pdata_len);
2387 STOP_EP_TIMER(ep);
2389 err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL);
2391 c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL);
2393 mutex_unlock(&ep->com.mutex);
2394 c4iw_put_ep(&ep->com);
2395 CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err);
2404 struct c4iw_ep *ep = to_ep(cm_id);
2409 mutex_lock(&ep->com.mutex);
2410 CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep);
2412 if ((ep->com.state == DEAD) ||
2413 (ep->com.state != MPA_REQ_RCVD)) {
2415 CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep);
2422 set_bit(ULP_ACCEPT, &ep->com.history);
2427 CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep);
2432 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) {
2434 CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep);
2436 if (conn_param->ord > ep->ird) {
2438 conn_param->ord = ep->ird;
2440 ep->ird = conn_param->ird;
2441 ep->ord = conn_param->ord;
2442 send_mpa_reject(ep, conn_param->private_data,
2448 if (conn_param->ird < ep->ord) {
2450 ep->ord <= h->rdev.adap->params.max_ordird_qp) {
2451 conn_param->ird = ep->ord;
2458 ep->ird = conn_param->ird;
2459 ep->ord = conn_param->ord;
2461 if (ep->mpa_attr.version == 1) {
2462 if (peer2peer && ep->ird == 0)
2463 ep->ird = 1;
2466 (ep->mpa_attr.p2p_type != FW_RI_INIT_P2PTYPE_DISABLED) &&
2467 (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) && ep->ird == 0)
2468 ep->ird = 1;
2472 ep->ird, ep->ord);
2474 ep->com.cm_id = cm_id;
2475 ref_cm_id(&ep->com);
2476 ep->com.qp = qp;
2477 ref_qp(ep);
2478 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq;
2481 attrs.mpa_attr = ep->mpa_attr;
2482 attrs.max_ird = ep->ird;
2483 attrs.max_ord = ep->ord;
2484 attrs.llp_stream_handle = ep;
2494 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1);
2496 CTR3(KTR_IW_CXGBE, "%s:caca %p, err: %d", __func__, ep, err);
2500 err = send_mpa_reply(ep, conn_param->private_data,
2503 CTR3(KTR_IW_CXGBE, "%s:cacb %p, err: %d", __func__, ep, err);
2507 ep->com.state = FPDU_MODE;
2508 established_upcall(ep);
2509 mutex_unlock(&ep->com.mutex);
2510 c4iw_put_ep(&ep->com);
2511 CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep);
2514 deref_cm_id(&ep->com);
2519 c4iw_ep_disconnect(ep, 1, GFP_KERNEL);
2520 mutex_unlock(&ep->com.mutex);
2521 c4iw_put_ep(&ep->com);
2522 CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep);
2595 struct c4iw_ep *ep = NULL;
2613 ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
2614 cm_id->provider_data = ep;
2616 init_timer(&ep->timer);
2617 ep->plen = conn_param->private_data_len;
2619 if (ep->plen) {
2621 CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep);
2622 memcpy(ep->mpa_pkt + sizeof(struct mpa_message),
2623 conn_param->private_data, ep->plen);
2625 ep->ird = conn_param->ird;
2626 ep->ord = conn_param->ord;
2628 if (peer2peer && ep->ord == 0) {
2630 CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep);
2631 ep->ord = 1;
2634 ep->com.dev = dev;
2635 ep->com.cm_id = cm_id;
2636 ref_cm_id(&ep->com);
2637 ep->com.qp = get_qhp(dev, conn_param->qpn);
2639 if (!ep->com.qp) {
2641 CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep);
2645 ref_qp(ep);
2646 ep->com.thread = curthread;
2656 CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep);
2667 ep->com.state = CONNECTING;
2668 ep->tos = 0;
2669 ep->com.local_addr = cm_id->local_addr;
2670 ep->com.remote_addr = cm_id->remote_addr;
2672 err = c4iw_sock_create(&cm_id->local_addr, &ep->com.so);
2676 setiwsockopt(ep->com.so);
2677 init_iwarp_socket(ep->com.so, &ep->com);
2678 err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr,
2679 ep->com.thread);
2682 CTR2(KTR_IW_CXGBE, "%s:ccE, ep %p", __func__, ep);
2686 uninit_iwarp_socket(ep->com.so);
2687 ep->com.state = DEAD;
2688 sock_release(ep->com.so);
2690 deref_cm_id(&ep->com);
2691 c4iw_put_ep(&ep->com);
2692 ep = NULL;
2804 int __c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
2807 mutex_lock(&ep->com.mutex);
2808 ret = c4iw_ep_disconnect(ep, abrupt, gfp);
2809 mutex_unlock(&ep->com.mutex);
2813 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
2820 CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep);
2822 rdev = &ep->com.dev->rdev;
2825 CTR3(KTR_IW_CXGBE, "%s:ced1 fatal error %p %s", __func__, ep,
2826 states[ep->com.state]);
2827 if (ep->com.state != DEAD) {
2828 send_abort(ep);
2829 ep->com.state = DEAD;
2831 close_complete_upcall(ep, -ECONNRESET);
2834 CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep,
2835 states[ep->com.state]);
2838 * Ref the ep here in case we have fatal errors causing the
2839 * ep to be released and freed.
2841 c4iw_get_ep(&ep->com);
2842 switch (ep->com.state) {
2851 ep->com.state = ABORTING;
2853 ep->com.state = CLOSING;
2854 START_EP_TIMER(ep);
2856 set_bit(CLOSE_SENT, &ep->com.flags);
2861 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) {
2865 STOP_EP_TIMER(ep);
2866 ep->com.state = ABORTING;
2868 ep->com.state = MORIBUND;
2876 "%s ignoring disconnect ep %p state %u", __func__,
2877 ep, ep->com.state);
2888 CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep);
2892 CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep);
2893 set_bit(EP_DISC_ABORT, &ep->com.history);
2894 close_complete_upcall(ep, -ECONNRESET);
2895 send_abort(ep);
2898 CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep);
2899 set_bit(EP_DISC_CLOSE, &ep->com.history);
2901 if (!ep->parent_ep)
2902 ep->com.state = MORIBUND;
2904 CURVNET_SET(ep->com.so->so_vnet);
2905 ret = sodisconnect(ep->com.so);
2908 CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep);
2909 STOP_EP_TIMER(ep);
2910 send_abort(ep);
2911 ep->com.state = DEAD;
2912 close_complete_upcall(ep, -ECONNRESET);
2913 set_bit(EP_DISC_FAIL, &ep->com.history);
2914 if (ep->com.qp) {
2919 ep->com.dev, ep->com.qp,
2923 __func__, ep, ret);
2928 c4iw_put_ep(&ep->com);
2929 CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep);
2937 struct c4iw_ep *ep = ctx;
2939 if (ep->dst != old)
2942 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new,
2945 cxgb4_l2t_release(ep->l2t);
2946 ep->l2t = l2t;
2948 ep->dst = new;
2957 struct c4iw_ep *ep = (struct c4iw_ep *)arg;
2959 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) {
2964 if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) {
2965 CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep);
2966 add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT);
3004 process_terminate(struct c4iw_ep *ep)
3008 CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep);
3010 if (ep && ep->com.qp) {
3013 ep->hwtid, ep->com.qp->wq.sq.qid);
3015 c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs,
3018 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n",
3019 ep->hwtid);
3020 CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep);