1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy***************************************************************************/ 29178786Skmacy#include <sys/cdefs.h> 30178786Skmacy__FBSDID("$FreeBSD: stable/10/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c 309378 2016-12-01 23:38:52Z jhb $"); 31178786Skmacy 32237263Snp#include "opt_inet.h" 33237263Snp 34237263Snp#ifdef TCP_OFFLOAD 35178786Skmacy#include <sys/param.h> 36178786Skmacy#include <sys/systm.h> 37178786Skmacy#include <sys/kernel.h> 38178786Skmacy#include <sys/bus.h> 39178786Skmacy#include <sys/pciio.h> 40178786Skmacy#include <sys/conf.h> 41178786Skmacy#include <machine/bus.h> 42178786Skmacy#include <machine/resource.h> 43178786Skmacy#include <sys/bus_dma.h> 44178786Skmacy#include <sys/rman.h> 45178786Skmacy#include <sys/ioccom.h> 46178786Skmacy#include <sys/mbuf.h> 47178786Skmacy#include <sys/rwlock.h> 48178786Skmacy#include <sys/linker.h> 49178786Skmacy#include <sys/firmware.h> 50178786Skmacy#include <sys/socket.h> 51178786Skmacy#include <sys/socketvar.h> 52178786Skmacy#include <sys/sockio.h> 53178786Skmacy#include <sys/smp.h> 54178786Skmacy#include <sys/sysctl.h> 55178786Skmacy#include <sys/syslog.h> 56178786Skmacy#include <sys/queue.h> 57178786Skmacy#include <sys/taskqueue.h> 58178786Skmacy#include <sys/proc.h> 59178786Skmacy#include <sys/uio.h> 60178786Skmacy 61178786Skmacy#include <net/route.h> 62178786Skmacy#include <netinet/in_systm.h> 63178786Skmacy#include <netinet/in.h> 64178786Skmacy#include <netinet/in_pcb.h> 65178786Skmacy#include <netinet/ip.h> 66178786Skmacy#include <netinet/ip_var.h> 67178786Skmacy#include <netinet/tcp_var.h> 68178786Skmacy#include <netinet/tcp.h> 69178786Skmacy#include <netinet/tcpip.h> 70178786Skmacy 71237263Snp#include <rdma/ib_verbs.h> 72237263Snp#include <linux/idr.h> 73237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 74178786Skmacy 75178786Skmacy#include <cxgb_include.h> 76178786Skmacy#include <ulp/tom/cxgb_tom.h> 77178786Skmacy#include <ulp/tom/cxgb_toepcb.h> 78237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 79237263Snp#include <rdma/ib_verbs.h> 80237263Snp#include <linux/idr.h> 81237263Snp 82178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 83178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 84178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 85178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 86178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 87178786Skmacy 88178786Skmacy#ifdef KTR 89178786Skmacystatic char *states[] = { 90178786Skmacy "idle", 91178786Skmacy "listen", 92178786Skmacy "connecting", 93178786Skmacy "mpa_wait_req", 94178786Skmacy "mpa_req_sent", 95178786Skmacy "mpa_req_rcvd", 96178786Skmacy "mpa_rep_sent", 97178786Skmacy "fpdu_mode", 98178786Skmacy "aborting", 99178786Skmacy "closing", 100178786Skmacy "moribund", 101178786Skmacy "dead", 102178786Skmacy NULL, 103178786Skmacy}; 104178786Skmacy#endif 105178786Skmacy 106237263SnpSYSCTL_NODE(_hw, OID_AUTO, iw_cxgb, CTLFLAG_RD, 0, "iw_cxgb driver parameters"); 107178786Skmacy 108237263Snpstatic int ep_timeout_secs = 60; 109178786SkmacyTUNABLE_INT("hw.iw_cxgb.ep_timeout_secs", &ep_timeout_secs); 110237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, ep_timeout_secs, CTLFLAG_RW, &ep_timeout_secs, 0, 111237263Snp "CM Endpoint operation timeout in seconds (default=60)"); 112178786Skmacy 113178786Skmacystatic int mpa_rev = 1; 114178786SkmacyTUNABLE_INT("hw.iw_cxgb.mpa_rev", &mpa_rev); 115237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, mpa_rev, CTLFLAG_RW, &mpa_rev, 0, 116178786Skmacy "MPA Revision, 0 supports amso1100, 1 is spec compliant. (default=1)"); 117178786Skmacy 118178786Skmacystatic int markers_enabled = 0; 119178786SkmacyTUNABLE_INT("hw.iw_cxgb.markers_enabled", &markers_enabled); 120237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, markers_enabled, CTLFLAG_RW, &markers_enabled, 0, 121178786Skmacy "Enable MPA MARKERS (default(0)=disabled)"); 122178786Skmacy 123178786Skmacystatic int crc_enabled = 1; 124178786SkmacyTUNABLE_INT("hw.iw_cxgb.crc_enabled", &crc_enabled); 125237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, crc_enabled, CTLFLAG_RW, &crc_enabled, 0, 126178786Skmacy "Enable MPA CRC (default(1)=enabled)"); 127178786Skmacy 128178786Skmacystatic int rcv_win = 256 * 1024; 129178786SkmacyTUNABLE_INT("hw.iw_cxgb.rcv_win", &rcv_win); 130237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, rcv_win, CTLFLAG_RW, &rcv_win, 0, 131178786Skmacy "TCP receive window in bytes (default=256KB)"); 132178786Skmacy 133178786Skmacystatic int snd_win = 32 * 1024; 134178786SkmacyTUNABLE_INT("hw.iw_cxgb.snd_win", &snd_win); 135237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, snd_win, CTLFLAG_RW, &snd_win, 0, 136178786Skmacy "TCP send window in bytes (default=32KB)"); 137178786Skmacy 138178786Skmacystatic unsigned int nocong = 0; 139178786SkmacyTUNABLE_INT("hw.iw_cxgb.nocong", &nocong); 140237263SnpSYSCTL_UINT(_hw_iw_cxgb, OID_AUTO, nocong, CTLFLAG_RW, &nocong, 0, 141178786Skmacy "Turn off congestion control (default=0)"); 142178786Skmacy 143178786Skmacystatic unsigned int cong_flavor = 1; 144178786SkmacyTUNABLE_INT("hw.iw_cxgb.cong_flavor", &cong_flavor); 145237263SnpSYSCTL_UINT(_hw_iw_cxgb, OID_AUTO, cong_flavor, CTLFLAG_RW, &cong_flavor, 0, 146178786Skmacy "TCP Congestion control flavor (default=1)"); 147178786Skmacy 148178786Skmacystatic void ep_timeout(void *arg); 149178786Skmacystatic void connect_reply_upcall(struct iwch_ep *ep, int status); 150193272Sjhbstatic int iwch_so_upcall(struct socket *so, void *arg, int waitflag); 151178786Skmacy 152178786Skmacy/* 153178786Skmacy * Cruft to offload socket upcalls onto thread. 154178786Skmacy */ 155178786Skmacystatic struct mtx req_lock; 156178786Skmacystatic TAILQ_HEAD(iwch_ep_list, iwch_ep_common) req_list; 157178786Skmacystatic struct task iw_cxgb_task; 158178786Skmacystatic struct taskqueue *iw_cxgb_taskq; 159178786Skmacystatic void process_req(void *ctx, int pending); 160178786Skmacy 161178786Skmacystatic void 162178786Skmacystart_ep_timer(struct iwch_ep *ep) 163178786Skmacy{ 164178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 165178786Skmacy if (callout_pending(&ep->timer)) { 166178786Skmacy CTR2(KTR_IW_CXGB, "%s stopped / restarted timer ep %p", __FUNCTION__, ep); 167178786Skmacy callout_deactivate(&ep->timer); 168178786Skmacy callout_drain(&ep->timer); 169178786Skmacy } else { 170178786Skmacy /* 171178786Skmacy * XXX this looks racy 172178786Skmacy */ 173178786Skmacy get_ep(&ep->com); 174178786Skmacy callout_init(&ep->timer, TRUE); 175178786Skmacy } 176178786Skmacy callout_reset(&ep->timer, ep_timeout_secs * hz, ep_timeout, ep); 177178786Skmacy} 178178786Skmacy 179178786Skmacystatic void 180178786Skmacystop_ep_timer(struct iwch_ep *ep) 181178786Skmacy{ 182178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 183237263Snp if (!callout_pending(&ep->timer)) { 184237263Snp CTR3(KTR_IW_CXGB, "%s timer stopped when its not running! ep %p state %u\n", 185237263Snp __func__, ep, ep->com.state); 186237263Snp return; 187237263Snp } 188178786Skmacy callout_drain(&ep->timer); 189178786Skmacy put_ep(&ep->com); 190178786Skmacy} 191178786Skmacy 192237263Snpstatic int 193237263Snpset_tcpinfo(struct iwch_ep *ep) 194178786Skmacy{ 195237263Snp struct socket *so = ep->com.so; 196237263Snp struct inpcb *inp = sotoinpcb(so); 197237263Snp struct tcpcb *tp; 198237263Snp struct toepcb *toep; 199237263Snp int rc = 0; 200178786Skmacy 201237263Snp INP_WLOCK(inp); 202237263Snp tp = intotcpcb(inp); 203237263Snp 204237263Snp if ((tp->t_flags & TF_TOE) == 0) { 205237263Snp rc = EINVAL; 206237263Snp printf("%s: connection NOT OFFLOADED!\n", __func__); 207237263Snp goto done; 208178786Skmacy } 209237263Snp toep = tp->t_toe; 210178786Skmacy 211237263Snp ep->hwtid = toep->tp_tid; 212237263Snp ep->snd_seq = tp->snd_nxt; 213237263Snp ep->rcv_seq = tp->rcv_nxt; 214237263Snp ep->emss = tp->t_maxseg; 215178786Skmacy if (ep->emss < 128) 216178786Skmacy ep->emss = 128; 217237263Snpdone: 218237263Snp INP_WUNLOCK(inp); 219237263Snp return (rc); 220237263Snp 221178786Skmacy} 222178786Skmacy 223178786Skmacystatic enum iwch_ep_state 224178786Skmacystate_read(struct iwch_ep_common *epc) 225178786Skmacy{ 226178786Skmacy enum iwch_ep_state state; 227178786Skmacy 228178786Skmacy mtx_lock(&epc->lock); 229178786Skmacy state = epc->state; 230178786Skmacy mtx_unlock(&epc->lock); 231178786Skmacy return state; 232178786Skmacy} 233178786Skmacy 234178786Skmacystatic void 235178786Skmacy__state_set(struct iwch_ep_common *epc, enum iwch_ep_state new) 236178786Skmacy{ 237178786Skmacy epc->state = new; 238178786Skmacy} 239178786Skmacy 240178786Skmacystatic void 241178786Skmacystate_set(struct iwch_ep_common *epc, enum iwch_ep_state new) 242178786Skmacy{ 243178786Skmacy 244178786Skmacy mtx_lock(&epc->lock); 245178786Skmacy CTR3(KTR_IW_CXGB, "%s - %s -> %s", __FUNCTION__, states[epc->state], states[new]); 246178786Skmacy __state_set(epc, new); 247178786Skmacy mtx_unlock(&epc->lock); 248178786Skmacy return; 249178786Skmacy} 250178786Skmacy 251178786Skmacystatic void * 252178786Skmacyalloc_ep(int size, int flags) 253178786Skmacy{ 254178786Skmacy struct iwch_ep_common *epc; 255178786Skmacy 256178786Skmacy epc = malloc(size, M_DEVBUF, flags); 257178786Skmacy if (epc) { 258178786Skmacy memset(epc, 0, size); 259178786Skmacy refcount_init(&epc->refcount, 1); 260178786Skmacy mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_DEF|MTX_DUPOK); 261178786Skmacy cv_init(&epc->waitq, "iwch_epc cv"); 262178786Skmacy } 263178786Skmacy CTR2(KTR_IW_CXGB, "%s alloc ep %p", __FUNCTION__, epc); 264178786Skmacy return epc; 265178786Skmacy} 266178786Skmacy 267178786Skmacyvoid __free_ep(struct iwch_ep_common *epc) 268178786Skmacy{ 269178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p state %s", __FUNCTION__, epc, states[state_read(epc)]); 270178786Skmacy KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list!\n", __FUNCTION__, epc)); 271178786Skmacy free(epc, M_DEVBUF); 272178786Skmacy} 273178786Skmacy 274178786Skmacystatic struct rtentry * 275178786Skmacyfind_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 276178786Skmacy __be16 peer_port, u8 tos) 277178786Skmacy{ 278178786Skmacy struct route iproute; 279178786Skmacy struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst; 280178786Skmacy 281178786Skmacy bzero(&iproute, sizeof iproute); 282178786Skmacy dst->sin_family = AF_INET; 283178786Skmacy dst->sin_len = sizeof *dst; 284178786Skmacy dst->sin_addr.s_addr = peer_ip; 285178786Skmacy 286178786Skmacy rtalloc(&iproute); 287178786Skmacy return iproute.ro_rt; 288178786Skmacy} 289178786Skmacy 290178786Skmacystatic void 291237263Snpclose_socket(struct iwch_ep_common *epc, int close) 292178786Skmacy{ 293178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]); 294178786Skmacy SOCK_LOCK(epc->so); 295193272Sjhb soupcall_clear(epc->so, SO_RCV); 296178786Skmacy SOCK_UNLOCK(epc->so); 297237263Snp if (close) 298237263Snp soclose(epc->so); 299237263Snp else 300237263Snp soshutdown(epc->so, SHUT_WR|SHUT_RD); 301178786Skmacy epc->so = NULL; 302178786Skmacy} 303178786Skmacy 304178786Skmacystatic void 305178786Skmacyshutdown_socket(struct iwch_ep_common *epc) 306178786Skmacy{ 307178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]); 308178786Skmacy soshutdown(epc->so, SHUT_WR); 309178786Skmacy} 310178786Skmacy 311178786Skmacystatic void 312178786Skmacyabort_socket(struct iwch_ep *ep) 313178786Skmacy{ 314178786Skmacy struct sockopt sopt; 315178786Skmacy int err; 316178786Skmacy struct linger l; 317178786Skmacy 318178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 319178786Skmacy l.l_onoff = 1; 320178786Skmacy l.l_linger = 0; 321178786Skmacy 322178786Skmacy /* linger_time of 0 forces RST to be sent */ 323178786Skmacy sopt.sopt_dir = SOPT_SET; 324178786Skmacy sopt.sopt_level = SOL_SOCKET; 325178786Skmacy sopt.sopt_name = SO_LINGER; 326178786Skmacy sopt.sopt_val = (caddr_t)&l; 327178786Skmacy sopt.sopt_valsize = sizeof l; 328178786Skmacy sopt.sopt_td = NULL; 329178786Skmacy err = sosetopt(ep->com.so, &sopt); 330178786Skmacy if (err) 331178786Skmacy printf("%s can't set linger to 0, no RST! err %d\n", __FUNCTION__, err); 332178786Skmacy} 333178786Skmacy 334178786Skmacystatic void 335178786Skmacysend_mpa_req(struct iwch_ep *ep) 336178786Skmacy{ 337178786Skmacy int mpalen; 338178786Skmacy struct mpa_message *mpa; 339178786Skmacy struct mbuf *m; 340178786Skmacy int err; 341178786Skmacy 342178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p pd_len %d", __FUNCTION__, ep, ep->plen); 343178786Skmacy 344178786Skmacy mpalen = sizeof(*mpa) + ep->plen; 345178786Skmacy m = m_gethdr(mpalen, M_NOWAIT); 346178786Skmacy if (m == NULL) { 347178786Skmacy connect_reply_upcall(ep, -ENOMEM); 348178786Skmacy return; 349178786Skmacy } 350178786Skmacy mpa = mtod(m, struct mpa_message *); 351178786Skmacy m->m_len = mpalen; 352178786Skmacy m->m_pkthdr.len = mpalen; 353178786Skmacy memset(mpa, 0, sizeof(*mpa)); 354178786Skmacy memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 355178786Skmacy mpa->flags = (crc_enabled ? MPA_CRC : 0) | 356178786Skmacy (markers_enabled ? MPA_MARKERS : 0); 357178786Skmacy mpa->private_data_size = htons(ep->plen); 358178786Skmacy mpa->revision = mpa_rev; 359178786Skmacy if (ep->plen) 360178786Skmacy memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen); 361178786Skmacy 362178786Skmacy err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 363178786Skmacy if (err) { 364178786Skmacy m_freem(m); 365178786Skmacy connect_reply_upcall(ep, -ENOMEM); 366178786Skmacy return; 367178786Skmacy } 368178786Skmacy 369178786Skmacy start_ep_timer(ep); 370178786Skmacy state_set(&ep->com, MPA_REQ_SENT); 371178786Skmacy return; 372178786Skmacy} 373178786Skmacy 374178786Skmacystatic int 375178786Skmacysend_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen) 376178786Skmacy{ 377178786Skmacy int mpalen; 378178786Skmacy struct mpa_message *mpa; 379178786Skmacy struct mbuf *m; 380178786Skmacy int err; 381178786Skmacy 382178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p plen %d", __FUNCTION__, ep, plen); 383178786Skmacy 384178786Skmacy mpalen = sizeof(*mpa) + plen; 385178786Skmacy 386178786Skmacy m = m_gethdr(mpalen, M_NOWAIT); 387178786Skmacy if (m == NULL) { 388178786Skmacy printf("%s - cannot alloc mbuf!\n", __FUNCTION__); 389178786Skmacy return (-ENOMEM); 390178786Skmacy } 391178786Skmacy mpa = mtod(m, struct mpa_message *); 392178786Skmacy m->m_len = mpalen; 393178786Skmacy m->m_pkthdr.len = mpalen; 394178786Skmacy memset(mpa, 0, sizeof(*mpa)); 395178786Skmacy memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 396178786Skmacy mpa->flags = MPA_REJECT; 397178786Skmacy mpa->revision = mpa_rev; 398178786Skmacy mpa->private_data_size = htons(plen); 399178786Skmacy if (plen) 400178786Skmacy memcpy(mpa->private_data, pdata, plen); 401178786Skmacy err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 402178786Skmacy PANIC_IF(err); 403178786Skmacy return 0; 404178786Skmacy} 405178786Skmacy 406178786Skmacystatic int 407178786Skmacysend_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen) 408178786Skmacy{ 409178786Skmacy int mpalen; 410178786Skmacy struct mpa_message *mpa; 411178786Skmacy struct mbuf *m; 412178786Skmacy 413178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p plen %d", __FUNCTION__, ep, ep->com.so, plen); 414178786Skmacy 415178786Skmacy mpalen = sizeof(*mpa) + plen; 416178786Skmacy 417178786Skmacy m = m_gethdr(mpalen, M_NOWAIT); 418178786Skmacy if (m == NULL) { 419178786Skmacy printf("%s - cannot alloc mbuf!\n", __FUNCTION__); 420178786Skmacy return (-ENOMEM); 421178786Skmacy } 422178786Skmacy mpa = mtod(m, struct mpa_message *); 423178786Skmacy m->m_len = mpalen; 424178786Skmacy m->m_pkthdr.len = mpalen; 425178786Skmacy memset(mpa, 0, sizeof(*mpa)); 426178786Skmacy memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 427178786Skmacy mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 428178786Skmacy (markers_enabled ? MPA_MARKERS : 0); 429178786Skmacy mpa->revision = mpa_rev; 430178786Skmacy mpa->private_data_size = htons(plen); 431178786Skmacy if (plen) 432178786Skmacy memcpy(mpa->private_data, pdata, plen); 433178786Skmacy 434178786Skmacy state_set(&ep->com, MPA_REP_SENT); 435178786Skmacy return sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 436178786Skmacy ep->com.thread); 437178786Skmacy} 438178786Skmacy 439178786Skmacystatic void 440178786Skmacyclose_complete_upcall(struct iwch_ep *ep) 441178786Skmacy{ 442178786Skmacy struct iw_cm_event event; 443178786Skmacy 444178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 445178786Skmacy memset(&event, 0, sizeof(event)); 446178786Skmacy event.event = IW_CM_EVENT_CLOSE; 447178786Skmacy if (ep->com.cm_id) { 448178786Skmacy CTR3(KTR_IW_CXGB, "close complete delivered ep %p cm_id %p tid %d", 449178786Skmacy ep, ep->com.cm_id, ep->hwtid); 450178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 451178786Skmacy ep->com.cm_id->rem_ref(ep->com.cm_id); 452178786Skmacy ep->com.cm_id = NULL; 453178786Skmacy ep->com.qp = NULL; 454178786Skmacy } 455178786Skmacy} 456178786Skmacy 457178786Skmacystatic void 458178786Skmacyabort_connection(struct iwch_ep *ep) 459178786Skmacy{ 460178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 461178786Skmacy state_set(&ep->com, ABORTING); 462178786Skmacy abort_socket(ep); 463237263Snp close_socket(&ep->com, 0); 464178786Skmacy close_complete_upcall(ep); 465178786Skmacy state_set(&ep->com, DEAD); 466178786Skmacy put_ep(&ep->com); 467178786Skmacy} 468178786Skmacy 469178786Skmacystatic void 470178786Skmacypeer_close_upcall(struct iwch_ep *ep) 471178786Skmacy{ 472178786Skmacy struct iw_cm_event event; 473178786Skmacy 474178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 475178786Skmacy memset(&event, 0, sizeof(event)); 476178786Skmacy event.event = IW_CM_EVENT_DISCONNECT; 477178786Skmacy if (ep->com.cm_id) { 478178786Skmacy CTR3(KTR_IW_CXGB, "peer close delivered ep %p cm_id %p tid %d", 479178786Skmacy ep, ep->com.cm_id, ep->hwtid); 480178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 481178786Skmacy } 482178786Skmacy} 483178786Skmacy 484178786Skmacystatic void 485178786Skmacypeer_abort_upcall(struct iwch_ep *ep) 486178786Skmacy{ 487178786Skmacy struct iw_cm_event event; 488178786Skmacy 489178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 490178786Skmacy memset(&event, 0, sizeof(event)); 491178786Skmacy event.event = IW_CM_EVENT_CLOSE; 492178786Skmacy event.status = ECONNRESET; 493178786Skmacy if (ep->com.cm_id) { 494178786Skmacy CTR3(KTR_IW_CXGB, "abort delivered ep %p cm_id %p tid %d", ep, 495178786Skmacy ep->com.cm_id, ep->hwtid); 496178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 497178786Skmacy ep->com.cm_id->rem_ref(ep->com.cm_id); 498178786Skmacy ep->com.cm_id = NULL; 499178786Skmacy ep->com.qp = NULL; 500178786Skmacy } 501178786Skmacy} 502178786Skmacy 503178786Skmacystatic void 504178786Skmacyconnect_reply_upcall(struct iwch_ep *ep, int status) 505178786Skmacy{ 506178786Skmacy struct iw_cm_event event; 507178786Skmacy 508178786Skmacy CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], status); 509178786Skmacy memset(&event, 0, sizeof(event)); 510178786Skmacy event.event = IW_CM_EVENT_CONNECT_REPLY; 511178786Skmacy event.status = status; 512178786Skmacy event.local_addr = ep->com.local_addr; 513178786Skmacy event.remote_addr = ep->com.remote_addr; 514178786Skmacy 515178786Skmacy if ((status == 0) || (status == ECONNREFUSED)) { 516178786Skmacy event.private_data_len = ep->plen; 517178786Skmacy event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 518178786Skmacy } 519178786Skmacy if (ep->com.cm_id) { 520178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p tid %d status %d", __FUNCTION__, ep, 521178786Skmacy ep->hwtid, status); 522178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 523178786Skmacy } 524178786Skmacy if (status < 0) { 525178786Skmacy ep->com.cm_id->rem_ref(ep->com.cm_id); 526178786Skmacy ep->com.cm_id = NULL; 527178786Skmacy ep->com.qp = NULL; 528178786Skmacy } 529178786Skmacy} 530178786Skmacy 531178786Skmacystatic void 532178786Skmacyconnect_request_upcall(struct iwch_ep *ep) 533178786Skmacy{ 534178786Skmacy struct iw_cm_event event; 535178786Skmacy 536178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 537178786Skmacy memset(&event, 0, sizeof(event)); 538178786Skmacy event.event = IW_CM_EVENT_CONNECT_REQUEST; 539178786Skmacy event.local_addr = ep->com.local_addr; 540178786Skmacy event.remote_addr = ep->com.remote_addr; 541178786Skmacy event.private_data_len = ep->plen; 542178786Skmacy event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 543178786Skmacy event.provider_data = ep; 544178786Skmacy event.so = ep->com.so; 545237263Snp if (state_read(&ep->parent_ep->com) != DEAD) { 546237263Snp get_ep(&ep->com); 547178786Skmacy ep->parent_ep->com.cm_id->event_handler( 548178786Skmacy ep->parent_ep->com.cm_id, 549178786Skmacy &event); 550237263Snp } 551178786Skmacy put_ep(&ep->parent_ep->com); 552178786Skmacy} 553178786Skmacy 554178786Skmacystatic void 555178786Skmacyestablished_upcall(struct iwch_ep *ep) 556178786Skmacy{ 557178786Skmacy struct iw_cm_event event; 558178786Skmacy 559178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 560178786Skmacy memset(&event, 0, sizeof(event)); 561178786Skmacy event.event = IW_CM_EVENT_ESTABLISHED; 562178786Skmacy if (ep->com.cm_id) { 563178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p tid %d", __FUNCTION__, ep, ep->hwtid); 564178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 565178786Skmacy } 566178786Skmacy} 567178786Skmacy 568178786Skmacystatic void 569178786Skmacyprocess_mpa_reply(struct iwch_ep *ep) 570178786Skmacy{ 571178786Skmacy struct mpa_message *mpa; 572178786Skmacy u16 plen; 573178786Skmacy struct iwch_qp_attributes attrs; 574178786Skmacy enum iwch_qp_attr_mask mask; 575178786Skmacy int err; 576178786Skmacy struct mbuf *top, *m; 577178786Skmacy int flags = MSG_DONTWAIT; 578178786Skmacy struct uio uio; 579178786Skmacy int len; 580178786Skmacy 581178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 582178786Skmacy 583178786Skmacy /* 584178786Skmacy * Stop mpa timer. If it expired, then the state has 585178786Skmacy * changed and we bail since ep_timeout already aborted 586178786Skmacy * the connection. 587178786Skmacy */ 588178786Skmacy stop_ep_timer(ep); 589178786Skmacy if (state_read(&ep->com) != MPA_REQ_SENT) 590178786Skmacy return; 591178786Skmacy 592178786Skmacy uio.uio_resid = len = 1000000; 593178786Skmacy uio.uio_td = ep->com.thread; 594178786Skmacy err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 595178786Skmacy if (err) { 596178786Skmacy if (err == EWOULDBLOCK) { 597178786Skmacy start_ep_timer(ep); 598178786Skmacy return; 599178786Skmacy } 600178786Skmacy err = -err; 601178786Skmacy goto err; 602178786Skmacy } 603178786Skmacy 604178786Skmacy if (ep->com.so->so_rcv.sb_mb) { 605178786Skmacy printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 606178786Skmacy __FUNCTION__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 607178786Skmacy } 608178786Skmacy 609178786Skmacy m = top; 610178786Skmacy do { 611178786Skmacy /* 612178786Skmacy * If we get more than the supported amount of private data 613178786Skmacy * then we must fail this connection. 614178786Skmacy */ 615178786Skmacy if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 616178786Skmacy err = (-EINVAL); 617178786Skmacy goto err; 618178786Skmacy } 619178786Skmacy 620178786Skmacy /* 621178786Skmacy * copy the new data into our accumulation buffer. 622178786Skmacy */ 623178786Skmacy m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 624178786Skmacy ep->mpa_pkt_len += m->m_len; 625178786Skmacy if (!m->m_next) 626178786Skmacy m = m->m_nextpkt; 627178786Skmacy else 628178786Skmacy m = m->m_next; 629178786Skmacy } while (m); 630178786Skmacy 631178786Skmacy m_freem(top); 632178786Skmacy 633178786Skmacy /* 634178786Skmacy * if we don't even have the mpa message, then bail. 635178786Skmacy */ 636178786Skmacy if (ep->mpa_pkt_len < sizeof(*mpa)) 637178786Skmacy return; 638178786Skmacy mpa = (struct mpa_message *)ep->mpa_pkt; 639178786Skmacy 640178786Skmacy /* Validate MPA header. */ 641178786Skmacy if (mpa->revision != mpa_rev) { 642178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision); 643178786Skmacy err = EPROTO; 644178786Skmacy goto err; 645178786Skmacy } 646178786Skmacy if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 647178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key); 648178786Skmacy err = EPROTO; 649178786Skmacy goto err; 650178786Skmacy } 651178786Skmacy 652178786Skmacy plen = ntohs(mpa->private_data_size); 653178786Skmacy 654178786Skmacy /* 655178786Skmacy * Fail if there's too much private data. 656178786Skmacy */ 657178786Skmacy if (plen > MPA_MAX_PRIVATE_DATA) { 658178786Skmacy CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen); 659178786Skmacy err = EPROTO; 660178786Skmacy goto err; 661178786Skmacy } 662178786Skmacy 663178786Skmacy /* 664178786Skmacy * If plen does not account for pkt size 665178786Skmacy */ 666178786Skmacy if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 667178786Skmacy CTR2(KTR_IW_CXGB, "%s pkt too big %d", __FUNCTION__, ep->mpa_pkt_len); 668178786Skmacy err = EPROTO; 669178786Skmacy goto err; 670178786Skmacy } 671178786Skmacy 672178786Skmacy ep->plen = (u8) plen; 673178786Skmacy 674178786Skmacy /* 675178786Skmacy * If we don't have all the pdata yet, then bail. 676178786Skmacy * We'll continue process when more data arrives. 677178786Skmacy */ 678178786Skmacy if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 679178786Skmacy return; 680178786Skmacy 681178786Skmacy if (mpa->flags & MPA_REJECT) { 682178786Skmacy err = ECONNREFUSED; 683178786Skmacy goto err; 684178786Skmacy } 685178786Skmacy 686178786Skmacy /* 687178786Skmacy * If we get here we have accumulated the entire mpa 688178786Skmacy * start reply message including private data. And 689178786Skmacy * the MPA header is valid. 690178786Skmacy */ 691178786Skmacy CTR1(KTR_IW_CXGB, "%s mpa rpl looks good!", __FUNCTION__); 692178786Skmacy state_set(&ep->com, FPDU_MODE); 693237263Snp ep->mpa_attr.initiator = 1; 694178786Skmacy ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 695178786Skmacy ep->mpa_attr.recv_marker_enabled = markers_enabled; 696178786Skmacy ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 697178786Skmacy ep->mpa_attr.version = mpa_rev; 698178786Skmacy if (set_tcpinfo(ep)) { 699178786Skmacy printf("%s set_tcpinfo error\n", __FUNCTION__); 700178786Skmacy goto err; 701178786Skmacy } 702178786Skmacy CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, " 703178786Skmacy "xmit_marker_enabled=%d, version=%d", __FUNCTION__, 704178786Skmacy ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 705178786Skmacy ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 706178786Skmacy 707178786Skmacy attrs.mpa_attr = ep->mpa_attr; 708178786Skmacy attrs.max_ird = ep->ird; 709178786Skmacy attrs.max_ord = ep->ord; 710178786Skmacy attrs.llp_stream_handle = ep; 711178786Skmacy attrs.next_state = IWCH_QP_STATE_RTS; 712178786Skmacy 713178786Skmacy mask = IWCH_QP_ATTR_NEXT_STATE | 714178786Skmacy IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR | 715178786Skmacy IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD; 716178786Skmacy 717178786Skmacy /* bind QP and TID with INIT_WR */ 718178786Skmacy err = iwch_modify_qp(ep->com.qp->rhp, 719178786Skmacy ep->com.qp, mask, &attrs, 1); 720178786Skmacy if (!err) 721178786Skmacy goto out; 722178786Skmacyerr: 723178786Skmacy abort_connection(ep); 724178786Skmacyout: 725178786Skmacy connect_reply_upcall(ep, err); 726178786Skmacy return; 727178786Skmacy} 728178786Skmacy 729178786Skmacystatic void 730178786Skmacyprocess_mpa_request(struct iwch_ep *ep) 731178786Skmacy{ 732178786Skmacy struct mpa_message *mpa; 733178786Skmacy u16 plen; 734178786Skmacy int flags = MSG_DONTWAIT; 735178786Skmacy struct mbuf *top, *m; 736178786Skmacy int err; 737178786Skmacy struct uio uio; 738178786Skmacy int len; 739178786Skmacy 740178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 741178786Skmacy 742178786Skmacy /* 743178786Skmacy * Stop mpa timer. If it expired, then the state has 744178786Skmacy * changed and we bail since ep_timeout already aborted 745178786Skmacy * the connection. 746178786Skmacy */ 747178786Skmacy stop_ep_timer(ep); 748178786Skmacy if (state_read(&ep->com) != MPA_REQ_WAIT) 749178786Skmacy return; 750178786Skmacy 751178786Skmacy uio.uio_resid = len = 1000000; 752178786Skmacy uio.uio_td = ep->com.thread; 753178786Skmacy err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 754178786Skmacy if (err) { 755178786Skmacy if (err == EWOULDBLOCK) { 756178786Skmacy start_ep_timer(ep); 757178786Skmacy return; 758178786Skmacy } 759178786Skmacy err = -err; 760178786Skmacy goto err; 761178786Skmacy } 762178786Skmacy 763178786Skmacy m = top; 764178786Skmacy do { 765178786Skmacy 766178786Skmacy /* 767178786Skmacy * If we get more than the supported amount of private data 768178786Skmacy * then we must fail this connection. 769178786Skmacy */ 770178786Skmacy if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 771178786Skmacy CTR2(KTR_IW_CXGB, "%s mpa message too big %d", __FUNCTION__, 772178786Skmacy ep->mpa_pkt_len + m->m_len); 773178786Skmacy goto err; 774178786Skmacy } 775178786Skmacy 776178786Skmacy 777178786Skmacy /* 778178786Skmacy * Copy the new data into our accumulation buffer. 779178786Skmacy */ 780178786Skmacy m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 781178786Skmacy ep->mpa_pkt_len += m->m_len; 782178786Skmacy 783178786Skmacy if (!m->m_next) 784178786Skmacy m = m->m_nextpkt; 785178786Skmacy else 786178786Skmacy m = m->m_next; 787178786Skmacy } while (m); 788178786Skmacy 789178786Skmacy m_freem(top); 790178786Skmacy 791178786Skmacy /* 792178786Skmacy * If we don't even have the mpa message, then bail. 793178786Skmacy * We'll continue process when more data arrives. 794178786Skmacy */ 795178786Skmacy if (ep->mpa_pkt_len < sizeof(*mpa)) { 796178786Skmacy start_ep_timer(ep); 797178786Skmacy CTR2(KTR_IW_CXGB, "%s not enough header %d...waiting...", __FUNCTION__, 798178786Skmacy ep->mpa_pkt_len); 799178786Skmacy return; 800178786Skmacy } 801178786Skmacy mpa = (struct mpa_message *) ep->mpa_pkt; 802178786Skmacy 803178786Skmacy /* 804178786Skmacy * Validate MPA Header. 805178786Skmacy */ 806178786Skmacy if (mpa->revision != mpa_rev) { 807178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision); 808178786Skmacy goto err; 809178786Skmacy } 810178786Skmacy 811178786Skmacy if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 812178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key); 813178786Skmacy goto err; 814178786Skmacy } 815178786Skmacy 816178786Skmacy plen = ntohs(mpa->private_data_size); 817178786Skmacy 818178786Skmacy /* 819178786Skmacy * Fail if there's too much private data. 820178786Skmacy */ 821178786Skmacy if (plen > MPA_MAX_PRIVATE_DATA) { 822178786Skmacy CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen); 823178786Skmacy goto err; 824178786Skmacy } 825178786Skmacy 826178786Skmacy /* 827178786Skmacy * If plen does not account for pkt size 828178786Skmacy */ 829178786Skmacy if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 830178786Skmacy CTR2(KTR_IW_CXGB, "%s more data after private data %d", __FUNCTION__, 831178786Skmacy ep->mpa_pkt_len); 832178786Skmacy goto err; 833178786Skmacy } 834178786Skmacy ep->plen = (u8) plen; 835178786Skmacy 836178786Skmacy /* 837178786Skmacy * If we don't have all the pdata yet, then bail. 838178786Skmacy */ 839178786Skmacy if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 840178786Skmacy start_ep_timer(ep); 841178786Skmacy CTR2(KTR_IW_CXGB, "%s more mpa msg to come %d", __FUNCTION__, 842178786Skmacy ep->mpa_pkt_len); 843178786Skmacy return; 844178786Skmacy } 845178786Skmacy 846178786Skmacy /* 847178786Skmacy * If we get here we have accumulated the entire mpa 848178786Skmacy * start reply message including private data. 849178786Skmacy */ 850237263Snp ep->mpa_attr.initiator = 0; 851178786Skmacy ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 852178786Skmacy ep->mpa_attr.recv_marker_enabled = markers_enabled; 853178786Skmacy ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 854178786Skmacy ep->mpa_attr.version = mpa_rev; 855178786Skmacy if (set_tcpinfo(ep)) { 856178786Skmacy printf("%s set_tcpinfo error\n", __FUNCTION__); 857178786Skmacy goto err; 858178786Skmacy } 859178786Skmacy CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, " 860178786Skmacy "xmit_marker_enabled=%d, version=%d", __FUNCTION__, 861178786Skmacy ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 862178786Skmacy ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 863178786Skmacy 864178786Skmacy state_set(&ep->com, MPA_REQ_RCVD); 865178786Skmacy 866178786Skmacy /* drive upcall */ 867178786Skmacy connect_request_upcall(ep); 868178786Skmacy return; 869178786Skmacyerr: 870178786Skmacy abort_connection(ep); 871178786Skmacy return; 872178786Skmacy} 873178786Skmacy 874178786Skmacystatic void 875178786Skmacyprocess_peer_close(struct iwch_ep *ep) 876178786Skmacy{ 877178786Skmacy struct iwch_qp_attributes attrs; 878178786Skmacy int disconnect = 1; 879178786Skmacy int release = 0; 880178786Skmacy 881178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 882178786Skmacy 883178786Skmacy mtx_lock(&ep->com.lock); 884178786Skmacy switch (ep->com.state) { 885178786Skmacy case MPA_REQ_WAIT: 886178786Skmacy __state_set(&ep->com, CLOSING); 887178786Skmacy break; 888178786Skmacy case MPA_REQ_SENT: 889178786Skmacy __state_set(&ep->com, CLOSING); 890178786Skmacy connect_reply_upcall(ep, -ECONNRESET); 891178786Skmacy break; 892178786Skmacy case MPA_REQ_RCVD: 893178786Skmacy 894178786Skmacy /* 895178786Skmacy * We're gonna mark this puppy DEAD, but keep 896178786Skmacy * the reference on it until the ULP accepts or 897178786Skmacy * rejects the CR. 898178786Skmacy */ 899178786Skmacy __state_set(&ep->com, CLOSING); 900178786Skmacy break; 901178786Skmacy case MPA_REP_SENT: 902178786Skmacy __state_set(&ep->com, CLOSING); 903178786Skmacy break; 904178786Skmacy case FPDU_MODE: 905178786Skmacy start_ep_timer(ep); 906178786Skmacy __state_set(&ep->com, CLOSING); 907178786Skmacy attrs.next_state = IWCH_QP_STATE_CLOSING; 908178786Skmacy iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, 909178786Skmacy IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); 910178786Skmacy peer_close_upcall(ep); 911178786Skmacy break; 912178786Skmacy case ABORTING: 913178786Skmacy disconnect = 0; 914178786Skmacy break; 915178786Skmacy case CLOSING: 916178786Skmacy __state_set(&ep->com, MORIBUND); 917178786Skmacy disconnect = 0; 918178786Skmacy break; 919178786Skmacy case MORIBUND: 920178786Skmacy stop_ep_timer(ep); 921178786Skmacy if (ep->com.cm_id && ep->com.qp) { 922178786Skmacy attrs.next_state = IWCH_QP_STATE_IDLE; 923178786Skmacy iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, 924178786Skmacy IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); 925178786Skmacy } 926237263Snp close_socket(&ep->com, 0); 927178786Skmacy close_complete_upcall(ep); 928178786Skmacy __state_set(&ep->com, DEAD); 929178786Skmacy release = 1; 930178786Skmacy disconnect = 0; 931178786Skmacy break; 932178786Skmacy case DEAD: 933178786Skmacy disconnect = 0; 934178786Skmacy break; 935178786Skmacy default: 936178786Skmacy PANIC_IF(1); 937178786Skmacy } 938178786Skmacy mtx_unlock(&ep->com.lock); 939178786Skmacy if (disconnect) 940178786Skmacy iwch_ep_disconnect(ep, 0, M_NOWAIT); 941178786Skmacy if (release) 942178786Skmacy put_ep(&ep->com); 943178786Skmacy return; 944178786Skmacy} 945178786Skmacy 946178786Skmacystatic void 947178786Skmacyprocess_conn_error(struct iwch_ep *ep) 948178786Skmacy{ 949178786Skmacy struct iwch_qp_attributes attrs; 950178786Skmacy int ret; 951178786Skmacy 952237263Snp mtx_lock(&ep->com.lock); 953237263Snp CTR3(KTR_IW_CXGB, "%s ep %p state %u", __func__, ep, ep->com.state); 954237263Snp switch (ep->com.state) { 955178786Skmacy case MPA_REQ_WAIT: 956178786Skmacy stop_ep_timer(ep); 957178786Skmacy break; 958178786Skmacy case MPA_REQ_SENT: 959178786Skmacy stop_ep_timer(ep); 960178786Skmacy connect_reply_upcall(ep, -ECONNRESET); 961178786Skmacy break; 962178786Skmacy case MPA_REP_SENT: 963178786Skmacy ep->com.rpl_err = ECONNRESET; 964178786Skmacy CTR1(KTR_IW_CXGB, "waking up ep %p", ep); 965178786Skmacy break; 966178786Skmacy case MPA_REQ_RCVD: 967178786Skmacy 968178786Skmacy /* 969178786Skmacy * We're gonna mark this puppy DEAD, but keep 970178786Skmacy * the reference on it until the ULP accepts or 971178786Skmacy * rejects the CR. 972178786Skmacy */ 973178786Skmacy break; 974178786Skmacy case MORIBUND: 975178786Skmacy case CLOSING: 976178786Skmacy stop_ep_timer(ep); 977178786Skmacy /*FALLTHROUGH*/ 978178786Skmacy case FPDU_MODE: 979178786Skmacy if (ep->com.cm_id && ep->com.qp) { 980178786Skmacy attrs.next_state = IWCH_QP_STATE_ERROR; 981178786Skmacy ret = iwch_modify_qp(ep->com.qp->rhp, 982178786Skmacy ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, 983178786Skmacy &attrs, 1); 984178786Skmacy if (ret) 985178786Skmacy log(LOG_ERR, 986178786Skmacy "%s - qp <- error failed!\n", 987178786Skmacy __FUNCTION__); 988178786Skmacy } 989178786Skmacy peer_abort_upcall(ep); 990178786Skmacy break; 991178786Skmacy case ABORTING: 992178786Skmacy break; 993178786Skmacy case DEAD: 994237263Snp mtx_unlock(&ep->com.lock); 995178786Skmacy CTR2(KTR_IW_CXGB, "%s so_error %d IN DEAD STATE!!!!", __FUNCTION__, 996178786Skmacy ep->com.so->so_error); 997178786Skmacy return; 998178786Skmacy default: 999178786Skmacy PANIC_IF(1); 1000178786Skmacy break; 1001178786Skmacy } 1002178786Skmacy 1003237263Snp if (ep->com.state != ABORTING) { 1004237263Snp close_socket(&ep->com, 0); 1005237263Snp __state_set(&ep->com, DEAD); 1006178786Skmacy put_ep(&ep->com); 1007178786Skmacy } 1008237263Snp mtx_unlock(&ep->com.lock); 1009178786Skmacy return; 1010178786Skmacy} 1011178786Skmacy 1012178786Skmacystatic void 1013178786Skmacyprocess_close_complete(struct iwch_ep *ep) 1014178786Skmacy{ 1015178786Skmacy struct iwch_qp_attributes attrs; 1016178786Skmacy int release = 0; 1017178786Skmacy 1018178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1019178786Skmacy PANIC_IF(!ep); 1020178786Skmacy 1021178786Skmacy /* The cm_id may be null if we failed to connect */ 1022178786Skmacy mtx_lock(&ep->com.lock); 1023178786Skmacy switch (ep->com.state) { 1024178786Skmacy case CLOSING: 1025178786Skmacy __state_set(&ep->com, MORIBUND); 1026178786Skmacy break; 1027178786Skmacy case MORIBUND: 1028178786Skmacy stop_ep_timer(ep); 1029178786Skmacy if ((ep->com.cm_id) && (ep->com.qp)) { 1030178786Skmacy attrs.next_state = IWCH_QP_STATE_IDLE; 1031178786Skmacy iwch_modify_qp(ep->com.qp->rhp, 1032178786Skmacy ep->com.qp, 1033178786Skmacy IWCH_QP_ATTR_NEXT_STATE, 1034178786Skmacy &attrs, 1); 1035178786Skmacy } 1036237263Snp if (ep->parent_ep) 1037237263Snp close_socket(&ep->com, 1); 1038237263Snp else 1039237263Snp close_socket(&ep->com, 0); 1040178786Skmacy close_complete_upcall(ep); 1041178786Skmacy __state_set(&ep->com, DEAD); 1042178786Skmacy release = 1; 1043178786Skmacy break; 1044178786Skmacy case ABORTING: 1045178786Skmacy break; 1046178786Skmacy case DEAD: 1047178786Skmacy default: 1048178786Skmacy PANIC_IF(1); 1049178786Skmacy break; 1050178786Skmacy } 1051178786Skmacy mtx_unlock(&ep->com.lock); 1052178786Skmacy if (release) 1053178786Skmacy put_ep(&ep->com); 1054178786Skmacy return; 1055178786Skmacy} 1056178786Skmacy 1057178786Skmacy/* 1058178786Skmacy * T3A does 3 things when a TERM is received: 1059178786Skmacy * 1) send up a CPL_RDMA_TERMINATE message with the TERM packet 1060178786Skmacy * 2) generate an async event on the QP with the TERMINATE opcode 1061178786Skmacy * 3) post a TERMINATE opcde cqe into the associated CQ. 1062178786Skmacy * 1063178786Skmacy * For (1), we save the message in the qp for later consumer consumption. 1064178786Skmacy * For (2), we move the QP into TERMINATE, post a QP event and disconnect. 1065178786Skmacy * For (3), we toss the CQE in cxio_poll_cq(). 1066178786Skmacy * 1067178786Skmacy * terminate() handles case (1)... 1068178786Skmacy */ 1069178786Skmacystatic int 1070237263Snpterminate(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 1071178786Skmacy{ 1072237263Snp struct adapter *sc = qs->adap; 1073237263Snp struct tom_data *td = sc->tom_softc; 1074237263Snp uint32_t hash = *((uint32_t *)r + 1); 1075237263Snp unsigned int tid = ntohl(hash) >> 8 & 0xfffff; 1076237263Snp struct toepcb *toep = lookup_tid(&td->tid_maps, tid); 1077237263Snp struct socket *so = toep->tp_inp->inp_socket; 1078193272Sjhb struct iwch_ep *ep = so->so_rcv.sb_upcallarg; 1079178786Skmacy 1080237263Snp if (state_read(&ep->com) != FPDU_MODE) 1081237263Snp goto done; 1082237263Snp 1083178786Skmacy m_adj(m, sizeof(struct cpl_rdma_terminate)); 1084237263Snp 1085237263Snp CTR4(KTR_IW_CXGB, "%s: tid %u, ep %p, saved %d bytes", 1086237263Snp __func__, tid, ep, m->m_len); 1087237263Snp 1088178786Skmacy m_copydata(m, 0, m->m_len, ep->com.qp->attr.terminate_buffer); 1089178786Skmacy ep->com.qp->attr.terminate_msg_len = m->m_len; 1090178786Skmacy ep->com.qp->attr.is_terminate_local = 0; 1091237263Snp 1092237263Snpdone: 1093237263Snp m_freem(m); 1094237263Snp return (0); 1095178786Skmacy} 1096178786Skmacy 1097178786Skmacystatic int 1098237263Snpec_status(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 1099178786Skmacy{ 1100237263Snp struct adapter *sc = qs->adap; 1101237263Snp struct tom_data *td = sc->tom_softc; 1102237263Snp struct cpl_rdma_ec_status *rep = mtod(m, void *); 1103237263Snp unsigned int tid = GET_TID(rep); 1104237263Snp struct toepcb *toep = lookup_tid(&td->tid_maps, tid); 1105237263Snp struct socket *so = toep->tp_inp->inp_socket; 1106237263Snp struct iwch_ep *ep = so->so_rcv.sb_upcallarg; 1107178786Skmacy 1108237263Snp if (rep->status) { 1109237263Snp struct iwch_qp_attributes attrs; 1110237263Snp 1111237263Snp CTR1(KTR_IW_CXGB, "%s BAD CLOSE - Aborting", __FUNCTION__); 1112178786Skmacy stop_ep_timer(ep); 1113178786Skmacy attrs.next_state = IWCH_QP_STATE_ERROR; 1114178786Skmacy iwch_modify_qp(ep->com.qp->rhp, 1115237263Snp ep->com.qp, 1116237263Snp IWCH_QP_ATTR_NEXT_STATE, 1117237263Snp &attrs, 1); 1118237263Snp abort_connection(ep); 1119178786Skmacy } 1120237263Snp 1121237263Snp m_freem(m); 1122237263Snp return (0); 1123178786Skmacy} 1124178786Skmacy 1125178786Skmacystatic void 1126178786Skmacyep_timeout(void *arg) 1127178786Skmacy{ 1128178786Skmacy struct iwch_ep *ep = (struct iwch_ep *)arg; 1129178786Skmacy struct iwch_qp_attributes attrs; 1130178786Skmacy int err = 0; 1131237263Snp int abort = 1; 1132178786Skmacy 1133178786Skmacy mtx_lock(&ep->com.lock); 1134178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1135178786Skmacy switch (ep->com.state) { 1136178786Skmacy case MPA_REQ_SENT: 1137237263Snp __state_set(&ep->com, ABORTING); 1138178786Skmacy connect_reply_upcall(ep, -ETIMEDOUT); 1139178786Skmacy break; 1140178786Skmacy case MPA_REQ_WAIT: 1141237263Snp __state_set(&ep->com, ABORTING); 1142178786Skmacy break; 1143178786Skmacy case CLOSING: 1144178786Skmacy case MORIBUND: 1145178786Skmacy if (ep->com.cm_id && ep->com.qp) 1146178786Skmacy err = 1; 1147237263Snp __state_set(&ep->com, ABORTING); 1148178786Skmacy break; 1149178786Skmacy default: 1150237263Snp CTR3(KTR_IW_CXGB, "%s unexpected state ep %p state %u\n", 1151237263Snp __func__, ep, ep->com.state); 1152237263Snp abort = 0; 1153178786Skmacy } 1154178786Skmacy mtx_unlock(&ep->com.lock); 1155178786Skmacy if (err){ 1156178786Skmacy attrs.next_state = IWCH_QP_STATE_ERROR; 1157178786Skmacy iwch_modify_qp(ep->com.qp->rhp, 1158178786Skmacy ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, 1159178786Skmacy &attrs, 1); 1160178786Skmacy } 1161237263Snp if (abort) 1162237263Snp abort_connection(ep); 1163178786Skmacy put_ep(&ep->com); 1164178786Skmacy} 1165178786Skmacy 1166178786Skmacyint 1167178786Skmacyiwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 1168178786Skmacy{ 1169178786Skmacy int err; 1170178786Skmacy struct iwch_ep *ep = to_ep(cm_id); 1171178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1172178786Skmacy 1173178786Skmacy if (state_read(&ep->com) == DEAD) { 1174178786Skmacy put_ep(&ep->com); 1175178786Skmacy return (-ECONNRESET); 1176178786Skmacy } 1177178786Skmacy PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD); 1178178786Skmacy if (mpa_rev == 0) { 1179178786Skmacy abort_connection(ep); 1180178786Skmacy } else { 1181178786Skmacy err = send_mpa_reject(ep, pdata, pdata_len); 1182178786Skmacy err = soshutdown(ep->com.so, 3); 1183178786Skmacy } 1184237263Snp put_ep(&ep->com); 1185178786Skmacy return 0; 1186178786Skmacy} 1187178786Skmacy 1188178786Skmacyint 1189178786Skmacyiwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 1190178786Skmacy{ 1191178786Skmacy int err; 1192178786Skmacy struct iwch_qp_attributes attrs; 1193178786Skmacy enum iwch_qp_attr_mask mask; 1194178786Skmacy struct iwch_ep *ep = to_ep(cm_id); 1195178786Skmacy struct iwch_dev *h = to_iwch_dev(cm_id->device); 1196178786Skmacy struct iwch_qp *qp = get_qhp(h, conn_param->qpn); 1197178786Skmacy 1198178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1199237263Snp if (state_read(&ep->com) == DEAD) { 1200237263Snp err = -ECONNRESET; 1201237263Snp goto err; 1202237263Snp } 1203178786Skmacy 1204178786Skmacy PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD); 1205178786Skmacy PANIC_IF(!qp); 1206178786Skmacy 1207178786Skmacy if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) || 1208178786Skmacy (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) { 1209178786Skmacy abort_connection(ep); 1210237263Snp err = -EINVAL; 1211237263Snp goto err; 1212178786Skmacy } 1213178786Skmacy 1214178786Skmacy cm_id->add_ref(cm_id); 1215178786Skmacy ep->com.cm_id = cm_id; 1216178786Skmacy ep->com.qp = qp; 1217178786Skmacy 1218178786Skmacy ep->com.rpl_err = 0; 1219178786Skmacy ep->com.rpl_done = 0; 1220178786Skmacy ep->ird = conn_param->ird; 1221178786Skmacy ep->ord = conn_param->ord; 1222178786Skmacy CTR3(KTR_IW_CXGB, "%s ird %d ord %d", __FUNCTION__, ep->ird, ep->ord); 1223178786Skmacy 1224178786Skmacy /* bind QP to EP and move to RTS */ 1225178786Skmacy attrs.mpa_attr = ep->mpa_attr; 1226237263Snp attrs.max_ird = ep->ird; 1227178786Skmacy attrs.max_ord = ep->ord; 1228178786Skmacy attrs.llp_stream_handle = ep; 1229178786Skmacy attrs.next_state = IWCH_QP_STATE_RTS; 1230178786Skmacy 1231178786Skmacy /* bind QP and TID with INIT_WR */ 1232178786Skmacy mask = IWCH_QP_ATTR_NEXT_STATE | 1233178786Skmacy IWCH_QP_ATTR_LLP_STREAM_HANDLE | 1234178786Skmacy IWCH_QP_ATTR_MPA_ATTR | 1235178786Skmacy IWCH_QP_ATTR_MAX_IRD | 1236178786Skmacy IWCH_QP_ATTR_MAX_ORD; 1237178786Skmacy 1238178786Skmacy err = iwch_modify_qp(ep->com.qp->rhp, 1239178786Skmacy ep->com.qp, mask, &attrs, 1); 1240178786Skmacy 1241178786Skmacy if (err) 1242237263Snp goto err1; 1243178786Skmacy 1244178786Skmacy err = send_mpa_reply(ep, conn_param->private_data, 1245178786Skmacy conn_param->private_data_len); 1246178786Skmacy if (err) 1247237263Snp goto err1; 1248178786Skmacy state_set(&ep->com, FPDU_MODE); 1249178786Skmacy established_upcall(ep); 1250178786Skmacy put_ep(&ep->com); 1251178786Skmacy return 0; 1252237263Snperr1: 1253178786Skmacy ep->com.cm_id = NULL; 1254178786Skmacy ep->com.qp = NULL; 1255178786Skmacy cm_id->rem_ref(cm_id); 1256237263Snperr: 1257178786Skmacy put_ep(&ep->com); 1258178786Skmacy return err; 1259178786Skmacy} 1260178786Skmacy 1261178786Skmacystatic int init_sock(struct iwch_ep_common *epc) 1262178786Skmacy{ 1263178786Skmacy int err; 1264178786Skmacy struct sockopt sopt; 1265178786Skmacy int on=1; 1266178786Skmacy 1267193272Sjhb SOCK_LOCK(epc->so); 1268193272Sjhb soupcall_set(epc->so, SO_RCV, iwch_so_upcall, epc); 1269178786Skmacy epc->so->so_state |= SS_NBIO; 1270193272Sjhb SOCK_UNLOCK(epc->so); 1271178786Skmacy sopt.sopt_dir = SOPT_SET; 1272178786Skmacy sopt.sopt_level = IPPROTO_TCP; 1273178786Skmacy sopt.sopt_name = TCP_NODELAY; 1274178786Skmacy sopt.sopt_val = (caddr_t)&on; 1275178786Skmacy sopt.sopt_valsize = sizeof on; 1276178786Skmacy sopt.sopt_td = NULL; 1277178786Skmacy err = sosetopt(epc->so, &sopt); 1278178786Skmacy if (err) 1279178786Skmacy printf("%s can't set TCP_NODELAY err %d\n", __FUNCTION__, err); 1280178786Skmacy 1281178786Skmacy return 0; 1282178786Skmacy} 1283178786Skmacy 1284178786Skmacystatic int 1285178786Skmacyis_loopback_dst(struct iw_cm_id *cm_id) 1286178786Skmacy{ 1287178786Skmacy uint16_t port = cm_id->remote_addr.sin_port; 1288194622Srwatson int ifa_present; 1289178786Skmacy 1290178786Skmacy cm_id->remote_addr.sin_port = 0; 1291194622Srwatson ifa_present = ifa_ifwithaddr_check( 1292194622Srwatson (struct sockaddr *)&cm_id->remote_addr); 1293178786Skmacy cm_id->remote_addr.sin_port = port; 1294194622Srwatson return (ifa_present); 1295178786Skmacy} 1296178786Skmacy 1297178786Skmacyint 1298178786Skmacyiwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 1299178786Skmacy{ 1300178786Skmacy int err = 0; 1301178786Skmacy struct iwch_dev *h = to_iwch_dev(cm_id->device); 1302178786Skmacy struct iwch_ep *ep; 1303178786Skmacy struct rtentry *rt; 1304178786Skmacy struct toedev *tdev; 1305178786Skmacy 1306178786Skmacy if (is_loopback_dst(cm_id)) { 1307178786Skmacy err = -ENOSYS; 1308178786Skmacy goto out; 1309178786Skmacy } 1310178786Skmacy 1311178786Skmacy ep = alloc_ep(sizeof(*ep), M_NOWAIT); 1312178786Skmacy if (!ep) { 1313178786Skmacy printf("%s - cannot alloc ep.\n", __FUNCTION__); 1314178786Skmacy err = (-ENOMEM); 1315178786Skmacy goto out; 1316178786Skmacy } 1317178786Skmacy callout_init(&ep->timer, TRUE); 1318178786Skmacy ep->plen = conn_param->private_data_len; 1319178786Skmacy if (ep->plen) 1320178786Skmacy memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 1321178786Skmacy conn_param->private_data, ep->plen); 1322178786Skmacy ep->ird = conn_param->ird; 1323178786Skmacy ep->ord = conn_param->ord; 1324178786Skmacy 1325178786Skmacy cm_id->add_ref(cm_id); 1326178786Skmacy ep->com.cm_id = cm_id; 1327178786Skmacy ep->com.qp = get_qhp(h, conn_param->qpn); 1328178786Skmacy ep->com.thread = curthread; 1329178786Skmacy PANIC_IF(!ep->com.qp); 1330178786Skmacy CTR4(KTR_IW_CXGB, "%s qpn 0x%x qp %p cm_id %p", __FUNCTION__, conn_param->qpn, 1331178786Skmacy ep->com.qp, cm_id); 1332178786Skmacy 1333178786Skmacy ep->com.so = cm_id->so; 1334178786Skmacy err = init_sock(&ep->com); 1335178786Skmacy if (err) 1336178786Skmacy goto fail2; 1337178786Skmacy 1338178786Skmacy /* find a route */ 1339178786Skmacy rt = find_route(cm_id->local_addr.sin_addr.s_addr, 1340178786Skmacy cm_id->remote_addr.sin_addr.s_addr, 1341178786Skmacy cm_id->local_addr.sin_port, 1342178786Skmacy cm_id->remote_addr.sin_port, IPTOS_LOWDELAY); 1343178786Skmacy if (!rt) { 1344178786Skmacy printf("%s - cannot find route.\n", __FUNCTION__); 1345178786Skmacy err = EHOSTUNREACH; 1346178786Skmacy goto fail2; 1347178786Skmacy } 1348178786Skmacy 1349178786Skmacy if (!(rt->rt_ifp->if_flags & IFCAP_TOE)) { 1350178786Skmacy printf("%s - interface not TOE capable.\n", __FUNCTION__); 1351237263Snp RTFREE(rt); 1352237263Snp goto fail2; 1353178786Skmacy } 1354178786Skmacy tdev = TOEDEV(rt->rt_ifp); 1355178786Skmacy if (tdev == NULL) { 1356178786Skmacy printf("%s - No toedev for interface.\n", __FUNCTION__); 1357237263Snp RTFREE(rt); 1358237263Snp goto fail2; 1359178786Skmacy } 1360178786Skmacy RTFREE(rt); 1361178786Skmacy 1362178786Skmacy state_set(&ep->com, CONNECTING); 1363178786Skmacy ep->com.local_addr = cm_id->local_addr; 1364178786Skmacy ep->com.remote_addr = cm_id->remote_addr; 1365178786Skmacy err = soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 1366178786Skmacy ep->com.thread); 1367178786Skmacy if (!err) 1368178786Skmacy goto out; 1369178786Skmacyfail2: 1370178786Skmacy put_ep(&ep->com); 1371178786Skmacyout: 1372178786Skmacy return err; 1373178786Skmacy} 1374178786Skmacy 1375178786Skmacyint 1376309378Sjhbiwch_create_listen_ep(struct iw_cm_id *cm_id, int backlog) 1377178786Skmacy{ 1378178786Skmacy int err = 0; 1379178786Skmacy struct iwch_listen_ep *ep; 1380178786Skmacy 1381178786Skmacy ep = alloc_ep(sizeof(*ep), M_NOWAIT); 1382178786Skmacy if (!ep) { 1383178786Skmacy printf("%s - cannot alloc ep.\n", __FUNCTION__); 1384178786Skmacy err = ENOMEM; 1385178786Skmacy goto out; 1386178786Skmacy } 1387178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 1388178786Skmacy cm_id->add_ref(cm_id); 1389178786Skmacy ep->com.cm_id = cm_id; 1390178786Skmacy ep->backlog = backlog; 1391178786Skmacy ep->com.local_addr = cm_id->local_addr; 1392178786Skmacy ep->com.thread = curthread; 1393178786Skmacy state_set(&ep->com, LISTEN); 1394178786Skmacy 1395178786Skmacy ep->com.so = cm_id->so; 1396309378Sjhb cm_id->provider_data = ep; 1397178786Skmacyout: 1398178786Skmacy return err; 1399178786Skmacy} 1400178786Skmacy 1401309378Sjhbvoid 1402309378Sjhbiwch_destroy_listen_ep(struct iw_cm_id *cm_id) 1403178786Skmacy{ 1404178786Skmacy struct iwch_listen_ep *ep = to_listen_ep(cm_id); 1405178786Skmacy 1406178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 1407178786Skmacy 1408178786Skmacy state_set(&ep->com, DEAD); 1409178786Skmacy cm_id->rem_ref(cm_id); 1410178786Skmacy put_ep(&ep->com); 1411309378Sjhb return; 1412178786Skmacy} 1413178786Skmacy 1414178786Skmacyint 1415178786Skmacyiwch_ep_disconnect(struct iwch_ep *ep, int abrupt, int flags) 1416178786Skmacy{ 1417178786Skmacy int close = 0; 1418178786Skmacy 1419178786Skmacy mtx_lock(&ep->com.lock); 1420178786Skmacy 1421178786Skmacy PANIC_IF(!ep); 1422178786Skmacy PANIC_IF(!ep->com.so); 1423178786Skmacy 1424178786Skmacy CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s, abrupt %d", __FUNCTION__, ep, 1425178786Skmacy ep->com.so, states[ep->com.state], abrupt); 1426178786Skmacy 1427178786Skmacy switch (ep->com.state) { 1428178786Skmacy case MPA_REQ_WAIT: 1429178786Skmacy case MPA_REQ_SENT: 1430178786Skmacy case MPA_REQ_RCVD: 1431178786Skmacy case MPA_REP_SENT: 1432178786Skmacy case FPDU_MODE: 1433178786Skmacy close = 1; 1434237263Snp if (abrupt) 1435237263Snp ep->com.state = ABORTING; 1436237263Snp else { 1437237263Snp ep->com.state = CLOSING; 1438237263Snp start_ep_timer(ep); 1439237263Snp } 1440178786Skmacy break; 1441178786Skmacy case CLOSING: 1442178786Skmacy close = 1; 1443237263Snp if (abrupt) { 1444237263Snp stop_ep_timer(ep); 1445237263Snp ep->com.state = ABORTING; 1446237263Snp } else 1447237263Snp ep->com.state = MORIBUND; 1448178786Skmacy break; 1449178786Skmacy case MORIBUND: 1450178786Skmacy case ABORTING: 1451237263Snp case DEAD: 1452237263Snp CTR3(KTR_IW_CXGB, "%s ignoring disconnect ep %p state %u\n", 1453237263Snp __func__, ep, ep->com.state); 1454178786Skmacy break; 1455178786Skmacy default: 1456178786Skmacy panic("unknown state: %d\n", ep->com.state); 1457178786Skmacy break; 1458178786Skmacy } 1459237263Snp 1460178786Skmacy mtx_unlock(&ep->com.lock); 1461178786Skmacy if (close) { 1462178786Skmacy if (abrupt) 1463178786Skmacy abort_connection(ep); 1464237263Snp else { 1465237263Snp if (!ep->parent_ep) 1466237263Snp __state_set(&ep->com, MORIBUND); 1467178786Skmacy shutdown_socket(&ep->com); 1468237263Snp } 1469178786Skmacy } 1470178786Skmacy return 0; 1471178786Skmacy} 1472178786Skmacy 1473178786Skmacystatic void 1474178786Skmacyprocess_data(struct iwch_ep *ep) 1475178786Skmacy{ 1476178786Skmacy struct sockaddr_in *local, *remote; 1477178786Skmacy 1478178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1479178786Skmacy 1480178786Skmacy switch (state_read(&ep->com)) { 1481178786Skmacy case MPA_REQ_SENT: 1482178786Skmacy process_mpa_reply(ep); 1483178786Skmacy break; 1484178786Skmacy case MPA_REQ_WAIT: 1485178786Skmacy 1486178786Skmacy /* 1487178786Skmacy * XXX 1488178786Skmacy * Set local and remote addrs here because when we 1489178786Skmacy * dequeue the newly accepted socket, they aren't set 1490178786Skmacy * yet in the pcb! 1491178786Skmacy */ 1492178786Skmacy in_getsockaddr(ep->com.so, (struct sockaddr **)&local); 1493178786Skmacy in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); 1494178786Skmacy CTR3(KTR_IW_CXGB, "%s local %s remote %s", __FUNCTION__, 1495178786Skmacy inet_ntoa(local->sin_addr), 1496178786Skmacy inet_ntoa(remote->sin_addr)); 1497178786Skmacy ep->com.local_addr = *local; 1498178786Skmacy ep->com.remote_addr = *remote; 1499178786Skmacy free(local, M_SONAME); 1500178786Skmacy free(remote, M_SONAME); 1501178786Skmacy process_mpa_request(ep); 1502178786Skmacy break; 1503178786Skmacy default: 1504178786Skmacy if (ep->com.so->so_rcv.sb_cc) 1505178786Skmacy printf("%s Unexpected streaming data." 1506178786Skmacy " ep %p state %d so %p so_state %x so_rcv.sb_cc %u so_rcv.sb_mb %p\n", 1507178786Skmacy __FUNCTION__, ep, state_read(&ep->com), ep->com.so, ep->com.so->so_state, 1508178786Skmacy ep->com.so->so_rcv.sb_cc, ep->com.so->so_rcv.sb_mb); 1509178786Skmacy break; 1510178786Skmacy } 1511178786Skmacy return; 1512178786Skmacy} 1513178786Skmacy 1514178786Skmacystatic void 1515178786Skmacyprocess_connected(struct iwch_ep *ep) 1516178786Skmacy{ 1517178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1518178786Skmacy if ((ep->com.so->so_state & SS_ISCONNECTED) && !ep->com.so->so_error) { 1519178786Skmacy send_mpa_req(ep); 1520178786Skmacy } else { 1521178786Skmacy connect_reply_upcall(ep, -ep->com.so->so_error); 1522237263Snp close_socket(&ep->com, 0); 1523178786Skmacy state_set(&ep->com, DEAD); 1524178786Skmacy put_ep(&ep->com); 1525178786Skmacy } 1526178786Skmacy} 1527178786Skmacy 1528309378Sjhbvoid 1529309378Sjhbprocess_newconn(struct iw_cm_id *parent_cm_id, struct socket *child_so) 1530178786Skmacy{ 1531178786Skmacy struct iwch_ep *child_ep; 1532309378Sjhb struct sockaddr_in *local; 1533178786Skmacy struct sockaddr_in *remote; 1534309378Sjhb struct iwch_ep *parent_ep = parent_cm_id->provider_data; 1535178786Skmacy 1536178786Skmacy CTR3(KTR_IW_CXGB, "%s parent ep %p so %p", __FUNCTION__, parent_ep, parent_ep->com.so); 1537309378Sjhb if (!child_so) { 1538309378Sjhb log(LOG_ERR, "%s - invalid child socket!\n", __func__); 1539309378Sjhb return; 1540309378Sjhb } 1541178786Skmacy child_ep = alloc_ep(sizeof(*child_ep), M_NOWAIT); 1542178786Skmacy if (!child_ep) { 1543178786Skmacy log(LOG_ERR, "%s - failed to allocate ep entry!\n", 1544178786Skmacy __FUNCTION__); 1545178786Skmacy return; 1546178786Skmacy } 1547309378Sjhb SOCKBUF_LOCK(&child_so->so_rcv); 1548309378Sjhb soupcall_set(child_so, SO_RCV, iwch_so_upcall, child_ep); 1549309378Sjhb SOCKBUF_UNLOCK(&child_so->so_rcv); 1550309378Sjhb 1551309378Sjhb in_getsockaddr(child_so, (struct sockaddr **)&local); 1552309378Sjhb in_getpeeraddr(child_so, (struct sockaddr **)&remote); 1553309378Sjhb 1554178786Skmacy CTR3(KTR_IW_CXGB, "%s remote addr %s port %d", __FUNCTION__, 1555178786Skmacy inet_ntoa(remote->sin_addr), ntohs(remote->sin_port)); 1556237263Snp child_ep->com.tdev = parent_ep->com.tdev; 1557237263Snp child_ep->com.local_addr.sin_family = parent_ep->com.local_addr.sin_family; 1558237263Snp child_ep->com.local_addr.sin_port = parent_ep->com.local_addr.sin_port; 1559237263Snp child_ep->com.local_addr.sin_addr.s_addr = parent_ep->com.local_addr.sin_addr.s_addr; 1560237263Snp child_ep->com.local_addr.sin_len = parent_ep->com.local_addr.sin_len; 1561237263Snp child_ep->com.remote_addr.sin_family = remote->sin_family; 1562237263Snp child_ep->com.remote_addr.sin_port = remote->sin_port; 1563237263Snp child_ep->com.remote_addr.sin_addr.s_addr = remote->sin_addr.s_addr; 1564237263Snp child_ep->com.remote_addr.sin_len = remote->sin_len; 1565178786Skmacy child_ep->com.so = child_so; 1566178786Skmacy child_ep->com.cm_id = NULL; 1567178786Skmacy child_ep->com.thread = parent_ep->com.thread; 1568178786Skmacy child_ep->parent_ep = parent_ep; 1569237263Snp 1570309378Sjhb free(local, M_SONAME); 1571178786Skmacy free(remote, M_SONAME); 1572178786Skmacy get_ep(&parent_ep->com); 1573178786Skmacy callout_init(&child_ep->timer, TRUE); 1574178786Skmacy state_set(&child_ep->com, MPA_REQ_WAIT); 1575178786Skmacy start_ep_timer(child_ep); 1576178786Skmacy 1577178786Skmacy /* maybe the request has already been queued up on the socket... */ 1578178786Skmacy process_mpa_request(child_ep); 1579178786Skmacy} 1580178786Skmacy 1581193272Sjhbstatic int 1582178786Skmacyiwch_so_upcall(struct socket *so, void *arg, int waitflag) 1583178786Skmacy{ 1584178786Skmacy struct iwch_ep *ep = arg; 1585178786Skmacy 1586178786Skmacy CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]); 1587178786Skmacy mtx_lock(&req_lock); 1588178786Skmacy if (ep && ep->com.so && !ep->com.entry.tqe_prev) { 1589178786Skmacy get_ep(&ep->com); 1590178786Skmacy TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 1591178786Skmacy taskqueue_enqueue(iw_cxgb_taskq, &iw_cxgb_task); 1592178786Skmacy } 1593178786Skmacy mtx_unlock(&req_lock); 1594193272Sjhb return (SU_OK); 1595178786Skmacy} 1596178786Skmacy 1597178786Skmacystatic void 1598178786Skmacyprocess_socket_event(struct iwch_ep *ep) 1599178786Skmacy{ 1600178786Skmacy int state = state_read(&ep->com); 1601178786Skmacy struct socket *so = ep->com.so; 1602178786Skmacy 1603178786Skmacy CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]); 1604178786Skmacy if (state == CONNECTING) { 1605178786Skmacy process_connected(ep); 1606178786Skmacy return; 1607178786Skmacy } 1608178786Skmacy 1609178786Skmacy if (state == LISTEN) { 1610309378Sjhb /* socket listening events are handled at IWCM */ 1611309378Sjhb CTR3(KTR_IW_CXGB, "%s Invalid ep state:%u, ep:%p", __func__, 1612309378Sjhb ep->com.state, ep); 1613309378Sjhb BUG(); 1614178786Skmacy return; 1615178786Skmacy } 1616178786Skmacy 1617178786Skmacy /* connection error */ 1618178786Skmacy if (so->so_error) { 1619178786Skmacy process_conn_error(ep); 1620178786Skmacy return; 1621178786Skmacy } 1622178786Skmacy 1623178786Skmacy /* peer close */ 1624178786Skmacy if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state < CLOSING) { 1625178786Skmacy process_peer_close(ep); 1626178786Skmacy return; 1627178786Skmacy } 1628178786Skmacy 1629178786Skmacy /* close complete */ 1630178786Skmacy if (so->so_state & (SS_ISDISCONNECTED)) { 1631178786Skmacy process_close_complete(ep); 1632178786Skmacy return; 1633178786Skmacy } 1634178786Skmacy 1635178786Skmacy /* rx data */ 1636178786Skmacy process_data(ep); 1637178786Skmacy return; 1638178786Skmacy} 1639178786Skmacy 1640178786Skmacystatic void 1641178786Skmacyprocess_req(void *ctx, int pending) 1642178786Skmacy{ 1643178786Skmacy struct iwch_ep_common *epc; 1644178786Skmacy 1645178786Skmacy CTR1(KTR_IW_CXGB, "%s enter", __FUNCTION__); 1646178786Skmacy mtx_lock(&req_lock); 1647178786Skmacy while (!TAILQ_EMPTY(&req_list)) { 1648178786Skmacy epc = TAILQ_FIRST(&req_list); 1649178786Skmacy TAILQ_REMOVE(&req_list, epc, entry); 1650178786Skmacy epc->entry.tqe_prev = NULL; 1651178786Skmacy mtx_unlock(&req_lock); 1652178786Skmacy if (epc->so) 1653178786Skmacy process_socket_event((struct iwch_ep *)epc); 1654178786Skmacy put_ep(epc); 1655178786Skmacy mtx_lock(&req_lock); 1656178786Skmacy } 1657178786Skmacy mtx_unlock(&req_lock); 1658178786Skmacy} 1659178786Skmacy 1660178786Skmacyint 1661178786Skmacyiwch_cm_init(void) 1662178786Skmacy{ 1663178786Skmacy TAILQ_INIT(&req_list); 1664178786Skmacy mtx_init(&req_lock, "iw_cxgb req_list lock", NULL, MTX_DEF); 1665178786Skmacy iw_cxgb_taskq = taskqueue_create("iw_cxgb_taskq", M_NOWAIT, 1666178786Skmacy taskqueue_thread_enqueue, &iw_cxgb_taskq); 1667178786Skmacy if (iw_cxgb_taskq == NULL) { 1668178786Skmacy printf("failed to allocate iw_cxgb taskqueue\n"); 1669178786Skmacy return (ENOMEM); 1670178786Skmacy } 1671178786Skmacy taskqueue_start_threads(&iw_cxgb_taskq, 1, PI_NET, "iw_cxgb taskq"); 1672178786Skmacy TASK_INIT(&iw_cxgb_task, 0, process_req, NULL); 1673237263Snp return (0); 1674178786Skmacy} 1675178786Skmacy 1676178786Skmacyvoid 1677178786Skmacyiwch_cm_term(void) 1678178786Skmacy{ 1679237263Snp 1680178786Skmacy taskqueue_drain(iw_cxgb_taskq, &iw_cxgb_task); 1681178786Skmacy taskqueue_free(iw_cxgb_taskq); 1682178786Skmacy} 1683178786Skmacy 1684237263Snpvoid 1685237263Snpiwch_cm_init_cpl(struct adapter *sc) 1686237263Snp{ 1687237263Snp 1688237263Snp t3_register_cpl_handler(sc, CPL_RDMA_TERMINATE, terminate); 1689237263Snp t3_register_cpl_handler(sc, CPL_RDMA_EC_STATUS, ec_status); 1690237263Snp} 1691237263Snp 1692237263Snpvoid 1693237263Snpiwch_cm_term_cpl(struct adapter *sc) 1694237263Snp{ 1695237263Snp 1696237263Snp t3_register_cpl_handler(sc, CPL_RDMA_TERMINATE, NULL); 1697237263Snp t3_register_cpl_handler(sc, CPL_RDMA_EC_STATUS, NULL); 1698237263Snp} 1699237263Snp#endif 1700