1256694Snp/* 2309378Sjhb * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. 3256694Snp * 4256694Snp * This software is available to you under a choice of one of two 5256694Snp * licenses. You may choose to be licensed under the terms of the GNU 6256694Snp * General Public License (GPL) Version 2, available from the file 7256694Snp * COPYING in the main directory of this source tree, or the 8256694Snp * OpenIB.org BSD license below: 9256694Snp * 10256694Snp * Redistribution and use in source and binary forms, with or 11256694Snp * without modification, are permitted provided that the following 12256694Snp * conditions are met: 13256694Snp * 14256694Snp * - Redistributions of source code must retain the above 15256694Snp * copyright notice, this list of conditions and the following 16256694Snp * disclaimer. 17256694Snp * 18256694Snp * - Redistributions in binary form must reproduce the above 19256694Snp * copyright notice, this list of conditions and the following 20256694Snp * disclaimer in the documentation and/or other materials 21256694Snp * provided with the distribution. 22256694Snp * 23256694Snp * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24256694Snp * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25256694Snp * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26256694Snp * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27256694Snp * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28256694Snp * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29256694Snp * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30256694Snp * SOFTWARE. 31256694Snp */ 32256694Snp#include <sys/cdefs.h> 33256694Snp__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/iw_cxgbe/cm.c 319272 2017-05-31 00:43:52Z np $"); 34256694Snp 35256694Snp#include "opt_inet.h" 36256694Snp 37256694Snp#ifdef TCP_OFFLOAD 38256694Snp#include <sys/types.h> 39256694Snp#include <sys/malloc.h> 40256694Snp#include <sys/socket.h> 41256694Snp#include <sys/socketvar.h> 42256694Snp#include <sys/sockio.h> 43256694Snp#include <sys/taskqueue.h> 44256694Snp#include <netinet/in.h> 45256694Snp#include <net/route.h> 46256694Snp 47256694Snp#include <netinet/in_systm.h> 48256694Snp#include <netinet/in_pcb.h> 49256694Snp#include <netinet/ip.h> 50256694Snp#include <netinet/ip_var.h> 51256694Snp#include <netinet/tcp_var.h> 52256694Snp#include <netinet/tcp.h> 53256694Snp#include <netinet/tcpip.h> 54256694Snp 55256694Snp#include <netinet/toecore.h> 56256694Snp 57256694Snpstruct sge_iq; 58256694Snpstruct rss_header; 59256694Snp#include <linux/types.h> 60256694Snp#include "offload.h" 61256694Snp#include "tom/t4_tom.h" 62256694Snp 63256694Snp#define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe)) 64256694Snp 65256694Snp#include "iw_cxgbe.h" 66256694Snp#include <linux/module.h> 67256694Snp#include <linux/workqueue.h> 68256694Snp#include <linux/notifier.h> 69256694Snp#include <linux/inetdevice.h> 70256694Snp#include <linux/if_vlan.h> 71256694Snp#include <net/netevent.h> 72256694Snp 73256694Snpstatic spinlock_t req_lock; 74256694Snpstatic TAILQ_HEAD(c4iw_ep_list, c4iw_ep_common) req_list; 75256694Snpstatic struct work_struct c4iw_task; 76256694Snpstatic struct workqueue_struct *c4iw_taskq; 77316123Snpstatic LIST_HEAD(err_cqe_list); 78316123Snpstatic spinlock_t err_cqe_lock; 79256694Snp 80256694Snpstatic void process_req(struct work_struct *ctx); 81256694Snpstatic void start_ep_timer(struct c4iw_ep *ep); 82309378Sjhbstatic int stop_ep_timer(struct c4iw_ep *ep); 83256694Snpstatic int set_tcpinfo(struct c4iw_ep *ep); 84309450Sjhbstatic void process_timeout(struct c4iw_ep *ep); 85316123Snpstatic void process_err_cqes(void); 86256694Snpstatic enum c4iw_ep_state state_read(struct c4iw_ep_common *epc); 87256694Snpstatic void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 88256694Snpstatic void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 89256694Snpstatic void *alloc_ep(int size, gfp_t flags); 90256694Snpstatic struct rtentry * find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 91256694Snp __be16 peer_port, u8 tos); 92314606Snpstatic void close_socket(struct socket *so); 93309450Sjhbstatic int send_mpa_req(struct c4iw_ep *ep); 94256694Snpstatic int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); 95256694Snpstatic int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); 96283856Snpstatic void close_complete_upcall(struct c4iw_ep *ep, int status); 97309378Sjhbstatic int send_abort(struct c4iw_ep *ep); 98256694Snpstatic void peer_close_upcall(struct c4iw_ep *ep); 99256694Snpstatic void peer_abort_upcall(struct c4iw_ep *ep); 100256694Snpstatic void connect_reply_upcall(struct c4iw_ep *ep, int status); 101309378Sjhbstatic int connect_request_upcall(struct c4iw_ep *ep); 102256694Snpstatic void established_upcall(struct c4iw_ep *ep); 103309378Sjhbstatic int process_mpa_reply(struct c4iw_ep *ep); 104309378Sjhbstatic int process_mpa_request(struct c4iw_ep *ep); 105256694Snpstatic void process_peer_close(struct c4iw_ep *ep); 106256694Snpstatic void process_conn_error(struct c4iw_ep *ep); 107256694Snpstatic void process_close_complete(struct c4iw_ep *ep); 108256694Snpstatic void ep_timeout(unsigned long arg); 109314606Snpstatic void init_iwarp_socket(struct socket *so, void *arg); 110314606Snpstatic void uninit_iwarp_socket(struct socket *so); 111256694Snpstatic void process_data(struct c4iw_ep *ep); 112256694Snpstatic void process_connected(struct c4iw_ep *ep); 113256694Snpstatic int c4iw_so_upcall(struct socket *so, void *arg, int waitflag); 114256694Snpstatic void process_socket_event(struct c4iw_ep *ep); 115256694Snpstatic void release_ep_resources(struct c4iw_ep *ep); 116316123Snpstatic int process_terminate(struct c4iw_ep *ep); 117316123Snpstatic int terminate(struct sge_iq *iq, const struct rss_header *rss, 118316123Snp struct mbuf *m); 119316123Snpstatic int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events); 120256694Snp#define START_EP_TIMER(ep) \ 121256694Snp do { \ 122256694Snp CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \ 123256694Snp __func__, __LINE__, (ep)); \ 124256694Snp start_ep_timer(ep); \ 125256694Snp } while (0) 126256694Snp 127256694Snp#define STOP_EP_TIMER(ep) \ 128309378Sjhb ({ \ 129256694Snp CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ 130256694Snp __func__, __LINE__, (ep)); \ 131256694Snp stop_ep_timer(ep); \ 132309378Sjhb }) 133256694Snp 134256694Snp#ifdef KTR 135256694Snpstatic char *states[] = { 136256694Snp "idle", 137256694Snp "listen", 138256694Snp "connecting", 139256694Snp "mpa_wait_req", 140256694Snp "mpa_req_sent", 141256694Snp "mpa_req_rcvd", 142256694Snp "mpa_rep_sent", 143256694Snp "fpdu_mode", 144256694Snp "aborting", 145256694Snp "closing", 146256694Snp "moribund", 147256694Snp "dead", 148256694Snp NULL, 149256694Snp}; 150256694Snp#endif 151256694Snp 152309378Sjhb 153309378Sjhbstatic void deref_cm_id(struct c4iw_ep_common *epc) 154309378Sjhb{ 155309378Sjhb epc->cm_id->rem_ref(epc->cm_id); 156309378Sjhb epc->cm_id = NULL; 157309378Sjhb set_bit(CM_ID_DEREFED, &epc->history); 158309378Sjhb} 159309378Sjhb 160309378Sjhbstatic void ref_cm_id(struct c4iw_ep_common *epc) 161309378Sjhb{ 162309378Sjhb set_bit(CM_ID_REFED, &epc->history); 163309378Sjhb epc->cm_id->add_ref(epc->cm_id); 164309378Sjhb} 165309378Sjhb 166309378Sjhbstatic void deref_qp(struct c4iw_ep *ep) 167309378Sjhb{ 168309378Sjhb c4iw_qp_rem_ref(&ep->com.qp->ibqp); 169309378Sjhb clear_bit(QP_REFERENCED, &ep->com.flags); 170309378Sjhb set_bit(QP_DEREFED, &ep->com.history); 171309378Sjhb} 172309378Sjhb 173309378Sjhbstatic void ref_qp(struct c4iw_ep *ep) 174309378Sjhb{ 175309378Sjhb set_bit(QP_REFERENCED, &ep->com.flags); 176309378Sjhb set_bit(QP_REFED, &ep->com.history); 177309378Sjhb c4iw_qp_add_ref(&ep->com.qp->ibqp); 178309378Sjhb} 179309378Sjhb 180309450Sjhbstatic void process_timeout(struct c4iw_ep *ep) 181309450Sjhb{ 182309450Sjhb struct c4iw_qp_attributes attrs; 183309450Sjhb int abort = 1; 184309450Sjhb 185309450Sjhb mutex_lock(&ep->com.mutex); 186309450Sjhb CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__, 187309450Sjhb ep, ep->hwtid, ep->com.state); 188309450Sjhb set_bit(TIMEDOUT, &ep->com.history); 189309450Sjhb switch (ep->com.state) { 190309450Sjhb case MPA_REQ_SENT: 191309450Sjhb connect_reply_upcall(ep, -ETIMEDOUT); 192309450Sjhb break; 193309450Sjhb case MPA_REQ_WAIT: 194309450Sjhb case MPA_REQ_RCVD: 195309450Sjhb case MPA_REP_SENT: 196309450Sjhb case FPDU_MODE: 197309450Sjhb break; 198309450Sjhb case CLOSING: 199309450Sjhb case MORIBUND: 200309450Sjhb if (ep->com.cm_id && ep->com.qp) { 201309450Sjhb attrs.next_state = C4IW_QP_STATE_ERROR; 202309450Sjhb c4iw_modify_qp(ep->com.dev, ep->com.qp, 203309450Sjhb C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 204309450Sjhb } 205309450Sjhb close_complete_upcall(ep, -ETIMEDOUT); 206309450Sjhb break; 207309450Sjhb case ABORTING: 208309450Sjhb case DEAD: 209309450Sjhb /* 210309450Sjhb * These states are expected if the ep timed out at the same 211309450Sjhb * time as another thread was calling stop_ep_timer(). 212309450Sjhb * So we silently do nothing for these states. 213309450Sjhb */ 214309450Sjhb abort = 0; 215309450Sjhb break; 216309450Sjhb default: 217309450Sjhb CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u\n" 218309450Sjhb , __func__, ep, ep->hwtid, ep->com.state); 219309450Sjhb abort = 0; 220309450Sjhb } 221309450Sjhb mutex_unlock(&ep->com.mutex); 222309450Sjhb if (abort) 223309450Sjhb c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 224309450Sjhb c4iw_put_ep(&ep->com); 225309450Sjhb return; 226309450Sjhb} 227309450Sjhb 228316123Snpstruct cqe_list_entry { 229316123Snp struct list_head entry; 230316123Snp struct c4iw_dev *rhp; 231316123Snp struct t4_cqe err_cqe; 232316123Snp}; 233316123Snp 234316123Snpstatic void 235316123Snpprocess_err_cqes(void) 236309450Sjhb{ 237316123Snp unsigned long flag; 238316123Snp struct cqe_list_entry *cle; 239309450Sjhb 240316123Snp spin_lock_irqsave(&err_cqe_lock, flag); 241316123Snp while (!list_empty(&err_cqe_list)) { 242309450Sjhb struct list_head *tmp; 243316123Snp tmp = err_cqe_list.next; 244309450Sjhb list_del(tmp); 245309450Sjhb tmp->next = tmp->prev = NULL; 246316123Snp spin_unlock_irqrestore(&err_cqe_lock, flag); 247316123Snp cle = list_entry(tmp, struct cqe_list_entry, entry); 248316123Snp c4iw_ev_dispatch(cle->rhp, &cle->err_cqe); 249316123Snp free(cle, M_CXGBE); 250316123Snp spin_lock_irqsave(&err_cqe_lock, flag); 251309450Sjhb } 252316123Snp spin_unlock_irqrestore(&err_cqe_lock, flag); 253316123Snp 254309450Sjhb return; 255309450Sjhb} 256309450Sjhb 257256694Snpstatic void 258256694Snpprocess_req(struct work_struct *ctx) 259256694Snp{ 260256694Snp struct c4iw_ep_common *epc; 261316123Snp unsigned long flag; 262316123Snp int ep_events; 263256694Snp 264316123Snp process_err_cqes(); 265316123Snp spin_lock_irqsave(&req_lock, flag); 266256694Snp while (!TAILQ_EMPTY(&req_list)) { 267256694Snp epc = TAILQ_FIRST(&req_list); 268256694Snp TAILQ_REMOVE(&req_list, epc, entry); 269256694Snp epc->entry.tqe_prev = NULL; 270316123Snp ep_events = epc->ep_events; 271316123Snp epc->ep_events = 0; 272316123Snp spin_unlock_irqrestore(&req_lock, flag); 273316123Snp CTR4(KTR_IW_CXGBE, "%s: so %p, ep %p, events 0x%x", __func__, 274316123Snp epc->so, epc, ep_events); 275316123Snp if (ep_events & C4IW_EVENT_TERM) 276316123Snp process_terminate((struct c4iw_ep *)epc); 277316123Snp if (ep_events & C4IW_EVENT_TIMEOUT) 278316123Snp process_timeout((struct c4iw_ep *)epc); 279316123Snp if (ep_events & C4IW_EVENT_SOCKET) 280256694Snp process_socket_event((struct c4iw_ep *)epc); 281256694Snp c4iw_put_ep(epc); 282316123Snp process_err_cqes(); 283316123Snp spin_lock_irqsave(&req_lock, flag); 284256694Snp } 285316123Snp spin_unlock_irqrestore(&req_lock, flag); 286256694Snp} 287256694Snp 288256694Snp/* 289256694Snp * XXX: doesn't belong here in the iWARP driver. 290256694Snp * XXX: assumes that the connection was offloaded by cxgbe/t4_tom if TF_TOE is 291256694Snp * set. Is this a valid assumption for active open? 292256694Snp */ 293256694Snpstatic int 294256694Snpset_tcpinfo(struct c4iw_ep *ep) 295256694Snp{ 296256694Snp struct socket *so = ep->com.so; 297256694Snp struct inpcb *inp = sotoinpcb(so); 298256694Snp struct tcpcb *tp; 299256694Snp struct toepcb *toep; 300256694Snp int rc = 0; 301256694Snp 302256694Snp INP_WLOCK(inp); 303256694Snp tp = intotcpcb(inp); 304256694Snp if ((tp->t_flags & TF_TOE) == 0) { 305256694Snp rc = EINVAL; 306256694Snp log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n", 307256694Snp __func__, so, ep); 308256694Snp goto done; 309256694Snp } 310256694Snp toep = TOEPCB(so); 311256694Snp 312256694Snp ep->hwtid = toep->tid; 313256694Snp ep->snd_seq = tp->snd_nxt; 314256694Snp ep->rcv_seq = tp->rcv_nxt; 315256694Snp ep->emss = max(tp->t_maxseg, 128); 316256694Snpdone: 317256694Snp INP_WUNLOCK(inp); 318256694Snp return (rc); 319256694Snp 320256694Snp} 321256694Snp 322256694Snpstatic struct rtentry * 323256694Snpfind_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 324256694Snp __be16 peer_port, u8 tos) 325256694Snp{ 326256694Snp struct route iproute; 327256694Snp struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst; 328256694Snp 329256694Snp CTR5(KTR_IW_CXGBE, "%s:frtB %x, %x, %d, %d", __func__, local_ip, 330256694Snp peer_ip, ntohs(local_port), ntohs(peer_port)); 331256694Snp bzero(&iproute, sizeof iproute); 332256694Snp dst->sin_family = AF_INET; 333256694Snp dst->sin_len = sizeof *dst; 334256694Snp dst->sin_addr.s_addr = peer_ip; 335256694Snp 336256694Snp rtalloc(&iproute); 337256694Snp CTR2(KTR_IW_CXGBE, "%s:frtE %p", __func__, (uint64_t)iproute.ro_rt); 338256694Snp return iproute.ro_rt; 339256694Snp} 340256694Snp 341256694Snpstatic void 342314606Snpclose_socket(struct socket *so) 343256694Snp{ 344256694Snp 345314606Snp uninit_iwarp_socket(so); 346314606Snp sodisconnect(so); 347256694Snp} 348256694Snp 349256694Snpstatic void 350256694Snpprocess_peer_close(struct c4iw_ep *ep) 351256694Snp{ 352256694Snp struct c4iw_qp_attributes attrs; 353256694Snp int disconnect = 1; 354256694Snp int release = 0; 355256694Snp 356256694Snp CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep, 357256694Snp ep->com.so, states[ep->com.state]); 358256694Snp 359256694Snp mutex_lock(&ep->com.mutex); 360256694Snp switch (ep->com.state) { 361256694Snp 362256694Snp case MPA_REQ_WAIT: 363256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT CLOSING", 364256694Snp __func__, ep); 365256694Snp __state_set(&ep->com, CLOSING); 366256694Snp break; 367256694Snp 368256694Snp case MPA_REQ_SENT: 369256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT CLOSING", 370256694Snp __func__, ep); 371256694Snp __state_set(&ep->com, DEAD); 372256694Snp connect_reply_upcall(ep, -ECONNABORTED); 373256694Snp 374256694Snp disconnect = 0; 375256694Snp STOP_EP_TIMER(ep); 376314606Snp close_socket(ep->com.so); 377309378Sjhb deref_cm_id(&ep->com); 378256694Snp release = 1; 379256694Snp break; 380256694Snp 381256694Snp case MPA_REQ_RCVD: 382256694Snp 383256694Snp /* 384256694Snp * We're gonna mark this puppy DEAD, but keep 385256694Snp * the reference on it until the ULP accepts or 386256694Snp * rejects the CR. 387256694Snp */ 388256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc3 %p MPA_REQ_RCVD CLOSING", 389256694Snp __func__, ep); 390256694Snp __state_set(&ep->com, CLOSING); 391256694Snp c4iw_get_ep(&ep->com); 392256694Snp break; 393256694Snp 394256694Snp case MPA_REP_SENT: 395256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc4 %p MPA_REP_SENT CLOSING", 396256694Snp __func__, ep); 397256694Snp __state_set(&ep->com, CLOSING); 398256694Snp break; 399256694Snp 400256694Snp case FPDU_MODE: 401256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc5 %p FPDU_MODE CLOSING", 402256694Snp __func__, ep); 403256694Snp START_EP_TIMER(ep); 404256694Snp __state_set(&ep->com, CLOSING); 405256694Snp attrs.next_state = C4IW_QP_STATE_CLOSING; 406256694Snp c4iw_modify_qp(ep->com.dev, ep->com.qp, 407256694Snp C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 408256694Snp peer_close_upcall(ep); 409256694Snp break; 410256694Snp 411256694Snp case ABORTING: 412256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc6 %p ABORTING (disconn)", 413256694Snp __func__, ep); 414256694Snp disconnect = 0; 415256694Snp break; 416256694Snp 417256694Snp case CLOSING: 418256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc7 %p CLOSING MORIBUND", 419256694Snp __func__, ep); 420256694Snp __state_set(&ep->com, MORIBUND); 421256694Snp disconnect = 0; 422256694Snp break; 423256694Snp 424256694Snp case MORIBUND: 425256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc8 %p MORIBUND DEAD", __func__, 426256694Snp ep); 427256694Snp STOP_EP_TIMER(ep); 428256694Snp if (ep->com.cm_id && ep->com.qp) { 429256694Snp attrs.next_state = C4IW_QP_STATE_IDLE; 430256694Snp c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 431256694Snp C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 432256694Snp } 433314606Snp close_socket(ep->com.so); 434283856Snp close_complete_upcall(ep, 0); 435256694Snp __state_set(&ep->com, DEAD); 436256694Snp release = 1; 437256694Snp disconnect = 0; 438256694Snp break; 439256694Snp 440256694Snp case DEAD: 441256694Snp CTR2(KTR_IW_CXGBE, "%s:ppc9 %p DEAD (disconn)", 442256694Snp __func__, ep); 443256694Snp disconnect = 0; 444256694Snp break; 445256694Snp 446256694Snp default: 447256694Snp panic("%s: ep %p state %d", __func__, ep, 448256694Snp ep->com.state); 449256694Snp break; 450256694Snp } 451256694Snp 452256694Snp mutex_unlock(&ep->com.mutex); 453256694Snp 454256694Snp if (disconnect) { 455256694Snp 456256694Snp CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep); 457256694Snp c4iw_ep_disconnect(ep, 0, M_NOWAIT); 458256694Snp } 459256694Snp if (release) { 460256694Snp 461256694Snp CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep); 462256694Snp c4iw_put_ep(&ep->com); 463256694Snp } 464256694Snp CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep); 465256694Snp return; 466256694Snp} 467256694Snp 468256694Snpstatic void 469256694Snpprocess_conn_error(struct c4iw_ep *ep) 470256694Snp{ 471256694Snp struct c4iw_qp_attributes attrs; 472256694Snp int ret; 473256694Snp int state; 474256694Snp 475309450Sjhb mutex_lock(&ep->com.mutex); 476309450Sjhb state = ep->com.state; 477256694Snp CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s", 478256694Snp __func__, ep, ep->com.so, ep->com.so->so_error, 479256694Snp states[ep->com.state]); 480256694Snp 481256694Snp switch (state) { 482256694Snp 483256694Snp case MPA_REQ_WAIT: 484256694Snp STOP_EP_TIMER(ep); 485256694Snp break; 486256694Snp 487256694Snp case MPA_REQ_SENT: 488256694Snp STOP_EP_TIMER(ep); 489256694Snp connect_reply_upcall(ep, -ECONNRESET); 490256694Snp break; 491256694Snp 492256694Snp case MPA_REP_SENT: 493256694Snp ep->com.rpl_err = ECONNRESET; 494256694Snp CTR1(KTR_IW_CXGBE, "waking up ep %p", ep); 495256694Snp break; 496256694Snp 497256694Snp case MPA_REQ_RCVD: 498256694Snp 499256694Snp /* 500256694Snp * We're gonna mark this puppy DEAD, but keep 501256694Snp * the reference on it until the ULP accepts or 502256694Snp * rejects the CR. 503256694Snp */ 504256694Snp c4iw_get_ep(&ep->com); 505256694Snp break; 506256694Snp 507256694Snp case MORIBUND: 508256694Snp case CLOSING: 509256694Snp STOP_EP_TIMER(ep); 510256694Snp /*FALLTHROUGH*/ 511256694Snp case FPDU_MODE: 512256694Snp 513256694Snp if (ep->com.cm_id && ep->com.qp) { 514256694Snp 515256694Snp attrs.next_state = C4IW_QP_STATE_ERROR; 516256694Snp ret = c4iw_modify_qp(ep->com.qp->rhp, 517256694Snp ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 518256694Snp &attrs, 1); 519256694Snp if (ret) 520256694Snp log(LOG_ERR, 521256694Snp "%s - qp <- error failed!\n", 522256694Snp __func__); 523256694Snp } 524256694Snp peer_abort_upcall(ep); 525256694Snp break; 526256694Snp 527256694Snp case ABORTING: 528256694Snp break; 529256694Snp 530256694Snp case DEAD: 531256694Snp CTR2(KTR_IW_CXGBE, "%s so_error %d IN DEAD STATE!!!!", 532256694Snp __func__, ep->com.so->so_error); 533309450Sjhb mutex_unlock(&ep->com.mutex); 534256694Snp return; 535256694Snp 536256694Snp default: 537256694Snp panic("%s: ep %p state %d", __func__, ep, state); 538256694Snp break; 539256694Snp } 540256694Snp 541256694Snp if (state != ABORTING) { 542314606Snp close_socket(ep->com.so); 543309450Sjhb __state_set(&ep->com, DEAD); 544256694Snp c4iw_put_ep(&ep->com); 545256694Snp } 546309450Sjhb mutex_unlock(&ep->com.mutex); 547256694Snp CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep); 548256694Snp return; 549256694Snp} 550256694Snp 551256694Snpstatic void 552256694Snpprocess_close_complete(struct c4iw_ep *ep) 553256694Snp{ 554256694Snp struct c4iw_qp_attributes attrs; 555256694Snp int release = 0; 556256694Snp 557256694Snp CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep, 558256694Snp ep->com.so, states[ep->com.state]); 559256694Snp 560256694Snp /* The cm_id may be null if we failed to connect */ 561256694Snp mutex_lock(&ep->com.mutex); 562309378Sjhb set_bit(CLOSE_CON_RPL, &ep->com.history); 563256694Snp 564256694Snp switch (ep->com.state) { 565256694Snp 566256694Snp case CLOSING: 567256694Snp CTR2(KTR_IW_CXGBE, "%s:pcc1 %p CLOSING MORIBUND", 568256694Snp __func__, ep); 569256694Snp __state_set(&ep->com, MORIBUND); 570256694Snp break; 571256694Snp 572256694Snp case MORIBUND: 573256694Snp CTR2(KTR_IW_CXGBE, "%s:pcc1 %p MORIBUND DEAD", __func__, 574256694Snp ep); 575256694Snp STOP_EP_TIMER(ep); 576256694Snp 577256694Snp if ((ep->com.cm_id) && (ep->com.qp)) { 578256694Snp 579256694Snp CTR2(KTR_IW_CXGBE, "%s:pcc2 %p QP_STATE_IDLE", 580256694Snp __func__, ep); 581256694Snp attrs.next_state = C4IW_QP_STATE_IDLE; 582256694Snp c4iw_modify_qp(ep->com.dev, 583256694Snp ep->com.qp, 584256694Snp C4IW_QP_ATTR_NEXT_STATE, 585256694Snp &attrs, 1); 586256694Snp } 587256694Snp 588314606Snp close_socket(ep->com.so); 589283856Snp close_complete_upcall(ep, 0); 590256694Snp __state_set(&ep->com, DEAD); 591256694Snp release = 1; 592256694Snp break; 593256694Snp 594256694Snp case ABORTING: 595256694Snp CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep); 596256694Snp break; 597256694Snp 598256694Snp case DEAD: 599256694Snp CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep); 600256694Snp break; 601309450Sjhb default: 602309450Sjhb CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state", 603309450Sjhb __func__, ep); 604309450Sjhb panic("%s:pcc6 %p unknown ep state", __func__, ep); 605309450Sjhb break; 606256694Snp } 607256694Snp mutex_unlock(&ep->com.mutex); 608256694Snp 609256694Snp if (release) { 610256694Snp 611309450Sjhb CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep); 612256694Snp c4iw_put_ep(&ep->com); 613256694Snp } 614256694Snp CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep); 615256694Snp return; 616256694Snp} 617256694Snp 618256694Snpstatic void 619314606Snpinit_iwarp_socket(struct socket *so, void *arg) 620256694Snp{ 621256694Snp int rc; 622256694Snp struct sockopt sopt; 623256694Snp int on = 1; 624256694Snp 625314606Snp /* Note that SOCK_LOCK(so) is same as SOCKBUF_LOCK(&so->so_rcv) */ 626256694Snp SOCK_LOCK(so); 627314606Snp soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); 628256694Snp so->so_state |= SS_NBIO; 629256694Snp SOCK_UNLOCK(so); 630256694Snp sopt.sopt_dir = SOPT_SET; 631256694Snp sopt.sopt_level = IPPROTO_TCP; 632256694Snp sopt.sopt_name = TCP_NODELAY; 633256694Snp sopt.sopt_val = (caddr_t)&on; 634256694Snp sopt.sopt_valsize = sizeof on; 635256694Snp sopt.sopt_td = NULL; 636256694Snp rc = sosetopt(so, &sopt); 637256694Snp if (rc) { 638256694Snp log(LOG_ERR, "%s: can't set TCP_NODELAY on so %p (%d)\n", 639256694Snp __func__, so, rc); 640256694Snp } 641256694Snp} 642256694Snp 643256694Snpstatic void 644314606Snpuninit_iwarp_socket(struct socket *so) 645314606Snp{ 646314606Snp 647314606Snp SOCKBUF_LOCK(&so->so_rcv); 648314606Snp soupcall_clear(so, SO_RCV); 649314606Snp SOCKBUF_UNLOCK(&so->so_rcv); 650314606Snp} 651314606Snp 652314606Snpstatic void 653256694Snpprocess_data(struct c4iw_ep *ep) 654256694Snp{ 655256694Snp struct sockaddr_in *local, *remote; 656309378Sjhb int disconnect = 0; 657256694Snp 658256694Snp CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sb_cc %d", __func__, 659256694Snp ep->com.so, ep, states[ep->com.state], ep->com.so->so_rcv.sb_cc); 660256694Snp 661256694Snp switch (state_read(&ep->com)) { 662256694Snp case MPA_REQ_SENT: 663309378Sjhb disconnect = process_mpa_reply(ep); 664256694Snp break; 665256694Snp case MPA_REQ_WAIT: 666256694Snp in_getsockaddr(ep->com.so, (struct sockaddr **)&local); 667256694Snp in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); 668256694Snp ep->com.local_addr = *local; 669256694Snp ep->com.remote_addr = *remote; 670256694Snp free(local, M_SONAME); 671256694Snp free(remote, M_SONAME); 672309378Sjhb disconnect = process_mpa_request(ep); 673256694Snp break; 674256694Snp default: 675256694Snp if (ep->com.so->so_rcv.sb_cc) 676256694Snp log(LOG_ERR, "%s: Unexpected streaming data. " 677256694Snp "ep %p, state %d, so %p, so_state 0x%x, sb_cc %u\n", 678256694Snp __func__, ep, state_read(&ep->com), ep->com.so, 679256694Snp ep->com.so->so_state, ep->com.so->so_rcv.sb_cc); 680256694Snp break; 681256694Snp } 682309378Sjhb if (disconnect) 683309378Sjhb c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); 684309378Sjhb 685256694Snp} 686256694Snp 687256694Snpstatic void 688256694Snpprocess_connected(struct c4iw_ep *ep) 689256694Snp{ 690314606Snp struct socket *so = ep->com.so; 691256694Snp 692314606Snp if ((so->so_state & SS_ISCONNECTED) && !so->so_error) { 693309450Sjhb if (send_mpa_req(ep)) 694309450Sjhb goto err; 695314606Snp } else { 696314606Snp connect_reply_upcall(ep, -so->so_error); 697309450Sjhb goto err; 698256694Snp } 699309450Sjhb return; 700309450Sjhberr: 701314606Snp close_socket(so); 702309450Sjhb state_set(&ep->com, DEAD); 703309450Sjhb c4iw_put_ep(&ep->com); 704309450Sjhb return; 705256694Snp} 706256694Snp 707309378Sjhbvoid 708309378Sjhbprocess_newconn(struct iw_cm_id *parent_cm_id, struct socket *child_so) 709256694Snp{ 710256694Snp struct c4iw_ep *child_ep; 711309378Sjhb struct sockaddr_in *local; 712256694Snp struct sockaddr_in *remote; 713309378Sjhb struct c4iw_ep *parent_ep = parent_cm_id->provider_data; 714309450Sjhb int ret = 0; 715256694Snp 716314606Snp MPASS(child_so != NULL); 717256694Snp 718316123Snp child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 719314606Snp 720256694Snp CTR5(KTR_IW_CXGBE, 721256694Snp "%s: parent so %p, parent ep %p, child so %p, child ep %p", 722256694Snp __func__, parent_ep->com.so, parent_ep, child_so, child_ep); 723256694Snp 724309378Sjhb in_getsockaddr(child_so, (struct sockaddr **)&local); 725309378Sjhb in_getpeeraddr(child_so, (struct sockaddr **)&remote); 726309378Sjhb 727309378Sjhb child_ep->com.local_addr = *local; 728256694Snp child_ep->com.remote_addr = *remote; 729256694Snp child_ep->com.dev = parent_ep->com.dev; 730256694Snp child_ep->com.so = child_so; 731256694Snp child_ep->com.cm_id = NULL; 732256694Snp child_ep->com.thread = parent_ep->com.thread; 733256694Snp child_ep->parent_ep = parent_ep; 734256694Snp 735309378Sjhb free(local, M_SONAME); 736256694Snp free(remote, M_SONAME); 737309378Sjhb 738314606Snp init_iwarp_socket(child_so, &child_ep->com); 739256694Snp c4iw_get_ep(&parent_ep->com); 740256694Snp init_timer(&child_ep->timer); 741256694Snp state_set(&child_ep->com, MPA_REQ_WAIT); 742256694Snp START_EP_TIMER(child_ep); 743256694Snp 744256694Snp /* maybe the request has already been queued up on the socket... */ 745309450Sjhb ret = process_mpa_request(child_ep); 746309450Sjhb if (ret == 2) 747309450Sjhb /* ABORT */ 748309450Sjhb c4iw_ep_disconnect(child_ep, 1, GFP_KERNEL); 749309450Sjhb else if (ret == 1) 750309450Sjhb /* CLOSE */ 751309450Sjhb c4iw_ep_disconnect(child_ep, 0, GFP_KERNEL); 752309450Sjhb 753309378Sjhb return; 754256694Snp} 755256694Snp 756256694Snpstatic int 757316123Snpadd_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event) 758316123Snp{ 759316123Snp unsigned long flag; 760316123Snp 761316123Snp spin_lock_irqsave(&req_lock, flag); 762316123Snp if (ep && ep->com.so) { 763316123Snp ep->com.ep_events |= new_ep_event; 764316123Snp if (!ep->com.entry.tqe_prev) { 765316123Snp c4iw_get_ep(&ep->com); 766316123Snp TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 767316123Snp queue_work(c4iw_taskq, &c4iw_task); 768316123Snp } 769316123Snp } 770316123Snp spin_unlock_irqrestore(&req_lock, flag); 771316123Snp 772316123Snp return (0); 773316123Snp} 774316123Snp 775316123Snpstatic int 776256694Snpc4iw_so_upcall(struct socket *so, void *arg, int waitflag) 777256694Snp{ 778256694Snp struct c4iw_ep *ep = arg; 779256694Snp 780256694Snp CTR6(KTR_IW_CXGBE, 781256694Snp "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p", 782256694Snp __func__, so, so->so_state, ep, states[ep->com.state], 783256694Snp ep->com.entry.tqe_prev); 784256694Snp 785316123Snp MPASS(ep->com.so == so); 786316123Snp add_ep_to_req_list(ep, C4IW_EVENT_SOCKET); 787256694Snp 788256694Snp return (SU_OK); 789256694Snp} 790256694Snp 791316123Snp 792316123Snpstatic int 793316123Snpterminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 794316123Snp{ 795316123Snp struct adapter *sc = iq->adapter; 796316123Snp const struct cpl_rdma_terminate *cpl = mtod(m, const void *); 797316123Snp unsigned int tid = GET_TID(cpl); 798316123Snp struct toepcb *toep = lookup_tid(sc, tid); 799316123Snp struct socket *so; 800316123Snp struct c4iw_ep *ep; 801316123Snp 802316123Snp INP_WLOCK(toep->inp); 803316123Snp so = inp_inpcbtosocket(toep->inp); 804316123Snp ep = so->so_rcv.sb_upcallarg; 805316123Snp INP_WUNLOCK(toep->inp); 806316123Snp 807316123Snp CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep); 808316123Snp add_ep_to_req_list(ep, C4IW_EVENT_TERM); 809316123Snp 810316123Snp return 0; 811316123Snp} 812316123Snp 813256694Snpstatic void 814256694Snpprocess_socket_event(struct c4iw_ep *ep) 815256694Snp{ 816256694Snp int state = state_read(&ep->com); 817256694Snp struct socket *so = ep->com.so; 818256694Snp 819256694Snp CTR6(KTR_IW_CXGBE, "process_socket_event: so %p, so_state 0x%x, " 820256694Snp "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state, 821256694Snp so->so_error, so->so_rcv.sb_state, ep, states[state]); 822256694Snp 823256694Snp if (state == CONNECTING) { 824256694Snp process_connected(ep); 825256694Snp return; 826256694Snp } 827256694Snp 828256694Snp if (state == LISTEN) { 829309378Sjhb /* socket listening events are handled at IWCM */ 830309378Sjhb CTR3(KTR_IW_CXGBE, "%s Invalid ep state:%u, ep:%p", __func__, 831309378Sjhb ep->com.state, ep); 832309378Sjhb BUG(); 833256694Snp return; 834256694Snp } 835256694Snp 836256694Snp /* connection error */ 837256694Snp if (so->so_error) { 838256694Snp process_conn_error(ep); 839256694Snp return; 840256694Snp } 841256694Snp 842256694Snp /* peer close */ 843309378Sjhb if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <= CLOSING) { 844256694Snp process_peer_close(ep); 845309450Sjhb /* 846309450Sjhb * check whether socket disconnect event is pending before 847309450Sjhb * returning. Fallthrough if yes. 848309450Sjhb */ 849309450Sjhb if (!(so->so_state & SS_ISDISCONNECTED)) 850309450Sjhb return; 851256694Snp } 852256694Snp 853256694Snp /* close complete */ 854256694Snp if (so->so_state & SS_ISDISCONNECTED) { 855256694Snp process_close_complete(ep); 856256694Snp return; 857256694Snp } 858256694Snp 859256694Snp /* rx data */ 860256694Snp process_data(ep); 861256694Snp} 862256694Snp 863256694SnpSYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD, 0, "iw_cxgbe driver parameters"); 864256694Snp 865309378Sjhbstatic int dack_mode = 0; 866256694SnpTUNABLE_INT("hw.iw_cxgbe.dack_mode", &dack_mode); 867256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RW, &dack_mode, 0, 868309378Sjhb "Delayed ack mode (default = 0)"); 869256694Snp 870256694Snpint c4iw_max_read_depth = 8; 871256694SnpTUNABLE_INT("hw.iw_cxgbe.c4iw_max_read_depth", &c4iw_max_read_depth); 872256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RW, &c4iw_max_read_depth, 0, 873256694Snp "Per-connection max ORD/IRD (default = 8)"); 874256694Snp 875256694Snpstatic int enable_tcp_timestamps; 876256694SnpTUNABLE_INT("hw.iw_cxgbe.enable_tcp_timestamps", &enable_tcp_timestamps); 877256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RW, &enable_tcp_timestamps, 0, 878256694Snp "Enable tcp timestamps (default = 0)"); 879256694Snp 880256694Snpstatic int enable_tcp_sack; 881256694SnpTUNABLE_INT("hw.iw_cxgbe.enable_tcp_sack", &enable_tcp_sack); 882256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RW, &enable_tcp_sack, 0, 883256694Snp "Enable tcp SACK (default = 0)"); 884256694Snp 885256694Snpstatic int enable_tcp_window_scaling = 1; 886256694SnpTUNABLE_INT("hw.iw_cxgbe.enable_tcp_window_scaling", &enable_tcp_window_scaling); 887256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RW, &enable_tcp_window_scaling, 0, 888256694Snp "Enable tcp window scaling (default = 1)"); 889256694Snp 890256694Snpint c4iw_debug = 1; 891256694SnpTUNABLE_INT("hw.iw_cxgbe.c4iw_debug", &c4iw_debug); 892256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RW, &c4iw_debug, 0, 893256694Snp "Enable debug logging (default = 0)"); 894256694Snp 895309378Sjhbstatic int peer2peer = 1; 896256694SnpTUNABLE_INT("hw.iw_cxgbe.peer2peer", &peer2peer); 897256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RW, &peer2peer, 0, 898309378Sjhb "Support peer2peer ULPs (default = 1)"); 899256694Snp 900256694Snpstatic int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 901256694SnpTUNABLE_INT("hw.iw_cxgbe.p2p_type", &p2p_type); 902256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RW, &p2p_type, 0, 903256694Snp "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)"); 904256694Snp 905256694Snpstatic int ep_timeout_secs = 60; 906256694SnpTUNABLE_INT("hw.iw_cxgbe.ep_timeout_secs", &ep_timeout_secs); 907256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RW, &ep_timeout_secs, 0, 908256694Snp "CM Endpoint operation timeout in seconds (default = 60)"); 909256694Snp 910256694Snpstatic int mpa_rev = 1; 911256694SnpTUNABLE_INT("hw.iw_cxgbe.mpa_rev", &mpa_rev); 912256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RW, &mpa_rev, 0, 913309378Sjhb "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)"); 914256694Snp 915256694Snpstatic int markers_enabled; 916256694SnpTUNABLE_INT("hw.iw_cxgbe.markers_enabled", &markers_enabled); 917256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RW, &markers_enabled, 0, 918256694Snp "Enable MPA MARKERS (default(0) = disabled)"); 919256694Snp 920256694Snpstatic int crc_enabled = 1; 921256694SnpTUNABLE_INT("hw.iw_cxgbe.crc_enabled", &crc_enabled); 922256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RW, &crc_enabled, 0, 923256694Snp "Enable MPA CRC (default(1) = enabled)"); 924256694Snp 925256694Snpstatic int rcv_win = 256 * 1024; 926256694SnpTUNABLE_INT("hw.iw_cxgbe.rcv_win", &rcv_win); 927256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RW, &rcv_win, 0, 928256694Snp "TCP receive window in bytes (default = 256KB)"); 929256694Snp 930256694Snpstatic int snd_win = 128 * 1024; 931256694SnpTUNABLE_INT("hw.iw_cxgbe.snd_win", &snd_win); 932256694SnpSYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RW, &snd_win, 0, 933256694Snp "TCP send window in bytes (default = 128KB)"); 934256694Snp 935256694Snpstatic void 936256694Snpstart_ep_timer(struct c4iw_ep *ep) 937256694Snp{ 938256694Snp 939256694Snp if (timer_pending(&ep->timer)) { 940256694Snp CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep); 941256694Snp printk(KERN_ERR "%s timer already started! ep %p\n", __func__, 942256694Snp ep); 943256694Snp return; 944256694Snp } 945256694Snp clear_bit(TIMEOUT, &ep->com.flags); 946256694Snp c4iw_get_ep(&ep->com); 947256694Snp ep->timer.expires = jiffies + ep_timeout_secs * HZ; 948256694Snp ep->timer.data = (unsigned long)ep; 949256694Snp ep->timer.function = ep_timeout; 950256694Snp add_timer(&ep->timer); 951256694Snp} 952256694Snp 953309378Sjhbstatic int 954256694Snpstop_ep_timer(struct c4iw_ep *ep) 955256694Snp{ 956256694Snp 957256694Snp del_timer_sync(&ep->timer); 958256694Snp if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 959256694Snp c4iw_put_ep(&ep->com); 960309378Sjhb return 0; 961256694Snp } 962309378Sjhb return 1; 963256694Snp} 964256694Snp 965256694Snpstatic enum 966256694Snpc4iw_ep_state state_read(struct c4iw_ep_common *epc) 967256694Snp{ 968256694Snp enum c4iw_ep_state state; 969256694Snp 970256694Snp mutex_lock(&epc->mutex); 971256694Snp state = epc->state; 972256694Snp mutex_unlock(&epc->mutex); 973256694Snp 974256694Snp return (state); 975256694Snp} 976256694Snp 977256694Snpstatic void 978256694Snp__state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 979256694Snp{ 980256694Snp 981256694Snp epc->state = new; 982256694Snp} 983256694Snp 984256694Snpstatic void 985256694Snpstate_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 986256694Snp{ 987256694Snp 988256694Snp mutex_lock(&epc->mutex); 989256694Snp __state_set(epc, new); 990256694Snp mutex_unlock(&epc->mutex); 991256694Snp} 992256694Snp 993256694Snpstatic void * 994256694Snpalloc_ep(int size, gfp_t gfp) 995256694Snp{ 996256694Snp struct c4iw_ep_common *epc; 997256694Snp 998256694Snp epc = kzalloc(size, gfp); 999256694Snp if (epc == NULL) 1000256694Snp return (NULL); 1001256694Snp 1002256694Snp kref_init(&epc->kref); 1003256694Snp mutex_init(&epc->mutex); 1004256694Snp c4iw_init_wr_wait(&epc->wr_wait); 1005256694Snp 1006256694Snp return (epc); 1007256694Snp} 1008256694Snp 1009256694Snpvoid _c4iw_free_ep(struct kref *kref) 1010256694Snp{ 1011256694Snp struct c4iw_ep *ep; 1012256694Snp struct c4iw_ep_common *epc; 1013256694Snp 1014256694Snp ep = container_of(kref, struct c4iw_ep, com.kref); 1015256694Snp epc = &ep->com; 1016256694Snp KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", 1017256694Snp __func__, epc)); 1018309378Sjhb if (test_bit(QP_REFERENCED, &ep->com.flags)) 1019309378Sjhb deref_qp(ep); 1020256694Snp kfree(ep); 1021256694Snp} 1022256694Snp 1023256694Snpstatic void release_ep_resources(struct c4iw_ep *ep) 1024256694Snp{ 1025256694Snp CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep); 1026256694Snp set_bit(RELEASE_RESOURCES, &ep->com.flags); 1027256694Snp c4iw_put_ep(&ep->com); 1028256694Snp CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep); 1029256694Snp} 1030256694Snp 1031309450Sjhbstatic int 1032256694Snpsend_mpa_req(struct c4iw_ep *ep) 1033256694Snp{ 1034256694Snp int mpalen; 1035256694Snp struct mpa_message *mpa; 1036256694Snp struct mpa_v2_conn_params mpa_v2_params; 1037256694Snp struct mbuf *m; 1038256694Snp char mpa_rev_to_use = mpa_rev; 1039309450Sjhb int err = 0; 1040256694Snp 1041256694Snp if (ep->retry_with_mpa_v1) 1042256694Snp mpa_rev_to_use = 1; 1043256694Snp mpalen = sizeof(*mpa) + ep->plen; 1044256694Snp if (mpa_rev_to_use == 2) 1045256694Snp mpalen += sizeof(struct mpa_v2_conn_params); 1046256694Snp 1047281247Snp mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1048281247Snp if (mpa == NULL) { 1049309450Sjhb err = -ENOMEM; 1050309450Sjhb CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d", 1051309450Sjhb __func__, ep, err); 1052309450Sjhb goto err; 1053256694Snp } 1054256694Snp 1055281247Snp memset(mpa, 0, mpalen); 1056256694Snp memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 1057256694Snp mpa->flags = (crc_enabled ? MPA_CRC : 0) | 1058256694Snp (markers_enabled ? MPA_MARKERS : 0) | 1059256694Snp (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 1060256694Snp mpa->private_data_size = htons(ep->plen); 1061256694Snp mpa->revision = mpa_rev_to_use; 1062256694Snp 1063256694Snp if (mpa_rev_to_use == 1) { 1064256694Snp ep->tried_with_mpa_v1 = 1; 1065256694Snp ep->retry_with_mpa_v1 = 0; 1066256694Snp } 1067256694Snp 1068256694Snp if (mpa_rev_to_use == 2) { 1069256694Snp mpa->private_data_size += 1070256694Snp htons(sizeof(struct mpa_v2_conn_params)); 1071256694Snp mpa_v2_params.ird = htons((u16)ep->ird); 1072256694Snp mpa_v2_params.ord = htons((u16)ep->ord); 1073256694Snp 1074256694Snp if (peer2peer) { 1075256694Snp mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1076256694Snp 1077256694Snp if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1078256694Snp mpa_v2_params.ord |= 1079256694Snp htons(MPA_V2_RDMA_WRITE_RTR); 1080256694Snp } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1081256694Snp mpa_v2_params.ord |= 1082256694Snp htons(MPA_V2_RDMA_READ_RTR); 1083256694Snp } 1084256694Snp } 1085256694Snp memcpy(mpa->private_data, &mpa_v2_params, 1086256694Snp sizeof(struct mpa_v2_conn_params)); 1087256694Snp 1088256694Snp if (ep->plen) { 1089256694Snp 1090256694Snp memcpy(mpa->private_data + 1091256694Snp sizeof(struct mpa_v2_conn_params), 1092256694Snp ep->mpa_pkt + sizeof(*mpa), ep->plen); 1093256694Snp } 1094256694Snp } else { 1095256694Snp 1096256694Snp if (ep->plen) 1097256694Snp memcpy(mpa->private_data, 1098256694Snp ep->mpa_pkt + sizeof(*mpa), ep->plen); 1099256694Snp CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep); 1100256694Snp } 1101256694Snp 1102281247Snp m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1103281247Snp if (m == NULL) { 1104309450Sjhb err = -ENOMEM; 1105309450Sjhb CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d", 1106309450Sjhb __func__, ep, err); 1107281247Snp free(mpa, M_CXGBE); 1108309450Sjhb goto err; 1109256694Snp } 1110281247Snp m_copyback(m, 0, mpalen, (void *)mpa); 1111281247Snp free(mpa, M_CXGBE); 1112256694Snp 1113309450Sjhb err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1114309450Sjhb ep->com.thread); 1115309450Sjhb if (err) { 1116309450Sjhb CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d", 1117309450Sjhb __func__, ep, err); 1118309450Sjhb goto err; 1119309450Sjhb } 1120281247Snp 1121256694Snp START_EP_TIMER(ep); 1122256694Snp state_set(&ep->com, MPA_REQ_SENT); 1123256694Snp ep->mpa_attr.initiator = 1; 1124309450Sjhb CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1125309450Sjhb return 0; 1126309450Sjhberr: 1127309450Sjhb connect_reply_upcall(ep, err); 1128309450Sjhb CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1129309450Sjhb return err; 1130256694Snp} 1131256694Snp 1132256694Snpstatic int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 1133256694Snp{ 1134256694Snp int mpalen ; 1135256694Snp struct mpa_message *mpa; 1136256694Snp struct mpa_v2_conn_params mpa_v2_params; 1137256694Snp struct mbuf *m; 1138256694Snp int err; 1139256694Snp 1140256694Snp CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid, 1141256694Snp ep->plen); 1142256694Snp 1143256694Snp mpalen = sizeof(*mpa) + plen; 1144256694Snp 1145256694Snp if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1146256694Snp 1147256694Snp mpalen += sizeof(struct mpa_v2_conn_params); 1148256694Snp CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep, 1149256694Snp ep->mpa_attr.version, mpalen); 1150256694Snp } 1151256694Snp 1152281247Snp mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1153281247Snp if (mpa == NULL) 1154256694Snp return (-ENOMEM); 1155256694Snp 1156281247Snp memset(mpa, 0, mpalen); 1157256694Snp memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1158256694Snp mpa->flags = MPA_REJECT; 1159256694Snp mpa->revision = mpa_rev; 1160256694Snp mpa->private_data_size = htons(plen); 1161256694Snp 1162256694Snp if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1163256694Snp 1164256694Snp mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1165256694Snp mpa->private_data_size += 1166256694Snp htons(sizeof(struct mpa_v2_conn_params)); 1167256694Snp mpa_v2_params.ird = htons(((u16)ep->ird) | 1168256694Snp (peer2peer ? MPA_V2_PEER2PEER_MODEL : 1169256694Snp 0)); 1170256694Snp mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 1171256694Snp (p2p_type == 1172256694Snp FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 1173256694Snp MPA_V2_RDMA_WRITE_RTR : p2p_type == 1174256694Snp FW_RI_INIT_P2PTYPE_READ_REQ ? 1175256694Snp MPA_V2_RDMA_READ_RTR : 0) : 0)); 1176256694Snp memcpy(mpa->private_data, &mpa_v2_params, 1177256694Snp sizeof(struct mpa_v2_conn_params)); 1178256694Snp 1179256694Snp if (ep->plen) 1180256694Snp memcpy(mpa->private_data + 1181256694Snp sizeof(struct mpa_v2_conn_params), pdata, plen); 1182256694Snp CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep, 1183256694Snp mpa_v2_params.ird, mpa_v2_params.ord, ep->plen); 1184256694Snp } else 1185256694Snp if (plen) 1186256694Snp memcpy(mpa->private_data, pdata, plen); 1187256694Snp 1188281247Snp m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1189281247Snp if (m == NULL) { 1190281247Snp free(mpa, M_CXGBE); 1191281247Snp return (-ENOMEM); 1192281247Snp } 1193281247Snp m_copyback(m, 0, mpalen, (void *)mpa); 1194281247Snp free(mpa, M_CXGBE); 1195281247Snp 1196281247Snp err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 1197256694Snp if (!err) 1198256694Snp ep->snd_seq += mpalen; 1199256694Snp CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err); 1200256694Snp return err; 1201256694Snp} 1202256694Snp 1203256694Snpstatic int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 1204256694Snp{ 1205256694Snp int mpalen; 1206256694Snp struct mpa_message *mpa; 1207256694Snp struct mbuf *m; 1208256694Snp struct mpa_v2_conn_params mpa_v2_params; 1209256694Snp int err; 1210256694Snp 1211256694Snp CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep); 1212256694Snp 1213256694Snp mpalen = sizeof(*mpa) + plen; 1214256694Snp 1215256694Snp if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1216256694Snp 1217256694Snp CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep, 1218256694Snp ep->mpa_attr.version); 1219256694Snp mpalen += sizeof(struct mpa_v2_conn_params); 1220256694Snp } 1221256694Snp 1222281247Snp mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1223281247Snp if (mpa == NULL) 1224256694Snp return (-ENOMEM); 1225256694Snp 1226256694Snp memset(mpa, 0, sizeof(*mpa)); 1227256694Snp memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1228256694Snp mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 1229256694Snp (markers_enabled ? MPA_MARKERS : 0); 1230256694Snp mpa->revision = ep->mpa_attr.version; 1231256694Snp mpa->private_data_size = htons(plen); 1232256694Snp 1233256694Snp if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1234256694Snp 1235256694Snp mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1236256694Snp mpa->private_data_size += 1237256694Snp htons(sizeof(struct mpa_v2_conn_params)); 1238256694Snp mpa_v2_params.ird = htons((u16)ep->ird); 1239256694Snp mpa_v2_params.ord = htons((u16)ep->ord); 1240256694Snp CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep, 1241256694Snp ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord); 1242256694Snp 1243256694Snp if (peer2peer && (ep->mpa_attr.p2p_type != 1244256694Snp FW_RI_INIT_P2PTYPE_DISABLED)) { 1245256694Snp 1246256694Snp mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1247256694Snp 1248256694Snp if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1249256694Snp 1250256694Snp mpa_v2_params.ord |= 1251256694Snp htons(MPA_V2_RDMA_WRITE_RTR); 1252256694Snp CTR5(KTR_IW_CXGBE, "%s:smrep4 %p %d %d %d", 1253256694Snp __func__, ep, p2p_type, mpa_v2_params.ird, 1254256694Snp mpa_v2_params.ord); 1255256694Snp } 1256256694Snp else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1257256694Snp 1258256694Snp mpa_v2_params.ord |= 1259256694Snp htons(MPA_V2_RDMA_READ_RTR); 1260256694Snp CTR5(KTR_IW_CXGBE, "%s:smrep5 %p %d %d %d", 1261256694Snp __func__, ep, p2p_type, mpa_v2_params.ird, 1262256694Snp mpa_v2_params.ord); 1263256694Snp } 1264256694Snp } 1265256694Snp 1266256694Snp memcpy(mpa->private_data, &mpa_v2_params, 1267256694Snp sizeof(struct mpa_v2_conn_params)); 1268256694Snp 1269256694Snp if (ep->plen) 1270256694Snp memcpy(mpa->private_data + 1271256694Snp sizeof(struct mpa_v2_conn_params), pdata, plen); 1272256694Snp } else 1273256694Snp if (plen) 1274256694Snp memcpy(mpa->private_data, pdata, plen); 1275256694Snp 1276281247Snp m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1277281247Snp if (m == NULL) { 1278281247Snp free(mpa, M_CXGBE); 1279281247Snp return (-ENOMEM); 1280281247Snp } 1281281247Snp m_copyback(m, 0, mpalen, (void *)mpa); 1282281247Snp free(mpa, M_CXGBE); 1283281247Snp 1284281247Snp 1285256694Snp state_set(&ep->com, MPA_REP_SENT); 1286256694Snp ep->snd_seq += mpalen; 1287281247Snp err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1288256694Snp ep->com.thread); 1289256694Snp CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err); 1290256694Snp return err; 1291256694Snp} 1292256694Snp 1293256694Snp 1294256694Snp 1295283856Snpstatic void close_complete_upcall(struct c4iw_ep *ep, int status) 1296256694Snp{ 1297256694Snp struct iw_cm_event event; 1298256694Snp 1299256694Snp CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep); 1300256694Snp memset(&event, 0, sizeof(event)); 1301256694Snp event.event = IW_CM_EVENT_CLOSE; 1302283856Snp event.status = status; 1303256694Snp 1304256694Snp if (ep->com.cm_id) { 1305256694Snp 1306256694Snp CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); 1307256694Snp ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1308309378Sjhb deref_cm_id(&ep->com); 1309256694Snp set_bit(CLOSE_UPCALL, &ep->com.history); 1310256694Snp } 1311256694Snp CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); 1312256694Snp} 1313256694Snp 1314314606Snpstatic int 1315314606Snpsend_abort(struct c4iw_ep *ep) 1316256694Snp{ 1317314606Snp struct socket *so = ep->com.so; 1318314606Snp struct sockopt sopt; 1319314606Snp int rc; 1320314606Snp struct linger l; 1321256694Snp 1322314606Snp CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so, 1323314606Snp states[ep->com.state], ep->hwtid); 1324309378Sjhb 1325314606Snp l.l_onoff = 1; 1326314606Snp l.l_linger = 0; 1327314606Snp 1328314606Snp /* linger_time of 0 forces RST to be sent */ 1329314606Snp sopt.sopt_dir = SOPT_SET; 1330314606Snp sopt.sopt_level = SOL_SOCKET; 1331314606Snp sopt.sopt_name = SO_LINGER; 1332314606Snp sopt.sopt_val = (caddr_t)&l; 1333314606Snp sopt.sopt_valsize = sizeof l; 1334314606Snp sopt.sopt_td = NULL; 1335314606Snp rc = sosetopt(so, &sopt); 1336314606Snp if (rc != 0) { 1337314606Snp log(LOG_ERR, "%s: sosetopt(%p, linger = 0) failed with %d.\n", 1338314606Snp __func__, so, rc); 1339314606Snp } 1340314606Snp 1341314606Snp uninit_iwarp_socket(so); 1342314606Snp sodisconnect(so); 1343256694Snp set_bit(ABORT_CONN, &ep->com.history); 1344309378Sjhb 1345309378Sjhb /* 1346314606Snp * TBD: iw_cxgbe driver should receive ABORT reply for every ABORT 1347309378Sjhb * request it has sent. But the current TOE driver is not propagating 1348309378Sjhb * this ABORT reply event (via do_abort_rpl) to iw_cxgbe. So as a work- 1349309378Sjhb * around de-refer 'ep' (which was refered before sending ABORT request) 1350309378Sjhb * here instead of doing it in abort_rpl() handler of iw_cxgbe driver. 1351309378Sjhb */ 1352309378Sjhb c4iw_put_ep(&ep->com); 1353314606Snp 1354314606Snp return (0); 1355256694Snp} 1356256694Snp 1357256694Snpstatic void peer_close_upcall(struct c4iw_ep *ep) 1358256694Snp{ 1359256694Snp struct iw_cm_event event; 1360256694Snp 1361256694Snp CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep); 1362256694Snp memset(&event, 0, sizeof(event)); 1363256694Snp event.event = IW_CM_EVENT_DISCONNECT; 1364256694Snp 1365256694Snp if (ep->com.cm_id) { 1366256694Snp 1367256694Snp CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep); 1368256694Snp ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1369256694Snp set_bit(DISCONN_UPCALL, &ep->com.history); 1370256694Snp } 1371256694Snp CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep); 1372256694Snp} 1373256694Snp 1374256694Snpstatic void peer_abort_upcall(struct c4iw_ep *ep) 1375256694Snp{ 1376256694Snp struct iw_cm_event event; 1377256694Snp 1378256694Snp CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep); 1379256694Snp memset(&event, 0, sizeof(event)); 1380256694Snp event.event = IW_CM_EVENT_CLOSE; 1381256694Snp event.status = -ECONNRESET; 1382256694Snp 1383256694Snp if (ep->com.cm_id) { 1384256694Snp 1385256694Snp CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); 1386256694Snp ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1387309378Sjhb deref_cm_id(&ep->com); 1388256694Snp set_bit(ABORT_UPCALL, &ep->com.history); 1389256694Snp } 1390256694Snp CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); 1391256694Snp} 1392256694Snp 1393256694Snpstatic void connect_reply_upcall(struct c4iw_ep *ep, int status) 1394256694Snp{ 1395256694Snp struct iw_cm_event event; 1396256694Snp 1397309450Sjhb CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status); 1398256694Snp memset(&event, 0, sizeof(event)); 1399256694Snp event.event = IW_CM_EVENT_CONNECT_REPLY; 1400309450Sjhb event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ? 1401309450Sjhb -ECONNRESET : status; 1402256694Snp event.local_addr = ep->com.local_addr; 1403256694Snp event.remote_addr = ep->com.remote_addr; 1404256694Snp 1405256694Snp if ((status == 0) || (status == -ECONNREFUSED)) { 1406256694Snp 1407256694Snp if (!ep->tried_with_mpa_v1) { 1408256694Snp 1409256694Snp CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep); 1410256694Snp /* this means MPA_v2 is used */ 1411256694Snp event.private_data_len = ep->plen - 1412256694Snp sizeof(struct mpa_v2_conn_params); 1413256694Snp event.private_data = ep->mpa_pkt + 1414256694Snp sizeof(struct mpa_message) + 1415256694Snp sizeof(struct mpa_v2_conn_params); 1416256694Snp } else { 1417256694Snp 1418256694Snp CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep); 1419256694Snp /* this means MPA_v1 is used */ 1420256694Snp event.private_data_len = ep->plen; 1421256694Snp event.private_data = ep->mpa_pkt + 1422256694Snp sizeof(struct mpa_message); 1423256694Snp } 1424256694Snp } 1425256694Snp 1426256694Snp if (ep->com.cm_id) { 1427256694Snp 1428256694Snp CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep); 1429256694Snp set_bit(CONN_RPL_UPCALL, &ep->com.history); 1430256694Snp ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1431256694Snp } 1432256694Snp 1433256694Snp if(status == -ECONNABORTED) { 1434256694Snp 1435256694Snp CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status); 1436256694Snp return; 1437256694Snp } 1438256694Snp 1439256694Snp if (status < 0) { 1440256694Snp 1441256694Snp CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); 1442309378Sjhb deref_cm_id(&ep->com); 1443256694Snp } 1444256694Snp 1445256694Snp CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); 1446256694Snp} 1447256694Snp 1448309378Sjhbstatic int connect_request_upcall(struct c4iw_ep *ep) 1449256694Snp{ 1450256694Snp struct iw_cm_event event; 1451309378Sjhb int ret; 1452256694Snp 1453256694Snp CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep, 1454256694Snp ep->tried_with_mpa_v1); 1455256694Snp 1456256694Snp memset(&event, 0, sizeof(event)); 1457256694Snp event.event = IW_CM_EVENT_CONNECT_REQUEST; 1458256694Snp event.local_addr = ep->com.local_addr; 1459256694Snp event.remote_addr = ep->com.remote_addr; 1460256694Snp event.provider_data = ep; 1461256694Snp event.so = ep->com.so; 1462256694Snp 1463256694Snp if (!ep->tried_with_mpa_v1) { 1464256694Snp /* this means MPA_v2 is used */ 1465256694Snp event.ord = ep->ord; 1466256694Snp event.ird = ep->ird; 1467256694Snp event.private_data_len = ep->plen - 1468256694Snp sizeof(struct mpa_v2_conn_params); 1469256694Snp event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1470256694Snp sizeof(struct mpa_v2_conn_params); 1471256694Snp } else { 1472256694Snp 1473256694Snp /* this means MPA_v1 is used. Send max supported */ 1474256694Snp event.ord = c4iw_max_read_depth; 1475256694Snp event.ird = c4iw_max_read_depth; 1476256694Snp event.private_data_len = ep->plen; 1477256694Snp event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1478256694Snp } 1479256694Snp 1480256694Snp c4iw_get_ep(&ep->com); 1481309378Sjhb ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1482256694Snp &event); 1483309378Sjhb if(ret) 1484309378Sjhb c4iw_put_ep(&ep->com); 1485309378Sjhb 1486256694Snp set_bit(CONNREQ_UPCALL, &ep->com.history); 1487256694Snp c4iw_put_ep(&ep->parent_ep->com); 1488309378Sjhb return ret; 1489256694Snp} 1490256694Snp 1491256694Snpstatic void established_upcall(struct c4iw_ep *ep) 1492256694Snp{ 1493256694Snp struct iw_cm_event event; 1494256694Snp 1495256694Snp CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep); 1496256694Snp memset(&event, 0, sizeof(event)); 1497256694Snp event.event = IW_CM_EVENT_ESTABLISHED; 1498256694Snp event.ird = ep->ird; 1499256694Snp event.ord = ep->ord; 1500309378Sjhb 1501256694Snp if (ep->com.cm_id) { 1502256694Snp 1503256694Snp CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep); 1504256694Snp ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1505256694Snp set_bit(ESTAB_UPCALL, &ep->com.history); 1506256694Snp } 1507256694Snp CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep); 1508256694Snp} 1509256694Snp 1510256694Snp 1511309378Sjhb/* 1512309378Sjhb * process_mpa_reply - process streaming mode MPA reply 1513309378Sjhb * 1514309378Sjhb * Returns: 1515309378Sjhb * 1516309378Sjhb * 0 upon success indicating a connect request was delivered to the ULP 1517309378Sjhb * or the mpa request is incomplete but valid so far. 1518309378Sjhb * 1519309378Sjhb * 1 if a failure requires the caller to close the connection. 1520309378Sjhb * 1521309378Sjhb * 2 if a failure requires the caller to abort the connection. 1522309378Sjhb */ 1523309378Sjhbstatic int process_mpa_reply(struct c4iw_ep *ep) 1524256694Snp{ 1525256694Snp struct mpa_message *mpa; 1526256694Snp struct mpa_v2_conn_params *mpa_v2_params; 1527256694Snp u16 plen; 1528256694Snp u16 resp_ird, resp_ord; 1529256694Snp u8 rtr_mismatch = 0, insuff_ird = 0; 1530256694Snp struct c4iw_qp_attributes attrs; 1531256694Snp enum c4iw_qp_attr_mask mask; 1532256694Snp int err; 1533256694Snp struct mbuf *top, *m; 1534256694Snp int flags = MSG_DONTWAIT; 1535256694Snp struct uio uio; 1536309378Sjhb int disconnect = 0; 1537256694Snp 1538256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); 1539256694Snp 1540256694Snp /* 1541309378Sjhb * Stop mpa timer. If it expired, then 1542309378Sjhb * we ignore the MPA reply. process_timeout() 1543309378Sjhb * will abort the connection. 1544256694Snp */ 1545309378Sjhb if (STOP_EP_TIMER(ep)) 1546309378Sjhb return 0; 1547256694Snp 1548256694Snp uio.uio_resid = 1000000; 1549256694Snp uio.uio_td = ep->com.thread; 1550256694Snp err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 1551256694Snp 1552256694Snp if (err) { 1553256694Snp 1554256694Snp if (err == EWOULDBLOCK) { 1555256694Snp 1556256694Snp CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); 1557256694Snp START_EP_TIMER(ep); 1558309378Sjhb return 0; 1559256694Snp } 1560256694Snp err = -err; 1561256694Snp CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); 1562256694Snp goto err; 1563256694Snp } 1564256694Snp 1565256694Snp if (ep->com.so->so_rcv.sb_mb) { 1566256694Snp 1567256694Snp CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep); 1568256694Snp printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 1569256694Snp __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 1570256694Snp } 1571256694Snp 1572256694Snp m = top; 1573256694Snp 1574256694Snp do { 1575256694Snp 1576256694Snp CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep); 1577256694Snp /* 1578256694Snp * If we get more than the supported amount of private data 1579256694Snp * then we must fail this connection. 1580256694Snp */ 1581256694Snp if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 1582256694Snp 1583256694Snp CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, 1584256694Snp ep->mpa_pkt_len + m->m_len); 1585256694Snp err = (-EINVAL); 1586309378Sjhb goto err_stop_timer; 1587256694Snp } 1588256694Snp 1589256694Snp /* 1590256694Snp * copy the new data into our accumulation buffer. 1591256694Snp */ 1592256694Snp m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 1593256694Snp ep->mpa_pkt_len += m->m_len; 1594256694Snp if (!m->m_next) 1595256694Snp m = m->m_nextpkt; 1596256694Snp else 1597256694Snp m = m->m_next; 1598256694Snp } while (m); 1599256694Snp 1600256694Snp m_freem(top); 1601256694Snp /* 1602256694Snp * if we don't even have the mpa message, then bail. 1603256694Snp */ 1604309378Sjhb if (ep->mpa_pkt_len < sizeof(*mpa)) { 1605309378Sjhb return 0; 1606309378Sjhb } 1607256694Snp mpa = (struct mpa_message *) ep->mpa_pkt; 1608256694Snp 1609256694Snp /* Validate MPA header. */ 1610256694Snp if (mpa->revision > mpa_rev) { 1611256694Snp 1612256694Snp CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep, 1613256694Snp mpa->revision, mpa_rev); 1614256694Snp printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " 1615256694Snp " Received = %d\n", __func__, mpa_rev, mpa->revision); 1616256694Snp err = -EPROTO; 1617309378Sjhb goto err_stop_timer; 1618256694Snp } 1619256694Snp 1620256694Snp if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1621256694Snp 1622256694Snp CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); 1623256694Snp err = -EPROTO; 1624309378Sjhb goto err_stop_timer; 1625256694Snp } 1626256694Snp 1627256694Snp plen = ntohs(mpa->private_data_size); 1628256694Snp 1629256694Snp /* 1630256694Snp * Fail if there's too much private data. 1631256694Snp */ 1632256694Snp if (plen > MPA_MAX_PRIVATE_DATA) { 1633256694Snp 1634256694Snp CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); 1635256694Snp err = -EPROTO; 1636309378Sjhb goto err_stop_timer; 1637256694Snp } 1638256694Snp 1639256694Snp /* 1640256694Snp * If plen does not account for pkt size 1641256694Snp */ 1642256694Snp if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1643256694Snp 1644256694Snp CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); 1645309378Sjhb STOP_EP_TIMER(ep); 1646256694Snp err = -EPROTO; 1647309378Sjhb goto err_stop_timer; 1648256694Snp } 1649256694Snp 1650256694Snp ep->plen = (u8) plen; 1651256694Snp 1652256694Snp /* 1653256694Snp * If we don't have all the pdata yet, then bail. 1654256694Snp * We'll continue process when more data arrives. 1655256694Snp */ 1656256694Snp if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 1657256694Snp 1658256694Snp CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); 1659309378Sjhb return 0; 1660256694Snp } 1661256694Snp 1662256694Snp if (mpa->flags & MPA_REJECT) { 1663256694Snp 1664256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); 1665256694Snp err = -ECONNREFUSED; 1666309378Sjhb goto err_stop_timer; 1667256694Snp } 1668256694Snp 1669256694Snp /* 1670256694Snp * If we get here we have accumulated the entire mpa 1671256694Snp * start reply message including private data. And 1672256694Snp * the MPA header is valid. 1673256694Snp */ 1674256694Snp state_set(&ep->com, FPDU_MODE); 1675256694Snp ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1676256694Snp ep->mpa_attr.recv_marker_enabled = markers_enabled; 1677256694Snp ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1678256694Snp ep->mpa_attr.version = mpa->revision; 1679256694Snp ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1680256694Snp 1681256694Snp if (mpa->revision == 2) { 1682256694Snp 1683256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep); 1684256694Snp ep->mpa_attr.enhanced_rdma_conn = 1685256694Snp mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1686256694Snp 1687256694Snp if (ep->mpa_attr.enhanced_rdma_conn) { 1688256694Snp 1689256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep); 1690256694Snp mpa_v2_params = (struct mpa_v2_conn_params *) 1691256694Snp (ep->mpa_pkt + sizeof(*mpa)); 1692256694Snp resp_ird = ntohs(mpa_v2_params->ird) & 1693256694Snp MPA_V2_IRD_ORD_MASK; 1694256694Snp resp_ord = ntohs(mpa_v2_params->ord) & 1695256694Snp MPA_V2_IRD_ORD_MASK; 1696256694Snp 1697256694Snp /* 1698256694Snp * This is a double-check. Ideally, below checks are 1699256694Snp * not required since ird/ord stuff has been taken 1700256694Snp * care of in c4iw_accept_cr 1701256694Snp */ 1702256694Snp if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1703256694Snp 1704256694Snp CTR2(KTR_IW_CXGBE, "%s:pmre %p", __func__, ep); 1705256694Snp err = -ENOMEM; 1706256694Snp ep->ird = resp_ord; 1707256694Snp ep->ord = resp_ird; 1708256694Snp insuff_ird = 1; 1709256694Snp } 1710256694Snp 1711256694Snp if (ntohs(mpa_v2_params->ird) & 1712256694Snp MPA_V2_PEER2PEER_MODEL) { 1713256694Snp 1714256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep); 1715256694Snp if (ntohs(mpa_v2_params->ord) & 1716256694Snp MPA_V2_RDMA_WRITE_RTR) { 1717256694Snp 1718256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep); 1719256694Snp ep->mpa_attr.p2p_type = 1720256694Snp FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1721256694Snp } 1722256694Snp else if (ntohs(mpa_v2_params->ord) & 1723256694Snp MPA_V2_RDMA_READ_RTR) { 1724256694Snp 1725256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep); 1726256694Snp ep->mpa_attr.p2p_type = 1727256694Snp FW_RI_INIT_P2PTYPE_READ_REQ; 1728256694Snp } 1729256694Snp } 1730256694Snp } 1731256694Snp } else { 1732256694Snp 1733256694Snp CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep); 1734256694Snp 1735256694Snp if (mpa->revision == 1) { 1736256694Snp 1737256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep); 1738256694Snp 1739256694Snp if (peer2peer) { 1740256694Snp 1741256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep); 1742256694Snp ep->mpa_attr.p2p_type = p2p_type; 1743256694Snp } 1744256694Snp } 1745256694Snp } 1746256694Snp 1747256694Snp if (set_tcpinfo(ep)) { 1748256694Snp 1749256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep); 1750256694Snp printf("%s set_tcpinfo error\n", __func__); 1751309450Sjhb err = -ECONNRESET; 1752256694Snp goto err; 1753256694Snp } 1754256694Snp 1755256694Snp CTR6(KTR_IW_CXGBE, "%s - crc_enabled = %d, recv_marker_enabled = %d, " 1756256694Snp "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__, 1757256694Snp ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1758256694Snp ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1759256694Snp ep->mpa_attr.p2p_type); 1760256694Snp 1761256694Snp /* 1762256694Snp * If responder's RTR does not match with that of initiator, assign 1763256694Snp * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1764256694Snp * generated when moving QP to RTS state. 1765256694Snp * A TERM message will be sent after QP has moved to RTS state 1766256694Snp */ 1767256694Snp if ((ep->mpa_attr.version == 2) && peer2peer && 1768256694Snp (ep->mpa_attr.p2p_type != p2p_type)) { 1769256694Snp 1770256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep); 1771256694Snp ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1772256694Snp rtr_mismatch = 1; 1773256694Snp } 1774256694Snp 1775256694Snp 1776256694Snp //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 1777256694Snp attrs.mpa_attr = ep->mpa_attr; 1778256694Snp attrs.max_ird = ep->ird; 1779256694Snp attrs.max_ord = ep->ord; 1780256694Snp attrs.llp_stream_handle = ep; 1781256694Snp attrs.next_state = C4IW_QP_STATE_RTS; 1782256694Snp 1783256694Snp mask = C4IW_QP_ATTR_NEXT_STATE | 1784256694Snp C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1785256694Snp C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1786256694Snp 1787256694Snp /* bind QP and TID with INIT_WR */ 1788256694Snp err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 1789256694Snp 1790256694Snp if (err) { 1791256694Snp 1792256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep); 1793256694Snp goto err; 1794256694Snp } 1795256694Snp 1796256694Snp /* 1797256694Snp * If responder's RTR requirement did not match with what initiator 1798256694Snp * supports, generate TERM message 1799256694Snp */ 1800256694Snp if (rtr_mismatch) { 1801256694Snp 1802256694Snp CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep); 1803256694Snp printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1804256694Snp attrs.layer_etype = LAYER_MPA | DDP_LLP; 1805256694Snp attrs.ecode = MPA_NOMATCH_RTR; 1806256694Snp attrs.next_state = C4IW_QP_STATE_TERMINATE; 1807256694Snp err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1808256694Snp C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1809256694Snp err = -ENOMEM; 1810309378Sjhb disconnect = 1; 1811256694Snp goto out; 1812256694Snp } 1813256694Snp 1814256694Snp /* 1815256694Snp * Generate TERM if initiator IRD is not sufficient for responder 1816256694Snp * provided ORD. Currently, we do the same behaviour even when 1817256694Snp * responder provided IRD is also not sufficient as regards to 1818256694Snp * initiator ORD. 1819256694Snp */ 1820256694Snp if (insuff_ird) { 1821256694Snp 1822256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep); 1823256694Snp printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1824256694Snp __func__); 1825256694Snp attrs.layer_etype = LAYER_MPA | DDP_LLP; 1826256694Snp attrs.ecode = MPA_INSUFF_IRD; 1827256694Snp attrs.next_state = C4IW_QP_STATE_TERMINATE; 1828256694Snp err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1829256694Snp C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1830256694Snp err = -ENOMEM; 1831309378Sjhb disconnect = 1; 1832256694Snp goto out; 1833256694Snp } 1834256694Snp goto out; 1835309378Sjhberr_stop_timer: 1836309378Sjhb STOP_EP_TIMER(ep); 1837256694Snperr: 1838309378Sjhb disconnect = 2; 1839256694Snpout: 1840256694Snp connect_reply_upcall(ep, err); 1841256694Snp CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); 1842309378Sjhb return disconnect; 1843256694Snp} 1844256694Snp 1845309378Sjhb/* 1846309378Sjhb * process_mpa_request - process streaming mode MPA request 1847309378Sjhb * 1848309378Sjhb * Returns: 1849309378Sjhb * 1850309378Sjhb * 0 upon success indicating a connect request was delivered to the ULP 1851309378Sjhb * or the mpa request is incomplete but valid so far. 1852309378Sjhb * 1853309378Sjhb * 1 if a failure requires the caller to close the connection. 1854309378Sjhb * 1855309378Sjhb * 2 if a failure requires the caller to abort the connection. 1856309378Sjhb */ 1857309378Sjhbstatic int 1858256694Snpprocess_mpa_request(struct c4iw_ep *ep) 1859256694Snp{ 1860256694Snp struct mpa_message *mpa; 1861256694Snp u16 plen; 1862256694Snp int flags = MSG_DONTWAIT; 1863256694Snp int rc; 1864256694Snp struct iovec iov; 1865256694Snp struct uio uio; 1866256694Snp enum c4iw_ep_state state = state_read(&ep->com); 1867256694Snp 1868256694Snp CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); 1869256694Snp 1870256694Snp if (state != MPA_REQ_WAIT) 1871309378Sjhb return 0; 1872256694Snp 1873256694Snp iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; 1874256694Snp iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1875256694Snp uio.uio_iov = &iov; 1876256694Snp uio.uio_iovcnt = 1; 1877256694Snp uio.uio_offset = 0; 1878256694Snp uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1879256694Snp uio.uio_segflg = UIO_SYSSPACE; 1880256694Snp uio.uio_rw = UIO_READ; 1881256694Snp uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */ 1882256694Snp 1883256694Snp rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); 1884256694Snp if (rc == EAGAIN) 1885309378Sjhb return 0; 1886309378Sjhb else if (rc) 1887309378Sjhb goto err_stop_timer; 1888309378Sjhb 1889256694Snp KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", 1890256694Snp __func__, ep->com.so)); 1891256694Snp ep->mpa_pkt_len += uio.uio_offset; 1892256694Snp 1893256694Snp /* 1894256694Snp * If we get more than the supported amount of private data then we must 1895256694Snp * fail this connection. XXX: check so_rcv->sb_cc, or peek with another 1896256694Snp * soreceive, or increase the size of mpa_pkt by 1 and abort if the last 1897256694Snp * byte is filled by the soreceive above. 1898256694Snp */ 1899256694Snp 1900256694Snp /* Don't even have the MPA message. Wait for more data to arrive. */ 1901256694Snp if (ep->mpa_pkt_len < sizeof(*mpa)) 1902309378Sjhb return 0; 1903256694Snp mpa = (struct mpa_message *) ep->mpa_pkt; 1904256694Snp 1905256694Snp /* 1906256694Snp * Validate MPA Header. 1907256694Snp */ 1908256694Snp if (mpa->revision > mpa_rev) { 1909256694Snp log(LOG_ERR, "%s: MPA version mismatch. Local = %d," 1910256694Snp " Received = %d\n", __func__, mpa_rev, mpa->revision); 1911309378Sjhb goto err_stop_timer; 1912256694Snp } 1913256694Snp 1914256694Snp if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) 1915309378Sjhb goto err_stop_timer; 1916256694Snp 1917256694Snp /* 1918256694Snp * Fail if there's too much private data. 1919256694Snp */ 1920256694Snp plen = ntohs(mpa->private_data_size); 1921256694Snp if (plen > MPA_MAX_PRIVATE_DATA) 1922309378Sjhb goto err_stop_timer; 1923256694Snp 1924256694Snp /* 1925256694Snp * If plen does not account for pkt size 1926256694Snp */ 1927256694Snp if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) 1928309378Sjhb goto err_stop_timer; 1929256694Snp 1930256694Snp ep->plen = (u8) plen; 1931256694Snp 1932256694Snp /* 1933256694Snp * If we don't have all the pdata yet, then bail. 1934256694Snp */ 1935256694Snp if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1936309378Sjhb return 0; 1937256694Snp 1938256694Snp /* 1939256694Snp * If we get here we have accumulated the entire mpa 1940256694Snp * start reply message including private data. 1941256694Snp */ 1942256694Snp ep->mpa_attr.initiator = 0; 1943256694Snp ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1944256694Snp ep->mpa_attr.recv_marker_enabled = markers_enabled; 1945256694Snp ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1946256694Snp ep->mpa_attr.version = mpa->revision; 1947256694Snp if (mpa->revision == 1) 1948256694Snp ep->tried_with_mpa_v1 = 1; 1949256694Snp ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1950256694Snp 1951256694Snp if (mpa->revision == 2) { 1952256694Snp ep->mpa_attr.enhanced_rdma_conn = 1953256694Snp mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1954256694Snp if (ep->mpa_attr.enhanced_rdma_conn) { 1955256694Snp struct mpa_v2_conn_params *mpa_v2_params; 1956256694Snp u16 ird, ord; 1957256694Snp 1958256694Snp mpa_v2_params = (void *)&ep->mpa_pkt[sizeof(*mpa)]; 1959256694Snp ird = ntohs(mpa_v2_params->ird); 1960256694Snp ord = ntohs(mpa_v2_params->ord); 1961256694Snp 1962256694Snp ep->ird = ird & MPA_V2_IRD_ORD_MASK; 1963256694Snp ep->ord = ord & MPA_V2_IRD_ORD_MASK; 1964256694Snp if (ird & MPA_V2_PEER2PEER_MODEL && peer2peer) { 1965256694Snp if (ord & MPA_V2_RDMA_WRITE_RTR) { 1966256694Snp ep->mpa_attr.p2p_type = 1967256694Snp FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1968256694Snp } else if (ord & MPA_V2_RDMA_READ_RTR) { 1969256694Snp ep->mpa_attr.p2p_type = 1970256694Snp FW_RI_INIT_P2PTYPE_READ_REQ; 1971256694Snp } 1972256694Snp } 1973256694Snp } 1974256694Snp } else if (mpa->revision == 1 && peer2peer) 1975256694Snp ep->mpa_attr.p2p_type = p2p_type; 1976256694Snp 1977256694Snp if (set_tcpinfo(ep)) 1978309378Sjhb goto err_stop_timer; 1979256694Snp 1980256694Snp CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " 1981256694Snp "xmit_marker_enabled = %d, version = %d", __func__, 1982256694Snp ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1983256694Snp ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 1984256694Snp 1985256694Snp state_set(&ep->com, MPA_REQ_RCVD); 1986256694Snp STOP_EP_TIMER(ep); 1987256694Snp 1988256694Snp /* drive upcall */ 1989256694Snp mutex_lock(&ep->parent_ep->com.mutex); 1990309378Sjhb if (ep->parent_ep->com.state != DEAD) { 1991309378Sjhb if (connect_request_upcall(ep)) 1992309378Sjhb goto err_unlock_parent; 1993309378Sjhb } else 1994309378Sjhb goto err_unlock_parent; 1995256694Snp mutex_unlock(&ep->parent_ep->com.mutex); 1996309378Sjhb return 0; 1997309378Sjhb 1998309378Sjhberr_unlock_parent: 1999309378Sjhb mutex_unlock(&ep->parent_ep->com.mutex); 2000309378Sjhb goto err_out; 2001309378Sjhberr_stop_timer: 2002309378Sjhb STOP_EP_TIMER(ep); 2003309378Sjhberr_out: 2004309378Sjhb return 2; 2005256694Snp} 2006256694Snp 2007256694Snp/* 2008256694Snp * Upcall from the adapter indicating data has been transmitted. 2009256694Snp * For us its just the single MPA request or reply. We can now free 2010256694Snp * the skb holding the mpa message. 2011256694Snp */ 2012256694Snpint c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2013256694Snp{ 2014256694Snp int err; 2015256694Snp struct c4iw_ep *ep = to_ep(cm_id); 2016256694Snp CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); 2017309450Sjhb int abort = 0; 2018256694Snp 2019309450Sjhb if ((state_read(&ep->com) == DEAD) || 2020309450Sjhb (state_read(&ep->com) != MPA_REQ_RCVD)) { 2021256694Snp 2022256694Snp CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep); 2023256694Snp c4iw_put_ep(&ep->com); 2024256694Snp return -ECONNRESET; 2025256694Snp } 2026256694Snp set_bit(ULP_REJECT, &ep->com.history); 2027256694Snp 2028256694Snp if (mpa_rev == 0) { 2029256694Snp 2030256694Snp CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); 2031309450Sjhb abort = 1; 2032256694Snp } 2033256694Snp else { 2034256694Snp 2035256694Snp CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep); 2036309450Sjhb abort = send_mpa_reject(ep, pdata, pdata_len); 2037256694Snp } 2038309450Sjhb stop_ep_timer(ep); 2039309450Sjhb err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL); 2040256694Snp c4iw_put_ep(&ep->com); 2041309450Sjhb CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err); 2042256694Snp return 0; 2043256694Snp} 2044256694Snp 2045256694Snpint c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2046256694Snp{ 2047256694Snp int err; 2048256694Snp struct c4iw_qp_attributes attrs; 2049256694Snp enum c4iw_qp_attr_mask mask; 2050256694Snp struct c4iw_ep *ep = to_ep(cm_id); 2051256694Snp struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2052256694Snp struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2053309378Sjhb int abort = 0; 2054256694Snp 2055256694Snp CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); 2056256694Snp 2057256694Snp if (state_read(&ep->com) == DEAD) { 2058256694Snp 2059256694Snp CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); 2060256694Snp err = -ECONNRESET; 2061309378Sjhb goto err_out; 2062256694Snp } 2063256694Snp 2064256694Snp BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2065256694Snp BUG_ON(!qp); 2066256694Snp 2067256694Snp set_bit(ULP_ACCEPT, &ep->com.history); 2068256694Snp 2069256694Snp if ((conn_param->ord > c4iw_max_read_depth) || 2070256694Snp (conn_param->ird > c4iw_max_read_depth)) { 2071256694Snp 2072256694Snp CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); 2073256694Snp err = -EINVAL; 2074309378Sjhb goto err_abort; 2075256694Snp } 2076256694Snp 2077256694Snp if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2078256694Snp 2079256694Snp CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep); 2080256694Snp 2081256694Snp if (conn_param->ord > ep->ird) { 2082256694Snp 2083256694Snp CTR2(KTR_IW_CXGBE, "%s:cac4 %p", __func__, ep); 2084256694Snp ep->ird = conn_param->ird; 2085256694Snp ep->ord = conn_param->ord; 2086256694Snp send_mpa_reject(ep, conn_param->private_data, 2087256694Snp conn_param->private_data_len); 2088256694Snp err = -ENOMEM; 2089309378Sjhb goto err_abort; 2090256694Snp } 2091256694Snp 2092256694Snp if (conn_param->ird > ep->ord) { 2093256694Snp 2094256694Snp CTR2(KTR_IW_CXGBE, "%s:cac5 %p", __func__, ep); 2095256694Snp 2096256694Snp if (!ep->ord) { 2097256694Snp 2098256694Snp CTR2(KTR_IW_CXGBE, "%s:cac6 %p", __func__, ep); 2099256694Snp conn_param->ird = 1; 2100256694Snp } 2101256694Snp else { 2102256694Snp CTR2(KTR_IW_CXGBE, "%s:cac7 %p", __func__, ep); 2103256694Snp err = -ENOMEM; 2104309378Sjhb goto err_abort; 2105256694Snp } 2106256694Snp } 2107256694Snp 2108256694Snp } 2109256694Snp ep->ird = conn_param->ird; 2110256694Snp ep->ord = conn_param->ord; 2111256694Snp 2112256694Snp if (ep->mpa_attr.version != 2) { 2113256694Snp 2114256694Snp CTR2(KTR_IW_CXGBE, "%s:cac8 %p", __func__, ep); 2115256694Snp 2116256694Snp if (peer2peer && ep->ird == 0) { 2117256694Snp 2118256694Snp CTR2(KTR_IW_CXGBE, "%s:cac9 %p", __func__, ep); 2119256694Snp ep->ird = 1; 2120256694Snp } 2121256694Snp } 2122256694Snp 2123256694Snp 2124256694Snp ep->com.cm_id = cm_id; 2125309378Sjhb ref_cm_id(&ep->com); 2126256694Snp ep->com.qp = qp; 2127309378Sjhb ref_qp(ep); 2128256694Snp //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 2129256694Snp 2130256694Snp /* bind QP to EP and move to RTS */ 2131256694Snp attrs.mpa_attr = ep->mpa_attr; 2132256694Snp attrs.max_ird = ep->ird; 2133256694Snp attrs.max_ord = ep->ord; 2134256694Snp attrs.llp_stream_handle = ep; 2135256694Snp attrs.next_state = C4IW_QP_STATE_RTS; 2136256694Snp 2137256694Snp /* bind QP and TID with INIT_WR */ 2138256694Snp mask = C4IW_QP_ATTR_NEXT_STATE | 2139256694Snp C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2140256694Snp C4IW_QP_ATTR_MPA_ATTR | 2141256694Snp C4IW_QP_ATTR_MAX_IRD | 2142256694Snp C4IW_QP_ATTR_MAX_ORD; 2143256694Snp 2144256694Snp err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 2145256694Snp 2146256694Snp if (err) { 2147256694Snp 2148256694Snp CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2149309378Sjhb goto err_defef_cm_id; 2150256694Snp } 2151256694Snp err = send_mpa_reply(ep, conn_param->private_data, 2152256694Snp conn_param->private_data_len); 2153256694Snp 2154256694Snp if (err) { 2155256694Snp 2156256694Snp CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2157309378Sjhb goto err_defef_cm_id; 2158256694Snp } 2159256694Snp 2160256694Snp state_set(&ep->com, FPDU_MODE); 2161256694Snp established_upcall(ep); 2162256694Snp c4iw_put_ep(&ep->com); 2163256694Snp CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); 2164256694Snp return 0; 2165309378Sjhberr_defef_cm_id: 2166309378Sjhb deref_cm_id(&ep->com); 2167309378Sjhberr_abort: 2168309378Sjhb abort = 1; 2169309378Sjhberr_out: 2170309378Sjhb if (abort) 2171309378Sjhb c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 2172256694Snp c4iw_put_ep(&ep->com); 2173256694Snp CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); 2174256694Snp return err; 2175256694Snp} 2176256694Snp 2177256694Snp 2178256694Snp 2179256694Snpint c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2180256694Snp{ 2181256694Snp int err = 0; 2182256694Snp struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2183256694Snp struct c4iw_ep *ep = NULL; 2184256694Snp struct rtentry *rt; 2185256694Snp 2186256694Snp CTR2(KTR_IW_CXGBE, "%s:ccB %p", __func__, cm_id); 2187256694Snp 2188256694Snp if ((conn_param->ord > c4iw_max_read_depth) || 2189256694Snp (conn_param->ird > c4iw_max_read_depth)) { 2190256694Snp 2191256694Snp CTR2(KTR_IW_CXGBE, "%s:cc1 %p", __func__, cm_id); 2192256694Snp err = -EINVAL; 2193256694Snp goto out; 2194256694Snp } 2195316123Snp ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2196256694Snp init_timer(&ep->timer); 2197256694Snp ep->plen = conn_param->private_data_len; 2198256694Snp 2199256694Snp if (ep->plen) { 2200256694Snp 2201256694Snp CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep); 2202256694Snp memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2203256694Snp conn_param->private_data, ep->plen); 2204256694Snp } 2205256694Snp ep->ird = conn_param->ird; 2206256694Snp ep->ord = conn_param->ord; 2207256694Snp 2208256694Snp if (peer2peer && ep->ord == 0) { 2209256694Snp 2210256694Snp CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep); 2211256694Snp ep->ord = 1; 2212256694Snp } 2213256694Snp 2214256694Snp ep->com.dev = dev; 2215256694Snp ep->com.cm_id = cm_id; 2216309378Sjhb ref_cm_id(&ep->com); 2217256694Snp ep->com.qp = get_qhp(dev, conn_param->qpn); 2218256694Snp 2219256694Snp if (!ep->com.qp) { 2220256694Snp 2221256694Snp CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep); 2222256694Snp err = -EINVAL; 2223256694Snp goto fail2; 2224256694Snp } 2225309378Sjhb ref_qp(ep); 2226256694Snp ep->com.thread = curthread; 2227256694Snp ep->com.so = cm_id->so; 2228256694Snp 2229256694Snp /* find a route */ 2230256694Snp rt = find_route( 2231256694Snp cm_id->local_addr.sin_addr.s_addr, 2232256694Snp cm_id->remote_addr.sin_addr.s_addr, 2233256694Snp cm_id->local_addr.sin_port, 2234256694Snp cm_id->remote_addr.sin_port, 0); 2235256694Snp 2236256694Snp if (!rt) { 2237256694Snp 2238256694Snp CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep); 2239256694Snp printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2240256694Snp err = -EHOSTUNREACH; 2241283856Snp goto fail2; 2242256694Snp } 2243256694Snp 2244314606Snp if (!(rt->rt_ifp->if_capenable & IFCAP_TOE) || 2245314606Snp TOEDEV(rt->rt_ifp) == NULL) { 2246283856Snp err = -ENOPROTOOPT; 2247256694Snp goto fail3; 2248256694Snp } 2249256694Snp RTFREE(rt); 2250256694Snp 2251256694Snp state_set(&ep->com, CONNECTING); 2252256694Snp ep->tos = 0; 2253256694Snp ep->com.local_addr = cm_id->local_addr; 2254256694Snp ep->com.remote_addr = cm_id->remote_addr; 2255316123Snp err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 2256256694Snp ep->com.thread); 2257256694Snp 2258256694Snp if (!err) { 2259314606Snp init_iwarp_socket(cm_id->so, &ep->com); 2260256694Snp goto out; 2261283856Snp } else { 2262283856Snp goto fail2; 2263256694Snp } 2264256694Snp 2265256694Snpfail3: 2266256694Snp RTFREE(rt); 2267256694Snpfail2: 2268309378Sjhb deref_cm_id(&ep->com); 2269256694Snp c4iw_put_ep(&ep->com); 2270314606Snp ep = NULL; /* CTR shouldn't display already-freed ep. */ 2271256694Snpout: 2272256694Snp CTR2(KTR_IW_CXGBE, "%s:ccE %p", __func__, ep); 2273256694Snp return err; 2274256694Snp} 2275256694Snp 2276256694Snp/* 2277309378Sjhb * iwcm->create_listen_ep. Returns -errno on failure. 2278256694Snp */ 2279256694Snpint 2280309378Sjhbc4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog) 2281256694Snp{ 2282256694Snp struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2283256694Snp struct c4iw_listen_ep *ep; 2284256694Snp struct socket *so = cm_id->so; 2285256694Snp 2286256694Snp ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2287256694Snp ep->com.cm_id = cm_id; 2288309378Sjhb ref_cm_id(&ep->com); 2289256694Snp ep->com.dev = dev; 2290256694Snp ep->backlog = backlog; 2291256694Snp ep->com.local_addr = cm_id->local_addr; 2292256694Snp ep->com.thread = curthread; 2293256694Snp state_set(&ep->com, LISTEN); 2294256694Snp ep->com.so = so; 2295256694Snp 2296256694Snp cm_id->provider_data = ep; 2297256694Snp return (0); 2298256694Snp} 2299256694Snp 2300309378Sjhbvoid 2301309378Sjhbc4iw_destroy_listen_ep(struct iw_cm_id *cm_id) 2302256694Snp{ 2303256694Snp struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2304256694Snp 2305309378Sjhb CTR4(KTR_IW_CXGBE, "%s: cm_id %p, so %p, state %s", __func__, cm_id, 2306309378Sjhb cm_id->so, states[ep->com.state]); 2307256694Snp 2308256694Snp state_set(&ep->com, DEAD); 2309309378Sjhb deref_cm_id(&ep->com); 2310256694Snp c4iw_put_ep(&ep->com); 2311256694Snp 2312309378Sjhb return; 2313256694Snp} 2314256694Snp 2315256694Snpint c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2316256694Snp{ 2317256694Snp int ret = 0; 2318256694Snp int close = 0; 2319256694Snp int fatal = 0; 2320256694Snp struct c4iw_rdev *rdev; 2321256694Snp 2322256694Snp mutex_lock(&ep->com.mutex); 2323256694Snp 2324256694Snp CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep); 2325256694Snp 2326256694Snp rdev = &ep->com.dev->rdev; 2327256694Snp 2328256694Snp if (c4iw_fatal_error(rdev)) { 2329256694Snp 2330256694Snp CTR2(KTR_IW_CXGBE, "%s:ced1 %p", __func__, ep); 2331256694Snp fatal = 1; 2332309378Sjhb close_complete_upcall(ep, -ECONNRESET); 2333256694Snp ep->com.state = DEAD; 2334256694Snp } 2335256694Snp CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep, 2336256694Snp states[ep->com.state]); 2337256694Snp 2338256694Snp switch (ep->com.state) { 2339256694Snp 2340256694Snp case MPA_REQ_WAIT: 2341256694Snp case MPA_REQ_SENT: 2342256694Snp case MPA_REQ_RCVD: 2343256694Snp case MPA_REP_SENT: 2344256694Snp case FPDU_MODE: 2345256694Snp close = 1; 2346256694Snp if (abrupt) 2347256694Snp ep->com.state = ABORTING; 2348256694Snp else { 2349256694Snp ep->com.state = CLOSING; 2350256694Snp START_EP_TIMER(ep); 2351256694Snp } 2352256694Snp set_bit(CLOSE_SENT, &ep->com.flags); 2353256694Snp break; 2354256694Snp 2355256694Snp case CLOSING: 2356256694Snp 2357256694Snp if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2358256694Snp 2359256694Snp close = 1; 2360256694Snp if (abrupt) { 2361256694Snp STOP_EP_TIMER(ep); 2362256694Snp ep->com.state = ABORTING; 2363256694Snp } else 2364256694Snp ep->com.state = MORIBUND; 2365256694Snp } 2366256694Snp break; 2367256694Snp 2368256694Snp case MORIBUND: 2369256694Snp case ABORTING: 2370256694Snp case DEAD: 2371256694Snp CTR3(KTR_IW_CXGBE, 2372256694Snp "%s ignoring disconnect ep %p state %u", __func__, 2373256694Snp ep, ep->com.state); 2374256694Snp break; 2375256694Snp 2376256694Snp default: 2377256694Snp BUG(); 2378256694Snp break; 2379256694Snp } 2380256694Snp 2381256694Snp mutex_unlock(&ep->com.mutex); 2382256694Snp 2383256694Snp if (close) { 2384256694Snp 2385256694Snp CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep); 2386256694Snp 2387256694Snp if (abrupt) { 2388256694Snp 2389256694Snp CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); 2390256694Snp set_bit(EP_DISC_ABORT, &ep->com.history); 2391309378Sjhb close_complete_upcall(ep, -ECONNRESET); 2392309378Sjhb ret = send_abort(ep); 2393319272Snp if (ret) 2394319272Snp fatal = 1; 2395256694Snp } else { 2396256694Snp 2397256694Snp CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); 2398256694Snp set_bit(EP_DISC_CLOSE, &ep->com.history); 2399256694Snp 2400256694Snp if (!ep->parent_ep) 2401256694Snp __state_set(&ep->com, MORIBUND); 2402319272Snp sodisconnect(ep->com.so); 2403256694Snp } 2404256694Snp 2405256694Snp } 2406256694Snp 2407256694Snp if (fatal) { 2408309378Sjhb set_bit(EP_DISC_FAIL, &ep->com.history); 2409309378Sjhb if (!abrupt) { 2410309378Sjhb STOP_EP_TIMER(ep); 2411309378Sjhb close_complete_upcall(ep, -EIO); 2412309378Sjhb } 2413309378Sjhb if (ep->com.qp) { 2414309378Sjhb struct c4iw_qp_attributes attrs; 2415256694Snp 2416309378Sjhb attrs.next_state = C4IW_QP_STATE_ERROR; 2417309378Sjhb ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, 2418309378Sjhb C4IW_QP_ATTR_NEXT_STATE, 2419309378Sjhb &attrs, 1); 2420309378Sjhb if (ret) { 2421309378Sjhb CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); 2422309378Sjhb printf("%s - qp <- error failed!\n", __func__); 2423309378Sjhb } 2424309378Sjhb } 2425256694Snp release_ep_resources(ep); 2426309378Sjhb ep->com.state = DEAD; 2427256694Snp CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); 2428256694Snp } 2429256694Snp CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); 2430256694Snp return ret; 2431256694Snp} 2432256694Snp 2433256694Snp#ifdef C4IW_EP_REDIRECT 2434256694Snpint c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 2435256694Snp struct l2t_entry *l2t) 2436256694Snp{ 2437256694Snp struct c4iw_ep *ep = ctx; 2438256694Snp 2439256694Snp if (ep->dst != old) 2440256694Snp return 0; 2441256694Snp 2442256694Snp PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, 2443256694Snp l2t); 2444256694Snp dst_hold(new); 2445256694Snp cxgb4_l2t_release(ep->l2t); 2446256694Snp ep->l2t = l2t; 2447256694Snp dst_release(old); 2448256694Snp ep->dst = new; 2449256694Snp return 1; 2450256694Snp} 2451256694Snp#endif 2452256694Snp 2453256694Snp 2454256694Snp 2455256694Snpstatic void ep_timeout(unsigned long arg) 2456256694Snp{ 2457256694Snp struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2458256694Snp 2459256694Snp if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 2460256694Snp 2461309378Sjhb /* 2462309378Sjhb * Only insert if it is not already on the list. 2463309378Sjhb */ 2464316123Snp if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) { 2465316123Snp CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep); 2466316123Snp add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT); 2467309378Sjhb } 2468256694Snp } 2469256694Snp} 2470256694Snp 2471256694Snpstatic int fw6_wr_rpl(struct adapter *sc, const __be64 *rpl) 2472256694Snp{ 2473256694Snp uint64_t val = be64toh(*rpl); 2474256694Snp int ret; 2475256694Snp struct c4iw_wr_wait *wr_waitp; 2476256694Snp 2477256694Snp ret = (int)((val >> 8) & 0xff); 2478256694Snp wr_waitp = (struct c4iw_wr_wait *)rpl[1]; 2479256694Snp CTR3(KTR_IW_CXGBE, "%s wr_waitp %p ret %u", __func__, wr_waitp, ret); 2480256694Snp if (wr_waitp) 2481256694Snp c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2482256694Snp 2483256694Snp return (0); 2484256694Snp} 2485256694Snp 2486256694Snpstatic int fw6_cqe_handler(struct adapter *sc, const __be64 *rpl) 2487256694Snp{ 2488316123Snp struct cqe_list_entry *cle; 2489316123Snp unsigned long flag; 2490256694Snp 2491316123Snp cle = malloc(sizeof(*cle), M_CXGBE, M_NOWAIT); 2492316123Snp cle->rhp = sc->iwarp_softc; 2493316123Snp cle->err_cqe = *(const struct t4_cqe *)(&rpl[0]); 2494256694Snp 2495316123Snp spin_lock_irqsave(&err_cqe_lock, flag); 2496316123Snp list_add_tail(&cle->entry, &err_cqe_list); 2497316123Snp queue_work(c4iw_taskq, &c4iw_task); 2498316123Snp spin_unlock_irqrestore(&err_cqe_lock, flag); 2499316123Snp 2500256694Snp return (0); 2501256694Snp} 2502256694Snp 2503316123Snpstatic int 2504316123Snpprocess_terminate(struct c4iw_ep *ep) 2505256694Snp{ 2506256694Snp struct c4iw_qp_attributes attrs; 2507256694Snp 2508256694Snp CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep); 2509256694Snp 2510256694Snp if (ep && ep->com.qp) { 2511256694Snp 2512316123Snp printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", 2513316123Snp ep->hwtid, ep->com.qp->wq.sq.qid); 2514256694Snp attrs.next_state = C4IW_QP_STATE_TERMINATE; 2515256694Snp c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 2516256694Snp 1); 2517256694Snp } else 2518316123Snp printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", 2519316123Snp ep->hwtid); 2520256694Snp CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep); 2521256694Snp 2522256694Snp return 0; 2523256694Snp} 2524256694Snp 2525309442Sjhbint __init c4iw_cm_init(void) 2526256694Snp{ 2527256694Snp 2528309442Sjhb t4_register_cpl_handler(CPL_RDMA_TERMINATE, terminate); 2529309442Sjhb t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, fw6_wr_rpl); 2530309442Sjhb t4_register_fw_msg_handler(FW6_TYPE_CQE, fw6_cqe_handler); 2531309442Sjhb t4_register_an_handler(c4iw_ev_handler); 2532256694Snp 2533256694Snp TAILQ_INIT(&req_list); 2534256694Snp spin_lock_init(&req_lock); 2535316123Snp INIT_LIST_HEAD(&err_cqe_list); 2536316123Snp spin_lock_init(&err_cqe_lock); 2537256694Snp 2538256694Snp INIT_WORK(&c4iw_task, process_req); 2539256694Snp 2540256694Snp c4iw_taskq = create_singlethread_workqueue("iw_cxgbe"); 2541256694Snp if (!c4iw_taskq) 2542256694Snp return -ENOMEM; 2543256694Snp 2544256694Snp return 0; 2545256694Snp} 2546256694Snp 2547256694Snpvoid __exit c4iw_cm_term(void) 2548256694Snp{ 2549256694Snp WARN_ON(!TAILQ_EMPTY(&req_list)); 2550316123Snp WARN_ON(!list_empty(&err_cqe_list)); 2551256694Snp flush_workqueue(c4iw_taskq); 2552256694Snp destroy_workqueue(c4iw_taskq); 2553309442Sjhb 2554309442Sjhb t4_register_cpl_handler(CPL_RDMA_TERMINATE, NULL); 2555309442Sjhb t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, NULL); 2556309442Sjhb t4_register_fw_msg_handler(FW6_TYPE_CQE, NULL); 2557309442Sjhb t4_register_an_handler(NULL); 2558256694Snp} 2559256694Snp#endif 2560