cm.c revision 355240
1/* 2 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/iw_cxgbe/cm.c 355240 2019-11-30 19:21:29Z np $"); 34 35#include "opt_inet.h" 36 37#ifdef TCP_OFFLOAD 38#include <sys/types.h> 39#include <sys/malloc.h> 40#include <sys/socket.h> 41#include <sys/socketvar.h> 42#include <sys/sockio.h> 43#include <sys/taskqueue.h> 44#include <netinet/in.h> 45#include <net/route.h> 46 47#include <netinet/in_systm.h> 48#include <netinet/in_pcb.h> 49#include <netinet6/in6_pcb.h> 50#include <netinet/ip.h> 51#include <netinet/in_fib.h> 52#include <netinet6/in6_fib.h> 53#include <netinet6/scope6_var.h> 54#include <netinet/ip_var.h> 55#include <netinet/tcp_var.h> 56#include <netinet/tcp.h> 57#include <netinet/tcpip.h> 58 59#include <netinet/toecore.h> 60 61struct sge_iq; 62struct rss_header; 63struct cpl_set_tcb_rpl; 64#include <linux/types.h> 65#include "offload.h" 66#include "tom/t4_tom.h" 67 68#define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe)) 69 70#include "iw_cxgbe.h" 71#include <linux/module.h> 72#include <linux/workqueue.h> 73#include <linux/notifier.h> 74#include <linux/inetdevice.h> 75#include <linux/if_vlan.h> 76#include <net/netevent.h> 77#include <rdma/rdma_cm.h> 78 79static spinlock_t req_lock; 80static TAILQ_HEAD(c4iw_ep_list, c4iw_ep_common) req_list; 81static struct work_struct c4iw_task; 82static struct workqueue_struct *c4iw_taskq; 83static LIST_HEAD(err_cqe_list); 84static spinlock_t err_cqe_lock; 85static LIST_HEAD(listen_port_list); 86static DEFINE_MUTEX(listen_port_mutex); 87 88static void process_req(struct work_struct *ctx); 89static void start_ep_timer(struct c4iw_ep *ep); 90static int stop_ep_timer(struct c4iw_ep *ep); 91static int set_tcpinfo(struct c4iw_ep *ep); 92static void process_timeout(struct c4iw_ep *ep); 93static void process_err_cqes(void); 94static void *alloc_ep(int size, gfp_t flags); 95static void close_socket(struct socket *so); 96static int send_mpa_req(struct c4iw_ep *ep); 97static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); 98static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); 99static void close_complete_upcall(struct c4iw_ep *ep, int status); 100static int send_abort(struct c4iw_ep *ep); 101static void peer_close_upcall(struct c4iw_ep *ep); 102static void peer_abort_upcall(struct c4iw_ep *ep); 103static void connect_reply_upcall(struct c4iw_ep *ep, int status); 104static int connect_request_upcall(struct c4iw_ep *ep); 105static void established_upcall(struct c4iw_ep *ep); 106static int process_mpa_reply(struct c4iw_ep *ep); 107static int process_mpa_request(struct c4iw_ep *ep); 108static void process_peer_close(struct c4iw_ep *ep); 109static void process_conn_error(struct c4iw_ep *ep); 110static void process_close_complete(struct c4iw_ep *ep); 111static void ep_timeout(unsigned long arg); 112static void setiwsockopt(struct socket *so); 113static void init_iwarp_socket(struct socket *so, void *arg); 114static void uninit_iwarp_socket(struct socket *so); 115static void process_data(struct c4iw_ep *ep); 116static void process_connected(struct c4iw_ep *ep); 117static int c4iw_so_upcall(struct socket *so, void *arg, int waitflag); 118static void process_socket_event(struct c4iw_ep *ep); 119static void release_ep_resources(struct c4iw_ep *ep); 120static int process_terminate(struct c4iw_ep *ep); 121static int terminate(struct sge_iq *iq, const struct rss_header *rss, 122 struct mbuf *m); 123static int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events); 124static struct listen_port_info * 125add_ep_to_listenlist(struct c4iw_listen_ep *lep); 126static int rem_ep_from_listenlist(struct c4iw_listen_ep *lep); 127static struct c4iw_listen_ep * 128find_real_listen_ep(struct c4iw_listen_ep *master_lep, struct socket *so); 129static int get_ifnet_from_raddr(struct sockaddr_storage *raddr, 130 struct ifnet **ifp); 131static void process_newconn(struct c4iw_listen_ep *master_lep, 132 struct socket *new_so); 133#define START_EP_TIMER(ep) \ 134 do { \ 135 CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \ 136 __func__, __LINE__, (ep)); \ 137 start_ep_timer(ep); \ 138 } while (0) 139 140#define STOP_EP_TIMER(ep) \ 141 ({ \ 142 CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ 143 __func__, __LINE__, (ep)); \ 144 stop_ep_timer(ep); \ 145 }) 146 147#define GET_LOCAL_ADDR(pladdr, so) \ 148 do { \ 149 struct sockaddr_storage *__a = NULL; \ 150 struct inpcb *__inp = sotoinpcb(so); \ 151 KASSERT(__inp != NULL, \ 152 ("GET_LOCAL_ADDR(%s):so:%p, inp = NULL", __func__, so)); \ 153 if (__inp->inp_vflag & INP_IPV4) \ 154 in_getsockaddr(so, (struct sockaddr **)&__a); \ 155 else \ 156 in6_getsockaddr(so, (struct sockaddr **)&__a); \ 157 *(pladdr) = *__a; \ 158 free(__a, M_SONAME); \ 159 } while (0) 160 161#define GET_REMOTE_ADDR(praddr, so) \ 162 do { \ 163 struct sockaddr_storage *__a = NULL; \ 164 struct inpcb *__inp = sotoinpcb(so); \ 165 KASSERT(__inp != NULL, \ 166 ("GET_REMOTE_ADDR(%s):so:%p, inp = NULL", __func__, so)); \ 167 if (__inp->inp_vflag & INP_IPV4) \ 168 in_getpeeraddr(so, (struct sockaddr **)&__a); \ 169 else \ 170 in6_getpeeraddr(so, (struct sockaddr **)&__a); \ 171 *(praddr) = *__a; \ 172 free(__a, M_SONAME); \ 173 } while (0) 174 175static char *states[] = { 176 "idle", 177 "listen", 178 "connecting", 179 "mpa_wait_req", 180 "mpa_req_sent", 181 "mpa_req_rcvd", 182 "mpa_rep_sent", 183 "fpdu_mode", 184 "aborting", 185 "closing", 186 "moribund", 187 "dead", 188 NULL, 189}; 190 191static void deref_cm_id(struct c4iw_ep_common *epc) 192{ 193 epc->cm_id->rem_ref(epc->cm_id); 194 epc->cm_id = NULL; 195 set_bit(CM_ID_DEREFED, &epc->history); 196} 197 198static void ref_cm_id(struct c4iw_ep_common *epc) 199{ 200 set_bit(CM_ID_REFED, &epc->history); 201 epc->cm_id->add_ref(epc->cm_id); 202} 203 204static void deref_qp(struct c4iw_ep *ep) 205{ 206 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 207 clear_bit(QP_REFERENCED, &ep->com.flags); 208 set_bit(QP_DEREFED, &ep->com.history); 209} 210 211static void ref_qp(struct c4iw_ep *ep) 212{ 213 set_bit(QP_REFERENCED, &ep->com.flags); 214 set_bit(QP_REFED, &ep->com.history); 215 c4iw_qp_add_ref(&ep->com.qp->ibqp); 216} 217/* allocated per TCP port while listening */ 218struct listen_port_info { 219 uint16_t port_num; /* TCP port address */ 220 struct list_head list; /* belongs to listen_port_list */ 221 struct list_head lep_list; /* per port lep list */ 222 uint32_t refcnt; /* number of lep's listening */ 223}; 224 225/* 226 * Following two lists are used to manage INADDR_ANY listeners: 227 * 1)listen_port_list 228 * 2)lep_list 229 * 230 * Below is the INADDR_ANY listener lists overview on a system with a two port 231 * adapter: 232 * |------------------| 233 * |listen_port_list | 234 * |------------------| 235 * | 236 * | |-----------| |-----------| 237 * | | port_num:X| | port_num:X| 238 * |--------------|-list------|-------|-list------|-------.... 239 * | lep_list----| | lep_list----| 240 * | refcnt | | | refcnt | | 241 * | | | | | | 242 * | | | | | | 243 * |-----------| | |-----------| | 244 * | | 245 * | | 246 * | | 247 * | | lep1 lep2 248 * | | |----------------| |----------------| 249 * | |----| listen_ep_list |----| listen_ep_list | 250 * | |----------------| |----------------| 251 * | 252 * | 253 * | lep1 lep2 254 * | |----------------| |----------------| 255 * |---| listen_ep_list |----| listen_ep_list | 256 * |----------------| |----------------| 257 * 258 * Because of two port adapter, the number of lep's are two(lep1 & lep2) for 259 * each TCP port number. 260 * 261 * Here 'lep1' is always marked as Master lep, because solisten() is always 262 * called through first lep. 263 * 264 */ 265static struct listen_port_info * 266add_ep_to_listenlist(struct c4iw_listen_ep *lep) 267{ 268 uint16_t port; 269 struct listen_port_info *port_info = NULL; 270 struct sockaddr_storage *laddr = &lep->com.local_addr; 271 272 port = (laddr->ss_family == AF_INET) ? 273 ((struct sockaddr_in *)laddr)->sin_port : 274 ((struct sockaddr_in6 *)laddr)->sin6_port; 275 276 mutex_lock(&listen_port_mutex); 277 278 list_for_each_entry(port_info, &listen_port_list, list) 279 if (port_info->port_num == port) 280 goto found_port; 281 282 port_info = malloc(sizeof(*port_info), M_CXGBE, M_WAITOK); 283 port_info->port_num = port; 284 port_info->refcnt = 0; 285 286 list_add_tail(&port_info->list, &listen_port_list); 287 INIT_LIST_HEAD(&port_info->lep_list); 288 289found_port: 290 port_info->refcnt++; 291 list_add_tail(&lep->listen_ep_list, &port_info->lep_list); 292 mutex_unlock(&listen_port_mutex); 293 return port_info; 294} 295 296static int 297rem_ep_from_listenlist(struct c4iw_listen_ep *lep) 298{ 299 uint16_t port; 300 struct listen_port_info *port_info = NULL; 301 struct sockaddr_storage *laddr = &lep->com.local_addr; 302 int refcnt = 0; 303 304 port = (laddr->ss_family == AF_INET) ? 305 ((struct sockaddr_in *)laddr)->sin_port : 306 ((struct sockaddr_in6 *)laddr)->sin6_port; 307 308 mutex_lock(&listen_port_mutex); 309 310 /* get the port_info structure based on the lep's port address */ 311 list_for_each_entry(port_info, &listen_port_list, list) { 312 if (port_info->port_num == port) { 313 port_info->refcnt--; 314 refcnt = port_info->refcnt; 315 /* remove the current lep from the listen list */ 316 list_del(&lep->listen_ep_list); 317 if (port_info->refcnt == 0) { 318 /* Remove this entry from the list as there 319 * are no more listeners for this port_num. 320 */ 321 list_del(&port_info->list); 322 kfree(port_info); 323 } 324 break; 325 } 326 } 327 mutex_unlock(&listen_port_mutex); 328 return refcnt; 329} 330 331/* 332 * Find the lep that belongs to the ifnet on which the SYN frame was received. 333 */ 334struct c4iw_listen_ep * 335find_real_listen_ep(struct c4iw_listen_ep *master_lep, struct socket *so) 336{ 337 struct adapter *adap = NULL; 338 struct c4iw_listen_ep *lep = NULL; 339 struct ifnet *ifp = NULL, *hw_ifp = NULL; 340 struct listen_port_info *port_info = NULL; 341 int i = 0, found_portinfo = 0, found_lep = 0; 342 uint16_t port; 343 344 /* 345 * STEP 1: Figure out 'ifp' of the physical interface, not pseudo 346 * interfaces like vlan, lagg, etc.. 347 * TBD: lagg support, lagg + vlan support. 348 */ 349 ifp = TOEPCB(so)->l2te->ifp; 350 if (ifp->if_type == IFT_L2VLAN) { 351 hw_ifp = VLAN_TRUNKDEV(ifp); 352 if (hw_ifp == NULL) { 353 CTR4(KTR_IW_CXGBE, "%s: Failed to get parent ifnet of " 354 "vlan ifnet %p, sock %p, master_lep %p", 355 __func__, ifp, so, master_lep); 356 return (NULL); 357 } 358 } else 359 hw_ifp = ifp; 360 361 /* STEP 2: Find 'port_info' with listener local port address. */ 362 port = (master_lep->com.local_addr.ss_family == AF_INET) ? 363 ((struct sockaddr_in *)&master_lep->com.local_addr)->sin_port : 364 ((struct sockaddr_in6 *)&master_lep->com.local_addr)->sin6_port; 365 366 367 mutex_lock(&listen_port_mutex); 368 list_for_each_entry(port_info, &listen_port_list, list) 369 if (port_info->port_num == port) { 370 found_portinfo =1; 371 break; 372 } 373 if (!found_portinfo) 374 goto out; 375 376 /* STEP 3: Traverse through list of lep's that are bound to the current 377 * TCP port address and find the lep that belongs to the ifnet on which 378 * the SYN frame was received. 379 */ 380 list_for_each_entry(lep, &port_info->lep_list, listen_ep_list) { 381 adap = lep->com.dev->rdev.adap; 382 for_each_port(adap, i) { 383 if (hw_ifp == adap->port[i]->vi[0].ifp) { 384 found_lep =1; 385 goto out; 386 } 387 } 388 } 389out: 390 mutex_unlock(&listen_port_mutex); 391 return found_lep ? lep : (NULL); 392} 393 394static void process_timeout(struct c4iw_ep *ep) 395{ 396 struct c4iw_qp_attributes attrs = {0}; 397 int abort = 1; 398 399 CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__, 400 ep, ep->hwtid, ep->com.state); 401 set_bit(TIMEDOUT, &ep->com.history); 402 switch (ep->com.state) { 403 case MPA_REQ_SENT: 404 connect_reply_upcall(ep, -ETIMEDOUT); 405 break; 406 case MPA_REQ_WAIT: 407 case MPA_REQ_RCVD: 408 case MPA_REP_SENT: 409 case FPDU_MODE: 410 break; 411 case CLOSING: 412 case MORIBUND: 413 if (ep->com.cm_id && ep->com.qp) { 414 attrs.next_state = C4IW_QP_STATE_ERROR; 415 c4iw_modify_qp(ep->com.dev, ep->com.qp, 416 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 417 } 418 close_complete_upcall(ep, -ETIMEDOUT); 419 break; 420 case ABORTING: 421 case DEAD: 422 /* 423 * These states are expected if the ep timed out at the same 424 * time as another thread was calling stop_ep_timer(). 425 * So we silently do nothing for these states. 426 */ 427 abort = 0; 428 break; 429 default: 430 CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u" 431 , __func__, ep, ep->hwtid, ep->com.state); 432 abort = 0; 433 } 434 if (abort) 435 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 436 c4iw_put_ep(&ep->com); 437 return; 438} 439 440struct cqe_list_entry { 441 struct list_head entry; 442 struct c4iw_dev *rhp; 443 struct t4_cqe err_cqe; 444}; 445 446static void 447process_err_cqes(void) 448{ 449 unsigned long flag; 450 struct cqe_list_entry *cle; 451 452 spin_lock_irqsave(&err_cqe_lock, flag); 453 while (!list_empty(&err_cqe_list)) { 454 struct list_head *tmp; 455 tmp = err_cqe_list.next; 456 list_del(tmp); 457 tmp->next = tmp->prev = NULL; 458 spin_unlock_irqrestore(&err_cqe_lock, flag); 459 cle = list_entry(tmp, struct cqe_list_entry, entry); 460 c4iw_ev_dispatch(cle->rhp, &cle->err_cqe); 461 free(cle, M_CXGBE); 462 spin_lock_irqsave(&err_cqe_lock, flag); 463 } 464 spin_unlock_irqrestore(&err_cqe_lock, flag); 465 466 return; 467} 468 469static void 470process_req(struct work_struct *ctx) 471{ 472 struct c4iw_ep_common *epc; 473 unsigned long flag; 474 int ep_events; 475 476 process_err_cqes(); 477 spin_lock_irqsave(&req_lock, flag); 478 while (!TAILQ_EMPTY(&req_list)) { 479 epc = TAILQ_FIRST(&req_list); 480 TAILQ_REMOVE(&req_list, epc, entry); 481 epc->entry.tqe_prev = NULL; 482 ep_events = epc->ep_events; 483 epc->ep_events = 0; 484 spin_unlock_irqrestore(&req_lock, flag); 485 mutex_lock(&epc->mutex); 486 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, ep_state %s events 0x%x", 487 __func__, epc->so, epc, states[epc->state], ep_events); 488 if (ep_events & C4IW_EVENT_TERM) 489 process_terminate((struct c4iw_ep *)epc); 490 if (ep_events & C4IW_EVENT_TIMEOUT) 491 process_timeout((struct c4iw_ep *)epc); 492 if (ep_events & C4IW_EVENT_SOCKET) 493 process_socket_event((struct c4iw_ep *)epc); 494 mutex_unlock(&epc->mutex); 495 c4iw_put_ep(epc); 496 process_err_cqes(); 497 spin_lock_irqsave(&req_lock, flag); 498 } 499 spin_unlock_irqrestore(&req_lock, flag); 500} 501 502/* 503 * XXX: doesn't belong here in the iWARP driver. 504 * XXX: assumes that the connection was offloaded by cxgbe/t4_tom if TF_TOE is 505 * set. Is this a valid assumption for active open? 506 */ 507static int 508set_tcpinfo(struct c4iw_ep *ep) 509{ 510 struct socket *so = ep->com.so; 511 struct inpcb *inp = sotoinpcb(so); 512 struct tcpcb *tp; 513 struct toepcb *toep; 514 int rc = 0; 515 516 INP_WLOCK(inp); 517 tp = intotcpcb(inp); 518 if ((tp->t_flags & TF_TOE) == 0) { 519 rc = EINVAL; 520 log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n", 521 __func__, so, ep); 522 goto done; 523 } 524 toep = TOEPCB(so); 525 526 ep->hwtid = toep->tid; 527 ep->snd_seq = tp->snd_nxt; 528 ep->rcv_seq = tp->rcv_nxt; 529done: 530 INP_WUNLOCK(inp); 531 return (rc); 532 533} 534static int 535get_ifnet_from_raddr(struct sockaddr_storage *raddr, struct ifnet **ifp) 536{ 537 int err = 0; 538 539 if (raddr->ss_family == AF_INET) { 540 struct sockaddr_in *raddr4 = (struct sockaddr_in *)raddr; 541 struct nhop4_extended nh4 = {0}; 542 543 err = fib4_lookup_nh_ext(RT_DEFAULT_FIB, raddr4->sin_addr, 544 NHR_REF, 0, &nh4); 545 *ifp = nh4.nh_ifp; 546 if (err) 547 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 548 } else { 549 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)raddr; 550 struct nhop6_extended nh6 = {0}; 551 struct in6_addr addr6; 552 uint32_t scopeid; 553 554 memset(&addr6, 0, sizeof(addr6)); 555 in6_splitscope((struct in6_addr *)&raddr6->sin6_addr, 556 &addr6, &scopeid); 557 err = fib6_lookup_nh_ext(RT_DEFAULT_FIB, &addr6, scopeid, 558 NHR_REF, 0, &nh6); 559 *ifp = nh6.nh_ifp; 560 if (err) 561 fib6_free_nh_ext(RT_DEFAULT_FIB, &nh6); 562 } 563 564 CTR2(KTR_IW_CXGBE, "%s: return: %d", __func__, err); 565 return err; 566} 567 568static void 569close_socket(struct socket *so) 570{ 571 uninit_iwarp_socket(so); 572 soclose(so); 573} 574 575static void 576process_peer_close(struct c4iw_ep *ep) 577{ 578 struct c4iw_qp_attributes attrs = {0}; 579 int disconnect = 1; 580 int release = 0; 581 582 CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep, 583 ep->com.so, states[ep->com.state]); 584 585 switch (ep->com.state) { 586 587 case MPA_REQ_WAIT: 588 CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT DEAD", 589 __func__, ep); 590 /* Fallthrough */ 591 case MPA_REQ_SENT: 592 CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT DEAD", 593 __func__, ep); 594 ep->com.state = DEAD; 595 connect_reply_upcall(ep, -ECONNABORTED); 596 597 disconnect = 0; 598 STOP_EP_TIMER(ep); 599 close_socket(ep->com.so); 600 deref_cm_id(&ep->com); 601 release = 1; 602 break; 603 604 case MPA_REQ_RCVD: 605 606 /* 607 * We're gonna mark this puppy DEAD, but keep 608 * the reference on it until the ULP accepts or 609 * rejects the CR. 610 */ 611 CTR2(KTR_IW_CXGBE, "%s:ppc3 %p MPA_REQ_RCVD CLOSING", 612 __func__, ep); 613 ep->com.state = CLOSING; 614 break; 615 616 case MPA_REP_SENT: 617 CTR2(KTR_IW_CXGBE, "%s:ppc4 %p MPA_REP_SENT CLOSING", 618 __func__, ep); 619 ep->com.state = CLOSING; 620 break; 621 622 case FPDU_MODE: 623 CTR2(KTR_IW_CXGBE, "%s:ppc5 %p FPDU_MODE CLOSING", 624 __func__, ep); 625 START_EP_TIMER(ep); 626 ep->com.state = CLOSING; 627 attrs.next_state = C4IW_QP_STATE_CLOSING; 628 c4iw_modify_qp(ep->com.dev, ep->com.qp, 629 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 630 peer_close_upcall(ep); 631 break; 632 633 case ABORTING: 634 CTR2(KTR_IW_CXGBE, "%s:ppc6 %p ABORTING (disconn)", 635 __func__, ep); 636 disconnect = 0; 637 break; 638 639 case CLOSING: 640 CTR2(KTR_IW_CXGBE, "%s:ppc7 %p CLOSING MORIBUND", 641 __func__, ep); 642 ep->com.state = MORIBUND; 643 disconnect = 0; 644 break; 645 646 case MORIBUND: 647 CTR2(KTR_IW_CXGBE, "%s:ppc8 %p MORIBUND DEAD", __func__, 648 ep); 649 STOP_EP_TIMER(ep); 650 if (ep->com.cm_id && ep->com.qp) { 651 attrs.next_state = C4IW_QP_STATE_IDLE; 652 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 653 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 654 } 655 close_socket(ep->com.so); 656 close_complete_upcall(ep, 0); 657 ep->com.state = DEAD; 658 release = 1; 659 disconnect = 0; 660 break; 661 662 case DEAD: 663 CTR2(KTR_IW_CXGBE, "%s:ppc9 %p DEAD (disconn)", 664 __func__, ep); 665 disconnect = 0; 666 break; 667 668 default: 669 panic("%s: ep %p state %d", __func__, ep, 670 ep->com.state); 671 break; 672 } 673 674 675 if (disconnect) { 676 677 CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep); 678 c4iw_ep_disconnect(ep, 0, M_NOWAIT); 679 } 680 if (release) { 681 682 CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep); 683 c4iw_put_ep(&ep->com); 684 } 685 CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep); 686 return; 687} 688 689static void 690process_conn_error(struct c4iw_ep *ep) 691{ 692 struct c4iw_qp_attributes attrs = {0}; 693 int ret; 694 int state; 695 696 state = ep->com.state; 697 CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s", 698 __func__, ep, ep->com.so, ep->com.so->so_error, 699 states[ep->com.state]); 700 701 switch (state) { 702 703 case MPA_REQ_WAIT: 704 STOP_EP_TIMER(ep); 705 c4iw_put_ep(&ep->parent_ep->com); 706 break; 707 708 case MPA_REQ_SENT: 709 STOP_EP_TIMER(ep); 710 connect_reply_upcall(ep, -ECONNRESET); 711 break; 712 713 case MPA_REP_SENT: 714 ep->com.rpl_err = ECONNRESET; 715 CTR1(KTR_IW_CXGBE, "waking up ep %p", ep); 716 break; 717 718 case MPA_REQ_RCVD: 719 break; 720 721 case MORIBUND: 722 case CLOSING: 723 STOP_EP_TIMER(ep); 724 /*FALLTHROUGH*/ 725 case FPDU_MODE: 726 727 if (ep->com.cm_id && ep->com.qp) { 728 729 attrs.next_state = C4IW_QP_STATE_ERROR; 730 ret = c4iw_modify_qp(ep->com.qp->rhp, 731 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 732 &attrs, 1); 733 if (ret) 734 log(LOG_ERR, 735 "%s - qp <- error failed!\n", 736 __func__); 737 } 738 peer_abort_upcall(ep); 739 break; 740 741 case ABORTING: 742 break; 743 744 case DEAD: 745 CTR2(KTR_IW_CXGBE, "%s so_error %d IN DEAD STATE!!!!", 746 __func__, ep->com.so->so_error); 747 return; 748 749 default: 750 panic("%s: ep %p state %d", __func__, ep, state); 751 break; 752 } 753 754 if (state != ABORTING) { 755 close_socket(ep->com.so); 756 ep->com.state = DEAD; 757 c4iw_put_ep(&ep->com); 758 } 759 CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep); 760 return; 761} 762 763static void 764process_close_complete(struct c4iw_ep *ep) 765{ 766 struct c4iw_qp_attributes attrs = {0}; 767 int release = 0; 768 769 CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep, 770 ep->com.so, states[ep->com.state]); 771 772 /* The cm_id may be null if we failed to connect */ 773 set_bit(CLOSE_CON_RPL, &ep->com.history); 774 775 switch (ep->com.state) { 776 777 case CLOSING: 778 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p CLOSING MORIBUND", 779 __func__, ep); 780 ep->com.state = MORIBUND; 781 break; 782 783 case MORIBUND: 784 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p MORIBUND DEAD", __func__, 785 ep); 786 STOP_EP_TIMER(ep); 787 788 if ((ep->com.cm_id) && (ep->com.qp)) { 789 790 CTR2(KTR_IW_CXGBE, "%s:pcc2 %p QP_STATE_IDLE", 791 __func__, ep); 792 attrs.next_state = C4IW_QP_STATE_IDLE; 793 c4iw_modify_qp(ep->com.dev, 794 ep->com.qp, 795 C4IW_QP_ATTR_NEXT_STATE, 796 &attrs, 1); 797 } 798 799 close_socket(ep->com.so); 800 close_complete_upcall(ep, 0); 801 ep->com.state = DEAD; 802 release = 1; 803 break; 804 805 case ABORTING: 806 CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep); 807 break; 808 809 case DEAD: 810 CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep); 811 break; 812 default: 813 CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state", 814 __func__, ep); 815 panic("%s:pcc6 %p unknown ep state", __func__, ep); 816 break; 817 } 818 819 if (release) { 820 821 CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep); 822 release_ep_resources(ep); 823 } 824 CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep); 825 return; 826} 827 828static void 829setiwsockopt(struct socket *so) 830{ 831 int rc; 832 struct sockopt sopt; 833 int on = 1; 834 835 sopt.sopt_dir = SOPT_SET; 836 sopt.sopt_level = IPPROTO_TCP; 837 sopt.sopt_name = TCP_NODELAY; 838 sopt.sopt_val = (caddr_t)&on; 839 sopt.sopt_valsize = sizeof on; 840 sopt.sopt_td = NULL; 841 rc = -sosetopt(so, &sopt); 842 if (rc) { 843 log(LOG_ERR, "%s: can't set TCP_NODELAY on so %p (%d)\n", 844 __func__, so, rc); 845 } 846} 847 848static void 849init_iwarp_socket(struct socket *so, void *arg) 850{ 851 852 SOCKBUF_LOCK(&so->so_rcv); 853 soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); 854 so->so_state |= SS_NBIO; 855 SOCKBUF_UNLOCK(&so->so_rcv); 856} 857 858static void 859uninit_iwarp_socket(struct socket *so) 860{ 861 862 SOCKBUF_LOCK(&so->so_rcv); 863 soupcall_clear(so, SO_RCV); 864 SOCKBUF_UNLOCK(&so->so_rcv); 865} 866 867static void 868process_data(struct c4iw_ep *ep) 869{ 870 int ret = 0; 871 int disconnect = 0; 872 struct c4iw_qp_attributes attrs = {0}; 873 874 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__, 875 ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv)); 876 877 switch (ep->com.state) { 878 case MPA_REQ_SENT: 879 disconnect = process_mpa_reply(ep); 880 break; 881 case MPA_REQ_WAIT: 882 disconnect = process_mpa_request(ep); 883 if (disconnect) 884 /* Refered in process_newconn() */ 885 c4iw_put_ep(&ep->parent_ep->com); 886 break; 887 case FPDU_MODE: 888 MPASS(ep->com.qp != NULL); 889 attrs.next_state = C4IW_QP_STATE_TERMINATE; 890 ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, 891 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 892 if (ret != -EINPROGRESS) 893 disconnect = 1; 894 break; 895 default: 896 log(LOG_ERR, "%s: Unexpected streaming data. ep %p, " 897 "state %d, so %p, so_state 0x%x, sbused %u\n", 898 __func__, ep, ep->com.state, ep->com.so, 899 ep->com.so->so_state, sbused(&ep->com.so->so_rcv)); 900 break; 901 } 902 if (disconnect) 903 c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); 904 905} 906 907static void 908process_connected(struct c4iw_ep *ep) 909{ 910 struct socket *so = ep->com.so; 911 912 if ((so->so_state & SS_ISCONNECTED) && !so->so_error) { 913 if (send_mpa_req(ep)) 914 goto err; 915 } else { 916 connect_reply_upcall(ep, -so->so_error); 917 goto err; 918 } 919 return; 920err: 921 close_socket(so); 922 ep->com.state = DEAD; 923 c4iw_put_ep(&ep->com); 924 return; 925} 926 927static inline int c4iw_zero_addr(struct sockaddr *addr) 928{ 929 struct in6_addr *ip6; 930 931 if (addr->sa_family == AF_INET) 932 return IN_ZERONET( 933 ntohl(((struct sockaddr_in *)addr)->sin_addr.s_addr)); 934 else { 935 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr; 936 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] | 937 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0; 938 } 939} 940 941static inline int c4iw_loopback_addr(struct sockaddr *addr) 942{ 943 if (addr->sa_family == AF_INET) 944 return IN_LOOPBACK( 945 ntohl(((struct sockaddr_in *) addr)->sin_addr.s_addr)); 946 else 947 return IN6_IS_ADDR_LOOPBACK( 948 &((struct sockaddr_in6 *) addr)->sin6_addr); 949} 950 951static inline int c4iw_any_addr(struct sockaddr *addr) 952{ 953 return c4iw_zero_addr(addr) || c4iw_loopback_addr(addr); 954} 955 956static void 957process_newconn(struct c4iw_listen_ep *master_lep, struct socket *new_so) 958{ 959 struct c4iw_listen_ep *real_lep = NULL; 960 struct c4iw_ep *new_ep = NULL; 961 struct sockaddr_in *remote = NULL; 962 int ret = 0; 963 964 MPASS(new_so != NULL); 965 966 if (c4iw_any_addr((struct sockaddr *)&master_lep->com.local_addr)) { 967 /* Here we need to find the 'real_lep' that belongs to the 968 * incomming socket's network interface, such that the newly 969 * created 'ep' can be attached to the real 'lep'. 970 */ 971 real_lep = find_real_listen_ep(master_lep, new_so); 972 if (real_lep == NULL) { 973 CTR2(KTR_IW_CXGBE, "%s: Could not find the real listen " 974 "ep for sock: %p", __func__, new_so); 975 log(LOG_ERR,"%s: Could not find the real listen ep for " 976 "sock: %p\n", __func__, new_so); 977 /* FIXME: properly free the 'new_so' in failure case. 978 * Use of soabort() and soclose() are not legal 979 * here(before soaccept()). 980 */ 981 return; 982 } 983 } else /* for Non-Wildcard address, master_lep is always the real_lep */ 984 real_lep = master_lep; 985 986 new_ep = alloc_ep(sizeof(*new_ep), GFP_KERNEL); 987 988 CTR6(KTR_IW_CXGBE, "%s: master_lep %p, real_lep: %p, new ep %p, " 989 "listening so %p, new so %p", __func__, master_lep, real_lep, 990 new_ep, master_lep->com.so, new_so); 991 992 new_ep->com.dev = real_lep->com.dev; 993 new_ep->com.so = new_so; 994 new_ep->com.cm_id = NULL; 995 new_ep->com.thread = real_lep->com.thread; 996 new_ep->parent_ep = real_lep; 997 998 GET_LOCAL_ADDR(&new_ep->com.local_addr, new_so); 999 GET_REMOTE_ADDR(&new_ep->com.remote_addr, new_so); 1000 c4iw_get_ep(&real_lep->com); 1001 init_timer(&new_ep->timer); 1002 new_ep->com.state = MPA_REQ_WAIT; 1003 START_EP_TIMER(new_ep); 1004 1005 setiwsockopt(new_so); 1006 ret = soaccept(new_so, (struct sockaddr **)&remote); 1007 if (ret != 0) { 1008 CTR4(KTR_IW_CXGBE, 1009 "%s:listen sock:%p, new sock:%p, ret:%d", 1010 __func__, master_lep->com.so, new_so, ret); 1011 if (remote != NULL) 1012 free(remote, M_SONAME); 1013 uninit_iwarp_socket(new_so); 1014 soclose(new_so); 1015 c4iw_put_ep(&new_ep->com); 1016 c4iw_put_ep(&real_lep->com); 1017 return; 1018 } 1019 free(remote, M_SONAME); 1020 1021 /* MPA request might have been queued up on the socket already, so we 1022 * initialize the socket/upcall_handler under lock to prevent processing 1023 * MPA request on another thread(via process_req()) simultaniously. 1024 */ 1025 c4iw_get_ep(&new_ep->com); /* Dereferenced at the end below, this is to 1026 avoid freeing of ep before ep unlock. */ 1027 mutex_lock(&new_ep->com.mutex); 1028 init_iwarp_socket(new_so, &new_ep->com); 1029 1030 ret = process_mpa_request(new_ep); 1031 if (ret) { 1032 /* ABORT */ 1033 c4iw_ep_disconnect(new_ep, 1, GFP_KERNEL); 1034 c4iw_put_ep(&real_lep->com); 1035 } 1036 mutex_unlock(&new_ep->com.mutex); 1037 c4iw_put_ep(&new_ep->com); 1038 return; 1039} 1040 1041static int 1042add_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event) 1043{ 1044 unsigned long flag; 1045 1046 spin_lock_irqsave(&req_lock, flag); 1047 if (ep && ep->com.so) { 1048 ep->com.ep_events |= new_ep_event; 1049 if (!ep->com.entry.tqe_prev) { 1050 c4iw_get_ep(&ep->com); 1051 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 1052 queue_work(c4iw_taskq, &c4iw_task); 1053 } 1054 } 1055 spin_unlock_irqrestore(&req_lock, flag); 1056 1057 return (0); 1058} 1059 1060static int 1061c4iw_so_upcall(struct socket *so, void *arg, int waitflag) 1062{ 1063 struct c4iw_ep *ep = arg; 1064 1065 CTR6(KTR_IW_CXGBE, 1066 "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p", 1067 __func__, so, so->so_state, ep, states[ep->com.state], 1068 ep->com.entry.tqe_prev); 1069 1070 MPASS(ep->com.so == so); 1071 /* 1072 * Wake up any threads waiting in rdma_init()/rdma_fini(), 1073 * with locks held. 1074 */ 1075 if (so->so_error) 1076 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1077 add_ep_to_req_list(ep, C4IW_EVENT_SOCKET); 1078 1079 return (SU_OK); 1080} 1081 1082 1083static int 1084terminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1085{ 1086 struct adapter *sc = iq->adapter; 1087 const struct cpl_rdma_terminate *cpl = mtod(m, const void *); 1088 unsigned int tid = GET_TID(cpl); 1089 struct toepcb *toep = lookup_tid(sc, tid); 1090 struct socket *so; 1091 struct c4iw_ep *ep; 1092 1093 INP_WLOCK(toep->inp); 1094 so = inp_inpcbtosocket(toep->inp); 1095 ep = so->so_rcv.sb_upcallarg; 1096 INP_WUNLOCK(toep->inp); 1097 1098 CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep); 1099 add_ep_to_req_list(ep, C4IW_EVENT_TERM); 1100 1101 return 0; 1102} 1103 1104static struct socket * 1105dequeue_socket(struct socket *head) 1106{ 1107 struct socket *so; 1108 struct sockaddr_in *remote; 1109 1110 ACCEPT_LOCK(); 1111 so = TAILQ_FIRST(&head->so_comp); 1112 if (!so) { 1113 ACCEPT_UNLOCK(); 1114 return NULL; 1115 } 1116 1117 SOCK_LOCK(so); 1118 /* 1119 * Before changing the flags on the socket, we have to bump the 1120 * reference count. Otherwise, if the protocol calls sofree(), 1121 * the socket will be released due to a zero refcount. 1122 */ 1123 soref(so); 1124 TAILQ_REMOVE(&head->so_comp, so, so_list); 1125 head->so_qlen--; 1126 so->so_qstate &= ~SQ_COMP; 1127 so->so_head = NULL; 1128 so->so_state |= SS_NBIO; 1129 SOCK_UNLOCK(so); 1130 ACCEPT_UNLOCK(); 1131 remote = NULL; 1132 if (soaccept(so, (struct sockaddr **)&remote) != 0) { 1133 soclose(so); 1134 so = NULL; 1135 } 1136 free(remote, M_SONAME); 1137 return so; 1138} 1139 1140static void 1141process_socket_event(struct c4iw_ep *ep) 1142{ 1143 int state = ep->com.state; 1144 struct socket *so = ep->com.so; 1145 1146 if (ep->com.state == DEAD) { 1147 CTR3(KTR_IW_CXGBE, "%s: Pending socket event discarded " 1148 "ep %p ep_state %s", __func__, ep, states[state]); 1149 return; 1150 } 1151 1152 CTR6(KTR_IW_CXGBE, "process_socket_event: so %p, so_state 0x%x, " 1153 "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state, 1154 so->so_error, so->so_rcv.sb_state, ep, states[state]); 1155 1156 if (state == CONNECTING) { 1157 process_connected(ep); 1158 return; 1159 } 1160 1161 if (state == LISTEN) { 1162 struct c4iw_listen_ep *lep = (struct c4iw_listen_ep *)ep; 1163 struct socket *new_so; 1164 1165 while ((new_so = dequeue_socket(so)) != NULL) { 1166 process_newconn(lep, new_so); 1167 } 1168 return; 1169 } 1170 1171 /* connection error */ 1172 if (so->so_error) { 1173 process_conn_error(ep); 1174 return; 1175 } 1176 1177 /* peer close */ 1178 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <= CLOSING) { 1179 process_peer_close(ep); 1180 /* 1181 * check whether socket disconnect event is pending before 1182 * returning. Fallthrough if yes. 1183 */ 1184 if (!(so->so_state & SS_ISDISCONNECTED)) 1185 return; 1186 } 1187 1188 /* close complete */ 1189 if (so->so_state & SS_ISDISCONNECTED) { 1190 process_close_complete(ep); 1191 return; 1192 } 1193 1194 /* rx data */ 1195 if (sbused(&ep->com.so->so_rcv)) { 1196 process_data(ep); 1197 return; 1198 } 1199 1200 /* Socket events for 'MPA Request Received' and 'Close Complete' 1201 * were already processed earlier in their previous events handlers. 1202 * Hence, these socket events are skipped. 1203 * And any other socket events must have handled above. 1204 */ 1205 MPASS((ep->com.state == MPA_REQ_RCVD) || (ep->com.state == MORIBUND)); 1206 1207 if ((ep->com.state != MPA_REQ_RCVD) && (ep->com.state != MORIBUND)) 1208 log(LOG_ERR, "%s: Unprocessed socket event so %p, " 1209 "so_state 0x%x, so_err %d, sb_state 0x%x, ep %p, ep_state %s\n", 1210 __func__, so, so->so_state, so->so_error, so->so_rcv.sb_state, 1211 ep, states[state]); 1212 1213} 1214 1215SYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD, 0, "iw_cxgbe driver parameters"); 1216 1217static int dack_mode = 0; 1218SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0, 1219 "Delayed ack mode (default = 0)"); 1220 1221int c4iw_max_read_depth = 8; 1222SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RWTUN, &c4iw_max_read_depth, 0, 1223 "Per-connection max ORD/IRD (default = 8)"); 1224 1225static int enable_tcp_timestamps; 1226SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RWTUN, &enable_tcp_timestamps, 0, 1227 "Enable tcp timestamps (default = 0)"); 1228 1229static int enable_tcp_sack; 1230SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RWTUN, &enable_tcp_sack, 0, 1231 "Enable tcp SACK (default = 0)"); 1232 1233static int enable_tcp_window_scaling = 1; 1234SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RWTUN, &enable_tcp_window_scaling, 0, 1235 "Enable tcp window scaling (default = 1)"); 1236 1237int c4iw_debug = 0; 1238SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RWTUN, &c4iw_debug, 0, 1239 "Enable debug logging (default = 0)"); 1240 1241static int peer2peer = 1; 1242SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0, 1243 "Support peer2peer ULPs (default = 1)"); 1244 1245static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 1246SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0, 1247 "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)"); 1248 1249static int ep_timeout_secs = 60; 1250SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0, 1251 "CM Endpoint operation timeout in seconds (default = 60)"); 1252 1253static int mpa_rev = 1; 1254SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0, 1255 "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)"); 1256 1257static int markers_enabled; 1258SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0, 1259 "Enable MPA MARKERS (default(0) = disabled)"); 1260 1261static int crc_enabled = 1; 1262SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0, 1263 "Enable MPA CRC (default(1) = enabled)"); 1264 1265static int rcv_win = 256 * 1024; 1266SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0, 1267 "TCP receive window in bytes (default = 256KB)"); 1268 1269static int snd_win = 128 * 1024; 1270SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0, 1271 "TCP send window in bytes (default = 128KB)"); 1272 1273int use_dsgl = 1; 1274SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, use_dsgl, CTLFLAG_RWTUN, &use_dsgl, 0, 1275 "Use DSGL for PBL/FastReg (default=1)"); 1276 1277int inline_threshold = 128; 1278SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, inline_threshold, CTLFLAG_RWTUN, &inline_threshold, 0, 1279 "inline vs dsgl threshold (default=128)"); 1280 1281static int reuseaddr = 0; 1282SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, reuseaddr, CTLFLAG_RWTUN, &reuseaddr, 0, 1283 "Enable SO_REUSEADDR & SO_REUSEPORT socket options on all iWARP client connections(default = 0)"); 1284 1285static void 1286start_ep_timer(struct c4iw_ep *ep) 1287{ 1288 1289 if (timer_pending(&ep->timer)) { 1290 CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep); 1291 printk(KERN_ERR "%s timer already started! ep %p\n", __func__, 1292 ep); 1293 return; 1294 } 1295 clear_bit(TIMEOUT, &ep->com.flags); 1296 c4iw_get_ep(&ep->com); 1297 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 1298 ep->timer.data = (unsigned long)ep; 1299 ep->timer.function = ep_timeout; 1300 add_timer(&ep->timer); 1301} 1302 1303static int 1304stop_ep_timer(struct c4iw_ep *ep) 1305{ 1306 1307 del_timer_sync(&ep->timer); 1308 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 1309 c4iw_put_ep(&ep->com); 1310 return 0; 1311 } 1312 return 1; 1313} 1314 1315static void * 1316alloc_ep(int size, gfp_t gfp) 1317{ 1318 struct c4iw_ep_common *epc; 1319 1320 epc = kzalloc(size, gfp); 1321 if (epc == NULL) 1322 return (NULL); 1323 1324 kref_init(&epc->kref); 1325 mutex_init(&epc->mutex); 1326 c4iw_init_wr_wait(&epc->wr_wait); 1327 1328 return (epc); 1329} 1330 1331void _c4iw_free_ep(struct kref *kref) 1332{ 1333 struct c4iw_ep *ep; 1334 struct c4iw_ep_common *epc; 1335 1336 ep = container_of(kref, struct c4iw_ep, com.kref); 1337 epc = &ep->com; 1338 KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", 1339 __func__, epc)); 1340 if (test_bit(QP_REFERENCED, &ep->com.flags)) 1341 deref_qp(ep); 1342 CTR4(KTR_IW_CXGBE, "%s: ep %p, history 0x%lx, flags 0x%lx", 1343 __func__, ep, epc->history, epc->flags); 1344 kfree(ep); 1345} 1346 1347static void release_ep_resources(struct c4iw_ep *ep) 1348{ 1349 CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep); 1350 set_bit(RELEASE_RESOURCES, &ep->com.flags); 1351 c4iw_put_ep(&ep->com); 1352 CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep); 1353} 1354 1355static int 1356send_mpa_req(struct c4iw_ep *ep) 1357{ 1358 int mpalen; 1359 struct mpa_message *mpa; 1360 struct mpa_v2_conn_params mpa_v2_params; 1361 struct mbuf *m; 1362 char mpa_rev_to_use = mpa_rev; 1363 int err = 0; 1364 1365 if (ep->retry_with_mpa_v1) 1366 mpa_rev_to_use = 1; 1367 mpalen = sizeof(*mpa) + ep->plen; 1368 if (mpa_rev_to_use == 2) 1369 mpalen += sizeof(struct mpa_v2_conn_params); 1370 1371 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1372 if (mpa == NULL) { 1373 err = -ENOMEM; 1374 CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d", 1375 __func__, ep, err); 1376 goto err; 1377 } 1378 1379 memset(mpa, 0, mpalen); 1380 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 1381 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 1382 (markers_enabled ? MPA_MARKERS : 0) | 1383 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 1384 mpa->private_data_size = htons(ep->plen); 1385 mpa->revision = mpa_rev_to_use; 1386 1387 if (mpa_rev_to_use == 1) { 1388 ep->tried_with_mpa_v1 = 1; 1389 ep->retry_with_mpa_v1 = 0; 1390 } 1391 1392 if (mpa_rev_to_use == 2) { 1393 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 1394 sizeof(struct mpa_v2_conn_params)); 1395 mpa_v2_params.ird = htons((u16)ep->ird); 1396 mpa_v2_params.ord = htons((u16)ep->ord); 1397 1398 if (peer2peer) { 1399 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1400 1401 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1402 mpa_v2_params.ord |= 1403 htons(MPA_V2_RDMA_WRITE_RTR); 1404 } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1405 mpa_v2_params.ord |= 1406 htons(MPA_V2_RDMA_READ_RTR); 1407 } 1408 } 1409 memcpy(mpa->private_data, &mpa_v2_params, 1410 sizeof(struct mpa_v2_conn_params)); 1411 1412 if (ep->plen) { 1413 1414 memcpy(mpa->private_data + 1415 sizeof(struct mpa_v2_conn_params), 1416 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1417 } 1418 } else { 1419 1420 if (ep->plen) 1421 memcpy(mpa->private_data, 1422 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1423 CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep); 1424 } 1425 1426 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1427 if (m == NULL) { 1428 err = -ENOMEM; 1429 CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d", 1430 __func__, ep, err); 1431 free(mpa, M_CXGBE); 1432 goto err; 1433 } 1434 m_copyback(m, 0, mpalen, (void *)mpa); 1435 free(mpa, M_CXGBE); 1436 1437 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1438 ep->com.thread); 1439 if (err) { 1440 CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d", 1441 __func__, ep, err); 1442 goto err; 1443 } 1444 1445 START_EP_TIMER(ep); 1446 ep->com.state = MPA_REQ_SENT; 1447 ep->mpa_attr.initiator = 1; 1448 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1449 return 0; 1450err: 1451 connect_reply_upcall(ep, err); 1452 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1453 return err; 1454} 1455 1456static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 1457{ 1458 int mpalen ; 1459 struct mpa_message *mpa; 1460 struct mpa_v2_conn_params mpa_v2_params; 1461 struct mbuf *m; 1462 int err; 1463 1464 CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid, 1465 ep->plen); 1466 1467 mpalen = sizeof(*mpa) + plen; 1468 1469 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1470 1471 mpalen += sizeof(struct mpa_v2_conn_params); 1472 CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep, 1473 ep->mpa_attr.version, mpalen); 1474 } 1475 1476 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1477 if (mpa == NULL) 1478 return (-ENOMEM); 1479 1480 memset(mpa, 0, mpalen); 1481 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1482 mpa->flags = MPA_REJECT; 1483 mpa->revision = mpa_rev; 1484 mpa->private_data_size = htons(plen); 1485 1486 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1487 1488 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1489 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 1490 sizeof(struct mpa_v2_conn_params)); 1491 mpa_v2_params.ird = htons(((u16)ep->ird) | 1492 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 1493 0)); 1494 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 1495 (p2p_type == 1496 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 1497 MPA_V2_RDMA_WRITE_RTR : p2p_type == 1498 FW_RI_INIT_P2PTYPE_READ_REQ ? 1499 MPA_V2_RDMA_READ_RTR : 0) : 0)); 1500 memcpy(mpa->private_data, &mpa_v2_params, 1501 sizeof(struct mpa_v2_conn_params)); 1502 1503 if (ep->plen) 1504 memcpy(mpa->private_data + 1505 sizeof(struct mpa_v2_conn_params), pdata, plen); 1506 CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep, 1507 mpa_v2_params.ird, mpa_v2_params.ord, ep->plen); 1508 } else 1509 if (plen) 1510 memcpy(mpa->private_data, pdata, plen); 1511 1512 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1513 if (m == NULL) { 1514 free(mpa, M_CXGBE); 1515 return (-ENOMEM); 1516 } 1517 m_copyback(m, 0, mpalen, (void *)mpa); 1518 free(mpa, M_CXGBE); 1519 1520 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 1521 if (!err) 1522 ep->snd_seq += mpalen; 1523 CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err); 1524 return err; 1525} 1526 1527static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 1528{ 1529 int mpalen; 1530 struct mpa_message *mpa; 1531 struct mbuf *m; 1532 struct mpa_v2_conn_params mpa_v2_params; 1533 int err; 1534 1535 CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep); 1536 1537 mpalen = sizeof(*mpa) + plen; 1538 1539 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1540 1541 CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep, 1542 ep->mpa_attr.version); 1543 mpalen += sizeof(struct mpa_v2_conn_params); 1544 } 1545 1546 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1547 if (mpa == NULL) 1548 return (-ENOMEM); 1549 1550 memset(mpa, 0, sizeof(*mpa)); 1551 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1552 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 1553 (markers_enabled ? MPA_MARKERS : 0); 1554 mpa->revision = ep->mpa_attr.version; 1555 mpa->private_data_size = htons(plen); 1556 1557 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1558 1559 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1560 mpa->private_data_size += 1561 htons(sizeof(struct mpa_v2_conn_params)); 1562 mpa_v2_params.ird = htons((u16)ep->ird); 1563 mpa_v2_params.ord = htons((u16)ep->ord); 1564 CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep, 1565 ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord); 1566 1567 if (peer2peer && (ep->mpa_attr.p2p_type != 1568 FW_RI_INIT_P2PTYPE_DISABLED)) { 1569 1570 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1571 1572 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1573 1574 mpa_v2_params.ord |= 1575 htons(MPA_V2_RDMA_WRITE_RTR); 1576 CTR5(KTR_IW_CXGBE, "%s:smrep4 %p %d %d %d", 1577 __func__, ep, p2p_type, mpa_v2_params.ird, 1578 mpa_v2_params.ord); 1579 } 1580 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1581 1582 mpa_v2_params.ord |= 1583 htons(MPA_V2_RDMA_READ_RTR); 1584 CTR5(KTR_IW_CXGBE, "%s:smrep5 %p %d %d %d", 1585 __func__, ep, p2p_type, mpa_v2_params.ird, 1586 mpa_v2_params.ord); 1587 } 1588 } 1589 1590 memcpy(mpa->private_data, &mpa_v2_params, 1591 sizeof(struct mpa_v2_conn_params)); 1592 1593 if (ep->plen) 1594 memcpy(mpa->private_data + 1595 sizeof(struct mpa_v2_conn_params), pdata, plen); 1596 } else 1597 if (plen) 1598 memcpy(mpa->private_data, pdata, plen); 1599 1600 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1601 if (m == NULL) { 1602 free(mpa, M_CXGBE); 1603 return (-ENOMEM); 1604 } 1605 m_copyback(m, 0, mpalen, (void *)mpa); 1606 free(mpa, M_CXGBE); 1607 1608 1609 ep->com.state = MPA_REP_SENT; 1610 ep->snd_seq += mpalen; 1611 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1612 ep->com.thread); 1613 CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err); 1614 return err; 1615} 1616 1617 1618 1619static void close_complete_upcall(struct c4iw_ep *ep, int status) 1620{ 1621 struct iw_cm_event event; 1622 1623 CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep); 1624 memset(&event, 0, sizeof(event)); 1625 event.event = IW_CM_EVENT_CLOSE; 1626 event.status = status; 1627 1628 if (ep->com.cm_id) { 1629 1630 CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); 1631 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1632 deref_cm_id(&ep->com); 1633 set_bit(CLOSE_UPCALL, &ep->com.history); 1634 } 1635 CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); 1636} 1637 1638static int 1639send_abort(struct c4iw_ep *ep) 1640{ 1641 struct socket *so = ep->com.so; 1642 struct sockopt sopt; 1643 int rc; 1644 struct linger l; 1645 1646 CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so, 1647 states[ep->com.state], ep->hwtid); 1648 1649 l.l_onoff = 1; 1650 l.l_linger = 0; 1651 1652 /* linger_time of 0 forces RST to be sent */ 1653 sopt.sopt_dir = SOPT_SET; 1654 sopt.sopt_level = SOL_SOCKET; 1655 sopt.sopt_name = SO_LINGER; 1656 sopt.sopt_val = (caddr_t)&l; 1657 sopt.sopt_valsize = sizeof l; 1658 sopt.sopt_td = NULL; 1659 rc = -sosetopt(so, &sopt); 1660 if (rc != 0) { 1661 log(LOG_ERR, "%s: sosetopt(%p, linger = 0) failed with %d.\n", 1662 __func__, so, rc); 1663 } 1664 1665 uninit_iwarp_socket(so); 1666 soclose(so); 1667 set_bit(ABORT_CONN, &ep->com.history); 1668 1669 /* 1670 * TBD: iw_cxgbe driver should receive ABORT reply for every ABORT 1671 * request it has sent. But the current TOE driver is not propagating 1672 * this ABORT reply event (via do_abort_rpl) to iw_cxgbe. So as a work- 1673 * around de-refererece 'ep' here instead of doing it in abort_rpl() 1674 * handler(not yet implemented) of iw_cxgbe driver. 1675 */ 1676 release_ep_resources(ep); 1677 ep->com.state = DEAD; 1678 1679 return (0); 1680} 1681 1682static void peer_close_upcall(struct c4iw_ep *ep) 1683{ 1684 struct iw_cm_event event; 1685 1686 CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep); 1687 memset(&event, 0, sizeof(event)); 1688 event.event = IW_CM_EVENT_DISCONNECT; 1689 1690 if (ep->com.cm_id) { 1691 1692 CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep); 1693 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1694 set_bit(DISCONN_UPCALL, &ep->com.history); 1695 } 1696 CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep); 1697} 1698 1699static void peer_abort_upcall(struct c4iw_ep *ep) 1700{ 1701 struct iw_cm_event event; 1702 1703 CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep); 1704 memset(&event, 0, sizeof(event)); 1705 event.event = IW_CM_EVENT_CLOSE; 1706 event.status = -ECONNRESET; 1707 1708 if (ep->com.cm_id) { 1709 1710 CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); 1711 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1712 deref_cm_id(&ep->com); 1713 set_bit(ABORT_UPCALL, &ep->com.history); 1714 } 1715 CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); 1716} 1717 1718static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1719{ 1720 struct iw_cm_event event; 1721 1722 CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status); 1723 memset(&event, 0, sizeof(event)); 1724 event.event = IW_CM_EVENT_CONNECT_REPLY; 1725 event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ? 1726 -ECONNRESET : status; 1727 event.local_addr = ep->com.local_addr; 1728 event.remote_addr = ep->com.remote_addr; 1729 1730 if ((status == 0) || (status == -ECONNREFUSED)) { 1731 1732 if (!ep->tried_with_mpa_v1) { 1733 1734 CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep); 1735 /* this means MPA_v2 is used */ 1736 event.ord = ep->ird; 1737 event.ird = ep->ord; 1738 event.private_data_len = ep->plen - 1739 sizeof(struct mpa_v2_conn_params); 1740 event.private_data = ep->mpa_pkt + 1741 sizeof(struct mpa_message) + 1742 sizeof(struct mpa_v2_conn_params); 1743 } else { 1744 1745 CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep); 1746 /* this means MPA_v1 is used */ 1747 event.ord = c4iw_max_read_depth; 1748 event.ird = c4iw_max_read_depth; 1749 event.private_data_len = ep->plen; 1750 event.private_data = ep->mpa_pkt + 1751 sizeof(struct mpa_message); 1752 } 1753 } 1754 1755 if (ep->com.cm_id) { 1756 1757 CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep); 1758 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1759 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1760 } 1761 1762 if(status == -ECONNABORTED) { 1763 1764 CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status); 1765 return; 1766 } 1767 1768 if (status < 0) { 1769 1770 CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); 1771 deref_cm_id(&ep->com); 1772 } 1773 1774 CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); 1775} 1776 1777static int connect_request_upcall(struct c4iw_ep *ep) 1778{ 1779 struct iw_cm_event event; 1780 int ret; 1781 1782 CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep, 1783 ep->tried_with_mpa_v1); 1784 1785 memset(&event, 0, sizeof(event)); 1786 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1787 event.local_addr = ep->com.local_addr; 1788 event.remote_addr = ep->com.remote_addr; 1789 event.provider_data = ep; 1790 1791 if (!ep->tried_with_mpa_v1) { 1792 /* this means MPA_v2 is used */ 1793 event.ord = ep->ord; 1794 event.ird = ep->ird; 1795 event.private_data_len = ep->plen - 1796 sizeof(struct mpa_v2_conn_params); 1797 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1798 sizeof(struct mpa_v2_conn_params); 1799 } else { 1800 1801 /* this means MPA_v1 is used. Send max supported */ 1802 event.ord = c4iw_max_read_depth; 1803 event.ird = c4iw_max_read_depth; 1804 event.private_data_len = ep->plen; 1805 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1806 } 1807 1808 c4iw_get_ep(&ep->com); 1809 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1810 &event); 1811 if(ret) { 1812 CTR3(KTR_IW_CXGBE, "%s: ep %p, Failure while notifying event to" 1813 " IWCM, err:%d", __func__, ep, ret); 1814 c4iw_put_ep(&ep->com); 1815 } else 1816 /* Dereference parent_ep only in success case. 1817 * In case of failure, parent_ep is dereferenced by the caller 1818 * of process_mpa_request(). 1819 */ 1820 c4iw_put_ep(&ep->parent_ep->com); 1821 1822 set_bit(CONNREQ_UPCALL, &ep->com.history); 1823 return ret; 1824} 1825 1826static void established_upcall(struct c4iw_ep *ep) 1827{ 1828 struct iw_cm_event event; 1829 1830 CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep); 1831 memset(&event, 0, sizeof(event)); 1832 event.event = IW_CM_EVENT_ESTABLISHED; 1833 event.ird = ep->ord; 1834 event.ord = ep->ird; 1835 1836 if (ep->com.cm_id) { 1837 1838 CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep); 1839 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1840 set_bit(ESTAB_UPCALL, &ep->com.history); 1841 } 1842 CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep); 1843} 1844 1845 1846#define RELAXED_IRD_NEGOTIATION 1 1847 1848/* 1849 * process_mpa_reply - process streaming mode MPA reply 1850 * 1851 * Returns: 1852 * 1853 * 0 upon success indicating a connect request was delivered to the ULP 1854 * or the mpa request is incomplete but valid so far. 1855 * 1856 * 1 if a failure requires the caller to close the connection. 1857 * 1858 * 2 if a failure requires the caller to abort the connection. 1859 */ 1860static int process_mpa_reply(struct c4iw_ep *ep) 1861{ 1862 struct mpa_message *mpa; 1863 struct mpa_v2_conn_params *mpa_v2_params; 1864 u16 plen; 1865 u16 resp_ird, resp_ord; 1866 u8 rtr_mismatch = 0, insuff_ird = 0; 1867 struct c4iw_qp_attributes attrs = {0}; 1868 enum c4iw_qp_attr_mask mask; 1869 int err; 1870 struct mbuf *top, *m; 1871 int flags = MSG_DONTWAIT; 1872 struct uio uio; 1873 int disconnect = 0; 1874 1875 CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); 1876 1877 /* 1878 * Stop mpa timer. If it expired, then 1879 * we ignore the MPA reply. process_timeout() 1880 * will abort the connection. 1881 */ 1882 if (STOP_EP_TIMER(ep)) 1883 return 0; 1884 1885 uio.uio_resid = 1000000; 1886 uio.uio_td = ep->com.thread; 1887 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 1888 1889 if (err) { 1890 1891 if (err == EWOULDBLOCK) { 1892 1893 CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); 1894 START_EP_TIMER(ep); 1895 return 0; 1896 } 1897 err = -err; 1898 CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); 1899 goto err; 1900 } 1901 1902 if (ep->com.so->so_rcv.sb_mb) { 1903 1904 CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep); 1905 printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 1906 __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 1907 } 1908 1909 m = top; 1910 1911 do { 1912 1913 CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep); 1914 /* 1915 * If we get more than the supported amount of private data 1916 * then we must fail this connection. 1917 */ 1918 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 1919 1920 CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, 1921 ep->mpa_pkt_len + m->m_len); 1922 err = (-EINVAL); 1923 goto err_stop_timer; 1924 } 1925 1926 /* 1927 * copy the new data into our accumulation buffer. 1928 */ 1929 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 1930 ep->mpa_pkt_len += m->m_len; 1931 if (!m->m_next) 1932 m = m->m_nextpkt; 1933 else 1934 m = m->m_next; 1935 } while (m); 1936 1937 m_freem(top); 1938 /* 1939 * if we don't even have the mpa message, then bail. 1940 */ 1941 if (ep->mpa_pkt_len < sizeof(*mpa)) { 1942 return 0; 1943 } 1944 mpa = (struct mpa_message *) ep->mpa_pkt; 1945 1946 /* Validate MPA header. */ 1947 if (mpa->revision > mpa_rev) { 1948 1949 CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep, 1950 mpa->revision, mpa_rev); 1951 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " 1952 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1953 err = -EPROTO; 1954 goto err_stop_timer; 1955 } 1956 1957 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1958 1959 CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); 1960 err = -EPROTO; 1961 goto err_stop_timer; 1962 } 1963 1964 plen = ntohs(mpa->private_data_size); 1965 1966 /* 1967 * Fail if there's too much private data. 1968 */ 1969 if (plen > MPA_MAX_PRIVATE_DATA) { 1970 1971 CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); 1972 err = -EPROTO; 1973 goto err_stop_timer; 1974 } 1975 1976 /* 1977 * If plen does not account for pkt size 1978 */ 1979 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1980 1981 CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); 1982 STOP_EP_TIMER(ep); 1983 err = -EPROTO; 1984 goto err_stop_timer; 1985 } 1986 1987 ep->plen = (u8) plen; 1988 1989 /* 1990 * If we don't have all the pdata yet, then bail. 1991 * We'll continue process when more data arrives. 1992 */ 1993 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 1994 1995 CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); 1996 return 0; 1997 } 1998 1999 if (mpa->flags & MPA_REJECT) { 2000 2001 CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); 2002 err = -ECONNREFUSED; 2003 goto err_stop_timer; 2004 } 2005 2006 /* 2007 * If we get here we have accumulated the entire mpa 2008 * start reply message including private data. And 2009 * the MPA header is valid. 2010 */ 2011 ep->com.state = FPDU_MODE; 2012 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 2013 ep->mpa_attr.recv_marker_enabled = markers_enabled; 2014 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 2015 ep->mpa_attr.version = mpa->revision; 2016 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 2017 2018 if (mpa->revision == 2) { 2019 2020 CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep); 2021 ep->mpa_attr.enhanced_rdma_conn = 2022 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 2023 2024 if (ep->mpa_attr.enhanced_rdma_conn) { 2025 2026 CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep); 2027 mpa_v2_params = (struct mpa_v2_conn_params *) 2028 (ep->mpa_pkt + sizeof(*mpa)); 2029 resp_ird = ntohs(mpa_v2_params->ird) & 2030 MPA_V2_IRD_ORD_MASK; 2031 resp_ord = ntohs(mpa_v2_params->ord) & 2032 MPA_V2_IRD_ORD_MASK; 2033 2034 /* 2035 * This is a double-check. Ideally, below checks are 2036 * not required since ird/ord stuff has been taken 2037 * care of in c4iw_accept_cr 2038 */ 2039 if (ep->ird < resp_ord) { 2040 if (RELAXED_IRD_NEGOTIATION && resp_ord <= 2041 ep->com.dev->rdev.adap->params.max_ordird_qp) 2042 ep->ird = resp_ord; 2043 else 2044 insuff_ird = 1; 2045 } else if (ep->ird > resp_ord) { 2046 ep->ird = resp_ord; 2047 } 2048 if (ep->ord > resp_ird) { 2049 if (RELAXED_IRD_NEGOTIATION) 2050 ep->ord = resp_ird; 2051 else 2052 insuff_ird = 1; 2053 } 2054 if (insuff_ird) { 2055 err = -ENOMEM; 2056 ep->ird = resp_ord; 2057 ep->ord = resp_ird; 2058 } 2059 2060 if (ntohs(mpa_v2_params->ird) & 2061 MPA_V2_PEER2PEER_MODEL) { 2062 2063 CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep); 2064 if (ntohs(mpa_v2_params->ord) & 2065 MPA_V2_RDMA_WRITE_RTR) { 2066 2067 CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep); 2068 ep->mpa_attr.p2p_type = 2069 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 2070 } 2071 else if (ntohs(mpa_v2_params->ord) & 2072 MPA_V2_RDMA_READ_RTR) { 2073 2074 CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep); 2075 ep->mpa_attr.p2p_type = 2076 FW_RI_INIT_P2PTYPE_READ_REQ; 2077 } 2078 } 2079 } 2080 } else { 2081 2082 CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep); 2083 2084 if (mpa->revision == 1) { 2085 2086 CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep); 2087 2088 if (peer2peer) { 2089 2090 CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep); 2091 ep->mpa_attr.p2p_type = p2p_type; 2092 } 2093 } 2094 } 2095 2096 if (set_tcpinfo(ep)) { 2097 2098 CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep); 2099 printf("%s set_tcpinfo error\n", __func__); 2100 err = -ECONNRESET; 2101 goto err; 2102 } 2103 2104 CTR6(KTR_IW_CXGBE, "%s - crc_enabled = %d, recv_marker_enabled = %d, " 2105 "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__, 2106 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 2107 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 2108 ep->mpa_attr.p2p_type); 2109 2110 /* 2111 * If responder's RTR does not match with that of initiator, assign 2112 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 2113 * generated when moving QP to RTS state. 2114 * A TERM message will be sent after QP has moved to RTS state 2115 */ 2116 if ((ep->mpa_attr.version == 2) && peer2peer && 2117 (ep->mpa_attr.p2p_type != p2p_type)) { 2118 2119 CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep); 2120 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 2121 rtr_mismatch = 1; 2122 } 2123 2124 2125 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 2126 attrs.mpa_attr = ep->mpa_attr; 2127 attrs.max_ird = ep->ird; 2128 attrs.max_ord = ep->ord; 2129 attrs.llp_stream_handle = ep; 2130 attrs.next_state = C4IW_QP_STATE_RTS; 2131 2132 mask = C4IW_QP_ATTR_NEXT_STATE | 2133 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 2134 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 2135 2136 /* bind QP and TID with INIT_WR */ 2137 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 2138 2139 if (err) { 2140 2141 CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep); 2142 goto err; 2143 } 2144 2145 /* 2146 * If responder's RTR requirement did not match with what initiator 2147 * supports, generate TERM message 2148 */ 2149 if (rtr_mismatch) { 2150 2151 CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep); 2152 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 2153 attrs.layer_etype = LAYER_MPA | DDP_LLP; 2154 attrs.ecode = MPA_NOMATCH_RTR; 2155 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2156 attrs.send_term = 1; 2157 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2158 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2159 err = -ENOMEM; 2160 disconnect = 1; 2161 goto out; 2162 } 2163 2164 /* 2165 * Generate TERM if initiator IRD is not sufficient for responder 2166 * provided ORD. Currently, we do the same behaviour even when 2167 * responder provided IRD is also not sufficient as regards to 2168 * initiator ORD. 2169 */ 2170 if (insuff_ird) { 2171 2172 CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep); 2173 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 2174 __func__); 2175 attrs.layer_etype = LAYER_MPA | DDP_LLP; 2176 attrs.ecode = MPA_INSUFF_IRD; 2177 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2178 attrs.send_term = 1; 2179 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2180 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2181 err = -ENOMEM; 2182 disconnect = 1; 2183 goto out; 2184 } 2185 goto out; 2186err_stop_timer: 2187 STOP_EP_TIMER(ep); 2188err: 2189 disconnect = 2; 2190out: 2191 connect_reply_upcall(ep, err); 2192 CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); 2193 return disconnect; 2194} 2195 2196/* 2197 * process_mpa_request - process streaming mode MPA request 2198 * 2199 * Returns: 2200 * 2201 * 0 upon success indicating a connect request was delivered to the ULP 2202 * or the mpa request is incomplete but valid so far. 2203 * 2204 * 1 if a failure requires the caller to close the connection. 2205 * 2206 * 2 if a failure requires the caller to abort the connection. 2207 */ 2208static int 2209process_mpa_request(struct c4iw_ep *ep) 2210{ 2211 struct mpa_message *mpa; 2212 struct mpa_v2_conn_params *mpa_v2_params; 2213 u16 plen; 2214 int flags = MSG_DONTWAIT; 2215 int rc; 2216 struct iovec iov; 2217 struct uio uio; 2218 enum c4iw_ep_state state = ep->com.state; 2219 2220 CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); 2221 2222 if (state != MPA_REQ_WAIT) 2223 return 0; 2224 2225 iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; 2226 iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 2227 uio.uio_iov = &iov; 2228 uio.uio_iovcnt = 1; 2229 uio.uio_offset = 0; 2230 uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 2231 uio.uio_segflg = UIO_SYSSPACE; 2232 uio.uio_rw = UIO_READ; 2233 uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */ 2234 2235 rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); 2236 if (rc == EAGAIN) 2237 return 0; 2238 else if (rc) 2239 goto err_stop_timer; 2240 2241 KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", 2242 __func__, ep->com.so)); 2243 ep->mpa_pkt_len += uio.uio_offset; 2244 2245 /* 2246 * If we get more than the supported amount of private data then we must 2247 * fail this connection. XXX: check so_rcv->sb_cc, or peek with another 2248 * soreceive, or increase the size of mpa_pkt by 1 and abort if the last 2249 * byte is filled by the soreceive above. 2250 */ 2251 2252 /* Don't even have the MPA message. Wait for more data to arrive. */ 2253 if (ep->mpa_pkt_len < sizeof(*mpa)) 2254 return 0; 2255 mpa = (struct mpa_message *) ep->mpa_pkt; 2256 2257 /* 2258 * Validate MPA Header. 2259 */ 2260 if (mpa->revision > mpa_rev) { 2261 log(LOG_ERR, "%s: MPA version mismatch. Local = %d," 2262 " Received = %d\n", __func__, mpa_rev, mpa->revision); 2263 goto err_stop_timer; 2264 } 2265 2266 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) 2267 goto err_stop_timer; 2268 2269 /* 2270 * Fail if there's too much private data. 2271 */ 2272 plen = ntohs(mpa->private_data_size); 2273 if (plen > MPA_MAX_PRIVATE_DATA) 2274 goto err_stop_timer; 2275 2276 /* 2277 * If plen does not account for pkt size 2278 */ 2279 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) 2280 goto err_stop_timer; 2281 2282 ep->plen = (u8) plen; 2283 2284 /* 2285 * If we don't have all the pdata yet, then bail. 2286 */ 2287 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 2288 return 0; 2289 2290 /* 2291 * If we get here we have accumulated the entire mpa 2292 * start reply message including private data. 2293 */ 2294 ep->mpa_attr.initiator = 0; 2295 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 2296 ep->mpa_attr.recv_marker_enabled = markers_enabled; 2297 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 2298 ep->mpa_attr.version = mpa->revision; 2299 if (mpa->revision == 1) 2300 ep->tried_with_mpa_v1 = 1; 2301 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 2302 2303 if (mpa->revision == 2) { 2304 ep->mpa_attr.enhanced_rdma_conn = 2305 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 2306 if (ep->mpa_attr.enhanced_rdma_conn) { 2307 mpa_v2_params = (struct mpa_v2_conn_params *) 2308 (ep->mpa_pkt + sizeof(*mpa)); 2309 ep->ird = ntohs(mpa_v2_params->ird) & 2310 MPA_V2_IRD_ORD_MASK; 2311 ep->ird = min_t(u32, ep->ird, 2312 cur_max_read_depth(ep->com.dev)); 2313 ep->ord = ntohs(mpa_v2_params->ord) & 2314 MPA_V2_IRD_ORD_MASK; 2315 ep->ord = min_t(u32, ep->ord, 2316 cur_max_read_depth(ep->com.dev)); 2317 CTR3(KTR_IW_CXGBE, "%s initiator ird %u ord %u", 2318 __func__, ep->ird, ep->ord); 2319 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 2320 if (peer2peer) { 2321 if (ntohs(mpa_v2_params->ord) & 2322 MPA_V2_RDMA_WRITE_RTR) 2323 ep->mpa_attr.p2p_type = 2324 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 2325 else if (ntohs(mpa_v2_params->ord) & 2326 MPA_V2_RDMA_READ_RTR) 2327 ep->mpa_attr.p2p_type = 2328 FW_RI_INIT_P2PTYPE_READ_REQ; 2329 } 2330 } 2331 } else if (mpa->revision == 1 && peer2peer) 2332 ep->mpa_attr.p2p_type = p2p_type; 2333 2334 if (set_tcpinfo(ep)) 2335 goto err_stop_timer; 2336 2337 CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " 2338 "xmit_marker_enabled = %d, version = %d", __func__, 2339 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 2340 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 2341 2342 ep->com.state = MPA_REQ_RCVD; 2343 STOP_EP_TIMER(ep); 2344 2345 /* drive upcall */ 2346 if (ep->parent_ep->com.state != DEAD) 2347 if (connect_request_upcall(ep)) 2348 goto err_out; 2349 return 0; 2350 2351err_stop_timer: 2352 STOP_EP_TIMER(ep); 2353err_out: 2354 return 2; 2355} 2356 2357/* 2358 * Upcall from the adapter indicating data has been transmitted. 2359 * For us its just the single MPA request or reply. We can now free 2360 * the skb holding the mpa message. 2361 */ 2362int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2363{ 2364 int err; 2365 struct c4iw_ep *ep = to_ep(cm_id); 2366 int abort = 0; 2367 2368 mutex_lock(&ep->com.mutex); 2369 CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); 2370 2371 if ((ep->com.state == DEAD) || 2372 (ep->com.state != MPA_REQ_RCVD)) { 2373 2374 CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep); 2375 mutex_unlock(&ep->com.mutex); 2376 c4iw_put_ep(&ep->com); 2377 return -ECONNRESET; 2378 } 2379 set_bit(ULP_REJECT, &ep->com.history); 2380 2381 if (mpa_rev == 0) { 2382 2383 CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); 2384 abort = 1; 2385 } 2386 else { 2387 2388 CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep); 2389 abort = send_mpa_reject(ep, pdata, pdata_len); 2390 } 2391 STOP_EP_TIMER(ep); 2392 err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL); 2393 mutex_unlock(&ep->com.mutex); 2394 c4iw_put_ep(&ep->com); 2395 CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err); 2396 return 0; 2397} 2398 2399int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2400{ 2401 int err; 2402 struct c4iw_qp_attributes attrs = {0}; 2403 enum c4iw_qp_attr_mask mask; 2404 struct c4iw_ep *ep = to_ep(cm_id); 2405 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2406 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2407 int abort = 0; 2408 2409 mutex_lock(&ep->com.mutex); 2410 CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); 2411 2412 if ((ep->com.state == DEAD) || 2413 (ep->com.state != MPA_REQ_RCVD)) { 2414 2415 CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); 2416 err = -ECONNRESET; 2417 goto err_out; 2418 } 2419 2420 BUG_ON(!qp); 2421 2422 set_bit(ULP_ACCEPT, &ep->com.history); 2423 2424 if ((conn_param->ord > c4iw_max_read_depth) || 2425 (conn_param->ird > c4iw_max_read_depth)) { 2426 2427 CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); 2428 err = -EINVAL; 2429 goto err_abort; 2430 } 2431 2432 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2433 2434 CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep); 2435 2436 if (conn_param->ord > ep->ird) { 2437 if (RELAXED_IRD_NEGOTIATION) { 2438 conn_param->ord = ep->ird; 2439 } else { 2440 ep->ird = conn_param->ird; 2441 ep->ord = conn_param->ord; 2442 send_mpa_reject(ep, conn_param->private_data, 2443 conn_param->private_data_len); 2444 err = -ENOMEM; 2445 goto err_abort; 2446 } 2447 } 2448 if (conn_param->ird < ep->ord) { 2449 if (RELAXED_IRD_NEGOTIATION && 2450 ep->ord <= h->rdev.adap->params.max_ordird_qp) { 2451 conn_param->ird = ep->ord; 2452 } else { 2453 err = -ENOMEM; 2454 goto err_abort; 2455 } 2456 } 2457 } 2458 ep->ird = conn_param->ird; 2459 ep->ord = conn_param->ord; 2460 2461 if (ep->mpa_attr.version == 1) { 2462 if (peer2peer && ep->ird == 0) 2463 ep->ird = 1; 2464 } else { 2465 if (peer2peer && 2466 (ep->mpa_attr.p2p_type != FW_RI_INIT_P2PTYPE_DISABLED) && 2467 (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) && ep->ird == 0) 2468 ep->ird = 1; 2469 } 2470 2471 CTR4(KTR_IW_CXGBE, "%s %d ird %d ord %d", __func__, __LINE__, 2472 ep->ird, ep->ord); 2473 2474 ep->com.cm_id = cm_id; 2475 ref_cm_id(&ep->com); 2476 ep->com.qp = qp; 2477 ref_qp(ep); 2478 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 2479 2480 /* bind QP to EP and move to RTS */ 2481 attrs.mpa_attr = ep->mpa_attr; 2482 attrs.max_ird = ep->ird; 2483 attrs.max_ord = ep->ord; 2484 attrs.llp_stream_handle = ep; 2485 attrs.next_state = C4IW_QP_STATE_RTS; 2486 2487 /* bind QP and TID with INIT_WR */ 2488 mask = C4IW_QP_ATTR_NEXT_STATE | 2489 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2490 C4IW_QP_ATTR_MPA_ATTR | 2491 C4IW_QP_ATTR_MAX_IRD | 2492 C4IW_QP_ATTR_MAX_ORD; 2493 2494 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 2495 if (err) { 2496 CTR3(KTR_IW_CXGBE, "%s:caca %p, err: %d", __func__, ep, err); 2497 goto err_defef_cm_id; 2498 } 2499 2500 err = send_mpa_reply(ep, conn_param->private_data, 2501 conn_param->private_data_len); 2502 if (err) { 2503 CTR3(KTR_IW_CXGBE, "%s:cacb %p, err: %d", __func__, ep, err); 2504 goto err_defef_cm_id; 2505 } 2506 2507 ep->com.state = FPDU_MODE; 2508 established_upcall(ep); 2509 mutex_unlock(&ep->com.mutex); 2510 c4iw_put_ep(&ep->com); 2511 CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); 2512 return 0; 2513err_defef_cm_id: 2514 deref_cm_id(&ep->com); 2515err_abort: 2516 abort = 1; 2517err_out: 2518 if (abort) 2519 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 2520 mutex_unlock(&ep->com.mutex); 2521 c4iw_put_ep(&ep->com); 2522 CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); 2523 return err; 2524} 2525 2526static int 2527c4iw_sock_create(struct sockaddr_storage *laddr, struct socket **so) 2528{ 2529 int ret; 2530 int size, on; 2531 struct socket *sock = NULL; 2532 struct sockopt sopt; 2533 2534 ret = sock_create_kern(laddr->ss_family, 2535 SOCK_STREAM, IPPROTO_TCP, &sock); 2536 if (ret) { 2537 CTR2(KTR_IW_CXGBE, "%s:Failed to create TCP socket. err %d", 2538 __func__, ret); 2539 return ret; 2540 } 2541 2542 if (reuseaddr) { 2543 bzero(&sopt, sizeof(struct sockopt)); 2544 sopt.sopt_dir = SOPT_SET; 2545 sopt.sopt_level = SOL_SOCKET; 2546 sopt.sopt_name = SO_REUSEADDR; 2547 on = 1; 2548 sopt.sopt_val = &on; 2549 sopt.sopt_valsize = sizeof(on); 2550 ret = -sosetopt(sock, &sopt); 2551 if (ret != 0) { 2552 log(LOG_ERR, "%s: sosetopt(%p, SO_REUSEADDR) " 2553 "failed with %d.\n", __func__, sock, ret); 2554 } 2555 bzero(&sopt, sizeof(struct sockopt)); 2556 sopt.sopt_dir = SOPT_SET; 2557 sopt.sopt_level = SOL_SOCKET; 2558 sopt.sopt_name = SO_REUSEPORT; 2559 on = 1; 2560 sopt.sopt_val = &on; 2561 sopt.sopt_valsize = sizeof(on); 2562 ret = -sosetopt(sock, &sopt); 2563 if (ret != 0) { 2564 log(LOG_ERR, "%s: sosetopt(%p, SO_REUSEPORT) " 2565 "failed with %d.\n", __func__, sock, ret); 2566 } 2567 } 2568 2569 ret = -sobind(sock, (struct sockaddr *)laddr, curthread); 2570 if (ret) { 2571 CTR2(KTR_IW_CXGBE, "%s:Failed to bind socket. err %p", 2572 __func__, ret); 2573 sock_release(sock); 2574 return ret; 2575 } 2576 2577 size = laddr->ss_family == AF_INET6 ? 2578 sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in); 2579 ret = sock_getname(sock, (struct sockaddr *)laddr, &size, 0); 2580 if (ret) { 2581 CTR2(KTR_IW_CXGBE, "%s:sock_getname failed. err %p", 2582 __func__, ret); 2583 sock_release(sock); 2584 return ret; 2585 } 2586 2587 *so = sock; 2588 return 0; 2589} 2590 2591int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2592{ 2593 int err = 0; 2594 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2595 struct c4iw_ep *ep = NULL; 2596 struct ifnet *nh_ifp; /* Logical egress interface */ 2597#ifdef VIMAGE 2598 struct rdma_cm_id *rdma_id = (struct rdma_cm_id*)cm_id->context; 2599 struct vnet *vnet = rdma_id->route.addr.dev_addr.net; 2600#endif 2601 2602 CTR2(KTR_IW_CXGBE, "%s:ccB %p", __func__, cm_id); 2603 2604 2605 if ((conn_param->ord > c4iw_max_read_depth) || 2606 (conn_param->ird > c4iw_max_read_depth)) { 2607 2608 CTR2(KTR_IW_CXGBE, "%s:cc1 %p", __func__, cm_id); 2609 err = -EINVAL; 2610 goto out; 2611 } 2612 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2613 cm_id->provider_data = ep; 2614 2615 init_timer(&ep->timer); 2616 ep->plen = conn_param->private_data_len; 2617 2618 if (ep->plen) { 2619 2620 CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep); 2621 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2622 conn_param->private_data, ep->plen); 2623 } 2624 ep->ird = conn_param->ird; 2625 ep->ord = conn_param->ord; 2626 2627 if (peer2peer && ep->ord == 0) { 2628 2629 CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep); 2630 ep->ord = 1; 2631 } 2632 2633 ep->com.dev = dev; 2634 ep->com.cm_id = cm_id; 2635 ref_cm_id(&ep->com); 2636 ep->com.qp = get_qhp(dev, conn_param->qpn); 2637 2638 if (!ep->com.qp) { 2639 2640 CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep); 2641 err = -EINVAL; 2642 goto fail; 2643 } 2644 ref_qp(ep); 2645 ep->com.thread = curthread; 2646 2647 CURVNET_SET(vnet); 2648 err = get_ifnet_from_raddr(&cm_id->remote_addr, &nh_ifp); 2649 CURVNET_RESTORE(); 2650 2651 if (err) { 2652 2653 CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep); 2654 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2655 err = EHOSTUNREACH; 2656 return err; 2657 } 2658 2659 if (!(nh_ifp->if_capenable & IFCAP_TOE) || 2660 TOEDEV(nh_ifp) == NULL) { 2661 err = -ENOPROTOOPT; 2662 goto fail; 2663 } 2664 ep->com.state = CONNECTING; 2665 ep->tos = 0; 2666 ep->com.local_addr = cm_id->local_addr; 2667 ep->com.remote_addr = cm_id->remote_addr; 2668 2669 err = c4iw_sock_create(&cm_id->local_addr, &ep->com.so); 2670 if (err) 2671 goto fail; 2672 2673 setiwsockopt(ep->com.so); 2674 init_iwarp_socket(ep->com.so, &ep->com); 2675 err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 2676 ep->com.thread); 2677 if (err) 2678 goto fail_free_so; 2679 CTR2(KTR_IW_CXGBE, "%s:ccE, ep %p", __func__, ep); 2680 return 0; 2681 2682fail_free_so: 2683 uninit_iwarp_socket(ep->com.so); 2684 ep->com.state = DEAD; 2685 sock_release(ep->com.so); 2686fail: 2687 deref_cm_id(&ep->com); 2688 c4iw_put_ep(&ep->com); 2689 ep = NULL; 2690out: 2691 CTR2(KTR_IW_CXGBE, "%s:ccE Error %d", __func__, err); 2692 return err; 2693} 2694 2695/* 2696 * iwcm->create_listen. Returns -errno on failure. 2697 */ 2698int 2699c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2700{ 2701 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2702 struct c4iw_listen_ep *lep = NULL; 2703 struct listen_port_info *port_info = NULL; 2704 int rc = 0; 2705 2706 CTR3(KTR_IW_CXGBE, "%s: cm_id %p, backlog %s", __func__, cm_id, 2707 backlog); 2708 lep = alloc_ep(sizeof(*lep), GFP_KERNEL); 2709 lep->com.cm_id = cm_id; 2710 ref_cm_id(&lep->com); 2711 lep->com.dev = dev; 2712 lep->backlog = backlog; 2713 lep->com.local_addr = cm_id->local_addr; 2714 lep->com.thread = curthread; 2715 cm_id->provider_data = lep; 2716 lep->com.state = LISTEN; 2717 2718 /* In case of INDADDR_ANY, ibcore creates cmid for each device and 2719 * invokes iw_cxgbe listener callbacks assuming that iw_cxgbe creates 2720 * HW listeners for each device seperately. But toecore expects single 2721 * solisten() call with INADDR_ANY address to create HW listeners on 2722 * all devices for a given port number. So iw_cxgbe driver calls 2723 * solisten() only once for INADDR_ANY(usually done at first time 2724 * listener callback from ibcore). And all the subsequent INADDR_ANY 2725 * listener callbacks from ibcore(for the same port address) do not 2726 * invoke solisten() as first listener callback has already created 2727 * listeners for all other devices(via solisten). 2728 */ 2729 if (c4iw_any_addr((struct sockaddr *)&lep->com.local_addr)) { 2730 port_info = add_ep_to_listenlist(lep); 2731 /* skip solisten() if refcnt > 1, as the listeners were 2732 * alredy created by 'Master lep' 2733 */ 2734 if (port_info->refcnt > 1) { 2735 /* As there will be only one listener socket for a TCP 2736 * port, copy Master lep's socket pointer to other lep's 2737 * that are belonging to same TCP port. 2738 */ 2739 struct c4iw_listen_ep *head_lep = 2740 container_of(port_info->lep_list.next, 2741 struct c4iw_listen_ep, listen_ep_list); 2742 lep->com.so = head_lep->com.so; 2743 goto out; 2744 } 2745 } 2746 rc = c4iw_sock_create(&cm_id->local_addr, &lep->com.so); 2747 if (rc) { 2748 CTR2(KTR_IW_CXGBE, "%s:Failed to create socket. err %d", 2749 __func__, rc); 2750 goto fail; 2751 } 2752 2753 rc = -solisten(lep->com.so, backlog, curthread); 2754 if (rc) { 2755 CTR3(KTR_IW_CXGBE, "%s:Failed to listen on sock:%p. err %d", 2756 __func__, lep->com.so, rc); 2757 goto fail_free_so; 2758 } 2759 init_iwarp_socket(lep->com.so, &lep->com); 2760out: 2761 return 0; 2762 2763fail_free_so: 2764 sock_release(lep->com.so); 2765fail: 2766 if (port_info) 2767 rem_ep_from_listenlist(lep); 2768 deref_cm_id(&lep->com); 2769 c4iw_put_ep(&lep->com); 2770 return rc; 2771} 2772 2773int 2774c4iw_destroy_listen(struct iw_cm_id *cm_id) 2775{ 2776 struct c4iw_listen_ep *lep = to_listen_ep(cm_id); 2777 2778 mutex_lock(&lep->com.mutex); 2779 CTR3(KTR_IW_CXGBE, "%s: cm_id %p, state %s", __func__, cm_id, 2780 states[lep->com.state]); 2781 2782 lep->com.state = DEAD; 2783 if (c4iw_any_addr((struct sockaddr *)&lep->com.local_addr)) { 2784 /* if no refcount then close listen socket */ 2785 if (!rem_ep_from_listenlist(lep)) 2786 close_socket(lep->com.so); 2787 } else 2788 close_socket(lep->com.so); 2789 deref_cm_id(&lep->com); 2790 mutex_unlock(&lep->com.mutex); 2791 c4iw_put_ep(&lep->com); 2792 return 0; 2793} 2794 2795int __c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2796{ 2797 int ret; 2798 mutex_lock(&ep->com.mutex); 2799 ret = c4iw_ep_disconnect(ep, abrupt, gfp); 2800 mutex_unlock(&ep->com.mutex); 2801 return ret; 2802} 2803 2804int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2805{ 2806 int ret = 0; 2807 int close = 0; 2808 int fatal = 0; 2809 struct c4iw_rdev *rdev; 2810 2811 2812 CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep); 2813 2814 rdev = &ep->com.dev->rdev; 2815 2816 if (c4iw_fatal_error(rdev)) { 2817 2818 CTR2(KTR_IW_CXGBE, "%s:ced1 %p", __func__, ep); 2819 fatal = 1; 2820 close_complete_upcall(ep, -ECONNRESET); 2821 send_abort(ep); 2822 ep->com.state = DEAD; 2823 } 2824 CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep, 2825 states[ep->com.state]); 2826 2827 /* 2828 * Ref the ep here in case we have fatal errors causing the 2829 * ep to be released and freed. 2830 */ 2831 c4iw_get_ep(&ep->com); 2832 switch (ep->com.state) { 2833 2834 case MPA_REQ_WAIT: 2835 case MPA_REQ_SENT: 2836 case MPA_REQ_RCVD: 2837 case MPA_REP_SENT: 2838 case FPDU_MODE: 2839 close = 1; 2840 if (abrupt) 2841 ep->com.state = ABORTING; 2842 else { 2843 ep->com.state = CLOSING; 2844 START_EP_TIMER(ep); 2845 } 2846 set_bit(CLOSE_SENT, &ep->com.flags); 2847 break; 2848 2849 case CLOSING: 2850 2851 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2852 2853 close = 1; 2854 if (abrupt) { 2855 STOP_EP_TIMER(ep); 2856 ep->com.state = ABORTING; 2857 } else 2858 ep->com.state = MORIBUND; 2859 } 2860 break; 2861 2862 case MORIBUND: 2863 case ABORTING: 2864 case DEAD: 2865 CTR3(KTR_IW_CXGBE, 2866 "%s ignoring disconnect ep %p state %u", __func__, 2867 ep, ep->com.state); 2868 break; 2869 2870 default: 2871 BUG(); 2872 break; 2873 } 2874 2875 2876 if (close) { 2877 2878 CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep); 2879 2880 if (abrupt) { 2881 2882 CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); 2883 set_bit(EP_DISC_ABORT, &ep->com.history); 2884 close_complete_upcall(ep, -ECONNRESET); 2885 ret = send_abort(ep); 2886 if (ret) 2887 fatal = 1; 2888 } else { 2889 2890 CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); 2891 set_bit(EP_DISC_CLOSE, &ep->com.history); 2892 2893 if (!ep->parent_ep) 2894 ep->com.state = MORIBUND; 2895 2896 CURVNET_SET(ep->com.so->so_vnet); 2897 sodisconnect(ep->com.so); 2898 CURVNET_RESTORE(); 2899 } 2900 2901 } 2902 2903 if (fatal) { 2904 set_bit(EP_DISC_FAIL, &ep->com.history); 2905 if (!abrupt) { 2906 STOP_EP_TIMER(ep); 2907 close_complete_upcall(ep, -EIO); 2908 } 2909 if (ep->com.qp) { 2910 struct c4iw_qp_attributes attrs = {0}; 2911 2912 attrs.next_state = C4IW_QP_STATE_ERROR; 2913 ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, 2914 C4IW_QP_ATTR_NEXT_STATE, 2915 &attrs, 1); 2916 if (ret) { 2917 CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); 2918 printf("%s - qp <- error failed!\n", __func__); 2919 } 2920 } 2921 release_ep_resources(ep); 2922 ep->com.state = DEAD; 2923 CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); 2924 } 2925 c4iw_put_ep(&ep->com); 2926 CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); 2927 return ret; 2928} 2929 2930#ifdef C4IW_EP_REDIRECT 2931int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 2932 struct l2t_entry *l2t) 2933{ 2934 struct c4iw_ep *ep = ctx; 2935 2936 if (ep->dst != old) 2937 return 0; 2938 2939 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, 2940 l2t); 2941 dst_hold(new); 2942 cxgb4_l2t_release(ep->l2t); 2943 ep->l2t = l2t; 2944 dst_release(old); 2945 ep->dst = new; 2946 return 1; 2947} 2948#endif 2949 2950 2951 2952static void ep_timeout(unsigned long arg) 2953{ 2954 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2955 2956 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 2957 2958 /* 2959 * Only insert if it is not already on the list. 2960 */ 2961 if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) { 2962 CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep); 2963 add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT); 2964 } 2965 } 2966} 2967 2968static int fw6_wr_rpl(struct adapter *sc, const __be64 *rpl) 2969{ 2970 uint64_t val = be64toh(*rpl); 2971 int ret; 2972 struct c4iw_wr_wait *wr_waitp; 2973 2974 ret = (int)((val >> 8) & 0xff); 2975 wr_waitp = (struct c4iw_wr_wait *)rpl[1]; 2976 CTR3(KTR_IW_CXGBE, "%s wr_waitp %p ret %u", __func__, wr_waitp, ret); 2977 if (wr_waitp) 2978 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2979 2980 return (0); 2981} 2982 2983static int fw6_cqe_handler(struct adapter *sc, const __be64 *rpl) 2984{ 2985 struct cqe_list_entry *cle; 2986 unsigned long flag; 2987 2988 cle = malloc(sizeof(*cle), M_CXGBE, M_NOWAIT); 2989 cle->rhp = sc->iwarp_softc; 2990 cle->err_cqe = *(const struct t4_cqe *)(&rpl[0]); 2991 2992 spin_lock_irqsave(&err_cqe_lock, flag); 2993 list_add_tail(&cle->entry, &err_cqe_list); 2994 queue_work(c4iw_taskq, &c4iw_task); 2995 spin_unlock_irqrestore(&err_cqe_lock, flag); 2996 2997 return (0); 2998} 2999 3000static int 3001process_terminate(struct c4iw_ep *ep) 3002{ 3003 struct c4iw_qp_attributes attrs = {0}; 3004 3005 CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep); 3006 3007 if (ep && ep->com.qp) { 3008 3009 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", 3010 ep->hwtid, ep->com.qp->wq.sq.qid); 3011 attrs.next_state = C4IW_QP_STATE_TERMINATE; 3012 c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 3013 1); 3014 } else 3015 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", 3016 ep->hwtid); 3017 CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep); 3018 3019 return 0; 3020} 3021 3022int __init c4iw_cm_init(void) 3023{ 3024 3025 t4_register_cpl_handler(CPL_RDMA_TERMINATE, terminate); 3026 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, fw6_wr_rpl); 3027 t4_register_fw_msg_handler(FW6_TYPE_CQE, fw6_cqe_handler); 3028 t4_register_an_handler(c4iw_ev_handler); 3029 3030 TAILQ_INIT(&req_list); 3031 spin_lock_init(&req_lock); 3032 INIT_LIST_HEAD(&err_cqe_list); 3033 spin_lock_init(&err_cqe_lock); 3034 3035 INIT_WORK(&c4iw_task, process_req); 3036 3037 c4iw_taskq = create_singlethread_workqueue("iw_cxgbe"); 3038 if (!c4iw_taskq) 3039 return -ENOMEM; 3040 3041 return 0; 3042} 3043 3044void __exit c4iw_cm_term(void) 3045{ 3046 WARN_ON(!TAILQ_EMPTY(&req_list)); 3047 WARN_ON(!list_empty(&err_cqe_list)); 3048 flush_workqueue(c4iw_taskq); 3049 destroy_workqueue(c4iw_taskq); 3050 3051 t4_register_cpl_handler(CPL_RDMA_TERMINATE, NULL); 3052 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, NULL); 3053 t4_register_fw_msg_handler(FW6_TYPE_CQE, NULL); 3054 t4_register_an_handler(NULL); 3055} 3056#endif 3057