cm.c revision 330307
1/* 2 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/iw_cxgbe/cm.c 330307 2018-03-03 02:30:52Z np $"); 34 35#include "opt_inet.h" 36 37#ifdef TCP_OFFLOAD 38#include <sys/types.h> 39#include <sys/malloc.h> 40#include <sys/socket.h> 41#include <sys/socketvar.h> 42#include <sys/sockio.h> 43#include <sys/taskqueue.h> 44#include <netinet/in.h> 45#include <net/route.h> 46 47#include <netinet/in_systm.h> 48#include <netinet/in_pcb.h> 49#include <netinet/ip.h> 50#include <netinet/in_fib.h> 51#include <netinet/ip_var.h> 52#include <netinet/tcp_var.h> 53#include <netinet/tcp.h> 54#include <netinet/tcpip.h> 55 56#include <netinet/toecore.h> 57 58struct sge_iq; 59struct rss_header; 60struct cpl_set_tcb_rpl; 61#include <linux/types.h> 62#include "offload.h" 63#include "tom/t4_tom.h" 64 65#define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe)) 66 67#include "iw_cxgbe.h" 68#include <linux/module.h> 69#include <linux/workqueue.h> 70#include <linux/notifier.h> 71#include <linux/inetdevice.h> 72#include <linux/if_vlan.h> 73#include <net/netevent.h> 74 75static spinlock_t req_lock; 76static TAILQ_HEAD(c4iw_ep_list, c4iw_ep_common) req_list; 77static struct work_struct c4iw_task; 78static struct workqueue_struct *c4iw_taskq; 79static LIST_HEAD(err_cqe_list); 80static spinlock_t err_cqe_lock; 81 82static void process_req(struct work_struct *ctx); 83static void start_ep_timer(struct c4iw_ep *ep); 84static int stop_ep_timer(struct c4iw_ep *ep); 85static int set_tcpinfo(struct c4iw_ep *ep); 86static void process_timeout(struct c4iw_ep *ep); 87static void process_err_cqes(void); 88static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc); 89static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 90static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 91static void *alloc_ep(int size, gfp_t flags); 92static int find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 93 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4); 94static void close_socket(struct socket *so); 95static int send_mpa_req(struct c4iw_ep *ep); 96static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); 97static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); 98static void close_complete_upcall(struct c4iw_ep *ep, int status); 99static int send_abort(struct c4iw_ep *ep); 100static void peer_close_upcall(struct c4iw_ep *ep); 101static void peer_abort_upcall(struct c4iw_ep *ep); 102static void connect_reply_upcall(struct c4iw_ep *ep, int status); 103static int connect_request_upcall(struct c4iw_ep *ep); 104static void established_upcall(struct c4iw_ep *ep); 105static int process_mpa_reply(struct c4iw_ep *ep); 106static int process_mpa_request(struct c4iw_ep *ep); 107static void process_peer_close(struct c4iw_ep *ep); 108static void process_conn_error(struct c4iw_ep *ep); 109static void process_close_complete(struct c4iw_ep *ep); 110static void ep_timeout(unsigned long arg); 111static void init_iwarp_socket(struct socket *so, void *arg); 112static void uninit_iwarp_socket(struct socket *so); 113static void process_data(struct c4iw_ep *ep); 114static void process_connected(struct c4iw_ep *ep); 115static int c4iw_so_upcall(struct socket *so, void *arg, int waitflag); 116static void process_socket_event(struct c4iw_ep *ep); 117static void release_ep_resources(struct c4iw_ep *ep); 118static int process_terminate(struct c4iw_ep *ep); 119static int terminate(struct sge_iq *iq, const struct rss_header *rss, 120 struct mbuf *m); 121static int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events); 122#define START_EP_TIMER(ep) \ 123 do { \ 124 CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \ 125 __func__, __LINE__, (ep)); \ 126 start_ep_timer(ep); \ 127 } while (0) 128 129#define STOP_EP_TIMER(ep) \ 130 ({ \ 131 CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ 132 __func__, __LINE__, (ep)); \ 133 stop_ep_timer(ep); \ 134 }) 135 136#ifdef KTR 137static char *states[] = { 138 "idle", 139 "listen", 140 "connecting", 141 "mpa_wait_req", 142 "mpa_req_sent", 143 "mpa_req_rcvd", 144 "mpa_rep_sent", 145 "fpdu_mode", 146 "aborting", 147 "closing", 148 "moribund", 149 "dead", 150 NULL, 151}; 152#endif 153 154 155static void deref_cm_id(struct c4iw_ep_common *epc) 156{ 157 epc->cm_id->rem_ref(epc->cm_id); 158 epc->cm_id = NULL; 159 set_bit(CM_ID_DEREFED, &epc->history); 160} 161 162static void ref_cm_id(struct c4iw_ep_common *epc) 163{ 164 set_bit(CM_ID_REFED, &epc->history); 165 epc->cm_id->add_ref(epc->cm_id); 166} 167 168static void deref_qp(struct c4iw_ep *ep) 169{ 170 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 171 clear_bit(QP_REFERENCED, &ep->com.flags); 172 set_bit(QP_DEREFED, &ep->com.history); 173} 174 175static void ref_qp(struct c4iw_ep *ep) 176{ 177 set_bit(QP_REFERENCED, &ep->com.flags); 178 set_bit(QP_REFED, &ep->com.history); 179 c4iw_qp_add_ref(&ep->com.qp->ibqp); 180} 181 182static void process_timeout(struct c4iw_ep *ep) 183{ 184 struct c4iw_qp_attributes attrs; 185 int abort = 1; 186 187 mutex_lock(&ep->com.mutex); 188 CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__, 189 ep, ep->hwtid, ep->com.state); 190 set_bit(TIMEDOUT, &ep->com.history); 191 switch (ep->com.state) { 192 case MPA_REQ_SENT: 193 connect_reply_upcall(ep, -ETIMEDOUT); 194 break; 195 case MPA_REQ_WAIT: 196 case MPA_REQ_RCVD: 197 case MPA_REP_SENT: 198 case FPDU_MODE: 199 break; 200 case CLOSING: 201 case MORIBUND: 202 if (ep->com.cm_id && ep->com.qp) { 203 attrs.next_state = C4IW_QP_STATE_ERROR; 204 c4iw_modify_qp(ep->com.dev, ep->com.qp, 205 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 206 } 207 close_complete_upcall(ep, -ETIMEDOUT); 208 break; 209 case ABORTING: 210 case DEAD: 211 /* 212 * These states are expected if the ep timed out at the same 213 * time as another thread was calling stop_ep_timer(). 214 * So we silently do nothing for these states. 215 */ 216 abort = 0; 217 break; 218 default: 219 CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u\n" 220 , __func__, ep, ep->hwtid, ep->com.state); 221 abort = 0; 222 } 223 mutex_unlock(&ep->com.mutex); 224 if (abort) 225 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 226 c4iw_put_ep(&ep->com); 227 return; 228} 229 230struct cqe_list_entry { 231 struct list_head entry; 232 struct c4iw_dev *rhp; 233 struct t4_cqe err_cqe; 234}; 235 236static void 237process_err_cqes(void) 238{ 239 unsigned long flag; 240 struct cqe_list_entry *cle; 241 242 spin_lock_irqsave(&err_cqe_lock, flag); 243 while (!list_empty(&err_cqe_list)) { 244 struct list_head *tmp; 245 tmp = err_cqe_list.next; 246 list_del(tmp); 247 tmp->next = tmp->prev = NULL; 248 spin_unlock_irqrestore(&err_cqe_lock, flag); 249 cle = list_entry(tmp, struct cqe_list_entry, entry); 250 c4iw_ev_dispatch(cle->rhp, &cle->err_cqe); 251 free(cle, M_CXGBE); 252 spin_lock_irqsave(&err_cqe_lock, flag); 253 } 254 spin_unlock_irqrestore(&err_cqe_lock, flag); 255 256 return; 257} 258 259static void 260process_req(struct work_struct *ctx) 261{ 262 struct c4iw_ep_common *epc; 263 unsigned long flag; 264 int ep_events; 265 266 process_err_cqes(); 267 spin_lock_irqsave(&req_lock, flag); 268 while (!TAILQ_EMPTY(&req_list)) { 269 epc = TAILQ_FIRST(&req_list); 270 TAILQ_REMOVE(&req_list, epc, entry); 271 epc->entry.tqe_prev = NULL; 272 ep_events = epc->ep_events; 273 epc->ep_events = 0; 274 spin_unlock_irqrestore(&req_lock, flag); 275 CTR4(KTR_IW_CXGBE, "%s: so %p, ep %p, events 0x%x", __func__, 276 epc->so, epc, ep_events); 277 if (ep_events & C4IW_EVENT_TERM) 278 process_terminate((struct c4iw_ep *)epc); 279 if (ep_events & C4IW_EVENT_TIMEOUT) 280 process_timeout((struct c4iw_ep *)epc); 281 if (ep_events & C4IW_EVENT_SOCKET) 282 process_socket_event((struct c4iw_ep *)epc); 283 c4iw_put_ep(epc); 284 process_err_cqes(); 285 spin_lock_irqsave(&req_lock, flag); 286 } 287 spin_unlock_irqrestore(&req_lock, flag); 288} 289 290/* 291 * XXX: doesn't belong here in the iWARP driver. 292 * XXX: assumes that the connection was offloaded by cxgbe/t4_tom if TF_TOE is 293 * set. Is this a valid assumption for active open? 294 */ 295static int 296set_tcpinfo(struct c4iw_ep *ep) 297{ 298 struct socket *so = ep->com.so; 299 struct inpcb *inp = sotoinpcb(so); 300 struct tcpcb *tp; 301 struct toepcb *toep; 302 int rc = 0; 303 304 INP_WLOCK(inp); 305 tp = intotcpcb(inp); 306 if ((tp->t_flags & TF_TOE) == 0) { 307 rc = EINVAL; 308 log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n", 309 __func__, so, ep); 310 goto done; 311 } 312 toep = TOEPCB(so); 313 314 ep->hwtid = toep->tid; 315 ep->snd_seq = tp->snd_nxt; 316 ep->rcv_seq = tp->rcv_nxt; 317 ep->emss = max(tp->t_maxseg, 128); 318done: 319 INP_WUNLOCK(inp); 320 return (rc); 321 322} 323 324static int 325find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 326 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4) 327{ 328 struct in_addr addr; 329 int err; 330 331 CTR5(KTR_IW_CXGBE, "%s:frtB %x, %x, %d, %d", __func__, local_ip, 332 peer_ip, ntohs(local_port), ntohs(peer_port)); 333 334 addr.s_addr = peer_ip; 335 err = fib4_lookup_nh_ext(RT_DEFAULT_FIB, addr, NHR_REF, 0, pnh4); 336 337 CTR2(KTR_IW_CXGBE, "%s:frtE %d", __func__, err); 338 return err; 339} 340 341static void 342close_socket(struct socket *so) 343{ 344 345 uninit_iwarp_socket(so); 346 sodisconnect(so); 347} 348 349static void 350process_peer_close(struct c4iw_ep *ep) 351{ 352 struct c4iw_qp_attributes attrs; 353 int disconnect = 1; 354 int release = 0; 355 356 CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep, 357 ep->com.so, states[ep->com.state]); 358 359 mutex_lock(&ep->com.mutex); 360 switch (ep->com.state) { 361 362 case MPA_REQ_WAIT: 363 CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT CLOSING", 364 __func__, ep); 365 __state_set(&ep->com, CLOSING); 366 break; 367 368 case MPA_REQ_SENT: 369 CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT CLOSING", 370 __func__, ep); 371 __state_set(&ep->com, DEAD); 372 connect_reply_upcall(ep, -ECONNABORTED); 373 374 disconnect = 0; 375 STOP_EP_TIMER(ep); 376 close_socket(ep->com.so); 377 deref_cm_id(&ep->com); 378 release = 1; 379 break; 380 381 case MPA_REQ_RCVD: 382 383 /* 384 * We're gonna mark this puppy DEAD, but keep 385 * the reference on it until the ULP accepts or 386 * rejects the CR. 387 */ 388 CTR2(KTR_IW_CXGBE, "%s:ppc3 %p MPA_REQ_RCVD CLOSING", 389 __func__, ep); 390 __state_set(&ep->com, CLOSING); 391 c4iw_get_ep(&ep->com); 392 break; 393 394 case MPA_REP_SENT: 395 CTR2(KTR_IW_CXGBE, "%s:ppc4 %p MPA_REP_SENT CLOSING", 396 __func__, ep); 397 __state_set(&ep->com, CLOSING); 398 break; 399 400 case FPDU_MODE: 401 CTR2(KTR_IW_CXGBE, "%s:ppc5 %p FPDU_MODE CLOSING", 402 __func__, ep); 403 START_EP_TIMER(ep); 404 __state_set(&ep->com, CLOSING); 405 attrs.next_state = C4IW_QP_STATE_CLOSING; 406 c4iw_modify_qp(ep->com.dev, ep->com.qp, 407 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 408 peer_close_upcall(ep); 409 break; 410 411 case ABORTING: 412 CTR2(KTR_IW_CXGBE, "%s:ppc6 %p ABORTING (disconn)", 413 __func__, ep); 414 disconnect = 0; 415 break; 416 417 case CLOSING: 418 CTR2(KTR_IW_CXGBE, "%s:ppc7 %p CLOSING MORIBUND", 419 __func__, ep); 420 __state_set(&ep->com, MORIBUND); 421 disconnect = 0; 422 break; 423 424 case MORIBUND: 425 CTR2(KTR_IW_CXGBE, "%s:ppc8 %p MORIBUND DEAD", __func__, 426 ep); 427 STOP_EP_TIMER(ep); 428 if (ep->com.cm_id && ep->com.qp) { 429 attrs.next_state = C4IW_QP_STATE_IDLE; 430 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 431 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 432 } 433 close_socket(ep->com.so); 434 close_complete_upcall(ep, 0); 435 __state_set(&ep->com, DEAD); 436 release = 1; 437 disconnect = 0; 438 break; 439 440 case DEAD: 441 CTR2(KTR_IW_CXGBE, "%s:ppc9 %p DEAD (disconn)", 442 __func__, ep); 443 disconnect = 0; 444 break; 445 446 default: 447 panic("%s: ep %p state %d", __func__, ep, 448 ep->com.state); 449 break; 450 } 451 452 mutex_unlock(&ep->com.mutex); 453 454 if (disconnect) { 455 456 CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep); 457 c4iw_ep_disconnect(ep, 0, M_NOWAIT); 458 } 459 if (release) { 460 461 CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep); 462 c4iw_put_ep(&ep->com); 463 } 464 CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep); 465 return; 466} 467 468static void 469process_conn_error(struct c4iw_ep *ep) 470{ 471 struct c4iw_qp_attributes attrs; 472 int ret; 473 int state; 474 475 mutex_lock(&ep->com.mutex); 476 state = ep->com.state; 477 CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s", 478 __func__, ep, ep->com.so, ep->com.so->so_error, 479 states[ep->com.state]); 480 481 switch (state) { 482 483 case MPA_REQ_WAIT: 484 STOP_EP_TIMER(ep); 485 break; 486 487 case MPA_REQ_SENT: 488 STOP_EP_TIMER(ep); 489 connect_reply_upcall(ep, -ECONNRESET); 490 break; 491 492 case MPA_REP_SENT: 493 ep->com.rpl_err = ECONNRESET; 494 CTR1(KTR_IW_CXGBE, "waking up ep %p", ep); 495 break; 496 497 case MPA_REQ_RCVD: 498 499 /* 500 * We're gonna mark this puppy DEAD, but keep 501 * the reference on it until the ULP accepts or 502 * rejects the CR. 503 */ 504 c4iw_get_ep(&ep->com); 505 break; 506 507 case MORIBUND: 508 case CLOSING: 509 STOP_EP_TIMER(ep); 510 /*FALLTHROUGH*/ 511 case FPDU_MODE: 512 513 if (ep->com.cm_id && ep->com.qp) { 514 515 attrs.next_state = C4IW_QP_STATE_ERROR; 516 ret = c4iw_modify_qp(ep->com.qp->rhp, 517 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 518 &attrs, 1); 519 if (ret) 520 log(LOG_ERR, 521 "%s - qp <- error failed!\n", 522 __func__); 523 } 524 peer_abort_upcall(ep); 525 break; 526 527 case ABORTING: 528 break; 529 530 case DEAD: 531 CTR2(KTR_IW_CXGBE, "%s so_error %d IN DEAD STATE!!!!", 532 __func__, ep->com.so->so_error); 533 mutex_unlock(&ep->com.mutex); 534 return; 535 536 default: 537 panic("%s: ep %p state %d", __func__, ep, state); 538 break; 539 } 540 541 if (state != ABORTING) { 542 close_socket(ep->com.so); 543 __state_set(&ep->com, DEAD); 544 c4iw_put_ep(&ep->com); 545 } 546 mutex_unlock(&ep->com.mutex); 547 CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep); 548 return; 549} 550 551static void 552process_close_complete(struct c4iw_ep *ep) 553{ 554 struct c4iw_qp_attributes attrs; 555 int release = 0; 556 557 CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep, 558 ep->com.so, states[ep->com.state]); 559 560 /* The cm_id may be null if we failed to connect */ 561 mutex_lock(&ep->com.mutex); 562 set_bit(CLOSE_CON_RPL, &ep->com.history); 563 564 switch (ep->com.state) { 565 566 case CLOSING: 567 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p CLOSING MORIBUND", 568 __func__, ep); 569 __state_set(&ep->com, MORIBUND); 570 break; 571 572 case MORIBUND: 573 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p MORIBUND DEAD", __func__, 574 ep); 575 STOP_EP_TIMER(ep); 576 577 if ((ep->com.cm_id) && (ep->com.qp)) { 578 579 CTR2(KTR_IW_CXGBE, "%s:pcc2 %p QP_STATE_IDLE", 580 __func__, ep); 581 attrs.next_state = C4IW_QP_STATE_IDLE; 582 c4iw_modify_qp(ep->com.dev, 583 ep->com.qp, 584 C4IW_QP_ATTR_NEXT_STATE, 585 &attrs, 1); 586 } 587 588 close_socket(ep->com.so); 589 close_complete_upcall(ep, 0); 590 __state_set(&ep->com, DEAD); 591 release = 1; 592 break; 593 594 case ABORTING: 595 CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep); 596 break; 597 598 case DEAD: 599 CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep); 600 break; 601 default: 602 CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state", 603 __func__, ep); 604 panic("%s:pcc6 %p unknown ep state", __func__, ep); 605 break; 606 } 607 mutex_unlock(&ep->com.mutex); 608 609 if (release) { 610 611 CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep); 612 c4iw_put_ep(&ep->com); 613 } 614 CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep); 615 return; 616} 617 618static void 619init_iwarp_socket(struct socket *so, void *arg) 620{ 621 int rc; 622 struct sockopt sopt; 623 int on = 1; 624 625 /* Note that SOCK_LOCK(so) is same as SOCKBUF_LOCK(&so->so_rcv) */ 626 SOCK_LOCK(so); 627 soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); 628 so->so_state |= SS_NBIO; 629 SOCK_UNLOCK(so); 630 sopt.sopt_dir = SOPT_SET; 631 sopt.sopt_level = IPPROTO_TCP; 632 sopt.sopt_name = TCP_NODELAY; 633 sopt.sopt_val = (caddr_t)&on; 634 sopt.sopt_valsize = sizeof on; 635 sopt.sopt_td = NULL; 636 rc = sosetopt(so, &sopt); 637 if (rc) { 638 log(LOG_ERR, "%s: can't set TCP_NODELAY on so %p (%d)\n", 639 __func__, so, rc); 640 } 641} 642 643static void 644uninit_iwarp_socket(struct socket *so) 645{ 646 647 SOCKBUF_LOCK(&so->so_rcv); 648 soupcall_clear(so, SO_RCV); 649 SOCKBUF_UNLOCK(&so->so_rcv); 650} 651 652static void 653process_data(struct c4iw_ep *ep) 654{ 655 struct sockaddr_in *local, *remote; 656 int disconnect = 0; 657 658 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__, 659 ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv)); 660 661 switch (state_read(&ep->com)) { 662 case MPA_REQ_SENT: 663 disconnect = process_mpa_reply(ep); 664 break; 665 case MPA_REQ_WAIT: 666 in_getsockaddr(ep->com.so, (struct sockaddr **)&local); 667 in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); 668 ep->com.local_addr = *local; 669 ep->com.remote_addr = *remote; 670 free(local, M_SONAME); 671 free(remote, M_SONAME); 672 disconnect = process_mpa_request(ep); 673 break; 674 default: 675 if (sbused(&ep->com.so->so_rcv)) 676 log(LOG_ERR, "%s: Unexpected streaming data. ep %p, " 677 "state %d, so %p, so_state 0x%x, sbused %u\n", 678 __func__, ep, state_read(&ep->com), ep->com.so, 679 ep->com.so->so_state, sbused(&ep->com.so->so_rcv)); 680 break; 681 } 682 if (disconnect) 683 c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); 684 685} 686 687static void 688process_connected(struct c4iw_ep *ep) 689{ 690 struct socket *so = ep->com.so; 691 692 if ((so->so_state & SS_ISCONNECTED) && !so->so_error) { 693 if (send_mpa_req(ep)) 694 goto err; 695 } else { 696 connect_reply_upcall(ep, -so->so_error); 697 goto err; 698 } 699 return; 700err: 701 close_socket(so); 702 state_set(&ep->com, DEAD); 703 c4iw_put_ep(&ep->com); 704 return; 705} 706 707void 708process_newconn(struct iw_cm_id *parent_cm_id, struct socket *child_so) 709{ 710 struct c4iw_ep *child_ep; 711 struct sockaddr_in *local; 712 struct sockaddr_in *remote; 713 struct c4iw_ep *parent_ep = parent_cm_id->provider_data; 714 int ret = 0; 715 716 MPASS(child_so != NULL); 717 718 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 719 720 CTR5(KTR_IW_CXGBE, 721 "%s: parent so %p, parent ep %p, child so %p, child ep %p", 722 __func__, parent_ep->com.so, parent_ep, child_so, child_ep); 723 724 in_getsockaddr(child_so, (struct sockaddr **)&local); 725 in_getpeeraddr(child_so, (struct sockaddr **)&remote); 726 727 child_ep->com.local_addr = *local; 728 child_ep->com.remote_addr = *remote; 729 child_ep->com.dev = parent_ep->com.dev; 730 child_ep->com.so = child_so; 731 child_ep->com.cm_id = NULL; 732 child_ep->com.thread = parent_ep->com.thread; 733 child_ep->parent_ep = parent_ep; 734 735 free(local, M_SONAME); 736 free(remote, M_SONAME); 737 738 init_iwarp_socket(child_so, &child_ep->com); 739 c4iw_get_ep(&parent_ep->com); 740 init_timer(&child_ep->timer); 741 state_set(&child_ep->com, MPA_REQ_WAIT); 742 START_EP_TIMER(child_ep); 743 744 /* maybe the request has already been queued up on the socket... */ 745 ret = process_mpa_request(child_ep); 746 if (ret == 2) 747 /* ABORT */ 748 c4iw_ep_disconnect(child_ep, 1, GFP_KERNEL); 749 else if (ret == 1) 750 /* CLOSE */ 751 c4iw_ep_disconnect(child_ep, 0, GFP_KERNEL); 752 753 return; 754} 755 756static int 757add_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event) 758{ 759 unsigned long flag; 760 761 spin_lock_irqsave(&req_lock, flag); 762 if (ep && ep->com.so) { 763 ep->com.ep_events |= new_ep_event; 764 if (!ep->com.entry.tqe_prev) { 765 c4iw_get_ep(&ep->com); 766 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 767 queue_work(c4iw_taskq, &c4iw_task); 768 } 769 } 770 spin_unlock_irqrestore(&req_lock, flag); 771 772 return (0); 773} 774 775static int 776c4iw_so_upcall(struct socket *so, void *arg, int waitflag) 777{ 778 struct c4iw_ep *ep = arg; 779 780 CTR6(KTR_IW_CXGBE, 781 "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p", 782 __func__, so, so->so_state, ep, states[ep->com.state], 783 ep->com.entry.tqe_prev); 784 785 MPASS(ep->com.so == so); 786 add_ep_to_req_list(ep, C4IW_EVENT_SOCKET); 787 788 return (SU_OK); 789} 790 791 792static int 793terminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 794{ 795 struct adapter *sc = iq->adapter; 796 const struct cpl_rdma_terminate *cpl = mtod(m, const void *); 797 unsigned int tid = GET_TID(cpl); 798 struct toepcb *toep = lookup_tid(sc, tid); 799 struct socket *so; 800 struct c4iw_ep *ep; 801 802 INP_WLOCK(toep->inp); 803 so = inp_inpcbtosocket(toep->inp); 804 ep = so->so_rcv.sb_upcallarg; 805 INP_WUNLOCK(toep->inp); 806 807 CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep); 808 add_ep_to_req_list(ep, C4IW_EVENT_TERM); 809 810 return 0; 811} 812 813static void 814process_socket_event(struct c4iw_ep *ep) 815{ 816 int state = state_read(&ep->com); 817 struct socket *so = ep->com.so; 818 819 CTR6(KTR_IW_CXGBE, "process_socket_event: so %p, so_state 0x%x, " 820 "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state, 821 so->so_error, so->so_rcv.sb_state, ep, states[state]); 822 823 if (state == CONNECTING) { 824 process_connected(ep); 825 return; 826 } 827 828 if (state == LISTEN) { 829 /* socket listening events are handled at IWCM */ 830 CTR3(KTR_IW_CXGBE, "%s Invalid ep state:%u, ep:%p", __func__, 831 ep->com.state, ep); 832 BUG(); 833 return; 834 } 835 836 /* connection error */ 837 if (so->so_error) { 838 process_conn_error(ep); 839 return; 840 } 841 842 /* peer close */ 843 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <= CLOSING) { 844 process_peer_close(ep); 845 /* 846 * check whether socket disconnect event is pending before 847 * returning. Fallthrough if yes. 848 */ 849 if (!(so->so_state & SS_ISDISCONNECTED)) 850 return; 851 } 852 853 /* close complete */ 854 if (so->so_state & SS_ISDISCONNECTED) { 855 process_close_complete(ep); 856 return; 857 } 858 859 /* rx data */ 860 process_data(ep); 861} 862 863SYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD, 0, "iw_cxgbe driver parameters"); 864 865static int dack_mode = 0; 866SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0, 867 "Delayed ack mode (default = 0)"); 868 869int c4iw_max_read_depth = 8; 870SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RWTUN, &c4iw_max_read_depth, 0, 871 "Per-connection max ORD/IRD (default = 8)"); 872 873static int enable_tcp_timestamps; 874SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RWTUN, &enable_tcp_timestamps, 0, 875 "Enable tcp timestamps (default = 0)"); 876 877static int enable_tcp_sack; 878SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RWTUN, &enable_tcp_sack, 0, 879 "Enable tcp SACK (default = 0)"); 880 881static int enable_tcp_window_scaling = 1; 882SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RWTUN, &enable_tcp_window_scaling, 0, 883 "Enable tcp window scaling (default = 1)"); 884 885int c4iw_debug = 1; 886SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RWTUN, &c4iw_debug, 0, 887 "Enable debug logging (default = 0)"); 888 889static int peer2peer = 1; 890SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0, 891 "Support peer2peer ULPs (default = 1)"); 892 893static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 894SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0, 895 "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)"); 896 897static int ep_timeout_secs = 60; 898SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0, 899 "CM Endpoint operation timeout in seconds (default = 60)"); 900 901static int mpa_rev = 1; 902SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0, 903 "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)"); 904 905static int markers_enabled; 906SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0, 907 "Enable MPA MARKERS (default(0) = disabled)"); 908 909static int crc_enabled = 1; 910SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0, 911 "Enable MPA CRC (default(1) = enabled)"); 912 913static int rcv_win = 256 * 1024; 914SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0, 915 "TCP receive window in bytes (default = 256KB)"); 916 917static int snd_win = 128 * 1024; 918SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0, 919 "TCP send window in bytes (default = 128KB)"); 920 921static void 922start_ep_timer(struct c4iw_ep *ep) 923{ 924 925 if (timer_pending(&ep->timer)) { 926 CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep); 927 printk(KERN_ERR "%s timer already started! ep %p\n", __func__, 928 ep); 929 return; 930 } 931 clear_bit(TIMEOUT, &ep->com.flags); 932 c4iw_get_ep(&ep->com); 933 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 934 ep->timer.data = (unsigned long)ep; 935 ep->timer.function = ep_timeout; 936 add_timer(&ep->timer); 937} 938 939static int 940stop_ep_timer(struct c4iw_ep *ep) 941{ 942 943 del_timer_sync(&ep->timer); 944 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 945 c4iw_put_ep(&ep->com); 946 return 0; 947 } 948 return 1; 949} 950 951static enum 952c4iw_ep_state state_read(struct c4iw_ep_common *epc) 953{ 954 enum c4iw_ep_state state; 955 956 mutex_lock(&epc->mutex); 957 state = epc->state; 958 mutex_unlock(&epc->mutex); 959 960 return (state); 961} 962 963static void 964__state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 965{ 966 967 epc->state = new; 968} 969 970static void 971state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 972{ 973 974 mutex_lock(&epc->mutex); 975 __state_set(epc, new); 976 mutex_unlock(&epc->mutex); 977} 978 979static void * 980alloc_ep(int size, gfp_t gfp) 981{ 982 struct c4iw_ep_common *epc; 983 984 epc = kzalloc(size, gfp); 985 if (epc == NULL) 986 return (NULL); 987 988 kref_init(&epc->kref); 989 mutex_init(&epc->mutex); 990 c4iw_init_wr_wait(&epc->wr_wait); 991 992 return (epc); 993} 994 995void _c4iw_free_ep(struct kref *kref) 996{ 997 struct c4iw_ep *ep; 998 struct c4iw_ep_common *epc; 999 1000 ep = container_of(kref, struct c4iw_ep, com.kref); 1001 epc = &ep->com; 1002 KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", 1003 __func__, epc)); 1004 if (test_bit(QP_REFERENCED, &ep->com.flags)) 1005 deref_qp(ep); 1006 CTR4(KTR_IW_CXGBE, "%s: ep %p, history 0x%lx, flags 0x%lx", 1007 __func__, ep, epc->history, epc->flags); 1008 kfree(ep); 1009} 1010 1011static void release_ep_resources(struct c4iw_ep *ep) 1012{ 1013 CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep); 1014 set_bit(RELEASE_RESOURCES, &ep->com.flags); 1015 c4iw_put_ep(&ep->com); 1016 CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep); 1017} 1018 1019static int 1020send_mpa_req(struct c4iw_ep *ep) 1021{ 1022 int mpalen; 1023 struct mpa_message *mpa; 1024 struct mpa_v2_conn_params mpa_v2_params; 1025 struct mbuf *m; 1026 char mpa_rev_to_use = mpa_rev; 1027 int err = 0; 1028 1029 if (ep->retry_with_mpa_v1) 1030 mpa_rev_to_use = 1; 1031 mpalen = sizeof(*mpa) + ep->plen; 1032 if (mpa_rev_to_use == 2) 1033 mpalen += sizeof(struct mpa_v2_conn_params); 1034 1035 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1036 if (mpa == NULL) { 1037 err = -ENOMEM; 1038 CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d", 1039 __func__, ep, err); 1040 goto err; 1041 } 1042 1043 memset(mpa, 0, mpalen); 1044 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 1045 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 1046 (markers_enabled ? MPA_MARKERS : 0) | 1047 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 1048 mpa->private_data_size = htons(ep->plen); 1049 mpa->revision = mpa_rev_to_use; 1050 1051 if (mpa_rev_to_use == 1) { 1052 ep->tried_with_mpa_v1 = 1; 1053 ep->retry_with_mpa_v1 = 0; 1054 } 1055 1056 if (mpa_rev_to_use == 2) { 1057 mpa->private_data_size += 1058 htons(sizeof(struct mpa_v2_conn_params)); 1059 mpa_v2_params.ird = htons((u16)ep->ird); 1060 mpa_v2_params.ord = htons((u16)ep->ord); 1061 1062 if (peer2peer) { 1063 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1064 1065 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1066 mpa_v2_params.ord |= 1067 htons(MPA_V2_RDMA_WRITE_RTR); 1068 } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1069 mpa_v2_params.ord |= 1070 htons(MPA_V2_RDMA_READ_RTR); 1071 } 1072 } 1073 memcpy(mpa->private_data, &mpa_v2_params, 1074 sizeof(struct mpa_v2_conn_params)); 1075 1076 if (ep->plen) { 1077 1078 memcpy(mpa->private_data + 1079 sizeof(struct mpa_v2_conn_params), 1080 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1081 } 1082 } else { 1083 1084 if (ep->plen) 1085 memcpy(mpa->private_data, 1086 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1087 CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep); 1088 } 1089 1090 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1091 if (m == NULL) { 1092 err = -ENOMEM; 1093 CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d", 1094 __func__, ep, err); 1095 free(mpa, M_CXGBE); 1096 goto err; 1097 } 1098 m_copyback(m, 0, mpalen, (void *)mpa); 1099 free(mpa, M_CXGBE); 1100 1101 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1102 ep->com.thread); 1103 if (err) { 1104 CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d", 1105 __func__, ep, err); 1106 goto err; 1107 } 1108 1109 START_EP_TIMER(ep); 1110 state_set(&ep->com, MPA_REQ_SENT); 1111 ep->mpa_attr.initiator = 1; 1112 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1113 return 0; 1114err: 1115 connect_reply_upcall(ep, err); 1116 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1117 return err; 1118} 1119 1120static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 1121{ 1122 int mpalen ; 1123 struct mpa_message *mpa; 1124 struct mpa_v2_conn_params mpa_v2_params; 1125 struct mbuf *m; 1126 int err; 1127 1128 CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid, 1129 ep->plen); 1130 1131 mpalen = sizeof(*mpa) + plen; 1132 1133 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1134 1135 mpalen += sizeof(struct mpa_v2_conn_params); 1136 CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep, 1137 ep->mpa_attr.version, mpalen); 1138 } 1139 1140 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1141 if (mpa == NULL) 1142 return (-ENOMEM); 1143 1144 memset(mpa, 0, mpalen); 1145 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1146 mpa->flags = MPA_REJECT; 1147 mpa->revision = mpa_rev; 1148 mpa->private_data_size = htons(plen); 1149 1150 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1151 1152 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1153 mpa->private_data_size += 1154 htons(sizeof(struct mpa_v2_conn_params)); 1155 mpa_v2_params.ird = htons(((u16)ep->ird) | 1156 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 1157 0)); 1158 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 1159 (p2p_type == 1160 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 1161 MPA_V2_RDMA_WRITE_RTR : p2p_type == 1162 FW_RI_INIT_P2PTYPE_READ_REQ ? 1163 MPA_V2_RDMA_READ_RTR : 0) : 0)); 1164 memcpy(mpa->private_data, &mpa_v2_params, 1165 sizeof(struct mpa_v2_conn_params)); 1166 1167 if (ep->plen) 1168 memcpy(mpa->private_data + 1169 sizeof(struct mpa_v2_conn_params), pdata, plen); 1170 CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep, 1171 mpa_v2_params.ird, mpa_v2_params.ord, ep->plen); 1172 } else 1173 if (plen) 1174 memcpy(mpa->private_data, pdata, plen); 1175 1176 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1177 if (m == NULL) { 1178 free(mpa, M_CXGBE); 1179 return (-ENOMEM); 1180 } 1181 m_copyback(m, 0, mpalen, (void *)mpa); 1182 free(mpa, M_CXGBE); 1183 1184 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 1185 if (!err) 1186 ep->snd_seq += mpalen; 1187 CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err); 1188 return err; 1189} 1190 1191static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 1192{ 1193 int mpalen; 1194 struct mpa_message *mpa; 1195 struct mbuf *m; 1196 struct mpa_v2_conn_params mpa_v2_params; 1197 int err; 1198 1199 CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep); 1200 1201 mpalen = sizeof(*mpa) + plen; 1202 1203 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1204 1205 CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep, 1206 ep->mpa_attr.version); 1207 mpalen += sizeof(struct mpa_v2_conn_params); 1208 } 1209 1210 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1211 if (mpa == NULL) 1212 return (-ENOMEM); 1213 1214 memset(mpa, 0, sizeof(*mpa)); 1215 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1216 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 1217 (markers_enabled ? MPA_MARKERS : 0); 1218 mpa->revision = ep->mpa_attr.version; 1219 mpa->private_data_size = htons(plen); 1220 1221 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1222 1223 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1224 mpa->private_data_size += 1225 htons(sizeof(struct mpa_v2_conn_params)); 1226 mpa_v2_params.ird = htons((u16)ep->ird); 1227 mpa_v2_params.ord = htons((u16)ep->ord); 1228 CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep, 1229 ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord); 1230 1231 if (peer2peer && (ep->mpa_attr.p2p_type != 1232 FW_RI_INIT_P2PTYPE_DISABLED)) { 1233 1234 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1235 1236 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1237 1238 mpa_v2_params.ord |= 1239 htons(MPA_V2_RDMA_WRITE_RTR); 1240 CTR5(KTR_IW_CXGBE, "%s:smrep4 %p %d %d %d", 1241 __func__, ep, p2p_type, mpa_v2_params.ird, 1242 mpa_v2_params.ord); 1243 } 1244 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1245 1246 mpa_v2_params.ord |= 1247 htons(MPA_V2_RDMA_READ_RTR); 1248 CTR5(KTR_IW_CXGBE, "%s:smrep5 %p %d %d %d", 1249 __func__, ep, p2p_type, mpa_v2_params.ird, 1250 mpa_v2_params.ord); 1251 } 1252 } 1253 1254 memcpy(mpa->private_data, &mpa_v2_params, 1255 sizeof(struct mpa_v2_conn_params)); 1256 1257 if (ep->plen) 1258 memcpy(mpa->private_data + 1259 sizeof(struct mpa_v2_conn_params), pdata, plen); 1260 } else 1261 if (plen) 1262 memcpy(mpa->private_data, pdata, plen); 1263 1264 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1265 if (m == NULL) { 1266 free(mpa, M_CXGBE); 1267 return (-ENOMEM); 1268 } 1269 m_copyback(m, 0, mpalen, (void *)mpa); 1270 free(mpa, M_CXGBE); 1271 1272 1273 state_set(&ep->com, MPA_REP_SENT); 1274 ep->snd_seq += mpalen; 1275 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1276 ep->com.thread); 1277 CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err); 1278 return err; 1279} 1280 1281 1282 1283static void close_complete_upcall(struct c4iw_ep *ep, int status) 1284{ 1285 struct iw_cm_event event; 1286 1287 CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep); 1288 memset(&event, 0, sizeof(event)); 1289 event.event = IW_CM_EVENT_CLOSE; 1290 event.status = status; 1291 1292 if (ep->com.cm_id) { 1293 1294 CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); 1295 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1296 deref_cm_id(&ep->com); 1297 set_bit(CLOSE_UPCALL, &ep->com.history); 1298 } 1299 CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); 1300} 1301 1302static int 1303send_abort(struct c4iw_ep *ep) 1304{ 1305 struct socket *so = ep->com.so; 1306 struct sockopt sopt; 1307 int rc; 1308 struct linger l; 1309 1310 CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so, 1311 states[ep->com.state], ep->hwtid); 1312 1313 l.l_onoff = 1; 1314 l.l_linger = 0; 1315 1316 /* linger_time of 0 forces RST to be sent */ 1317 sopt.sopt_dir = SOPT_SET; 1318 sopt.sopt_level = SOL_SOCKET; 1319 sopt.sopt_name = SO_LINGER; 1320 sopt.sopt_val = (caddr_t)&l; 1321 sopt.sopt_valsize = sizeof l; 1322 sopt.sopt_td = NULL; 1323 rc = sosetopt(so, &sopt); 1324 if (rc != 0) { 1325 log(LOG_ERR, "%s: sosetopt(%p, linger = 0) failed with %d.\n", 1326 __func__, so, rc); 1327 } 1328 1329 uninit_iwarp_socket(so); 1330 sodisconnect(so); 1331 set_bit(ABORT_CONN, &ep->com.history); 1332 1333 /* 1334 * TBD: iw_cxgbe driver should receive ABORT reply for every ABORT 1335 * request it has sent. But the current TOE driver is not propagating 1336 * this ABORT reply event (via do_abort_rpl) to iw_cxgbe. So as a work- 1337 * around de-refer 'ep' (which was refered before sending ABORT request) 1338 * here instead of doing it in abort_rpl() handler of iw_cxgbe driver. 1339 */ 1340 c4iw_put_ep(&ep->com); 1341 1342 return (0); 1343} 1344 1345static void peer_close_upcall(struct c4iw_ep *ep) 1346{ 1347 struct iw_cm_event event; 1348 1349 CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep); 1350 memset(&event, 0, sizeof(event)); 1351 event.event = IW_CM_EVENT_DISCONNECT; 1352 1353 if (ep->com.cm_id) { 1354 1355 CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep); 1356 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1357 set_bit(DISCONN_UPCALL, &ep->com.history); 1358 } 1359 CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep); 1360} 1361 1362static void peer_abort_upcall(struct c4iw_ep *ep) 1363{ 1364 struct iw_cm_event event; 1365 1366 CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep); 1367 memset(&event, 0, sizeof(event)); 1368 event.event = IW_CM_EVENT_CLOSE; 1369 event.status = -ECONNRESET; 1370 1371 if (ep->com.cm_id) { 1372 1373 CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); 1374 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1375 deref_cm_id(&ep->com); 1376 set_bit(ABORT_UPCALL, &ep->com.history); 1377 } 1378 CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); 1379} 1380 1381static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1382{ 1383 struct iw_cm_event event; 1384 1385 CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status); 1386 memset(&event, 0, sizeof(event)); 1387 event.event = IW_CM_EVENT_CONNECT_REPLY; 1388 event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ? 1389 -ECONNRESET : status; 1390 event.local_addr = ep->com.local_addr; 1391 event.remote_addr = ep->com.remote_addr; 1392 1393 if ((status == 0) || (status == -ECONNREFUSED)) { 1394 1395 if (!ep->tried_with_mpa_v1) { 1396 1397 CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep); 1398 /* this means MPA_v2 is used */ 1399 event.private_data_len = ep->plen - 1400 sizeof(struct mpa_v2_conn_params); 1401 event.private_data = ep->mpa_pkt + 1402 sizeof(struct mpa_message) + 1403 sizeof(struct mpa_v2_conn_params); 1404 } else { 1405 1406 CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep); 1407 /* this means MPA_v1 is used */ 1408 event.private_data_len = ep->plen; 1409 event.private_data = ep->mpa_pkt + 1410 sizeof(struct mpa_message); 1411 } 1412 } 1413 1414 if (ep->com.cm_id) { 1415 1416 CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep); 1417 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1418 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1419 } 1420 1421 if(status == -ECONNABORTED) { 1422 1423 CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status); 1424 return; 1425 } 1426 1427 if (status < 0) { 1428 1429 CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); 1430 deref_cm_id(&ep->com); 1431 } 1432 1433 CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); 1434} 1435 1436static int connect_request_upcall(struct c4iw_ep *ep) 1437{ 1438 struct iw_cm_event event; 1439 int ret; 1440 1441 CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep, 1442 ep->tried_with_mpa_v1); 1443 1444 memset(&event, 0, sizeof(event)); 1445 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1446 event.local_addr = ep->com.local_addr; 1447 event.remote_addr = ep->com.remote_addr; 1448 event.provider_data = ep; 1449 event.so = ep->com.so; 1450 1451 if (!ep->tried_with_mpa_v1) { 1452 /* this means MPA_v2 is used */ 1453 event.ord = ep->ord; 1454 event.ird = ep->ird; 1455 event.private_data_len = ep->plen - 1456 sizeof(struct mpa_v2_conn_params); 1457 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1458 sizeof(struct mpa_v2_conn_params); 1459 } else { 1460 1461 /* this means MPA_v1 is used. Send max supported */ 1462 event.ord = c4iw_max_read_depth; 1463 event.ird = c4iw_max_read_depth; 1464 event.private_data_len = ep->plen; 1465 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1466 } 1467 1468 c4iw_get_ep(&ep->com); 1469 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1470 &event); 1471 if(ret) 1472 c4iw_put_ep(&ep->com); 1473 1474 set_bit(CONNREQ_UPCALL, &ep->com.history); 1475 c4iw_put_ep(&ep->parent_ep->com); 1476 return ret; 1477} 1478 1479static void established_upcall(struct c4iw_ep *ep) 1480{ 1481 struct iw_cm_event event; 1482 1483 CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep); 1484 memset(&event, 0, sizeof(event)); 1485 event.event = IW_CM_EVENT_ESTABLISHED; 1486 event.ird = ep->ird; 1487 event.ord = ep->ord; 1488 1489 if (ep->com.cm_id) { 1490 1491 CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep); 1492 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1493 set_bit(ESTAB_UPCALL, &ep->com.history); 1494 } 1495 CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep); 1496} 1497 1498 1499/* 1500 * process_mpa_reply - process streaming mode MPA reply 1501 * 1502 * Returns: 1503 * 1504 * 0 upon success indicating a connect request was delivered to the ULP 1505 * or the mpa request is incomplete but valid so far. 1506 * 1507 * 1 if a failure requires the caller to close the connection. 1508 * 1509 * 2 if a failure requires the caller to abort the connection. 1510 */ 1511static int process_mpa_reply(struct c4iw_ep *ep) 1512{ 1513 struct mpa_message *mpa; 1514 struct mpa_v2_conn_params *mpa_v2_params; 1515 u16 plen; 1516 u16 resp_ird, resp_ord; 1517 u8 rtr_mismatch = 0, insuff_ird = 0; 1518 struct c4iw_qp_attributes attrs; 1519 enum c4iw_qp_attr_mask mask; 1520 int err; 1521 struct mbuf *top, *m; 1522 int flags = MSG_DONTWAIT; 1523 struct uio uio; 1524 int disconnect = 0; 1525 1526 CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); 1527 1528 /* 1529 * Stop mpa timer. If it expired, then 1530 * we ignore the MPA reply. process_timeout() 1531 * will abort the connection. 1532 */ 1533 if (STOP_EP_TIMER(ep)) 1534 return 0; 1535 1536 uio.uio_resid = 1000000; 1537 uio.uio_td = ep->com.thread; 1538 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 1539 1540 if (err) { 1541 1542 if (err == EWOULDBLOCK) { 1543 1544 CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); 1545 START_EP_TIMER(ep); 1546 return 0; 1547 } 1548 err = -err; 1549 CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); 1550 goto err; 1551 } 1552 1553 if (ep->com.so->so_rcv.sb_mb) { 1554 1555 CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep); 1556 printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 1557 __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 1558 } 1559 1560 m = top; 1561 1562 do { 1563 1564 CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep); 1565 /* 1566 * If we get more than the supported amount of private data 1567 * then we must fail this connection. 1568 */ 1569 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 1570 1571 CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, 1572 ep->mpa_pkt_len + m->m_len); 1573 err = (-EINVAL); 1574 goto err_stop_timer; 1575 } 1576 1577 /* 1578 * copy the new data into our accumulation buffer. 1579 */ 1580 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 1581 ep->mpa_pkt_len += m->m_len; 1582 if (!m->m_next) 1583 m = m->m_nextpkt; 1584 else 1585 m = m->m_next; 1586 } while (m); 1587 1588 m_freem(top); 1589 /* 1590 * if we don't even have the mpa message, then bail. 1591 */ 1592 if (ep->mpa_pkt_len < sizeof(*mpa)) { 1593 return 0; 1594 } 1595 mpa = (struct mpa_message *) ep->mpa_pkt; 1596 1597 /* Validate MPA header. */ 1598 if (mpa->revision > mpa_rev) { 1599 1600 CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep, 1601 mpa->revision, mpa_rev); 1602 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " 1603 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1604 err = -EPROTO; 1605 goto err_stop_timer; 1606 } 1607 1608 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1609 1610 CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); 1611 err = -EPROTO; 1612 goto err_stop_timer; 1613 } 1614 1615 plen = ntohs(mpa->private_data_size); 1616 1617 /* 1618 * Fail if there's too much private data. 1619 */ 1620 if (plen > MPA_MAX_PRIVATE_DATA) { 1621 1622 CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); 1623 err = -EPROTO; 1624 goto err_stop_timer; 1625 } 1626 1627 /* 1628 * If plen does not account for pkt size 1629 */ 1630 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1631 1632 CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); 1633 STOP_EP_TIMER(ep); 1634 err = -EPROTO; 1635 goto err_stop_timer; 1636 } 1637 1638 ep->plen = (u8) plen; 1639 1640 /* 1641 * If we don't have all the pdata yet, then bail. 1642 * We'll continue process when more data arrives. 1643 */ 1644 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 1645 1646 CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); 1647 return 0; 1648 } 1649 1650 if (mpa->flags & MPA_REJECT) { 1651 1652 CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); 1653 err = -ECONNREFUSED; 1654 goto err_stop_timer; 1655 } 1656 1657 /* 1658 * If we get here we have accumulated the entire mpa 1659 * start reply message including private data. And 1660 * the MPA header is valid. 1661 */ 1662 state_set(&ep->com, FPDU_MODE); 1663 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1664 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1665 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1666 ep->mpa_attr.version = mpa->revision; 1667 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1668 1669 if (mpa->revision == 2) { 1670 1671 CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep); 1672 ep->mpa_attr.enhanced_rdma_conn = 1673 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1674 1675 if (ep->mpa_attr.enhanced_rdma_conn) { 1676 1677 CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep); 1678 mpa_v2_params = (struct mpa_v2_conn_params *) 1679 (ep->mpa_pkt + sizeof(*mpa)); 1680 resp_ird = ntohs(mpa_v2_params->ird) & 1681 MPA_V2_IRD_ORD_MASK; 1682 resp_ord = ntohs(mpa_v2_params->ord) & 1683 MPA_V2_IRD_ORD_MASK; 1684 1685 /* 1686 * This is a double-check. Ideally, below checks are 1687 * not required since ird/ord stuff has been taken 1688 * care of in c4iw_accept_cr 1689 */ 1690 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1691 1692 CTR2(KTR_IW_CXGBE, "%s:pmre %p", __func__, ep); 1693 err = -ENOMEM; 1694 ep->ird = resp_ord; 1695 ep->ord = resp_ird; 1696 insuff_ird = 1; 1697 } 1698 1699 if (ntohs(mpa_v2_params->ird) & 1700 MPA_V2_PEER2PEER_MODEL) { 1701 1702 CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep); 1703 if (ntohs(mpa_v2_params->ord) & 1704 MPA_V2_RDMA_WRITE_RTR) { 1705 1706 CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep); 1707 ep->mpa_attr.p2p_type = 1708 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1709 } 1710 else if (ntohs(mpa_v2_params->ord) & 1711 MPA_V2_RDMA_READ_RTR) { 1712 1713 CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep); 1714 ep->mpa_attr.p2p_type = 1715 FW_RI_INIT_P2PTYPE_READ_REQ; 1716 } 1717 } 1718 } 1719 } else { 1720 1721 CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep); 1722 1723 if (mpa->revision == 1) { 1724 1725 CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep); 1726 1727 if (peer2peer) { 1728 1729 CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep); 1730 ep->mpa_attr.p2p_type = p2p_type; 1731 } 1732 } 1733 } 1734 1735 if (set_tcpinfo(ep)) { 1736 1737 CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep); 1738 printf("%s set_tcpinfo error\n", __func__); 1739 err = -ECONNRESET; 1740 goto err; 1741 } 1742 1743 CTR6(KTR_IW_CXGBE, "%s - crc_enabled = %d, recv_marker_enabled = %d, " 1744 "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__, 1745 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1746 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1747 ep->mpa_attr.p2p_type); 1748 1749 /* 1750 * If responder's RTR does not match with that of initiator, assign 1751 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1752 * generated when moving QP to RTS state. 1753 * A TERM message will be sent after QP has moved to RTS state 1754 */ 1755 if ((ep->mpa_attr.version == 2) && peer2peer && 1756 (ep->mpa_attr.p2p_type != p2p_type)) { 1757 1758 CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep); 1759 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1760 rtr_mismatch = 1; 1761 } 1762 1763 1764 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 1765 attrs.mpa_attr = ep->mpa_attr; 1766 attrs.max_ird = ep->ird; 1767 attrs.max_ord = ep->ord; 1768 attrs.llp_stream_handle = ep; 1769 attrs.next_state = C4IW_QP_STATE_RTS; 1770 1771 mask = C4IW_QP_ATTR_NEXT_STATE | 1772 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1773 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1774 1775 /* bind QP and TID with INIT_WR */ 1776 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 1777 1778 if (err) { 1779 1780 CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep); 1781 goto err; 1782 } 1783 1784 /* 1785 * If responder's RTR requirement did not match with what initiator 1786 * supports, generate TERM message 1787 */ 1788 if (rtr_mismatch) { 1789 1790 CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep); 1791 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1792 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1793 attrs.ecode = MPA_NOMATCH_RTR; 1794 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1795 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1796 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1797 err = -ENOMEM; 1798 disconnect = 1; 1799 goto out; 1800 } 1801 1802 /* 1803 * Generate TERM if initiator IRD is not sufficient for responder 1804 * provided ORD. Currently, we do the same behaviour even when 1805 * responder provided IRD is also not sufficient as regards to 1806 * initiator ORD. 1807 */ 1808 if (insuff_ird) { 1809 1810 CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep); 1811 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1812 __func__); 1813 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1814 attrs.ecode = MPA_INSUFF_IRD; 1815 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1816 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1817 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1818 err = -ENOMEM; 1819 disconnect = 1; 1820 goto out; 1821 } 1822 goto out; 1823err_stop_timer: 1824 STOP_EP_TIMER(ep); 1825err: 1826 disconnect = 2; 1827out: 1828 connect_reply_upcall(ep, err); 1829 CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); 1830 return disconnect; 1831} 1832 1833/* 1834 * process_mpa_request - process streaming mode MPA request 1835 * 1836 * Returns: 1837 * 1838 * 0 upon success indicating a connect request was delivered to the ULP 1839 * or the mpa request is incomplete but valid so far. 1840 * 1841 * 1 if a failure requires the caller to close the connection. 1842 * 1843 * 2 if a failure requires the caller to abort the connection. 1844 */ 1845static int 1846process_mpa_request(struct c4iw_ep *ep) 1847{ 1848 struct mpa_message *mpa; 1849 u16 plen; 1850 int flags = MSG_DONTWAIT; 1851 int rc; 1852 struct iovec iov; 1853 struct uio uio; 1854 enum c4iw_ep_state state = state_read(&ep->com); 1855 1856 CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); 1857 1858 if (state != MPA_REQ_WAIT) 1859 return 0; 1860 1861 iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; 1862 iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1863 uio.uio_iov = &iov; 1864 uio.uio_iovcnt = 1; 1865 uio.uio_offset = 0; 1866 uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1867 uio.uio_segflg = UIO_SYSSPACE; 1868 uio.uio_rw = UIO_READ; 1869 uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */ 1870 1871 rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); 1872 if (rc == EAGAIN) 1873 return 0; 1874 else if (rc) 1875 goto err_stop_timer; 1876 1877 KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", 1878 __func__, ep->com.so)); 1879 ep->mpa_pkt_len += uio.uio_offset; 1880 1881 /* 1882 * If we get more than the supported amount of private data then we must 1883 * fail this connection. XXX: check so_rcv->sb_cc, or peek with another 1884 * soreceive, or increase the size of mpa_pkt by 1 and abort if the last 1885 * byte is filled by the soreceive above. 1886 */ 1887 1888 /* Don't even have the MPA message. Wait for more data to arrive. */ 1889 if (ep->mpa_pkt_len < sizeof(*mpa)) 1890 return 0; 1891 mpa = (struct mpa_message *) ep->mpa_pkt; 1892 1893 /* 1894 * Validate MPA Header. 1895 */ 1896 if (mpa->revision > mpa_rev) { 1897 log(LOG_ERR, "%s: MPA version mismatch. Local = %d," 1898 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1899 goto err_stop_timer; 1900 } 1901 1902 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) 1903 goto err_stop_timer; 1904 1905 /* 1906 * Fail if there's too much private data. 1907 */ 1908 plen = ntohs(mpa->private_data_size); 1909 if (plen > MPA_MAX_PRIVATE_DATA) 1910 goto err_stop_timer; 1911 1912 /* 1913 * If plen does not account for pkt size 1914 */ 1915 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) 1916 goto err_stop_timer; 1917 1918 ep->plen = (u8) plen; 1919 1920 /* 1921 * If we don't have all the pdata yet, then bail. 1922 */ 1923 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1924 return 0; 1925 1926 /* 1927 * If we get here we have accumulated the entire mpa 1928 * start reply message including private data. 1929 */ 1930 ep->mpa_attr.initiator = 0; 1931 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1932 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1933 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1934 ep->mpa_attr.version = mpa->revision; 1935 if (mpa->revision == 1) 1936 ep->tried_with_mpa_v1 = 1; 1937 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1938 1939 if (mpa->revision == 2) { 1940 ep->mpa_attr.enhanced_rdma_conn = 1941 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1942 if (ep->mpa_attr.enhanced_rdma_conn) { 1943 struct mpa_v2_conn_params *mpa_v2_params; 1944 u16 ird, ord; 1945 1946 mpa_v2_params = (void *)&ep->mpa_pkt[sizeof(*mpa)]; 1947 ird = ntohs(mpa_v2_params->ird); 1948 ord = ntohs(mpa_v2_params->ord); 1949 1950 ep->ird = ird & MPA_V2_IRD_ORD_MASK; 1951 ep->ord = ord & MPA_V2_IRD_ORD_MASK; 1952 if (ird & MPA_V2_PEER2PEER_MODEL && peer2peer) { 1953 if (ord & MPA_V2_RDMA_WRITE_RTR) { 1954 ep->mpa_attr.p2p_type = 1955 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1956 } else if (ord & MPA_V2_RDMA_READ_RTR) { 1957 ep->mpa_attr.p2p_type = 1958 FW_RI_INIT_P2PTYPE_READ_REQ; 1959 } 1960 } 1961 } 1962 } else if (mpa->revision == 1 && peer2peer) 1963 ep->mpa_attr.p2p_type = p2p_type; 1964 1965 if (set_tcpinfo(ep)) 1966 goto err_stop_timer; 1967 1968 CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " 1969 "xmit_marker_enabled = %d, version = %d", __func__, 1970 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1971 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 1972 1973 state_set(&ep->com, MPA_REQ_RCVD); 1974 STOP_EP_TIMER(ep); 1975 1976 /* drive upcall */ 1977 mutex_lock(&ep->parent_ep->com.mutex); 1978 if (ep->parent_ep->com.state != DEAD) { 1979 if (connect_request_upcall(ep)) 1980 goto err_unlock_parent; 1981 } else 1982 goto err_unlock_parent; 1983 mutex_unlock(&ep->parent_ep->com.mutex); 1984 return 0; 1985 1986err_unlock_parent: 1987 mutex_unlock(&ep->parent_ep->com.mutex); 1988 goto err_out; 1989err_stop_timer: 1990 STOP_EP_TIMER(ep); 1991err_out: 1992 return 2; 1993} 1994 1995/* 1996 * Upcall from the adapter indicating data has been transmitted. 1997 * For us its just the single MPA request or reply. We can now free 1998 * the skb holding the mpa message. 1999 */ 2000int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2001{ 2002 int err; 2003 struct c4iw_ep *ep = to_ep(cm_id); 2004 CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); 2005 int abort = 0; 2006 2007 if ((state_read(&ep->com) == DEAD) || 2008 (state_read(&ep->com) != MPA_REQ_RCVD)) { 2009 2010 CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep); 2011 c4iw_put_ep(&ep->com); 2012 return -ECONNRESET; 2013 } 2014 set_bit(ULP_REJECT, &ep->com.history); 2015 2016 if (mpa_rev == 0) { 2017 2018 CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); 2019 abort = 1; 2020 } 2021 else { 2022 2023 CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep); 2024 abort = send_mpa_reject(ep, pdata, pdata_len); 2025 } 2026 stop_ep_timer(ep); 2027 err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL); 2028 c4iw_put_ep(&ep->com); 2029 CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err); 2030 return 0; 2031} 2032 2033int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2034{ 2035 int err; 2036 struct c4iw_qp_attributes attrs; 2037 enum c4iw_qp_attr_mask mask; 2038 struct c4iw_ep *ep = to_ep(cm_id); 2039 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2040 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2041 int abort = 0; 2042 2043 CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); 2044 2045 if (state_read(&ep->com) == DEAD) { 2046 2047 CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); 2048 err = -ECONNRESET; 2049 goto err_out; 2050 } 2051 2052 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2053 BUG_ON(!qp); 2054 2055 set_bit(ULP_ACCEPT, &ep->com.history); 2056 2057 if ((conn_param->ord > c4iw_max_read_depth) || 2058 (conn_param->ird > c4iw_max_read_depth)) { 2059 2060 CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); 2061 err = -EINVAL; 2062 goto err_abort; 2063 } 2064 2065 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2066 2067 CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep); 2068 2069 if (conn_param->ord > ep->ird) { 2070 2071 CTR2(KTR_IW_CXGBE, "%s:cac4 %p", __func__, ep); 2072 ep->ird = conn_param->ird; 2073 ep->ord = conn_param->ord; 2074 send_mpa_reject(ep, conn_param->private_data, 2075 conn_param->private_data_len); 2076 err = -ENOMEM; 2077 goto err_abort; 2078 } 2079 2080 if (conn_param->ird > ep->ord) { 2081 2082 CTR2(KTR_IW_CXGBE, "%s:cac5 %p", __func__, ep); 2083 2084 if (!ep->ord) { 2085 2086 CTR2(KTR_IW_CXGBE, "%s:cac6 %p", __func__, ep); 2087 conn_param->ird = 1; 2088 } 2089 else { 2090 CTR2(KTR_IW_CXGBE, "%s:cac7 %p", __func__, ep); 2091 err = -ENOMEM; 2092 goto err_abort; 2093 } 2094 } 2095 2096 } 2097 ep->ird = conn_param->ird; 2098 ep->ord = conn_param->ord; 2099 2100 if (ep->mpa_attr.version != 2) { 2101 2102 CTR2(KTR_IW_CXGBE, "%s:cac8 %p", __func__, ep); 2103 2104 if (peer2peer && ep->ird == 0) { 2105 2106 CTR2(KTR_IW_CXGBE, "%s:cac9 %p", __func__, ep); 2107 ep->ird = 1; 2108 } 2109 } 2110 2111 2112 ep->com.cm_id = cm_id; 2113 ref_cm_id(&ep->com); 2114 ep->com.qp = qp; 2115 ref_qp(ep); 2116 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 2117 2118 /* bind QP to EP and move to RTS */ 2119 attrs.mpa_attr = ep->mpa_attr; 2120 attrs.max_ird = ep->ird; 2121 attrs.max_ord = ep->ord; 2122 attrs.llp_stream_handle = ep; 2123 attrs.next_state = C4IW_QP_STATE_RTS; 2124 2125 /* bind QP and TID with INIT_WR */ 2126 mask = C4IW_QP_ATTR_NEXT_STATE | 2127 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2128 C4IW_QP_ATTR_MPA_ATTR | 2129 C4IW_QP_ATTR_MAX_IRD | 2130 C4IW_QP_ATTR_MAX_ORD; 2131 2132 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 2133 2134 if (err) { 2135 2136 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2137 goto err_defef_cm_id; 2138 } 2139 err = send_mpa_reply(ep, conn_param->private_data, 2140 conn_param->private_data_len); 2141 2142 if (err) { 2143 2144 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2145 goto err_defef_cm_id; 2146 } 2147 2148 state_set(&ep->com, FPDU_MODE); 2149 established_upcall(ep); 2150 c4iw_put_ep(&ep->com); 2151 CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); 2152 return 0; 2153err_defef_cm_id: 2154 deref_cm_id(&ep->com); 2155err_abort: 2156 abort = 1; 2157err_out: 2158 if (abort) 2159 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 2160 c4iw_put_ep(&ep->com); 2161 CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); 2162 return err; 2163} 2164 2165 2166 2167int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2168{ 2169 int err = 0; 2170 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2171 struct c4iw_ep *ep = NULL; 2172 struct nhop4_extended nh4; 2173 2174 CTR2(KTR_IW_CXGBE, "%s:ccB %p", __func__, cm_id); 2175 2176 if ((conn_param->ord > c4iw_max_read_depth) || 2177 (conn_param->ird > c4iw_max_read_depth)) { 2178 2179 CTR2(KTR_IW_CXGBE, "%s:cc1 %p", __func__, cm_id); 2180 err = -EINVAL; 2181 goto out; 2182 } 2183 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2184 init_timer(&ep->timer); 2185 ep->plen = conn_param->private_data_len; 2186 2187 if (ep->plen) { 2188 2189 CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep); 2190 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2191 conn_param->private_data, ep->plen); 2192 } 2193 ep->ird = conn_param->ird; 2194 ep->ord = conn_param->ord; 2195 2196 if (peer2peer && ep->ord == 0) { 2197 2198 CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep); 2199 ep->ord = 1; 2200 } 2201 2202 ep->com.dev = dev; 2203 ep->com.cm_id = cm_id; 2204 ref_cm_id(&ep->com); 2205 ep->com.qp = get_qhp(dev, conn_param->qpn); 2206 2207 if (!ep->com.qp) { 2208 2209 CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep); 2210 err = -EINVAL; 2211 goto fail2; 2212 } 2213 ref_qp(ep); 2214 ep->com.thread = curthread; 2215 ep->com.so = cm_id->so; 2216 2217 /* find a route */ 2218 err = find_route( 2219 cm_id->local_addr.sin_addr.s_addr, 2220 cm_id->remote_addr.sin_addr.s_addr, 2221 cm_id->local_addr.sin_port, 2222 cm_id->remote_addr.sin_port, 0, &nh4); 2223 2224 if (err) { 2225 2226 CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep); 2227 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2228 err = -EHOSTUNREACH; 2229 goto fail2; 2230 } 2231 2232 if (!(nh4.nh_ifp->if_capenable & IFCAP_TOE) || 2233 TOEDEV(nh4.nh_ifp) == NULL) { 2234 err = -ENOPROTOOPT; 2235 goto fail3; 2236 } 2237 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2238 2239 state_set(&ep->com, CONNECTING); 2240 ep->tos = 0; 2241 ep->com.local_addr = cm_id->local_addr; 2242 ep->com.remote_addr = cm_id->remote_addr; 2243 err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 2244 ep->com.thread); 2245 2246 if (!err) { 2247 init_iwarp_socket(cm_id->so, &ep->com); 2248 goto out; 2249 } else { 2250 goto fail2; 2251 } 2252 2253fail3: 2254 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2255fail2: 2256 deref_cm_id(&ep->com); 2257 c4iw_put_ep(&ep->com); 2258 ep = NULL; /* CTR shouldn't display already-freed ep. */ 2259out: 2260 CTR2(KTR_IW_CXGBE, "%s:ccE %p", __func__, ep); 2261 return err; 2262} 2263 2264/* 2265 * iwcm->create_listen_ep. Returns -errno on failure. 2266 */ 2267int 2268c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog) 2269{ 2270 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2271 struct c4iw_listen_ep *ep; 2272 struct socket *so = cm_id->so; 2273 2274 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2275 ep->com.cm_id = cm_id; 2276 ref_cm_id(&ep->com); 2277 ep->com.dev = dev; 2278 ep->backlog = backlog; 2279 ep->com.local_addr = cm_id->local_addr; 2280 ep->com.thread = curthread; 2281 state_set(&ep->com, LISTEN); 2282 ep->com.so = so; 2283 2284 cm_id->provider_data = ep; 2285 return (0); 2286} 2287 2288void 2289c4iw_destroy_listen_ep(struct iw_cm_id *cm_id) 2290{ 2291 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2292 2293 CTR4(KTR_IW_CXGBE, "%s: cm_id %p, so %p, state %s", __func__, cm_id, 2294 cm_id->so, states[ep->com.state]); 2295 2296 state_set(&ep->com, DEAD); 2297 deref_cm_id(&ep->com); 2298 c4iw_put_ep(&ep->com); 2299 2300 return; 2301} 2302 2303int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2304{ 2305 int ret = 0; 2306 int close = 0; 2307 int fatal = 0; 2308 struct c4iw_rdev *rdev; 2309 2310 mutex_lock(&ep->com.mutex); 2311 2312 CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep); 2313 2314 rdev = &ep->com.dev->rdev; 2315 2316 if (c4iw_fatal_error(rdev)) { 2317 2318 CTR2(KTR_IW_CXGBE, "%s:ced1 %p", __func__, ep); 2319 fatal = 1; 2320 close_complete_upcall(ep, -ECONNRESET); 2321 ep->com.state = DEAD; 2322 } 2323 CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep, 2324 states[ep->com.state]); 2325 2326 switch (ep->com.state) { 2327 2328 case MPA_REQ_WAIT: 2329 case MPA_REQ_SENT: 2330 case MPA_REQ_RCVD: 2331 case MPA_REP_SENT: 2332 case FPDU_MODE: 2333 close = 1; 2334 if (abrupt) 2335 ep->com.state = ABORTING; 2336 else { 2337 ep->com.state = CLOSING; 2338 START_EP_TIMER(ep); 2339 } 2340 set_bit(CLOSE_SENT, &ep->com.flags); 2341 break; 2342 2343 case CLOSING: 2344 2345 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2346 2347 close = 1; 2348 if (abrupt) { 2349 STOP_EP_TIMER(ep); 2350 ep->com.state = ABORTING; 2351 } else 2352 ep->com.state = MORIBUND; 2353 } 2354 break; 2355 2356 case MORIBUND: 2357 case ABORTING: 2358 case DEAD: 2359 CTR3(KTR_IW_CXGBE, 2360 "%s ignoring disconnect ep %p state %u", __func__, 2361 ep, ep->com.state); 2362 break; 2363 2364 default: 2365 BUG(); 2366 break; 2367 } 2368 2369 mutex_unlock(&ep->com.mutex); 2370 2371 if (close) { 2372 2373 CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep); 2374 2375 if (abrupt) { 2376 2377 CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); 2378 set_bit(EP_DISC_ABORT, &ep->com.history); 2379 close_complete_upcall(ep, -ECONNRESET); 2380 ret = send_abort(ep); 2381 if (ret) 2382 fatal = 1; 2383 } else { 2384 2385 CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); 2386 set_bit(EP_DISC_CLOSE, &ep->com.history); 2387 2388 if (!ep->parent_ep) 2389 __state_set(&ep->com, MORIBUND); 2390 sodisconnect(ep->com.so); 2391 } 2392 2393 } 2394 2395 if (fatal) { 2396 set_bit(EP_DISC_FAIL, &ep->com.history); 2397 if (!abrupt) { 2398 STOP_EP_TIMER(ep); 2399 close_complete_upcall(ep, -EIO); 2400 } 2401 if (ep->com.qp) { 2402 struct c4iw_qp_attributes attrs; 2403 2404 attrs.next_state = C4IW_QP_STATE_ERROR; 2405 ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, 2406 C4IW_QP_ATTR_NEXT_STATE, 2407 &attrs, 1); 2408 if (ret) { 2409 CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); 2410 printf("%s - qp <- error failed!\n", __func__); 2411 } 2412 } 2413 release_ep_resources(ep); 2414 ep->com.state = DEAD; 2415 CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); 2416 } 2417 CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); 2418 return ret; 2419} 2420 2421#ifdef C4IW_EP_REDIRECT 2422int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 2423 struct l2t_entry *l2t) 2424{ 2425 struct c4iw_ep *ep = ctx; 2426 2427 if (ep->dst != old) 2428 return 0; 2429 2430 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, 2431 l2t); 2432 dst_hold(new); 2433 cxgb4_l2t_release(ep->l2t); 2434 ep->l2t = l2t; 2435 dst_release(old); 2436 ep->dst = new; 2437 return 1; 2438} 2439#endif 2440 2441 2442 2443static void ep_timeout(unsigned long arg) 2444{ 2445 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2446 2447 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 2448 2449 /* 2450 * Only insert if it is not already on the list. 2451 */ 2452 if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) { 2453 CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep); 2454 add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT); 2455 } 2456 } 2457} 2458 2459static int fw6_wr_rpl(struct adapter *sc, const __be64 *rpl) 2460{ 2461 uint64_t val = be64toh(*rpl); 2462 int ret; 2463 struct c4iw_wr_wait *wr_waitp; 2464 2465 ret = (int)((val >> 8) & 0xff); 2466 wr_waitp = (struct c4iw_wr_wait *)rpl[1]; 2467 CTR3(KTR_IW_CXGBE, "%s wr_waitp %p ret %u", __func__, wr_waitp, ret); 2468 if (wr_waitp) 2469 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2470 2471 return (0); 2472} 2473 2474static int fw6_cqe_handler(struct adapter *sc, const __be64 *rpl) 2475{ 2476 struct cqe_list_entry *cle; 2477 unsigned long flag; 2478 2479 cle = malloc(sizeof(*cle), M_CXGBE, M_NOWAIT); 2480 cle->rhp = sc->iwarp_softc; 2481 cle->err_cqe = *(const struct t4_cqe *)(&rpl[0]); 2482 2483 spin_lock_irqsave(&err_cqe_lock, flag); 2484 list_add_tail(&cle->entry, &err_cqe_list); 2485 queue_work(c4iw_taskq, &c4iw_task); 2486 spin_unlock_irqrestore(&err_cqe_lock, flag); 2487 2488 return (0); 2489} 2490 2491static int 2492process_terminate(struct c4iw_ep *ep) 2493{ 2494 struct c4iw_qp_attributes attrs; 2495 2496 CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep); 2497 2498 if (ep && ep->com.qp) { 2499 2500 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", 2501 ep->hwtid, ep->com.qp->wq.sq.qid); 2502 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2503 c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 2504 1); 2505 } else 2506 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", 2507 ep->hwtid); 2508 CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep); 2509 2510 return 0; 2511} 2512 2513int __init c4iw_cm_init(void) 2514{ 2515 2516 t4_register_cpl_handler(CPL_RDMA_TERMINATE, terminate); 2517 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, fw6_wr_rpl); 2518 t4_register_fw_msg_handler(FW6_TYPE_CQE, fw6_cqe_handler); 2519 t4_register_an_handler(c4iw_ev_handler); 2520 2521 TAILQ_INIT(&req_list); 2522 spin_lock_init(&req_lock); 2523 INIT_LIST_HEAD(&err_cqe_list); 2524 spin_lock_init(&err_cqe_lock); 2525 2526 INIT_WORK(&c4iw_task, process_req); 2527 2528 c4iw_taskq = create_singlethread_workqueue("iw_cxgbe"); 2529 if (!c4iw_taskq) 2530 return -ENOMEM; 2531 2532 return 0; 2533} 2534 2535void __exit c4iw_cm_term(void) 2536{ 2537 WARN_ON(!TAILQ_EMPTY(&req_list)); 2538 WARN_ON(!list_empty(&err_cqe_list)); 2539 flush_workqueue(c4iw_taskq); 2540 destroy_workqueue(c4iw_taskq); 2541 2542 t4_register_cpl_handler(CPL_RDMA_TERMINATE, NULL); 2543 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, NULL); 2544 t4_register_fw_msg_handler(FW6_TYPE_CQE, NULL); 2545 t4_register_an_handler(NULL); 2546} 2547#endif 2548