1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4/* Kai Shen <kaishen@linux.alibaba.com> */ 5/* Copyright (c) 2020-2022, Alibaba Group. */ 6 7/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 8/* Fredy Neeser */ 9/* Greg Joyce <greg@opengridcomputing.com> */ 10/* Copyright (c) 2008-2019, IBM Corporation */ 11/* Copyright (c) 2017, Open Grid Computing, Inc. */ 12 13#include <linux/workqueue.h> 14#include <trace/events/sock.h> 15 16#include "erdma.h" 17#include "erdma_cm.h" 18#include "erdma_verbs.h" 19 20static struct workqueue_struct *erdma_cm_wq; 21 22static void erdma_cm_llp_state_change(struct sock *sk); 23static void erdma_cm_llp_data_ready(struct sock *sk); 24static void erdma_cm_llp_error_report(struct sock *sk); 25 26static void erdma_sk_assign_cm_upcalls(struct sock *sk) 27{ 28 write_lock_bh(&sk->sk_callback_lock); 29 sk->sk_state_change = erdma_cm_llp_state_change; 30 sk->sk_data_ready = erdma_cm_llp_data_ready; 31 sk->sk_error_report = erdma_cm_llp_error_report; 32 write_unlock_bh(&sk->sk_callback_lock); 33} 34 35static void erdma_sk_save_upcalls(struct sock *sk) 36{ 37 struct erdma_cep *cep = sk_to_cep(sk); 38 39 write_lock_bh(&sk->sk_callback_lock); 40 cep->sk_state_change = sk->sk_state_change; 41 cep->sk_data_ready = sk->sk_data_ready; 42 cep->sk_error_report = sk->sk_error_report; 43 write_unlock_bh(&sk->sk_callback_lock); 44} 45 46static void erdma_sk_restore_upcalls(struct sock *sk, struct erdma_cep *cep) 47{ 48 sk->sk_state_change = cep->sk_state_change; 49 sk->sk_data_ready = cep->sk_data_ready; 50 sk->sk_error_report = cep->sk_error_report; 51 sk->sk_user_data = NULL; 52} 53 54static void erdma_socket_disassoc(struct socket *s) 55{ 56 struct sock *sk = s->sk; 57 struct erdma_cep *cep; 58 59 if (sk) { 60 write_lock_bh(&sk->sk_callback_lock); 61 cep = sk_to_cep(sk); 62 if (cep) { 63 erdma_sk_restore_upcalls(sk, cep); 64 erdma_cep_put(cep); 65 } else { 66 WARN_ON_ONCE(1); 67 } 68 write_unlock_bh(&sk->sk_callback_lock); 69 } else { 70 WARN_ON_ONCE(1); 71 } 72} 73 74static void erdma_cep_socket_assoc(struct erdma_cep *cep, struct socket *s) 75{ 76 cep->sock = s; 77 erdma_cep_get(cep); 78 s->sk->sk_user_data = cep; 79 80 erdma_sk_save_upcalls(s->sk); 81 erdma_sk_assign_cm_upcalls(s->sk); 82} 83 84static void erdma_disassoc_listen_cep(struct erdma_cep *cep) 85{ 86 if (cep->listen_cep) { 87 erdma_cep_put(cep->listen_cep); 88 cep->listen_cep = NULL; 89 } 90} 91 92static struct erdma_cep *erdma_cep_alloc(struct erdma_dev *dev) 93{ 94 struct erdma_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL); 95 unsigned long flags; 96 97 if (!cep) 98 return NULL; 99 100 INIT_LIST_HEAD(&cep->listenq); 101 INIT_LIST_HEAD(&cep->devq); 102 INIT_LIST_HEAD(&cep->work_freelist); 103 104 kref_init(&cep->ref); 105 cep->state = ERDMA_EPSTATE_IDLE; 106 init_waitqueue_head(&cep->waitq); 107 spin_lock_init(&cep->lock); 108 cep->dev = dev; 109 110 spin_lock_irqsave(&dev->lock, flags); 111 list_add_tail(&cep->devq, &dev->cep_list); 112 spin_unlock_irqrestore(&dev->lock, flags); 113 114 return cep; 115} 116 117static void erdma_cm_free_work(struct erdma_cep *cep) 118{ 119 struct list_head *w, *tmp; 120 struct erdma_cm_work *work; 121 122 list_for_each_safe(w, tmp, &cep->work_freelist) { 123 work = list_entry(w, struct erdma_cm_work, list); 124 list_del(&work->list); 125 kfree(work); 126 } 127} 128 129static void erdma_cancel_mpatimer(struct erdma_cep *cep) 130{ 131 spin_lock_bh(&cep->lock); 132 if (cep->mpa_timer) { 133 if (cancel_delayed_work(&cep->mpa_timer->work)) { 134 erdma_cep_put(cep); 135 kfree(cep->mpa_timer); 136 } 137 cep->mpa_timer = NULL; 138 } 139 spin_unlock_bh(&cep->lock); 140} 141 142static void erdma_put_work(struct erdma_cm_work *work) 143{ 144 INIT_LIST_HEAD(&work->list); 145 spin_lock_bh(&work->cep->lock); 146 list_add(&work->list, &work->cep->work_freelist); 147 spin_unlock_bh(&work->cep->lock); 148} 149 150static void erdma_cep_set_inuse(struct erdma_cep *cep) 151{ 152 unsigned long flags; 153 154 spin_lock_irqsave(&cep->lock, flags); 155 while (cep->in_use) { 156 spin_unlock_irqrestore(&cep->lock, flags); 157 wait_event_interruptible(cep->waitq, !cep->in_use); 158 if (signal_pending(current)) 159 flush_signals(current); 160 161 spin_lock_irqsave(&cep->lock, flags); 162 } 163 164 cep->in_use = 1; 165 spin_unlock_irqrestore(&cep->lock, flags); 166} 167 168static void erdma_cep_set_free(struct erdma_cep *cep) 169{ 170 unsigned long flags; 171 172 spin_lock_irqsave(&cep->lock, flags); 173 cep->in_use = 0; 174 spin_unlock_irqrestore(&cep->lock, flags); 175 176 wake_up(&cep->waitq); 177} 178 179static void __erdma_cep_dealloc(struct kref *ref) 180{ 181 struct erdma_cep *cep = container_of(ref, struct erdma_cep, ref); 182 struct erdma_dev *dev = cep->dev; 183 unsigned long flags; 184 185 WARN_ON(cep->listen_cep); 186 187 kfree(cep->private_data); 188 kfree(cep->mpa.pdata); 189 spin_lock_bh(&cep->lock); 190 if (!list_empty(&cep->work_freelist)) 191 erdma_cm_free_work(cep); 192 spin_unlock_bh(&cep->lock); 193 194 spin_lock_irqsave(&dev->lock, flags); 195 list_del(&cep->devq); 196 spin_unlock_irqrestore(&dev->lock, flags); 197 kfree(cep); 198} 199 200static struct erdma_cm_work *erdma_get_work(struct erdma_cep *cep) 201{ 202 struct erdma_cm_work *work = NULL; 203 204 spin_lock_bh(&cep->lock); 205 if (!list_empty(&cep->work_freelist)) { 206 work = list_entry(cep->work_freelist.next, struct erdma_cm_work, 207 list); 208 list_del_init(&work->list); 209 } 210 211 spin_unlock_bh(&cep->lock); 212 return work; 213} 214 215static int erdma_cm_alloc_work(struct erdma_cep *cep, int num) 216{ 217 struct erdma_cm_work *work; 218 219 while (num--) { 220 work = kmalloc(sizeof(*work), GFP_KERNEL); 221 if (!work) { 222 if (!(list_empty(&cep->work_freelist))) 223 erdma_cm_free_work(cep); 224 return -ENOMEM; 225 } 226 work->cep = cep; 227 INIT_LIST_HEAD(&work->list); 228 list_add(&work->list, &cep->work_freelist); 229 } 230 231 return 0; 232} 233 234static int erdma_cm_upcall(struct erdma_cep *cep, enum iw_cm_event_type reason, 235 int status) 236{ 237 struct iw_cm_event event; 238 struct iw_cm_id *cm_id; 239 240 memset(&event, 0, sizeof(event)); 241 event.status = status; 242 event.event = reason; 243 244 if (reason == IW_CM_EVENT_CONNECT_REQUEST) { 245 event.provider_data = cep; 246 cm_id = cep->listen_cep->cm_id; 247 248 event.ird = cep->dev->attrs.max_ird; 249 event.ord = cep->dev->attrs.max_ord; 250 } else { 251 cm_id = cep->cm_id; 252 } 253 254 if (reason == IW_CM_EVENT_CONNECT_REQUEST || 255 reason == IW_CM_EVENT_CONNECT_REPLY) { 256 u16 pd_len = be16_to_cpu(cep->mpa.hdr.params.pd_len); 257 258 if (pd_len && cep->mpa.pdata) { 259 event.private_data_len = pd_len; 260 event.private_data = cep->mpa.pdata; 261 } 262 263 getname_local(cep->sock, &event.local_addr); 264 getname_peer(cep->sock, &event.remote_addr); 265 } 266 267 return cm_id->event_handler(cm_id, &event); 268} 269 270void erdma_qp_cm_drop(struct erdma_qp *qp) 271{ 272 struct erdma_cep *cep = qp->cep; 273 274 if (!qp->cep) 275 return; 276 277 erdma_cep_set_inuse(cep); 278 279 /* already closed. */ 280 if (cep->state == ERDMA_EPSTATE_CLOSED) 281 goto out; 282 283 if (cep->cm_id) { 284 switch (cep->state) { 285 case ERDMA_EPSTATE_AWAIT_MPAREP: 286 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 287 -EINVAL); 288 break; 289 case ERDMA_EPSTATE_RDMA_MODE: 290 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); 291 break; 292 case ERDMA_EPSTATE_IDLE: 293 case ERDMA_EPSTATE_LISTENING: 294 case ERDMA_EPSTATE_CONNECTING: 295 case ERDMA_EPSTATE_AWAIT_MPAREQ: 296 case ERDMA_EPSTATE_RECVD_MPAREQ: 297 case ERDMA_EPSTATE_CLOSED: 298 default: 299 break; 300 } 301 cep->cm_id->rem_ref(cep->cm_id); 302 cep->cm_id = NULL; 303 erdma_cep_put(cep); 304 } 305 cep->state = ERDMA_EPSTATE_CLOSED; 306 307 if (cep->sock) { 308 erdma_socket_disassoc(cep->sock); 309 sock_release(cep->sock); 310 cep->sock = NULL; 311 } 312 313 if (cep->qp) { 314 cep->qp = NULL; 315 erdma_qp_put(qp); 316 } 317out: 318 erdma_cep_set_free(cep); 319} 320 321void erdma_cep_put(struct erdma_cep *cep) 322{ 323 WARN_ON(kref_read(&cep->ref) < 1); 324 kref_put(&cep->ref, __erdma_cep_dealloc); 325} 326 327void erdma_cep_get(struct erdma_cep *cep) 328{ 329 kref_get(&cep->ref); 330} 331 332static int erdma_send_mpareqrep(struct erdma_cep *cep, const void *pdata, 333 u8 pd_len) 334{ 335 struct socket *s = cep->sock; 336 struct mpa_rr *rr = &cep->mpa.hdr; 337 struct kvec iov[3]; 338 struct msghdr msg; 339 int iovec_num = 0; 340 int ret; 341 int mpa_len; 342 343 memset(&msg, 0, sizeof(msg)); 344 345 rr->params.pd_len = cpu_to_be16(pd_len); 346 347 iov[iovec_num].iov_base = rr; 348 iov[iovec_num].iov_len = sizeof(*rr); 349 iovec_num++; 350 mpa_len = sizeof(*rr); 351 352 iov[iovec_num].iov_base = &cep->mpa.ext_data; 353 iov[iovec_num].iov_len = sizeof(cep->mpa.ext_data); 354 iovec_num++; 355 mpa_len += sizeof(cep->mpa.ext_data); 356 357 if (pd_len) { 358 iov[iovec_num].iov_base = (char *)pdata; 359 iov[iovec_num].iov_len = pd_len; 360 mpa_len += pd_len; 361 iovec_num++; 362 } 363 364 ret = kernel_sendmsg(s, &msg, iov, iovec_num, mpa_len); 365 366 return ret < 0 ? ret : 0; 367} 368 369static inline int ksock_recv(struct socket *sock, char *buf, size_t size, 370 int flags) 371{ 372 struct kvec iov = { buf, size }; 373 struct msghdr msg = { .msg_name = NULL, .msg_flags = flags }; 374 375 return kernel_recvmsg(sock, &msg, &iov, 1, size, flags); 376} 377 378static int __recv_mpa_hdr(struct erdma_cep *cep, int hdr_rcvd, char *hdr, 379 int hdr_size, int *rcvd_out) 380{ 381 struct socket *s = cep->sock; 382 int rcvd; 383 384 *rcvd_out = 0; 385 if (hdr_rcvd < hdr_size) { 386 rcvd = ksock_recv(s, hdr + hdr_rcvd, hdr_size - hdr_rcvd, 387 MSG_DONTWAIT); 388 if (rcvd == -EAGAIN) 389 return -EAGAIN; 390 391 if (rcvd <= 0) 392 return -ECONNABORTED; 393 394 hdr_rcvd += rcvd; 395 *rcvd_out = rcvd; 396 397 if (hdr_rcvd < hdr_size) 398 return -EAGAIN; 399 } 400 401 return 0; 402} 403 404static void __mpa_rr_set_revision(__be16 *bits, u8 rev) 405{ 406 *bits = (*bits & ~MPA_RR_MASK_REVISION) | 407 (cpu_to_be16(rev) & MPA_RR_MASK_REVISION); 408} 409 410static u8 __mpa_rr_revision(__be16 mpa_rr_bits) 411{ 412 __be16 rev = mpa_rr_bits & MPA_RR_MASK_REVISION; 413 414 return (u8)be16_to_cpu(rev); 415} 416 417static void __mpa_ext_set_cc(__be32 *bits, u32 cc) 418{ 419 *bits = (*bits & ~MPA_EXT_FLAG_CC) | 420 (cpu_to_be32(cc) & MPA_EXT_FLAG_CC); 421} 422 423static u8 __mpa_ext_cc(__be32 mpa_ext_bits) 424{ 425 __be32 cc = mpa_ext_bits & MPA_EXT_FLAG_CC; 426 427 return (u8)be32_to_cpu(cc); 428} 429 430/* 431 * Receive MPA Request/Reply header. 432 * 433 * Returns 0 if complete MPA Request/Reply haeder including 434 * eventual private data was received. Returns -EAGAIN if 435 * header was partially received or negative error code otherwise. 436 * 437 * Context: May be called in process context only 438 */ 439static int erdma_recv_mpa_rr(struct erdma_cep *cep) 440{ 441 struct mpa_rr *hdr = &cep->mpa.hdr; 442 struct socket *s = cep->sock; 443 u16 pd_len; 444 int rcvd, to_rcv, ret, pd_rcvd; 445 446 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) { 447 ret = __recv_mpa_hdr(cep, cep->mpa.bytes_rcvd, 448 (char *)&cep->mpa.hdr, 449 sizeof(struct mpa_rr), &rcvd); 450 cep->mpa.bytes_rcvd += rcvd; 451 if (ret) 452 return ret; 453 } 454 455 if (be16_to_cpu(hdr->params.pd_len) > MPA_MAX_PRIVDATA || 456 __mpa_rr_revision(hdr->params.bits) != MPA_REVISION_EXT_1) 457 return -EPROTO; 458 459 if (cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) < 460 sizeof(struct erdma_mpa_ext)) { 461 ret = __recv_mpa_hdr( 462 cep, cep->mpa.bytes_rcvd - sizeof(struct mpa_rr), 463 (char *)&cep->mpa.ext_data, 464 sizeof(struct erdma_mpa_ext), &rcvd); 465 cep->mpa.bytes_rcvd += rcvd; 466 if (ret) 467 return ret; 468 } 469 470 pd_len = be16_to_cpu(hdr->params.pd_len); 471 pd_rcvd = cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) - 472 sizeof(struct erdma_mpa_ext); 473 to_rcv = pd_len - pd_rcvd; 474 475 if (!to_rcv) { 476 /* 477 * We have received the whole MPA Request/Reply message. 478 * Check against peer protocol violation. 479 */ 480 u32 word; 481 482 ret = __recv_mpa_hdr(cep, 0, (char *)&word, sizeof(word), 483 &rcvd); 484 if (ret == -EAGAIN && rcvd == 0) 485 return 0; 486 487 if (ret) 488 return ret; 489 490 return -EPROTO; 491 } 492 493 /* 494 * At this point, MPA header has been fully received, and pd_len != 0. 495 * So, begin to receive private data. 496 */ 497 if (!cep->mpa.pdata) { 498 cep->mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL); 499 if (!cep->mpa.pdata) 500 return -ENOMEM; 501 } 502 503 rcvd = ksock_recv(s, cep->mpa.pdata + pd_rcvd, to_rcv + 4, 504 MSG_DONTWAIT); 505 if (rcvd < 0) 506 return rcvd; 507 508 if (rcvd > to_rcv) 509 return -EPROTO; 510 511 cep->mpa.bytes_rcvd += rcvd; 512 513 if (to_rcv == rcvd) 514 return 0; 515 516 return -EAGAIN; 517} 518 519/* 520 * erdma_proc_mpareq() 521 * 522 * Read MPA Request from socket and signal new connection to IWCM 523 * if success. Caller must hold lock on corresponding listening CEP. 524 */ 525static int erdma_proc_mpareq(struct erdma_cep *cep) 526{ 527 struct mpa_rr *req; 528 int ret; 529 530 ret = erdma_recv_mpa_rr(cep); 531 if (ret) 532 return ret; 533 534 req = &cep->mpa.hdr; 535 536 if (memcmp(req->key, MPA_KEY_REQ, MPA_KEY_SIZE)) 537 return -EPROTO; 538 539 memcpy(req->key, MPA_KEY_REP, MPA_KEY_SIZE); 540 541 /* Currently does not support marker and crc. */ 542 if (req->params.bits & MPA_RR_FLAG_MARKERS || 543 req->params.bits & MPA_RR_FLAG_CRC) 544 goto reject_conn; 545 546 cep->state = ERDMA_EPSTATE_RECVD_MPAREQ; 547 548 /* Keep reference until IWCM accepts/rejects */ 549 erdma_cep_get(cep); 550 ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REQUEST, 0); 551 if (ret) 552 erdma_cep_put(cep); 553 554 return ret; 555 556reject_conn: 557 req->params.bits &= ~MPA_RR_FLAG_MARKERS; 558 req->params.bits |= MPA_RR_FLAG_REJECT; 559 req->params.bits &= ~MPA_RR_FLAG_CRC; 560 561 kfree(cep->mpa.pdata); 562 cep->mpa.pdata = NULL; 563 erdma_send_mpareqrep(cep, NULL, 0); 564 565 return -EOPNOTSUPP; 566} 567 568static int erdma_proc_mpareply(struct erdma_cep *cep) 569{ 570 struct erdma_qp_attrs qp_attrs; 571 struct erdma_qp *qp = cep->qp; 572 struct mpa_rr *rep; 573 int ret; 574 575 ret = erdma_recv_mpa_rr(cep); 576 if (ret) 577 goto out_err; 578 579 erdma_cancel_mpatimer(cep); 580 581 rep = &cep->mpa.hdr; 582 583 if (memcmp(rep->key, MPA_KEY_REP, MPA_KEY_SIZE)) { 584 ret = -EPROTO; 585 goto out_err; 586 } 587 588 if (rep->params.bits & MPA_RR_FLAG_REJECT) { 589 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET); 590 return -ECONNRESET; 591 } 592 593 /* Currently does not support marker and crc. */ 594 if ((rep->params.bits & MPA_RR_FLAG_MARKERS) || 595 (rep->params.bits & MPA_RR_FLAG_CRC)) { 596 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); 597 return -EINVAL; 598 } 599 600 memset(&qp_attrs, 0, sizeof(qp_attrs)); 601 qp_attrs.irq_size = cep->ird; 602 qp_attrs.orq_size = cep->ord; 603 qp_attrs.state = ERDMA_QP_STATE_RTS; 604 605 down_write(&qp->state_lock); 606 if (qp->attrs.state > ERDMA_QP_STATE_RTR) { 607 ret = -EINVAL; 608 up_write(&qp->state_lock); 609 goto out_err; 610 } 611 612 qp->attrs.qp_type = ERDMA_QP_ACTIVE; 613 if (__mpa_ext_cc(cep->mpa.ext_data.bits) != qp->attrs.cc) 614 qp->attrs.cc = COMPROMISE_CC; 615 616 ret = erdma_modify_qp_internal(qp, &qp_attrs, 617 ERDMA_QP_ATTR_STATE | 618 ERDMA_QP_ATTR_LLP_HANDLE | 619 ERDMA_QP_ATTR_MPA); 620 621 up_write(&qp->state_lock); 622 623 if (!ret) { 624 ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 0); 625 if (!ret) 626 cep->state = ERDMA_EPSTATE_RDMA_MODE; 627 628 return 0; 629 } 630 631out_err: 632 if (ret != -EAGAIN) 633 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EINVAL); 634 635 return ret; 636} 637 638static void erdma_accept_newconn(struct erdma_cep *cep) 639{ 640 struct socket *s = cep->sock; 641 struct socket *new_s = NULL; 642 struct erdma_cep *new_cep = NULL; 643 int ret = 0; 644 645 if (cep->state != ERDMA_EPSTATE_LISTENING) 646 goto error; 647 648 new_cep = erdma_cep_alloc(cep->dev); 649 if (!new_cep) 650 goto error; 651 652 /* 653 * 4: Allocate a sufficient number of work elements 654 * to allow concurrent handling of local + peer close 655 * events, MPA header processing + MPA timeout. 656 */ 657 if (erdma_cm_alloc_work(new_cep, 4) != 0) 658 goto error; 659 660 /* 661 * Copy saved socket callbacks from listening CEP 662 * and assign new socket with new CEP 663 */ 664 new_cep->sk_state_change = cep->sk_state_change; 665 new_cep->sk_data_ready = cep->sk_data_ready; 666 new_cep->sk_error_report = cep->sk_error_report; 667 668 ret = kernel_accept(s, &new_s, O_NONBLOCK); 669 if (ret != 0) 670 goto error; 671 672 new_cep->sock = new_s; 673 erdma_cep_get(new_cep); 674 new_s->sk->sk_user_data = new_cep; 675 676 tcp_sock_set_nodelay(new_s->sk); 677 new_cep->state = ERDMA_EPSTATE_AWAIT_MPAREQ; 678 679 ret = erdma_cm_queue_work(new_cep, ERDMA_CM_WORK_MPATIMEOUT); 680 if (ret) 681 goto error; 682 683 new_cep->listen_cep = cep; 684 erdma_cep_get(cep); 685 686 if (atomic_read(&new_s->sk->sk_rmem_alloc)) { 687 /* MPA REQ already queued */ 688 erdma_cep_set_inuse(new_cep); 689 ret = erdma_proc_mpareq(new_cep); 690 if (ret != -EAGAIN) { 691 erdma_cep_put(cep); 692 new_cep->listen_cep = NULL; 693 if (ret) { 694 erdma_cep_set_free(new_cep); 695 goto error; 696 } 697 } 698 erdma_cep_set_free(new_cep); 699 } 700 return; 701 702error: 703 if (new_cep) { 704 new_cep->state = ERDMA_EPSTATE_CLOSED; 705 erdma_cancel_mpatimer(new_cep); 706 707 erdma_cep_put(new_cep); 708 new_cep->sock = NULL; 709 } 710 711 if (new_s) { 712 erdma_socket_disassoc(new_s); 713 sock_release(new_s); 714 } 715} 716 717static int erdma_newconn_connected(struct erdma_cep *cep) 718{ 719 int ret = 0; 720 721 cep->mpa.hdr.params.bits = 0; 722 __mpa_rr_set_revision(&cep->mpa.hdr.params.bits, MPA_REVISION_EXT_1); 723 724 memcpy(cep->mpa.hdr.key, MPA_KEY_REQ, MPA_KEY_SIZE); 725 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.cookie); 726 __mpa_ext_set_cc(&cep->mpa.ext_data.bits, cep->qp->attrs.cc); 727 728 ret = erdma_send_mpareqrep(cep, cep->private_data, cep->pd_len); 729 cep->state = ERDMA_EPSTATE_AWAIT_MPAREP; 730 cep->mpa.hdr.params.pd_len = 0; 731 732 if (ret >= 0) 733 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_MPATIMEOUT); 734 735 return ret; 736} 737 738static void erdma_cm_work_handler(struct work_struct *w) 739{ 740 struct erdma_cm_work *work; 741 struct erdma_cep *cep; 742 int release_cep = 0, ret = 0; 743 744 work = container_of(w, struct erdma_cm_work, work.work); 745 cep = work->cep; 746 747 erdma_cep_set_inuse(cep); 748 749 switch (work->type) { 750 case ERDMA_CM_WORK_CONNECTED: 751 erdma_cancel_mpatimer(cep); 752 if (cep->state == ERDMA_EPSTATE_CONNECTING) { 753 ret = erdma_newconn_connected(cep); 754 if (ret) { 755 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 756 -EIO); 757 release_cep = 1; 758 } 759 } 760 break; 761 case ERDMA_CM_WORK_CONNECTTIMEOUT: 762 if (cep->state == ERDMA_EPSTATE_CONNECTING) { 763 cep->mpa_timer = NULL; 764 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 765 -ETIMEDOUT); 766 release_cep = 1; 767 } 768 break; 769 case ERDMA_CM_WORK_ACCEPT: 770 erdma_accept_newconn(cep); 771 break; 772 case ERDMA_CM_WORK_READ_MPAHDR: 773 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) { 774 if (cep->listen_cep) { 775 erdma_cep_set_inuse(cep->listen_cep); 776 777 if (cep->listen_cep->state == 778 ERDMA_EPSTATE_LISTENING) 779 ret = erdma_proc_mpareq(cep); 780 else 781 ret = -EFAULT; 782 783 erdma_cep_set_free(cep->listen_cep); 784 785 if (ret != -EAGAIN) { 786 erdma_cep_put(cep->listen_cep); 787 cep->listen_cep = NULL; 788 if (ret) 789 erdma_cep_put(cep); 790 } 791 } 792 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) { 793 ret = erdma_proc_mpareply(cep); 794 } 795 796 if (ret && ret != -EAGAIN) 797 release_cep = 1; 798 break; 799 case ERDMA_CM_WORK_CLOSE_LLP: 800 if (cep->cm_id) 801 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); 802 release_cep = 1; 803 break; 804 case ERDMA_CM_WORK_PEER_CLOSE: 805 if (cep->cm_id) { 806 if (cep->state == ERDMA_EPSTATE_CONNECTING || 807 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) { 808 /* 809 * MPA reply not received, but connection drop 810 */ 811 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 812 -ECONNRESET); 813 } else if (cep->state == ERDMA_EPSTATE_RDMA_MODE) { 814 /* 815 * NOTE: IW_CM_EVENT_DISCONNECT is given just 816 * to transition IWCM into CLOSING. 817 */ 818 erdma_cm_upcall(cep, IW_CM_EVENT_DISCONNECT, 0); 819 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); 820 } 821 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) { 822 /* Socket close before MPA request received. */ 823 erdma_disassoc_listen_cep(cep); 824 erdma_cep_put(cep); 825 } 826 release_cep = 1; 827 break; 828 case ERDMA_CM_WORK_MPATIMEOUT: 829 cep->mpa_timer = NULL; 830 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) { 831 /* 832 * MPA request timed out: 833 * Hide any partially received private data and signal 834 * timeout 835 */ 836 cep->mpa.hdr.params.pd_len = 0; 837 838 if (cep->cm_id) 839 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 840 -ETIMEDOUT); 841 release_cep = 1; 842 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) { 843 /* No MPA req received after peer TCP stream setup. */ 844 erdma_disassoc_listen_cep(cep); 845 846 erdma_cep_put(cep); 847 release_cep = 1; 848 } 849 break; 850 default: 851 WARN(1, "Undefined CM work type: %d\n", work->type); 852 } 853 854 if (release_cep) { 855 erdma_cancel_mpatimer(cep); 856 cep->state = ERDMA_EPSTATE_CLOSED; 857 if (cep->qp) { 858 struct erdma_qp *qp = cep->qp; 859 /* 860 * Serialize a potential race with application 861 * closing the QP and calling erdma_qp_cm_drop() 862 */ 863 erdma_qp_get(qp); 864 erdma_cep_set_free(cep); 865 866 erdma_qp_llp_close(qp); 867 erdma_qp_put(qp); 868 869 erdma_cep_set_inuse(cep); 870 cep->qp = NULL; 871 erdma_qp_put(qp); 872 } 873 874 if (cep->sock) { 875 erdma_socket_disassoc(cep->sock); 876 sock_release(cep->sock); 877 cep->sock = NULL; 878 } 879 880 if (cep->cm_id) { 881 cep->cm_id->rem_ref(cep->cm_id); 882 cep->cm_id = NULL; 883 if (cep->state != ERDMA_EPSTATE_LISTENING) 884 erdma_cep_put(cep); 885 } 886 } 887 erdma_cep_set_free(cep); 888 erdma_put_work(work); 889 erdma_cep_put(cep); 890} 891 892int erdma_cm_queue_work(struct erdma_cep *cep, enum erdma_work_type type) 893{ 894 struct erdma_cm_work *work = erdma_get_work(cep); 895 unsigned long delay = 0; 896 897 if (!work) 898 return -ENOMEM; 899 900 work->type = type; 901 work->cep = cep; 902 903 erdma_cep_get(cep); 904 905 INIT_DELAYED_WORK(&work->work, erdma_cm_work_handler); 906 907 if (type == ERDMA_CM_WORK_MPATIMEOUT) { 908 cep->mpa_timer = work; 909 910 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) 911 delay = MPAREP_TIMEOUT; 912 else 913 delay = MPAREQ_TIMEOUT; 914 } else if (type == ERDMA_CM_WORK_CONNECTTIMEOUT) { 915 cep->mpa_timer = work; 916 917 delay = CONNECT_TIMEOUT; 918 } 919 920 queue_delayed_work(erdma_cm_wq, &work->work, delay); 921 922 return 0; 923} 924 925static void erdma_cm_llp_data_ready(struct sock *sk) 926{ 927 struct erdma_cep *cep; 928 929 trace_sk_data_ready(sk); 930 931 read_lock(&sk->sk_callback_lock); 932 933 cep = sk_to_cep(sk); 934 if (!cep) 935 goto out; 936 937 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ || 938 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) 939 erdma_cm_queue_work(cep, ERDMA_CM_WORK_READ_MPAHDR); 940 941out: 942 read_unlock(&sk->sk_callback_lock); 943} 944 945static void erdma_cm_llp_error_report(struct sock *sk) 946{ 947 struct erdma_cep *cep = sk_to_cep(sk); 948 949 if (cep) 950 cep->sk_error_report(sk); 951} 952 953static void erdma_cm_llp_state_change(struct sock *sk) 954{ 955 struct erdma_cep *cep; 956 void (*orig_state_change)(struct sock *sk); 957 958 read_lock(&sk->sk_callback_lock); 959 960 cep = sk_to_cep(sk); 961 if (!cep) { 962 read_unlock(&sk->sk_callback_lock); 963 return; 964 } 965 orig_state_change = cep->sk_state_change; 966 967 switch (sk->sk_state) { 968 case TCP_ESTABLISHED: 969 if (cep->state == ERDMA_EPSTATE_CONNECTING) 970 erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED); 971 else 972 erdma_cm_queue_work(cep, ERDMA_CM_WORK_ACCEPT); 973 break; 974 case TCP_CLOSE: 975 case TCP_CLOSE_WAIT: 976 if (cep->state != ERDMA_EPSTATE_LISTENING) 977 erdma_cm_queue_work(cep, ERDMA_CM_WORK_PEER_CLOSE); 978 break; 979 default: 980 break; 981 } 982 read_unlock(&sk->sk_callback_lock); 983 orig_state_change(sk); 984} 985 986static int kernel_bindconnect(struct socket *s, struct sockaddr *laddr, 987 int laddrlen, struct sockaddr *raddr, 988 int raddrlen, int flags) 989{ 990 int ret; 991 992 sock_set_reuseaddr(s->sk); 993 ret = s->ops->bind(s, laddr, laddrlen); 994 if (ret) 995 return ret; 996 ret = s->ops->connect(s, raddr, raddrlen, flags); 997 return ret < 0 ? ret : 0; 998} 999 1000int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) 1001{ 1002 struct erdma_dev *dev = to_edev(id->device); 1003 struct erdma_qp *qp; 1004 struct erdma_cep *cep = NULL; 1005 struct socket *s = NULL; 1006 struct sockaddr *laddr = (struct sockaddr *)&id->m_local_addr; 1007 struct sockaddr *raddr = (struct sockaddr *)&id->m_remote_addr; 1008 u16 pd_len = params->private_data_len; 1009 int ret; 1010 1011 if (pd_len > MPA_MAX_PRIVDATA) 1012 return -EINVAL; 1013 1014 if (params->ird > dev->attrs.max_ird || 1015 params->ord > dev->attrs.max_ord) 1016 return -EINVAL; 1017 1018 if (laddr->sa_family != AF_INET || raddr->sa_family != AF_INET) 1019 return -EAFNOSUPPORT; 1020 1021 qp = find_qp_by_qpn(dev, params->qpn); 1022 if (!qp) 1023 return -ENOENT; 1024 erdma_qp_get(qp); 1025 1026 ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &s); 1027 if (ret < 0) 1028 goto error_put_qp; 1029 1030 cep = erdma_cep_alloc(dev); 1031 if (!cep) { 1032 ret = -ENOMEM; 1033 goto error_release_sock; 1034 } 1035 1036 erdma_cep_set_inuse(cep); 1037 1038 /* Associate QP with CEP */ 1039 erdma_cep_get(cep); 1040 qp->cep = cep; 1041 cep->qp = qp; 1042 1043 /* Associate cm_id with CEP */ 1044 id->add_ref(id); 1045 cep->cm_id = id; 1046 1047 /* 1048 * 6: Allocate a sufficient number of work elements 1049 * to allow concurrent handling of local + peer close 1050 * events, MPA header processing + MPA timeout, connected event 1051 * and connect timeout. 1052 */ 1053 ret = erdma_cm_alloc_work(cep, 6); 1054 if (ret != 0) { 1055 ret = -ENOMEM; 1056 goto error_release_cep; 1057 } 1058 1059 cep->ird = params->ird; 1060 cep->ord = params->ord; 1061 cep->state = ERDMA_EPSTATE_CONNECTING; 1062 1063 erdma_cep_socket_assoc(cep, s); 1064 1065 if (pd_len) { 1066 cep->pd_len = pd_len; 1067 cep->private_data = kmalloc(pd_len, GFP_KERNEL); 1068 if (!cep->private_data) { 1069 ret = -ENOMEM; 1070 goto error_disassoc; 1071 } 1072 1073 memcpy(cep->private_data, params->private_data, 1074 params->private_data_len); 1075 } 1076 1077 ret = kernel_bindconnect(s, laddr, sizeof(*laddr), raddr, 1078 sizeof(*raddr), O_NONBLOCK); 1079 if (ret != -EINPROGRESS && ret != 0) { 1080 goto error_disassoc; 1081 } else if (ret == 0) { 1082 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED); 1083 if (ret) 1084 goto error_disassoc; 1085 } else { 1086 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTTIMEOUT); 1087 if (ret) 1088 goto error_disassoc; 1089 } 1090 1091 erdma_cep_set_free(cep); 1092 return 0; 1093 1094error_disassoc: 1095 kfree(cep->private_data); 1096 cep->private_data = NULL; 1097 cep->pd_len = 0; 1098 1099 erdma_socket_disassoc(s); 1100 1101error_release_cep: 1102 /* disassoc with cm_id */ 1103 cep->cm_id = NULL; 1104 id->rem_ref(id); 1105 1106 /* disassoc with qp */ 1107 qp->cep = NULL; 1108 erdma_cep_put(cep); 1109 cep->qp = NULL; 1110 1111 cep->state = ERDMA_EPSTATE_CLOSED; 1112 1113 erdma_cep_set_free(cep); 1114 1115 /* release the cep. */ 1116 erdma_cep_put(cep); 1117 1118error_release_sock: 1119 if (s) 1120 sock_release(s); 1121error_put_qp: 1122 erdma_qp_put(qp); 1123 1124 return ret; 1125} 1126 1127int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) 1128{ 1129 struct erdma_dev *dev = to_edev(id->device); 1130 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data; 1131 struct erdma_qp *qp; 1132 struct erdma_qp_attrs qp_attrs; 1133 int ret; 1134 1135 erdma_cep_set_inuse(cep); 1136 erdma_cep_put(cep); 1137 1138 /* Free lingering inbound private data */ 1139 if (cep->mpa.hdr.params.pd_len) { 1140 cep->mpa.hdr.params.pd_len = 0; 1141 kfree(cep->mpa.pdata); 1142 cep->mpa.pdata = NULL; 1143 } 1144 erdma_cancel_mpatimer(cep); 1145 1146 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) { 1147 erdma_cep_set_free(cep); 1148 erdma_cep_put(cep); 1149 1150 return -ECONNRESET; 1151 } 1152 1153 qp = find_qp_by_qpn(dev, params->qpn); 1154 if (!qp) 1155 return -ENOENT; 1156 erdma_qp_get(qp); 1157 1158 down_write(&qp->state_lock); 1159 if (qp->attrs.state > ERDMA_QP_STATE_RTR) { 1160 ret = -EINVAL; 1161 up_write(&qp->state_lock); 1162 goto error; 1163 } 1164 1165 if (params->ord > dev->attrs.max_ord || 1166 params->ird > dev->attrs.max_ord) { 1167 ret = -EINVAL; 1168 up_write(&qp->state_lock); 1169 goto error; 1170 } 1171 1172 if (params->private_data_len > MPA_MAX_PRIVDATA) { 1173 ret = -EINVAL; 1174 up_write(&qp->state_lock); 1175 goto error; 1176 } 1177 1178 cep->ird = params->ird; 1179 cep->ord = params->ord; 1180 1181 cep->cm_id = id; 1182 id->add_ref(id); 1183 1184 memset(&qp_attrs, 0, sizeof(qp_attrs)); 1185 qp_attrs.orq_size = params->ord; 1186 qp_attrs.irq_size = params->ird; 1187 1188 qp_attrs.state = ERDMA_QP_STATE_RTS; 1189 1190 /* Associate QP with CEP */ 1191 erdma_cep_get(cep); 1192 qp->cep = cep; 1193 cep->qp = qp; 1194 1195 cep->state = ERDMA_EPSTATE_RDMA_MODE; 1196 1197 qp->attrs.qp_type = ERDMA_QP_PASSIVE; 1198 qp->attrs.pd_len = params->private_data_len; 1199 1200 if (qp->attrs.cc != __mpa_ext_cc(cep->mpa.ext_data.bits)) 1201 qp->attrs.cc = COMPROMISE_CC; 1202 1203 /* move to rts */ 1204 ret = erdma_modify_qp_internal(qp, &qp_attrs, 1205 ERDMA_QP_ATTR_STATE | 1206 ERDMA_QP_ATTR_ORD | 1207 ERDMA_QP_ATTR_LLP_HANDLE | 1208 ERDMA_QP_ATTR_IRD | 1209 ERDMA_QP_ATTR_MPA); 1210 up_write(&qp->state_lock); 1211 1212 if (ret) 1213 goto error; 1214 1215 cep->mpa.ext_data.bits = 0; 1216 __mpa_ext_set_cc(&cep->mpa.ext_data.bits, qp->attrs.cc); 1217 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.cookie); 1218 1219 ret = erdma_send_mpareqrep(cep, params->private_data, 1220 params->private_data_len); 1221 if (!ret) { 1222 ret = erdma_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0); 1223 if (ret) 1224 goto error; 1225 1226 erdma_cep_set_free(cep); 1227 1228 return 0; 1229 } 1230 1231error: 1232 erdma_socket_disassoc(cep->sock); 1233 sock_release(cep->sock); 1234 cep->sock = NULL; 1235 1236 cep->state = ERDMA_EPSTATE_CLOSED; 1237 1238 if (cep->cm_id) { 1239 cep->cm_id->rem_ref(id); 1240 cep->cm_id = NULL; 1241 } 1242 1243 if (qp->cep) { 1244 erdma_cep_put(cep); 1245 qp->cep = NULL; 1246 } 1247 1248 cep->qp = NULL; 1249 erdma_qp_put(qp); 1250 1251 erdma_cep_set_free(cep); 1252 erdma_cep_put(cep); 1253 1254 return ret; 1255} 1256 1257int erdma_reject(struct iw_cm_id *id, const void *pdata, u8 plen) 1258{ 1259 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data; 1260 1261 erdma_cep_set_inuse(cep); 1262 erdma_cep_put(cep); 1263 1264 erdma_cancel_mpatimer(cep); 1265 1266 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) { 1267 erdma_cep_set_free(cep); 1268 erdma_cep_put(cep); 1269 1270 return -ECONNRESET; 1271 } 1272 1273 if (__mpa_rr_revision(cep->mpa.hdr.params.bits) == MPA_REVISION_EXT_1) { 1274 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */ 1275 erdma_send_mpareqrep(cep, pdata, plen); 1276 } 1277 1278 erdma_socket_disassoc(cep->sock); 1279 sock_release(cep->sock); 1280 cep->sock = NULL; 1281 1282 cep->state = ERDMA_EPSTATE_CLOSED; 1283 1284 erdma_cep_set_free(cep); 1285 erdma_cep_put(cep); 1286 1287 return 0; 1288} 1289 1290int erdma_create_listen(struct iw_cm_id *id, int backlog) 1291{ 1292 struct socket *s; 1293 struct erdma_cep *cep = NULL; 1294 int ret = 0; 1295 struct erdma_dev *dev = to_edev(id->device); 1296 int addr_family = id->local_addr.ss_family; 1297 struct sockaddr_in *laddr = &to_sockaddr_in(id->local_addr); 1298 1299 if (addr_family != AF_INET) 1300 return -EAFNOSUPPORT; 1301 1302 ret = sock_create(addr_family, SOCK_STREAM, IPPROTO_TCP, &s); 1303 if (ret < 0) 1304 return ret; 1305 1306 sock_set_reuseaddr(s->sk); 1307 1308 /* For wildcard addr, limit binding to current device only */ 1309 if (ipv4_is_zeronet(laddr->sin_addr.s_addr)) 1310 s->sk->sk_bound_dev_if = dev->netdev->ifindex; 1311 1312 ret = s->ops->bind(s, (struct sockaddr *)laddr, 1313 sizeof(struct sockaddr_in)); 1314 if (ret) 1315 goto error; 1316 1317 cep = erdma_cep_alloc(dev); 1318 if (!cep) { 1319 ret = -ENOMEM; 1320 goto error; 1321 } 1322 erdma_cep_socket_assoc(cep, s); 1323 1324 ret = erdma_cm_alloc_work(cep, backlog); 1325 if (ret) 1326 goto error; 1327 1328 ret = s->ops->listen(s, backlog); 1329 if (ret) 1330 goto error; 1331 1332 cep->cm_id = id; 1333 id->add_ref(id); 1334 1335 if (!id->provider_data) { 1336 id->provider_data = 1337 kmalloc(sizeof(struct list_head), GFP_KERNEL); 1338 if (!id->provider_data) { 1339 ret = -ENOMEM; 1340 goto error; 1341 } 1342 INIT_LIST_HEAD((struct list_head *)id->provider_data); 1343 } 1344 1345 list_add_tail(&cep->listenq, (struct list_head *)id->provider_data); 1346 cep->state = ERDMA_EPSTATE_LISTENING; 1347 1348 return 0; 1349 1350error: 1351 if (cep) { 1352 erdma_cep_set_inuse(cep); 1353 1354 if (cep->cm_id) { 1355 cep->cm_id->rem_ref(cep->cm_id); 1356 cep->cm_id = NULL; 1357 } 1358 cep->sock = NULL; 1359 erdma_socket_disassoc(s); 1360 cep->state = ERDMA_EPSTATE_CLOSED; 1361 1362 erdma_cep_set_free(cep); 1363 erdma_cep_put(cep); 1364 } 1365 sock_release(s); 1366 1367 return ret; 1368} 1369 1370static void erdma_drop_listeners(struct iw_cm_id *id) 1371{ 1372 struct list_head *p, *tmp; 1373 /* 1374 * In case of a wildcard rdma_listen on a multi-homed device, 1375 * a listener's IWCM id is associated with more than one listening CEP. 1376 */ 1377 list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) { 1378 struct erdma_cep *cep = 1379 list_entry(p, struct erdma_cep, listenq); 1380 1381 list_del(p); 1382 1383 erdma_cep_set_inuse(cep); 1384 1385 if (cep->cm_id) { 1386 cep->cm_id->rem_ref(cep->cm_id); 1387 cep->cm_id = NULL; 1388 } 1389 if (cep->sock) { 1390 erdma_socket_disassoc(cep->sock); 1391 sock_release(cep->sock); 1392 cep->sock = NULL; 1393 } 1394 cep->state = ERDMA_EPSTATE_CLOSED; 1395 erdma_cep_set_free(cep); 1396 erdma_cep_put(cep); 1397 } 1398} 1399 1400int erdma_destroy_listen(struct iw_cm_id *id) 1401{ 1402 if (!id->provider_data) 1403 return 0; 1404 1405 erdma_drop_listeners(id); 1406 kfree(id->provider_data); 1407 id->provider_data = NULL; 1408 1409 return 0; 1410} 1411 1412int erdma_cm_init(void) 1413{ 1414 erdma_cm_wq = create_singlethread_workqueue("erdma_cm_wq"); 1415 if (!erdma_cm_wq) 1416 return -ENOMEM; 1417 1418 return 0; 1419} 1420 1421void erdma_cm_exit(void) 1422{ 1423 if (erdma_cm_wq) 1424 destroy_workqueue(erdma_cm_wq); 1425} 1426