1/*- 2 * Copyright (c) 2012 The FreeBSD Foundation 3 * Copyright (c) 2015 Chelsio Communications, Inc. 4 * All rights reserved. 5 * 6 * This software was developed by Edward Tomasz Napierala under sponsorship 7 * from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32/* 33 * cxgbei implementation of iSCSI Common Layer kobj(9) interface. 34 */ 35 36#include <sys/cdefs.h> 37#include "opt_inet.h" 38#include "opt_inet6.h" 39 40#ifdef TCP_OFFLOAD 41#include <sys/param.h> 42#include <sys/bio.h> 43#include <sys/capsicum.h> 44#include <sys/condvar.h> 45#include <sys/conf.h> 46#include <sys/file.h> 47#include <sys/kernel.h> 48#include <sys/kthread.h> 49#include <sys/ktr.h> 50#include <sys/lock.h> 51#include <sys/mbuf.h> 52#include <sys/mutex.h> 53#include <sys/module.h> 54#include <sys/protosw.h> 55#include <sys/socket.h> 56#include <sys/socketvar.h> 57#include <sys/sysctl.h> 58#include <sys/systm.h> 59#include <sys/sx.h> 60#include <sys/uio.h> 61#include <machine/bus.h> 62#include <vm/vm.h> 63#include <vm/vm_page.h> 64#include <vm/pmap.h> 65#include <netinet/in.h> 66#include <netinet/in_pcb.h> 67#include <netinet/tcp.h> 68#include <netinet/tcp_var.h> 69#include <netinet/toecore.h> 70 71#include <dev/iscsi/icl.h> 72#include <dev/iscsi/iscsi_proto.h> 73#include <icl_conn_if.h> 74 75#include <cam/scsi/scsi_all.h> 76#include <cam/scsi/scsi_da.h> 77#include <cam/ctl/ctl_io.h> 78#include <cam/ctl/ctl.h> 79#include <cam/ctl/ctl_backend.h> 80#include <cam/ctl/ctl_error.h> 81#include <cam/ctl/ctl_frontend.h> 82#include <cam/ctl/ctl_debug.h> 83#include <cam/ctl/ctl_ha.h> 84#include <cam/ctl/ctl_ioctl.h> 85 86#include <cam/cam.h> 87#include <cam/cam_ccb.h> 88#include <cam/cam_xpt.h> 89#include <cam/cam_debug.h> 90#include <cam/cam_sim.h> 91#include <cam/cam_xpt_sim.h> 92#include <cam/cam_xpt_periph.h> 93#include <cam/cam_periph.h> 94#include <cam/cam_compat.h> 95#include <cam/scsi/scsi_message.h> 96 97#include "common/common.h" 98#include "common/t4_regs.h" 99#include "common/t4_tcb.h" 100#include "tom/t4_tom.h" 101#include "cxgbei.h" 102 103/* 104 * Use the page pod tag for the TT hash. 105 */ 106#define TT_HASH(icc, tt) (G_PPOD_TAG(tt) & (icc)->cmp_hash_mask) 107 108struct cxgbei_ddp_state { 109 struct ppod_reservation prsv; 110 struct cxgbei_cmp cmp; 111}; 112 113static MALLOC_DEFINE(M_CXGBEI, "cxgbei", "cxgbei(4)"); 114 115SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 116 "Chelsio iSCSI offload"); 117static int first_burst_length = 8192; 118SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, first_burst_length, CTLFLAG_RWTUN, 119 &first_burst_length, 0, "First burst length"); 120static int max_burst_length = 2 * 1024 * 1024; 121SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, max_burst_length, CTLFLAG_RWTUN, 122 &max_burst_length, 0, "Maximum burst length"); 123static int sendspace = 1048576; 124SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN, 125 &sendspace, 0, "Default send socket buffer size"); 126static int recvspace = 1048576; 127SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN, 128 &recvspace, 0, "Default receive socket buffer size"); 129 130static volatile u_int icl_cxgbei_ncons; 131 132static icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu; 133static icl_conn_pdu_data_segment_length_t 134 icl_cxgbei_conn_pdu_data_segment_length; 135static icl_conn_pdu_append_bio_t icl_cxgbei_conn_pdu_append_bio; 136static icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data; 137static icl_conn_pdu_get_bio_t icl_cxgbei_conn_pdu_get_bio; 138static icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data; 139static icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue; 140static icl_conn_pdu_queue_cb_t icl_cxgbei_conn_pdu_queue_cb; 141static icl_conn_handoff_t icl_cxgbei_conn_handoff; 142static icl_conn_free_t icl_cxgbei_conn_free; 143static icl_conn_close_t icl_cxgbei_conn_close; 144static icl_conn_task_setup_t icl_cxgbei_conn_task_setup; 145static icl_conn_task_done_t icl_cxgbei_conn_task_done; 146static icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup; 147static icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done; 148 149static kobj_method_t icl_cxgbei_methods[] = { 150 KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu), 151 KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free), 152 KOBJMETHOD(icl_conn_pdu_data_segment_length, 153 icl_cxgbei_conn_pdu_data_segment_length), 154 KOBJMETHOD(icl_conn_pdu_append_bio, icl_cxgbei_conn_pdu_append_bio), 155 KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data), 156 KOBJMETHOD(icl_conn_pdu_get_bio, icl_cxgbei_conn_pdu_get_bio), 157 KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data), 158 KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue), 159 KOBJMETHOD(icl_conn_pdu_queue_cb, icl_cxgbei_conn_pdu_queue_cb), 160 KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff), 161 KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free), 162 KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close), 163 KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup), 164 KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done), 165 KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup), 166 KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done), 167 { 0, 0 } 168}; 169 170DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn)); 171 172void 173icl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 174{ 175 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 176 177 KASSERT(icp->ref_cnt != 0, ("freeing deleted PDU")); 178 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 179 MPASS(ic == ip->ip_conn); 180 181 m_freem(ip->ip_ahs_mbuf); 182 m_freem(ip->ip_data_mbuf); 183 m_freem(ip->ip_bhs_mbuf); 184 185 KASSERT(ic != NULL || icp->ref_cnt == 1, 186 ("orphaned PDU has oustanding references")); 187 188 if (atomic_fetchadd_int(&icp->ref_cnt, -1) != 1) 189 return; 190 191 free(icp, M_CXGBEI); 192#ifdef DIAGNOSTIC 193 if (__predict_true(ic != NULL)) 194 refcount_release(&ic->ic_outstanding_pdus); 195#endif 196} 197 198static void 199icl_cxgbei_pdu_call_cb(struct icl_pdu *ip) 200{ 201 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 202 203 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 204 205 if (icp->cb != NULL) 206 icp->cb(ip, icp->error); 207#ifdef DIAGNOSTIC 208 if (__predict_true(ip->ip_conn != NULL)) 209 refcount_release(&ip->ip_conn->ic_outstanding_pdus); 210#endif 211 free(icp, M_CXGBEI); 212} 213 214static void 215icl_cxgbei_pdu_done(struct icl_pdu *ip, int error) 216{ 217 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 218 219 if (error != 0) 220 icp->error = error; 221 222 m_freem(ip->ip_ahs_mbuf); 223 ip->ip_ahs_mbuf = NULL; 224 m_freem(ip->ip_data_mbuf); 225 ip->ip_data_mbuf = NULL; 226 m_freem(ip->ip_bhs_mbuf); 227 ip->ip_bhs_mbuf = NULL; 228 229 /* 230 * All other references to this PDU should have been dropped 231 * by the m_freem() of ip_data_mbuf. 232 */ 233 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) 234 icl_cxgbei_pdu_call_cb(ip); 235 else 236 __assert_unreachable(); 237} 238 239static void 240icl_cxgbei_mbuf_done(struct mbuf *mb) 241{ 242 243 struct icl_cxgbei_pdu *icp = (struct icl_cxgbei_pdu *)mb->m_ext.ext_arg1; 244 245 /* 246 * NB: mb_free_mext() might leave ref_cnt as 1 without 247 * decrementing it if it hits the fast path in the ref_cnt 248 * check. 249 */ 250 icl_cxgbei_pdu_call_cb(&icp->ip); 251} 252 253struct icl_pdu * 254icl_cxgbei_new_pdu(int flags) 255{ 256 struct icl_cxgbei_pdu *icp; 257 struct icl_pdu *ip; 258 struct mbuf *m; 259 260 icp = malloc(sizeof(*icp), M_CXGBEI, flags | M_ZERO); 261 if (__predict_false(icp == NULL)) 262 return (NULL); 263 264 icp->icp_signature = CXGBEI_PDU_SIGNATURE; 265 icp->ref_cnt = 1; 266 ip = &icp->ip; 267 268 m = m_gethdr(flags, MT_DATA); 269 if (__predict_false(m == NULL)) { 270 free(icp, M_CXGBEI); 271 return (NULL); 272 } 273 274 ip->ip_bhs_mbuf = m; 275 ip->ip_bhs = mtod(m, struct iscsi_bhs *); 276 memset(ip->ip_bhs, 0, sizeof(*ip->ip_bhs)); 277 m->m_len = sizeof(struct iscsi_bhs); 278 m->m_pkthdr.len = m->m_len; 279 280 return (ip); 281} 282 283void 284icl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic) 285{ 286 287 ip->ip_conn = ic; 288#ifdef DIAGNOSTIC 289 refcount_acquire(&ic->ic_outstanding_pdus); 290#endif 291} 292 293/* 294 * Allocate icl_pdu with empty BHS to fill up by the caller. 295 */ 296static struct icl_pdu * 297icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags) 298{ 299 struct icl_pdu *ip; 300 301 ip = icl_cxgbei_new_pdu(flags); 302 if (__predict_false(ip == NULL)) 303 return (NULL); 304 icl_cxgbei_new_pdu_set_conn(ip, ic); 305 306 return (ip); 307} 308 309static size_t 310icl_pdu_data_segment_length(const struct icl_pdu *request) 311{ 312 uint32_t len = 0; 313 314 len += request->ip_bhs->bhs_data_segment_len[0]; 315 len <<= 8; 316 len += request->ip_bhs->bhs_data_segment_len[1]; 317 len <<= 8; 318 len += request->ip_bhs->bhs_data_segment_len[2]; 319 320 return (len); 321} 322 323size_t 324icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic, 325 const struct icl_pdu *request) 326{ 327 328 return (icl_pdu_data_segment_length(request)); 329} 330 331static struct mbuf * 332finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp) 333{ 334 struct icl_pdu *ip = &icp->ip; 335 uint8_t ulp_submode, padding; 336 struct mbuf *m, *last; 337 struct iscsi_bhs *bhs; 338 int data_len; 339 340 /* 341 * Fix up the data segment mbuf first. 342 */ 343 m = ip->ip_data_mbuf; 344 ulp_submode = icc->ulp_submode; 345 if (m != NULL) { 346 last = m_last(m); 347 348 /* 349 * Round up the data segment to a 4B boundary. Pad with 0 if 350 * necessary. There will definitely be room in the mbuf. 351 */ 352 padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len; 353 if (padding != 0) { 354 MPASS(padding <= M_TRAILINGSPACE(last)); 355 bzero(mtod(last, uint8_t *) + last->m_len, padding); 356 last->m_len += padding; 357 } 358 } else { 359 MPASS(ip->ip_data_len == 0); 360 ulp_submode &= ~ULP_CRC_DATA; 361 padding = 0; 362 } 363 364 /* 365 * Now the header mbuf that has the BHS. 366 */ 367 m = ip->ip_bhs_mbuf; 368 MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs)); 369 MPASS(m->m_len == sizeof(struct iscsi_bhs)); 370 371 bhs = ip->ip_bhs; 372 data_len = ip->ip_data_len; 373 if (data_len > icc->ic.ic_max_send_data_segment_length) { 374 struct iscsi_bhs_data_in *bhsdi; 375 int flags; 376 377 KASSERT(padding == 0, ("%s: ISO with padding %d for icp %p", 378 __func__, padding, icp)); 379 switch (bhs->bhs_opcode) { 380 case ISCSI_BHS_OPCODE_SCSI_DATA_OUT: 381 flags = 1; 382 break; 383 case ISCSI_BHS_OPCODE_SCSI_DATA_IN: 384 flags = 2; 385 break; 386 default: 387 panic("invalid opcode %#x for ISO", bhs->bhs_opcode); 388 } 389 data_len = icc->ic.ic_max_send_data_segment_length; 390 bhsdi = (struct iscsi_bhs_data_in *)bhs; 391 if (bhsdi->bhsdi_flags & BHSDI_FLAGS_F) { 392 /* 393 * Firmware will set F on the final PDU in the 394 * burst. 395 */ 396 flags |= CXGBE_ISO_F; 397 bhsdi->bhsdi_flags &= ~BHSDI_FLAGS_F; 398 } 399 set_mbuf_iscsi_iso(m, true); 400 set_mbuf_iscsi_iso_flags(m, flags); 401 set_mbuf_iscsi_iso_mss(m, data_len); 402 } 403 404 bhs->bhs_data_segment_len[2] = data_len; 405 bhs->bhs_data_segment_len[1] = data_len >> 8; 406 bhs->bhs_data_segment_len[0] = data_len >> 16; 407 408 /* 409 * Extract mbuf chain from PDU. 410 */ 411 m->m_pkthdr.len += ip->ip_data_len + padding; 412 m->m_next = ip->ip_data_mbuf; 413 set_mbuf_ulp_submode(m, ulp_submode); 414 ip->ip_bhs_mbuf = NULL; 415 ip->ip_data_mbuf = NULL; 416 ip->ip_bhs = NULL; 417 418 /* 419 * Drop PDU reference on icp. Additional references might 420 * still be held by zero-copy PDU buffers (ICL_NOCOPY). 421 */ 422 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) 423 icl_cxgbei_pdu_call_cb(ip); 424 425 return (m); 426} 427 428static void 429icl_cxgbei_tx_main(void *arg) 430{ 431 struct epoch_tracker et; 432 struct icl_cxgbei_conn *icc = arg; 433 struct icl_conn *ic = &icc->ic; 434 struct toepcb *toep = icc->toep; 435 struct socket *so = ic->ic_socket; 436 struct inpcb *inp = sotoinpcb(so); 437 struct icl_pdu *ip; 438 struct mbuf *m; 439 struct mbufq mq; 440 STAILQ_HEAD(, icl_pdu) tx_pdus = STAILQ_HEAD_INITIALIZER(tx_pdus); 441 442 mbufq_init(&mq, INT_MAX); 443 444 ICL_CONN_LOCK(ic); 445 while (__predict_true(!ic->ic_disconnecting)) { 446 while (STAILQ_EMPTY(&icc->sent_pdus)) { 447 icc->tx_active = false; 448 mtx_sleep(&icc->tx_active, ic->ic_lock, 0, "-", 0); 449 if (__predict_false(ic->ic_disconnecting)) 450 goto out; 451 MPASS(icc->tx_active); 452 } 453 454 STAILQ_SWAP(&icc->sent_pdus, &tx_pdus, icl_pdu); 455 ICL_CONN_UNLOCK(ic); 456 457 while ((ip = STAILQ_FIRST(&tx_pdus)) != NULL) { 458 STAILQ_REMOVE_HEAD(&tx_pdus, ip_next); 459 460 m = finalize_pdu(icc, ip_to_icp(ip)); 461 M_ASSERTPKTHDR(m); 462 MPASS((m->m_pkthdr.len & 3) == 0); 463 464 mbufq_enqueue(&mq, m); 465 } 466 467 ICL_CONN_LOCK(ic); 468 if (__predict_false(ic->ic_disconnecting) || 469 __predict_false(ic->ic_socket == NULL)) { 470 mbufq_drain(&mq); 471 break; 472 } 473 474 CURVNET_SET(toep->vnet); 475 NET_EPOCH_ENTER(et); 476 INP_WLOCK(inp); 477 478 ICL_CONN_UNLOCK(ic); 479 if (__predict_false(inp->inp_flags & INP_DROPPED) || 480 __predict_false((toep->flags & TPF_ATTACHED) == 0)) { 481 mbufq_drain(&mq); 482 } else { 483 mbufq_concat(&toep->ulp_pduq, &mq); 484 t4_push_pdus(icc->sc, toep, 0); 485 } 486 INP_WUNLOCK(inp); 487 NET_EPOCH_EXIT(et); 488 CURVNET_RESTORE(); 489 490 ICL_CONN_LOCK(ic); 491 } 492out: 493 ICL_CONN_UNLOCK(ic); 494 495 kthread_exit(); 496} 497 498static void 499icl_cxgbei_rx_main(void *arg) 500{ 501 struct icl_cxgbei_conn *icc = arg; 502 struct icl_conn *ic = &icc->ic; 503 struct icl_pdu *ip; 504 struct sockbuf *sb; 505 STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus); 506 bool cantrcvmore; 507 508 sb = &ic->ic_socket->so_rcv; 509 SOCKBUF_LOCK(sb); 510 while (__predict_true(!ic->ic_disconnecting)) { 511 while (STAILQ_EMPTY(&icc->rcvd_pdus)) { 512 icc->rx_active = false; 513 mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0); 514 if (__predict_false(ic->ic_disconnecting)) 515 goto out; 516 MPASS(icc->rx_active); 517 } 518 519 if (__predict_false(sbused(sb)) != 0) { 520 /* 521 * PDUs were received before the tid 522 * transitioned to ULP mode. Convert 523 * them to icl_cxgbei_pdus and insert 524 * them into the head of rcvd_pdus. 525 */ 526 parse_pdus(icc, sb); 527 } 528 cantrcvmore = (sb->sb_state & SBS_CANTRCVMORE) != 0; 529 MPASS(STAILQ_EMPTY(&rx_pdus)); 530 STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu); 531 SOCKBUF_UNLOCK(sb); 532 533 /* Hand over PDUs to ICL. */ 534 while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) { 535 STAILQ_REMOVE_HEAD(&rx_pdus, ip_next); 536 if (cantrcvmore) 537 icl_cxgbei_pdu_done(ip, ENOTCONN); 538 else 539 ic->ic_receive(ip); 540 } 541 542 SOCKBUF_LOCK(sb); 543 } 544out: 545 /* 546 * Since ic_disconnecting is set before the SOCKBUF_MTX is 547 * locked in icl_cxgbei_conn_close, the loop above can exit 548 * before icl_cxgbei_conn_close can lock SOCKBUF_MTX and block 549 * waiting for the thread exit. 550 */ 551 while (!icc->rx_exiting) 552 mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0); 553 SOCKBUF_UNLOCK(sb); 554 555 kthread_exit(); 556} 557 558static void 559cxgbei_free_mext_pg(struct mbuf *m) 560{ 561 struct icl_cxgbei_pdu *icp; 562 563 M_ASSERTEXTPG(m); 564 565 /* 566 * Nothing to do for the pages; they are owned by the PDU / 567 * I/O request. 568 */ 569 570 /* Drop reference on the PDU. */ 571 icp = m->m_ext.ext_arg1; 572 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) 573 icl_cxgbei_pdu_call_cb(&icp->ip); 574} 575 576static struct mbuf * 577cxgbei_getm(size_t len, int flags) 578{ 579 struct mbuf *m, *m0, *m_tail; 580 581 m_tail = m0 = NULL; 582 583 /* Allocate as jumbo mbufs of size MJUM16BYTES. */ 584 while (len >= MJUM16BYTES) { 585 m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES); 586 if (__predict_false(m == NULL)) { 587 if ((flags & M_WAITOK) != 0) { 588 /* Fall back to non-jumbo mbufs. */ 589 break; 590 } 591 return (NULL); 592 } 593 if (m0 == NULL) { 594 m0 = m_tail = m; 595 } else { 596 m_tail->m_next = m; 597 m_tail = m; 598 } 599 len -= MJUM16BYTES; 600 } 601 602 /* Allocate mbuf chain for the remaining data. */ 603 if (len != 0) { 604 m = m_getm2(NULL, len, flags, MT_DATA, 0); 605 if (__predict_false(m == NULL)) { 606 m_freem(m0); 607 return (NULL); 608 } 609 if (m0 == NULL) 610 m0 = m; 611 else 612 m_tail->m_next = m; 613 } 614 615 return (m0); 616} 617 618int 619icl_cxgbei_conn_pdu_append_bio(struct icl_conn *ic, struct icl_pdu *ip, 620 struct bio *bp, size_t offset, size_t len, int flags) 621{ 622 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 623 struct mbuf *m, *m_tail; 624 vm_offset_t vaddr; 625 size_t page_offset, todo, mtodo; 626 bool mapped; 627 int i; 628 629 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 630 MPASS(ic == ip->ip_conn); 631 KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len)); 632 633 m_tail = ip->ip_data_mbuf; 634 if (m_tail != NULL) 635 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next) 636 ; 637 638 MPASS(bp->bio_flags & BIO_UNMAPPED); 639 if (offset < PAGE_SIZE - bp->bio_ma_offset) { 640 page_offset = bp->bio_ma_offset + offset; 641 i = 0; 642 } else { 643 offset -= PAGE_SIZE - bp->bio_ma_offset; 644 for (i = 1; offset >= PAGE_SIZE; i++) 645 offset -= PAGE_SIZE; 646 page_offset = offset; 647 } 648 649 if (flags & ICL_NOCOPY) { 650 m = NULL; 651 while (len > 0) { 652 if (m == NULL) { 653 m = mb_alloc_ext_pgs(flags & ~ICL_NOCOPY, 654 cxgbei_free_mext_pg); 655 if (__predict_false(m == NULL)) 656 return (ENOMEM); 657 atomic_add_int(&icp->ref_cnt, 1); 658 m->m_ext.ext_arg1 = icp; 659 m->m_epg_1st_off = page_offset; 660 } 661 662 todo = MIN(len, PAGE_SIZE - page_offset); 663 664 m->m_epg_pa[m->m_epg_npgs] = 665 VM_PAGE_TO_PHYS(bp->bio_ma[i]); 666 m->m_epg_npgs++; 667 m->m_epg_last_len = todo; 668 m->m_len += todo; 669 m->m_ext.ext_size += PAGE_SIZE; 670 MBUF_EXT_PGS_ASSERT_SANITY(m); 671 672 if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) { 673 if (m_tail != NULL) 674 m_tail->m_next = m; 675 else 676 ip->ip_data_mbuf = m; 677 m_tail = m; 678 ip->ip_data_len += m->m_len; 679 m = NULL; 680 } 681 682 page_offset = 0; 683 len -= todo; 684 i++; 685 } 686 687 if (m != NULL) { 688 if (m_tail != NULL) 689 m_tail->m_next = m; 690 else 691 ip->ip_data_mbuf = m; 692 ip->ip_data_len += m->m_len; 693 } 694 return (0); 695 } 696 697 m = cxgbei_getm(len, flags); 698 if (__predict_false(m == NULL)) 699 return (ENOMEM); 700 701 if (ip->ip_data_mbuf == NULL) { 702 ip->ip_data_mbuf = m; 703 ip->ip_data_len = len; 704 } else { 705 m_tail->m_next = m; 706 ip->ip_data_len += len; 707 } 708 709 while (len > 0) { 710 todo = MIN(len, PAGE_SIZE - page_offset); 711 712 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1, 713 false); 714 715 do { 716 mtodo = min(todo, M_SIZE(m) - m->m_len); 717 memcpy(mtod(m, char *) + m->m_len, (char *)vaddr + 718 page_offset, mtodo); 719 m->m_len += mtodo; 720 if (m->m_len == M_SIZE(m)) 721 m = m->m_next; 722 page_offset += mtodo; 723 todo -= mtodo; 724 } while (todo > 0); 725 726 if (__predict_false(mapped)) 727 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1, 728 false); 729 730 page_offset = 0; 731 len -= todo; 732 i++; 733 } 734 735 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length, 736 ic->ic_hw_isomax)); 737 738 return (0); 739} 740 741int 742icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip, 743 const void *addr, size_t len, int flags) 744{ 745 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 746 struct mbuf *m, *m_tail; 747 const char *src; 748 749 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 750 MPASS(ic == ip->ip_conn); 751 KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len)); 752 753 m_tail = ip->ip_data_mbuf; 754 if (m_tail != NULL) 755 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next) 756 ; 757 758 if (flags & ICL_NOCOPY) { 759 m = m_get(flags & ~ICL_NOCOPY, MT_DATA); 760 if (m == NULL) { 761 ICL_WARN("failed to allocate mbuf"); 762 return (ENOMEM); 763 } 764 765 m->m_flags |= M_RDONLY; 766 m_extaddref(m, __DECONST(char *, addr), len, &icp->ref_cnt, 767 icl_cxgbei_mbuf_done, icp, NULL); 768 m->m_len = len; 769 if (ip->ip_data_mbuf == NULL) { 770 ip->ip_data_mbuf = m; 771 ip->ip_data_len = len; 772 } else { 773 m_tail->m_next = m; 774 m_tail = m_tail->m_next; 775 ip->ip_data_len += len; 776 } 777 778 return (0); 779 } 780 781 m = cxgbei_getm(len, flags); 782 if (__predict_false(m == NULL)) 783 return (ENOMEM); 784 785 if (ip->ip_data_mbuf == NULL) { 786 ip->ip_data_mbuf = m; 787 ip->ip_data_len = len; 788 } else { 789 m_tail->m_next = m; 790 ip->ip_data_len += len; 791 } 792 src = (const char *)addr; 793 for (; m != NULL; m = m->m_next) { 794 m->m_len = min(len, M_SIZE(m)); 795 memcpy(mtod(m, void *), src, m->m_len); 796 src += m->m_len; 797 len -= m->m_len; 798 } 799 MPASS(len == 0); 800 801 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length, 802 ic->ic_hw_isomax)); 803 804 return (0); 805} 806 807void 808icl_cxgbei_conn_pdu_get_bio(struct icl_conn *ic, struct icl_pdu *ip, 809 size_t pdu_off, struct bio *bp, size_t bio_off, size_t len) 810{ 811 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 812 vm_offset_t vaddr; 813 size_t page_offset, todo; 814 bool mapped; 815 int i; 816 817 if (icp->icp_flags & ICPF_RX_DDP) 818 return; /* data is DDP'ed, no need to copy */ 819 820 MPASS(bp->bio_flags & BIO_UNMAPPED); 821 if (bio_off < PAGE_SIZE - bp->bio_ma_offset) { 822 page_offset = bp->bio_ma_offset + bio_off; 823 i = 0; 824 } else { 825 bio_off -= PAGE_SIZE - bp->bio_ma_offset; 826 for (i = 1; bio_off >= PAGE_SIZE; i++) 827 bio_off -= PAGE_SIZE; 828 page_offset = bio_off; 829 } 830 831 while (len > 0) { 832 todo = MIN(len, PAGE_SIZE - page_offset); 833 834 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1, 835 false); 836 m_copydata(ip->ip_data_mbuf, pdu_off, todo, (char *)vaddr + 837 page_offset); 838 if (__predict_false(mapped)) 839 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1, 840 false); 841 842 page_offset = 0; 843 pdu_off += todo; 844 len -= todo; 845 i++; 846 } 847} 848 849void 850icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 851 size_t off, void *addr, size_t len) 852{ 853 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 854 855 if (icp->icp_flags & ICPF_RX_DDP) 856 return; /* data is DDP'ed, no need to copy */ 857 m_copydata(ip->ip_data_mbuf, off, len, addr); 858} 859 860void 861icl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 862{ 863 icl_cxgbei_conn_pdu_queue_cb(ic, ip, NULL); 864} 865 866void 867icl_cxgbei_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip, 868 icl_pdu_cb cb) 869{ 870 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 871 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 872 struct socket *so = ic->ic_socket; 873 874 MPASS(ic == ip->ip_conn); 875 MPASS(ip->ip_bhs_mbuf != NULL); 876 /* The kernel doesn't generate PDUs with AHS. */ 877 MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0); 878 879 ICL_CONN_LOCK_ASSERT(ic); 880 881 icp->cb = cb; 882 883 /* NOTE: sowriteable without so_snd lock is a mostly harmless race. */ 884 if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) { 885 icl_cxgbei_pdu_done(ip, ENOTCONN); 886 return; 887 } 888 889 STAILQ_INSERT_TAIL(&icc->sent_pdus, ip, ip_next); 890 if (!icc->tx_active) { 891 icc->tx_active = true; 892 wakeup(&icc->tx_active); 893 } 894} 895 896static struct icl_conn * 897icl_cxgbei_new_conn(const char *name, struct mtx *lock) 898{ 899 struct icl_cxgbei_conn *icc; 900 struct icl_conn *ic; 901 902 refcount_acquire(&icl_cxgbei_ncons); 903 904 icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE, 905 M_WAITOK | M_ZERO); 906 icc->icc_signature = CXGBEI_CONN_SIGNATURE; 907 STAILQ_INIT(&icc->rcvd_pdus); 908 STAILQ_INIT(&icc->sent_pdus); 909 910 icc->cmp_table = hashinit(64, M_CXGBEI, &icc->cmp_hash_mask); 911 mtx_init(&icc->cmp_lock, "cxgbei_cmp", NULL, MTX_DEF); 912 913 ic = &icc->ic; 914 ic->ic_lock = lock; 915 916#ifdef DIAGNOSTIC 917 refcount_init(&ic->ic_outstanding_pdus, 0); 918#endif 919 ic->ic_name = name; 920 ic->ic_offload = "cxgbei"; 921 ic->ic_unmapped = true; 922 923 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); 924 925 return (ic); 926} 927 928void 929icl_cxgbei_conn_free(struct icl_conn *ic) 930{ 931 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 932 933 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 934 935 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); 936 937 mtx_destroy(&icc->cmp_lock); 938 hashdestroy(icc->cmp_table, M_CXGBEI, icc->cmp_hash_mask); 939 kobj_delete((struct kobj *)icc, M_CXGBE); 940 refcount_release(&icl_cxgbei_ncons); 941} 942 943static int 944icl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so, int sspace, 945 int rspace) 946{ 947 struct sockopt opt; 948 int error, one = 1, ss, rs; 949 950 ss = max(sendspace, sspace); 951 rs = max(recvspace, rspace); 952 953 error = soreserve(so, ss, rs); 954 if (error != 0) 955 return (error); 956 SOCKBUF_LOCK(&so->so_snd); 957 so->so_snd.sb_flags |= SB_AUTOSIZE; 958 SOCKBUF_UNLOCK(&so->so_snd); 959 SOCKBUF_LOCK(&so->so_rcv); 960 so->so_rcv.sb_flags |= SB_AUTOSIZE; 961 SOCKBUF_UNLOCK(&so->so_rcv); 962 963 /* 964 * Disable Nagle. 965 */ 966 bzero(&opt, sizeof(opt)); 967 opt.sopt_dir = SOPT_SET; 968 opt.sopt_level = IPPROTO_TCP; 969 opt.sopt_name = TCP_NODELAY; 970 opt.sopt_val = &one; 971 opt.sopt_valsize = sizeof(one); 972 error = sosetopt(so, &opt); 973 if (error != 0) 974 return (error); 975 976 return (0); 977} 978 979/* 980 * Request/response structure used to find out the adapter offloading a socket. 981 */ 982struct find_ofld_adapter_rr { 983 struct socket *so; 984 struct adapter *sc; /* result */ 985}; 986 987static void 988find_offload_adapter(struct adapter *sc, void *arg) 989{ 990 struct find_ofld_adapter_rr *fa = arg; 991 struct socket *so = fa->so; 992 struct tom_data *td = sc->tom_softc; 993 struct tcpcb *tp; 994 struct inpcb *inp; 995 996 /* Non-TCP were filtered out earlier. */ 997 MPASS(so->so_proto->pr_protocol == IPPROTO_TCP); 998 999 if (fa->sc != NULL) 1000 return; /* Found already. */ 1001 1002 if (td == NULL) 1003 return; /* TOE not enabled on this adapter. */ 1004 1005 inp = sotoinpcb(so); 1006 INP_WLOCK(inp); 1007 if ((inp->inp_flags & INP_DROPPED) == 0) { 1008 tp = intotcpcb(inp); 1009 if (tp->t_flags & TF_TOE && tp->tod == &td->tod) 1010 fa->sc = sc; /* Found. */ 1011 } 1012 INP_WUNLOCK(inp); 1013} 1014 1015static bool 1016is_memfree(struct adapter *sc) 1017{ 1018 uint32_t em; 1019 1020 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1021 if ((em & F_EXT_MEM_ENABLE) != 0) 1022 return (false); 1023 if (is_t5(sc) && (em & F_EXT_MEM1_ENABLE) != 0) 1024 return (false); 1025 return (true); 1026} 1027 1028/* XXXNP: move this to t4_tom. */ 1029static void 1030send_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen) 1031{ 1032 struct wrqe *wr; 1033 struct fw_flowc_wr *flowc; 1034 const u_int nparams = 1; 1035 u_int flowclen; 1036 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 1037 1038 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); 1039 1040 wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq); 1041 if (wr == NULL) { 1042 /* XXX */ 1043 panic("%s: allocation failure.", __func__); 1044 } 1045 flowc = wrtod(wr); 1046 memset(flowc, 0, wr->wr_len); 1047 1048 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 1049 V_FW_FLOWC_WR_NPARAMS(nparams)); 1050 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | 1051 V_FW_WR_FLOWID(toep->tid)); 1052 1053 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; 1054 flowc->mnemval[0].val = htobe32(maxlen); 1055 1056 txsd->tx_credits = howmany(flowclen, 16); 1057 txsd->plen = 0; 1058 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 1059 ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 1060 toep->tx_credits -= txsd->tx_credits; 1061 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 1062 toep->txsd_pidx = 0; 1063 toep->txsd_avail--; 1064 1065 t4_wrq_tx(sc, wr); 1066} 1067 1068static void 1069set_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, u_int ulp_submode) 1070{ 1071 uint64_t val; 1072 1073 CTR3(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, submode=%#x", 1074 __func__, toep->tid, ulp_submode); 1075 1076 val = V_TCB_ULP_TYPE(ULP_MODE_ISCSI) | V_TCB_ULP_RAW(ulp_submode); 1077 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_ULP_TYPE, 1078 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE) | V_TCB_ULP_RAW(M_TCB_ULP_RAW), val, 1079 0, 0); 1080 1081 val = V_TF_RX_FLOW_CONTROL_DISABLE(1ULL); 1082 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, val, val, 0, 0); 1083} 1084 1085/* 1086 * XXXNP: Who is responsible for cleaning up the socket if this returns with an 1087 * error? Review all error paths. 1088 * 1089 * XXXNP: What happens to the socket's fd reference if the operation is 1090 * successful, and how does that affect the socket's life cycle? 1091 */ 1092int 1093icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd) 1094{ 1095 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1096 struct find_ofld_adapter_rr fa; 1097 struct file *fp; 1098 struct socket *so; 1099 struct inpcb *inp; 1100 struct tcpcb *tp; 1101 struct toepcb *toep; 1102 cap_rights_t rights; 1103 u_int max_iso_payload, max_rx_pdu_len, max_tx_pdu_len; 1104 int error, max_iso_pdus; 1105 1106 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 1107 ICL_CONN_LOCK_ASSERT_NOT(ic); 1108 1109 /* 1110 * Steal the socket from userland. 1111 */ 1112 error = fget(curthread, fd, 1113 cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); 1114 if (error != 0) 1115 return (error); 1116 if (fp->f_type != DTYPE_SOCKET) { 1117 fdrop(fp, curthread); 1118 return (EINVAL); 1119 } 1120 so = fp->f_data; 1121 if (so->so_type != SOCK_STREAM || 1122 so->so_proto->pr_protocol != IPPROTO_TCP) { 1123 fdrop(fp, curthread); 1124 return (EINVAL); 1125 } 1126 1127 ICL_CONN_LOCK(ic); 1128 if (ic->ic_socket != NULL) { 1129 ICL_CONN_UNLOCK(ic); 1130 fdrop(fp, curthread); 1131 return (EBUSY); 1132 } 1133 ic->ic_disconnecting = false; 1134 ic->ic_socket = so; 1135 fp->f_ops = &badfileops; 1136 fp->f_data = NULL; 1137 fdrop(fp, curthread); 1138 ICL_CONN_UNLOCK(ic); 1139 1140 /* Find the adapter offloading this socket. */ 1141 fa.sc = NULL; 1142 fa.so = so; 1143 t4_iterate(find_offload_adapter, &fa); 1144 if (fa.sc == NULL) { 1145 error = EINVAL; 1146 goto out; 1147 } 1148 icc->sc = fa.sc; 1149 1150 max_rx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_recv_data_segment_length; 1151 max_tx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_send_data_segment_length; 1152 if (ic->ic_header_crc32c) { 1153 max_rx_pdu_len += ISCSI_HEADER_DIGEST_SIZE; 1154 max_tx_pdu_len += ISCSI_HEADER_DIGEST_SIZE; 1155 } 1156 if (ic->ic_data_crc32c) { 1157 max_rx_pdu_len += ISCSI_DATA_DIGEST_SIZE; 1158 max_tx_pdu_len += ISCSI_DATA_DIGEST_SIZE; 1159 } 1160 1161 inp = sotoinpcb(so); 1162 INP_WLOCK(inp); 1163 tp = intotcpcb(inp); 1164 if (inp->inp_flags & INP_DROPPED) { 1165 INP_WUNLOCK(inp); 1166 error = ENOTCONN; 1167 goto out; 1168 } 1169 1170 /* 1171 * socket could not have been "unoffloaded" if here. 1172 */ 1173 MPASS(tp->t_flags & TF_TOE); 1174 MPASS(tp->tod != NULL); 1175 MPASS(tp->t_toe != NULL); 1176 toep = tp->t_toe; 1177 MPASS(toep->vi->adapter == icc->sc); 1178 1179 if (ulp_mode(toep) != ULP_MODE_NONE) { 1180 INP_WUNLOCK(inp); 1181 error = EINVAL; 1182 goto out; 1183 } 1184 1185 icc->toep = toep; 1186 1187 icc->ulp_submode = 0; 1188 if (ic->ic_header_crc32c) 1189 icc->ulp_submode |= ULP_CRC_HEADER; 1190 if (ic->ic_data_crc32c) 1191 icc->ulp_submode |= ULP_CRC_DATA; 1192 1193 if (icc->sc->tt.iso && chip_id(icc->sc) >= CHELSIO_T5 && 1194 !is_memfree(icc->sc)) { 1195 max_iso_payload = rounddown(CXGBEI_MAX_ISO_PAYLOAD, 1196 tp->t_maxseg); 1197 max_iso_pdus = max_iso_payload / max_tx_pdu_len; 1198 ic->ic_hw_isomax = max_iso_pdus * 1199 ic->ic_max_send_data_segment_length; 1200 } else 1201 max_iso_pdus = 1; 1202 1203 toep->params.ulp_mode = ULP_MODE_ISCSI; 1204 toep->ulpcb = icc; 1205 1206 send_iscsi_flowc_wr(icc->sc, toep, 1207 roundup(max_iso_pdus * max_tx_pdu_len, tp->t_maxseg)); 1208 set_ulp_mode_iscsi(icc->sc, toep, icc->ulp_submode); 1209 INP_WUNLOCK(inp); 1210 1211 error = kthread_add(icl_cxgbei_tx_main, icc, NULL, &icc->tx_thread, 0, 1212 0, "%stx (cxgbei)", ic->ic_name); 1213 if (error != 0) 1214 goto out; 1215 1216 error = kthread_add(icl_cxgbei_rx_main, icc, NULL, &icc->rx_thread, 0, 1217 0, "%srx (cxgbei)", ic->ic_name); 1218 if (error != 0) 1219 goto out; 1220 1221 error = icl_cxgbei_setsockopt(ic, so, max_tx_pdu_len, max_rx_pdu_len); 1222out: 1223 if (error != 0) 1224 icl_cxgbei_conn_close(ic); 1225 return (error); 1226} 1227 1228void 1229icl_cxgbei_conn_close(struct icl_conn *ic) 1230{ 1231 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1232 struct icl_pdu *ip; 1233 struct socket *so; 1234 struct sockbuf *sb; 1235 struct inpcb *inp; 1236 struct toepcb *toep = icc->toep; 1237 1238 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 1239 ICL_CONN_LOCK_ASSERT_NOT(ic); 1240 1241 ICL_CONN_LOCK(ic); 1242 so = ic->ic_socket; 1243 if (ic->ic_disconnecting || so == NULL) { 1244 CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p", 1245 __func__, icc, ic->ic_disconnecting, so); 1246 ICL_CONN_UNLOCK(ic); 1247 return; 1248 } 1249 ic->ic_disconnecting = true; 1250 1251#ifdef DIAGNOSTIC 1252 KASSERT(ic->ic_outstanding_pdus == 0, 1253 ("destroying session with %d outstanding PDUs", 1254 ic->ic_outstanding_pdus)); 1255#endif 1256 1257 CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1, 1258 icc); 1259 1260 /* 1261 * Wait for the transmit thread to stop processing 1262 * this connection. 1263 */ 1264 if (icc->tx_thread != NULL) { 1265 wakeup(&icc->tx_active); 1266 mtx_sleep(icc->tx_thread, ic->ic_lock, 0, "conclo", 0); 1267 } 1268 1269 /* Discard PDUs queued for TX. */ 1270 while (!STAILQ_EMPTY(&icc->sent_pdus)) { 1271 ip = STAILQ_FIRST(&icc->sent_pdus); 1272 STAILQ_REMOVE_HEAD(&icc->sent_pdus, ip_next); 1273 icl_cxgbei_pdu_done(ip, ENOTCONN); 1274 } 1275 ICL_CONN_UNLOCK(ic); 1276 1277 inp = sotoinpcb(so); 1278 sb = &so->so_rcv; 1279 1280 /* 1281 * Wait for the receive thread to stop processing this 1282 * connection. 1283 */ 1284 SOCKBUF_LOCK(sb); 1285 if (icc->rx_thread != NULL) { 1286 icc->rx_exiting = true; 1287 wakeup(&icc->rx_active); 1288 mtx_sleep(icc->rx_thread, SOCKBUF_MTX(sb), 0, "conclo", 0); 1289 } 1290 1291 /* 1292 * Discard received PDUs not passed to the iSCSI layer. 1293 */ 1294 while (!STAILQ_EMPTY(&icc->rcvd_pdus)) { 1295 ip = STAILQ_FIRST(&icc->rcvd_pdus); 1296 STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next); 1297 icl_cxgbei_pdu_done(ip, ENOTCONN); 1298 } 1299 SOCKBUF_UNLOCK(sb); 1300 1301 INP_WLOCK(inp); 1302 if (toep != NULL) { /* NULL if connection was never offloaded. */ 1303 toep->ulpcb = NULL; 1304 1305 /* Discard mbufs queued for TX. */ 1306 mbufq_drain(&toep->ulp_pduq); 1307 1308 /* 1309 * Grab a reference to use when waiting for the final 1310 * CPL to be received. If toep->inp is NULL, then 1311 * final_cpl_received() has already been called (e.g. 1312 * due to the peer sending a RST). 1313 */ 1314 if (toep->inp != NULL) { 1315 toep = hold_toepcb(toep); 1316 toep->flags |= TPF_WAITING_FOR_FINAL; 1317 } else 1318 toep = NULL; 1319 } 1320 INP_WUNLOCK(inp); 1321 1322 ICL_CONN_LOCK(ic); 1323 ic->ic_socket = NULL; 1324 ICL_CONN_UNLOCK(ic); 1325 1326 /* 1327 * XXXNP: we should send RST instead of FIN when PDUs held in various 1328 * queues were purged instead of delivered reliably but soabort isn't 1329 * really general purpose and wouldn't do the right thing here. 1330 */ 1331 soclose(so); 1332 1333 /* 1334 * Wait for the socket to fully close. This ensures any 1335 * pending received data has been received (and in particular, 1336 * any data that would be received by DDP has been handled). 1337 * Callers assume that it is safe to free buffers for tasks 1338 * and transfers after this function returns. 1339 */ 1340 if (toep != NULL) { 1341 struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep); 1342 1343 mtx_lock(lock); 1344 while ((toep->flags & TPF_WAITING_FOR_FINAL) != 0) 1345 mtx_sleep(toep, lock, PSOCK, "conclo2", 0); 1346 mtx_unlock(lock); 1347 free_toepcb(toep); 1348 } 1349} 1350 1351static void 1352cxgbei_insert_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp, 1353 uint32_t tt) 1354{ 1355#ifdef INVARIANTS 1356 struct cxgbei_cmp *cmp2; 1357#endif 1358 1359 cmp->tt = tt; 1360 1361 mtx_lock(&icc->cmp_lock); 1362#ifdef INVARIANTS 1363 LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, tt)], link) { 1364 KASSERT(cmp2->tt != tt, ("%s: duplicate cmp", __func__)); 1365 } 1366#endif 1367 LIST_INSERT_HEAD(&icc->cmp_table[TT_HASH(icc, tt)], cmp, link); 1368 mtx_unlock(&icc->cmp_lock); 1369} 1370 1371struct cxgbei_cmp * 1372cxgbei_find_cmp(struct icl_cxgbei_conn *icc, uint32_t tt) 1373{ 1374 struct cxgbei_cmp *cmp; 1375 1376 mtx_lock(&icc->cmp_lock); 1377 LIST_FOREACH(cmp, &icc->cmp_table[TT_HASH(icc, tt)], link) { 1378 if (cmp->tt == tt) 1379 break; 1380 } 1381 mtx_unlock(&icc->cmp_lock); 1382 return (cmp); 1383} 1384 1385static void 1386cxgbei_rm_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp) 1387{ 1388#ifdef INVARIANTS 1389 struct cxgbei_cmp *cmp2; 1390#endif 1391 1392 mtx_lock(&icc->cmp_lock); 1393 1394#ifdef INVARIANTS 1395 LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, cmp->tt)], link) { 1396 if (cmp2 == cmp) 1397 goto found; 1398 } 1399 panic("%s: could not find cmp", __func__); 1400found: 1401#endif 1402 LIST_REMOVE(cmp, link); 1403 mtx_unlock(&icc->cmp_lock); 1404} 1405 1406int 1407icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 1408 struct ccb_scsiio *csio, uint32_t *ittp, void **arg) 1409{ 1410 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1411 struct toepcb *toep = icc->toep; 1412 struct adapter *sc = icc->sc; 1413 struct cxgbei_data *ci = sc->iscsi_ulp_softc; 1414 struct ppod_region *pr = &ci->pr; 1415 struct cxgbei_ddp_state *ddp; 1416 struct ppod_reservation *prsv; 1417 struct inpcb *inp; 1418 struct mbufq mq; 1419 uint32_t itt; 1420 int rc = 0; 1421 1422 ICL_CONN_LOCK_ASSERT(ic); 1423 1424 /* This is for the offload driver's state. Must not be set already. */ 1425 MPASS(arg != NULL); 1426 MPASS(*arg == NULL); 1427 1428 if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN || 1429 csio->dxfer_len < ci->ddp_threshold || ic->ic_disconnecting || 1430 ic->ic_socket == NULL) { 1431no_ddp: 1432 /* 1433 * No DDP for this I/O. Allocate an ITT (based on the one 1434 * passed in) that cannot be a valid hardware DDP tag in the 1435 * iSCSI region. 1436 */ 1437 itt = *ittp & M_PPOD_TAG; 1438 itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit; 1439 *ittp = htobe32(itt); 1440 MPASS(*arg == NULL); /* State is maintained for DDP only. */ 1441 if (rc != 0) 1442 counter_u64_add( 1443 toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1); 1444 return (0); 1445 } 1446 1447 /* 1448 * Reserve resources for DDP, update the itt that should be used in the 1449 * PDU, and save DDP specific state for this I/O in *arg. 1450 */ 1451 ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO); 1452 if (ddp == NULL) { 1453 rc = ENOMEM; 1454 goto no_ddp; 1455 } 1456 prsv = &ddp->prsv; 1457 1458 mbufq_init(&mq, INT_MAX); 1459 switch (csio->ccb_h.flags & CAM_DATA_MASK) { 1460 case CAM_DATA_BIO: 1461 rc = t4_alloc_page_pods_for_bio(pr, 1462 (struct bio *)csio->data_ptr, prsv); 1463 if (rc != 0) { 1464 free(ddp, M_CXGBEI); 1465 goto no_ddp; 1466 } 1467 1468 rc = t4_write_page_pods_for_bio(sc, toep, prsv, 1469 (struct bio *)csio->data_ptr, &mq); 1470 if (__predict_false(rc != 0)) { 1471 mbufq_drain(&mq); 1472 t4_free_page_pods(prsv); 1473 free(ddp, M_CXGBEI); 1474 goto no_ddp; 1475 } 1476 break; 1477 case CAM_DATA_VADDR: 1478 rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr, 1479 csio->dxfer_len, prsv); 1480 if (rc != 0) { 1481 free(ddp, M_CXGBEI); 1482 goto no_ddp; 1483 } 1484 1485 rc = t4_write_page_pods_for_buf(sc, toep, prsv, 1486 (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq); 1487 if (__predict_false(rc != 0)) { 1488 mbufq_drain(&mq); 1489 t4_free_page_pods(prsv); 1490 free(ddp, M_CXGBEI); 1491 goto no_ddp; 1492 } 1493 break; 1494 default: 1495 free(ddp, M_CXGBEI); 1496 rc = EINVAL; 1497 goto no_ddp; 1498 } 1499 1500 /* 1501 * Do not get inp from toep->inp as the toepcb might have 1502 * detached already. 1503 */ 1504 inp = sotoinpcb(ic->ic_socket); 1505 INP_WLOCK(inp); 1506 if ((inp->inp_flags & INP_DROPPED) != 0) { 1507 INP_WUNLOCK(inp); 1508 mbufq_drain(&mq); 1509 t4_free_page_pods(prsv); 1510 free(ddp, M_CXGBEI); 1511 goto no_ddp; 1512 } 1513 mbufq_concat(&toep->ulp_pduq, &mq); 1514 INP_WUNLOCK(inp); 1515 1516 ddp->cmp.last_datasn = -1; 1517 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); 1518 *ittp = htobe32(prsv->prsv_tag); 1519 *arg = prsv; 1520 counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1); 1521 return (0); 1522} 1523 1524void 1525icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg) 1526{ 1527 1528 if (arg != NULL) { 1529 struct cxgbei_ddp_state *ddp = arg; 1530 1531 cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp); 1532 t4_free_page_pods(&ddp->prsv); 1533 free(ddp, M_CXGBEI); 1534 } 1535} 1536 1537static inline bool 1538ddp_sgl_check(struct ctl_sg_entry *sg, int entries, int xferlen) 1539{ 1540#ifdef INVARIANTS 1541 int total_len = 0; 1542#endif 1543 1544 MPASS(entries > 0); 1545 if (((vm_offset_t)sg[--entries].addr & 3U) != 0) 1546 return (false); 1547 1548#ifdef INVARIANTS 1549 total_len += sg[entries].len; 1550#endif 1551 1552 while (--entries >= 0) { 1553 if (((vm_offset_t)sg[entries].addr & PAGE_MASK) != 0 || 1554 (sg[entries].len % PAGE_SIZE) != 0) 1555 return (false); 1556#ifdef INVARIANTS 1557 total_len += sg[entries].len; 1558#endif 1559 } 1560 1561 MPASS(total_len == xferlen); 1562 return (true); 1563} 1564 1565#define io_to_ddp_state(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr) 1566 1567int 1568icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, struct icl_pdu *ip, 1569 union ctl_io *io, uint32_t *tttp, void **arg) 1570{ 1571 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1572 struct toepcb *toep = icc->toep; 1573 struct ctl_scsiio *ctsio = &io->scsiio; 1574 struct adapter *sc = icc->sc; 1575 struct cxgbei_data *ci = sc->iscsi_ulp_softc; 1576 struct ppod_region *pr = &ci->pr; 1577 struct cxgbei_ddp_state *ddp; 1578 struct ppod_reservation *prsv; 1579 struct ctl_sg_entry *sgl, sg_entry; 1580 struct inpcb *inp; 1581 struct mbufq mq; 1582 int sg_entries = ctsio->kern_sg_entries; 1583 uint32_t ttt; 1584 int xferlen, rc = 0, alias; 1585 1586 /* This is for the offload driver's state. Must not be set already. */ 1587 MPASS(arg != NULL); 1588 MPASS(*arg == NULL); 1589 1590 if (ctsio->ext_data_filled == 0) { 1591 int first_burst; 1592#ifdef INVARIANTS 1593 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 1594 1595 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 1596 MPASS(ic == ip->ip_conn); 1597 MPASS(ip->ip_bhs_mbuf != NULL); 1598#endif 1599 first_burst = icl_pdu_data_segment_length(ip); 1600 1601 /* 1602 * Note that ICL calls conn_transfer_setup even if the first 1603 * burst had everything and there's nothing left to transfer. 1604 * 1605 * NB: The CTL frontend might have provided a buffer 1606 * whose length (kern_data_len) is smaller than the 1607 * FirstBurstLength of unsolicited data. Treat those 1608 * as an empty transfer. 1609 */ 1610 xferlen = ctsio->kern_data_len; 1611 if (xferlen < first_burst || 1612 xferlen - first_burst < ci->ddp_threshold) { 1613no_ddp: 1614 /* 1615 * No DDP for this transfer. Allocate a TTT (based on 1616 * the one passed in) that cannot be a valid hardware 1617 * DDP tag in the iSCSI region. 1618 */ 1619 ttt = *tttp & M_PPOD_TAG; 1620 ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit; 1621 *tttp = htobe32(ttt); 1622 MPASS(io_to_ddp_state(io) == NULL); 1623 if (rc != 0) 1624 counter_u64_add( 1625 toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1); 1626 return (0); 1627 } 1628 1629 if (sg_entries == 0) { 1630 sgl = &sg_entry; 1631 sgl->len = xferlen; 1632 sgl->addr = (void *)ctsio->kern_data_ptr; 1633 sg_entries = 1; 1634 } else 1635 sgl = (void *)ctsio->kern_data_ptr; 1636 1637 if (!ddp_sgl_check(sgl, sg_entries, xferlen)) 1638 goto no_ddp; 1639 1640 /* 1641 * Reserve resources for DDP, update the ttt that should be used 1642 * in the PDU, and save DDP specific state for this I/O. 1643 */ 1644 MPASS(io_to_ddp_state(io) == NULL); 1645 ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO); 1646 if (ddp == NULL) { 1647 rc = ENOMEM; 1648 goto no_ddp; 1649 } 1650 prsv = &ddp->prsv; 1651 1652 rc = t4_alloc_page_pods_for_sgl(pr, sgl, sg_entries, prsv); 1653 if (rc != 0) { 1654 free(ddp, M_CXGBEI); 1655 goto no_ddp; 1656 } 1657 1658 mbufq_init(&mq, INT_MAX); 1659 rc = t4_write_page_pods_for_sgl(sc, toep, prsv, sgl, sg_entries, 1660 xferlen, &mq); 1661 if (__predict_false(rc != 0)) { 1662 mbufq_drain(&mq); 1663 t4_free_page_pods(prsv); 1664 free(ddp, M_CXGBEI); 1665 goto no_ddp; 1666 } 1667 1668 /* 1669 * Do not get inp from toep->inp as the toepcb might 1670 * have detached already. 1671 */ 1672 ICL_CONN_LOCK(ic); 1673 if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1674 ICL_CONN_UNLOCK(ic); 1675 mbufq_drain(&mq); 1676 t4_free_page_pods(prsv); 1677 free(ddp, M_CXGBEI); 1678 return (ECONNRESET); 1679 } 1680 inp = sotoinpcb(ic->ic_socket); 1681 INP_WLOCK(inp); 1682 ICL_CONN_UNLOCK(ic); 1683 if ((inp->inp_flags & INP_DROPPED) != 0) { 1684 INP_WUNLOCK(inp); 1685 mbufq_drain(&mq); 1686 t4_free_page_pods(prsv); 1687 free(ddp, M_CXGBEI); 1688 return (ECONNRESET); 1689 } 1690 mbufq_concat(&toep->ulp_pduq, &mq); 1691 INP_WUNLOCK(inp); 1692 1693 ddp->cmp.next_buffer_offset = ctsio->kern_rel_offset + 1694 first_burst; 1695 ddp->cmp.last_datasn = -1; 1696 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); 1697 *tttp = htobe32(prsv->prsv_tag); 1698 io_to_ddp_state(io) = ddp; 1699 *arg = ctsio; 1700 counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1); 1701 return (0); 1702 } 1703 1704 /* 1705 * In the middle of an I/O. A non-NULL page pod reservation indicates 1706 * that a DDP buffer is being used for the I/O. 1707 */ 1708 ddp = io_to_ddp_state(ctsio); 1709 if (ddp == NULL) 1710 goto no_ddp; 1711 prsv = &ddp->prsv; 1712 1713 alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift; 1714 alias++; 1715 prsv->prsv_tag &= ~pr->pr_alias_mask; 1716 prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask; 1717 1718 ddp->cmp.last_datasn = -1; 1719 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); 1720 *tttp = htobe32(prsv->prsv_tag); 1721 *arg = ctsio; 1722 1723 return (0); 1724} 1725 1726void 1727icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg) 1728{ 1729 struct ctl_scsiio *ctsio = arg; 1730 1731 if (ctsio != NULL) { 1732 struct cxgbei_ddp_state *ddp; 1733 1734 ddp = io_to_ddp_state(ctsio); 1735 MPASS(ddp != NULL); 1736 1737 cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp); 1738 if (ctsio->kern_data_len == ctsio->ext_data_filled || 1739 ic->ic_disconnecting) { 1740 t4_free_page_pods(&ddp->prsv); 1741 free(ddp, M_CXGBEI); 1742 io_to_ddp_state(ctsio) = NULL; 1743 } 1744 } 1745} 1746 1747#ifdef COMPAT_FREEBSD13 1748static void 1749cxgbei_limits(struct adapter *sc, void *arg) 1750{ 1751 struct icl_drv_limits *idl = arg; 1752 struct cxgbei_data *ci; 1753 int max_dsl; 1754 1755 if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims") != 0) 1756 return; 1757 1758 if (uld_active(sc, ULD_ISCSI)) { 1759 ci = sc->iscsi_ulp_softc; 1760 MPASS(ci != NULL); 1761 1762 1763 max_dsl = ci->max_rx_data_len; 1764 if (idl->idl_max_recv_data_segment_length > max_dsl) 1765 idl->idl_max_recv_data_segment_length = max_dsl; 1766 1767 max_dsl = ci->max_tx_data_len; 1768 if (idl->idl_max_send_data_segment_length > max_dsl) 1769 idl->idl_max_send_data_segment_length = max_dsl; 1770 } 1771 1772 end_synchronized_op(sc, LOCK_HELD); 1773} 1774#endif 1775 1776static int 1777cxgbei_limits_fd(struct icl_drv_limits *idl, int fd) 1778{ 1779 struct find_ofld_adapter_rr fa; 1780 struct file *fp; 1781 struct socket *so; 1782 struct adapter *sc; 1783 struct cxgbei_data *ci; 1784 cap_rights_t rights; 1785 int error; 1786 1787 error = fget(curthread, fd, 1788 cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); 1789 if (error != 0) 1790 return (error); 1791 if (fp->f_type != DTYPE_SOCKET) { 1792 fdrop(fp, curthread); 1793 return (EINVAL); 1794 } 1795 so = fp->f_data; 1796 if (so->so_type != SOCK_STREAM || 1797 so->so_proto->pr_protocol != IPPROTO_TCP) { 1798 fdrop(fp, curthread); 1799 return (EINVAL); 1800 } 1801 1802 /* Find the adapter offloading this socket. */ 1803 fa.sc = NULL; 1804 fa.so = so; 1805 t4_iterate(find_offload_adapter, &fa); 1806 if (fa.sc == NULL) { 1807 fdrop(fp, curthread); 1808 return (ENXIO); 1809 } 1810 fdrop(fp, curthread); 1811 1812 sc = fa.sc; 1813 error = begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims"); 1814 if (error != 0) 1815 return (error); 1816 1817 if (uld_active(sc, ULD_ISCSI)) { 1818 ci = sc->iscsi_ulp_softc; 1819 MPASS(ci != NULL); 1820 1821 idl->idl_max_recv_data_segment_length = ci->max_rx_data_len; 1822 idl->idl_max_send_data_segment_length = ci->max_tx_data_len; 1823 } else 1824 error = ENXIO; 1825 1826 end_synchronized_op(sc, LOCK_HELD); 1827 1828 return (error); 1829} 1830 1831static int 1832icl_cxgbei_limits(struct icl_drv_limits *idl, int socket) 1833{ 1834 1835 /* Maximum allowed by the RFC. cxgbei_limits will clip them. */ 1836 idl->idl_max_recv_data_segment_length = (1 << 24) - 1; 1837 idl->idl_max_send_data_segment_length = (1 << 24) - 1; 1838 1839 /* These are somewhat arbitrary. */ 1840 idl->idl_max_burst_length = max_burst_length; 1841 idl->idl_first_burst_length = first_burst_length; 1842 1843#ifdef COMPAT_FREEBSD13 1844 if (socket == 0) { 1845 t4_iterate(cxgbei_limits, idl); 1846 return (0); 1847 } 1848#endif 1849 1850 return (cxgbei_limits_fd(idl, socket)); 1851} 1852 1853int 1854icl_cxgbei_mod_load(void) 1855{ 1856 int rc; 1857 1858 refcount_init(&icl_cxgbei_ncons, 0); 1859 1860 rc = icl_register("cxgbei", false, -100, icl_cxgbei_limits, 1861 icl_cxgbei_new_conn); 1862 1863 return (rc); 1864} 1865 1866int 1867icl_cxgbei_mod_unload(void) 1868{ 1869 1870 if (icl_cxgbei_ncons != 0) 1871 return (EBUSY); 1872 1873 icl_unregister("cxgbei", false); 1874 1875 return (0); 1876} 1877#endif 1878