icl_cxgbei.c revision 301119
1292740Snp/*- 2292740Snp * Copyright (c) 2012 The FreeBSD Foundation 3292740Snp * Copyright (c) 2015 Chelsio Communications, Inc. 4292740Snp * All rights reserved. 5292740Snp * 6292740Snp * This software was developed by Edward Tomasz Napierala under sponsorship 7292740Snp * from the FreeBSD Foundation. 8292740Snp * 9292740Snp * Redistribution and use in source and binary forms, with or without 10292740Snp * modification, are permitted provided that the following conditions 11292740Snp * are met: 12292740Snp * 1. Redistributions of source code must retain the above copyright 13292740Snp * notice, this list of conditions and the following disclaimer. 14292740Snp * 2. Redistributions in binary form must reproduce the above copyright 15292740Snp * notice, this list of conditions and the following disclaimer in the 16292740Snp * documentation and/or other materials provided with the distribution. 17292740Snp * 18292740Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19292740Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20292740Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21292740Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22292740Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23292740Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24292740Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25292740Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26292740Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27292740Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28292740Snp * SUCH DAMAGE. 29292740Snp * 30292740Snp */ 31292740Snp 32292740Snp/* 33292740Snp * cxgbei implementation of iSCSI Common Layer kobj(9) interface. 34292740Snp */ 35292740Snp 36292740Snp#include <sys/cdefs.h> 37292740Snp__FBSDID("$FreeBSD: head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c 301119 2016-06-01 12:04:04Z trasz $"); 38292740Snp 39292740Snp#include "opt_inet.h" 40292740Snp#include "opt_inet6.h" 41292740Snp 42292740Snp#ifdef TCP_OFFLOAD 43292740Snp#include <sys/param.h> 44292740Snp#include <sys/capsicum.h> 45292740Snp#include <sys/condvar.h> 46292740Snp#include <sys/conf.h> 47292740Snp#include <sys/file.h> 48292740Snp#include <sys/kernel.h> 49292740Snp#include <sys/kthread.h> 50292740Snp#include <sys/lock.h> 51292740Snp#include <sys/mbuf.h> 52292740Snp#include <sys/mutex.h> 53292740Snp#include <sys/module.h> 54292740Snp#include <sys/protosw.h> 55292740Snp#include <sys/socket.h> 56292740Snp#include <sys/socketvar.h> 57292740Snp#include <sys/sysctl.h> 58292740Snp#include <sys/systm.h> 59292740Snp#include <sys/sx.h> 60292740Snp#include <sys/uio.h> 61292740Snp#include <machine/bus.h> 62292740Snp#include <vm/uma.h> 63292740Snp#include <netinet/in.h> 64292740Snp#include <netinet/in_pcb.h> 65292740Snp#include <netinet/tcp.h> 66292740Snp#include <netinet/tcp_var.h> 67292740Snp#include <netinet/toecore.h> 68292740Snp 69292740Snp#include <dev/iscsi/icl.h> 70292740Snp#include <dev/iscsi/iscsi_proto.h> 71292740Snp#include <icl_conn_if.h> 72292740Snp 73292740Snp#include "common/common.h" 74292740Snp#include "tom/t4_tom.h" 75292740Snp#include "cxgbei.h" 76292740Snp 77292740SnpSYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD, 0, "Chelsio iSCSI offload"); 78292740Snpstatic int coalesce = 1; 79292740SnpSYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, coalesce, CTLFLAG_RWTUN, 80292740Snp &coalesce, 0, "Try to coalesce PDUs before sending"); 81292740Snpstatic int partial_receive_len = 128 * 1024; 82292740SnpSYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, 83292740Snp &partial_receive_len, 0, "Minimum read size for partially received " 84292740Snp "data segment"); 85292740Snpstatic int sendspace = 1048576; 86292740SnpSYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN, 87292740Snp &sendspace, 0, "Default send socket buffer size"); 88292740Snpstatic int recvspace = 1048576; 89292740SnpSYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN, 90292740Snp &recvspace, 0, "Default receive socket buffer size"); 91292740Snp 92292740Snpstatic uma_zone_t icl_transfer_zone; 93292740Snp 94292740Snpstatic volatile u_int icl_cxgbei_ncons; 95292740Snp 96292740Snp#define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) 97292740Snp#define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) 98292740Snp#define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) 99292740Snp#define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) 100292740Snp 101292740Snpstruct icl_pdu *icl_cxgbei_new_pdu(int); 102292740Snpvoid icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *); 103292740Snp 104292740Snpstatic icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu; 105292740Snpicl_conn_pdu_free_t icl_cxgbei_conn_pdu_free; 106292740Snpstatic icl_conn_pdu_data_segment_length_t 107292740Snp icl_cxgbei_conn_pdu_data_segment_length; 108292740Snpstatic icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data; 109292740Snpstatic icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data; 110292740Snpstatic icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue; 111292740Snpstatic icl_conn_handoff_t icl_cxgbei_conn_handoff; 112292740Snpstatic icl_conn_free_t icl_cxgbei_conn_free; 113292740Snpstatic icl_conn_close_t icl_cxgbei_conn_close; 114292740Snpstatic icl_conn_task_setup_t icl_cxgbei_conn_task_setup; 115292740Snpstatic icl_conn_task_done_t icl_cxgbei_conn_task_done; 116292740Snpstatic icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup; 117292740Snpstatic icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done; 118292740Snp 119292740Snpstatic kobj_method_t icl_cxgbei_methods[] = { 120292740Snp KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu), 121292740Snp KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free), 122292740Snp KOBJMETHOD(icl_conn_pdu_data_segment_length, 123292740Snp icl_cxgbei_conn_pdu_data_segment_length), 124292740Snp KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data), 125292740Snp KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data), 126292740Snp KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue), 127292740Snp KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff), 128292740Snp KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free), 129292740Snp KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close), 130292740Snp KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup), 131292740Snp KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done), 132292740Snp KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup), 133292740Snp KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done), 134292740Snp { 0, 0 } 135292740Snp}; 136292740Snp 137292740SnpDEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn)); 138292740Snp 139292740Snp#if 0 140292740Snp/* 141292740Snp * Subtract another 256 for AHS from MAX_DSL if AHS could be used. 142292740Snp */ 143292740Snp#define CXGBEI_MAX_PDU 16224 144292740Snp#define CXGBEI_MAX_DSL (CXGBEI_MAX_PDU - sizeof(struct iscsi_bhs) - 8) 145292740Snp#endif 146292740Snp#define CXGBEI_MAX_DSL 8192 147292740Snp#define CXGBEI_MAX_PDU (CXGBEI_MAX_DSL + sizeof(struct iscsi_bhs) + 8) 148292740Snp 149292740Snpvoid 150292740Snpicl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 151292740Snp{ 152292740Snp#ifdef INVARIANTS 153292740Snp struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 154292740Snp#endif 155292740Snp 156292740Snp MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 157292740Snp MPASS(ic == ip->ip_conn); 158292740Snp MPASS(ip->ip_bhs_mbuf != NULL); 159292740Snp 160292740Snp m_freem(ip->ip_ahs_mbuf); 161292740Snp m_freem(ip->ip_data_mbuf); 162292740Snp m_freem(ip->ip_bhs_mbuf); /* storage for icl_cxgbei_pdu itself */ 163292740Snp 164292740Snp#ifdef DIAGNOSTIC 165292740Snp if (__predict_true(ic != NULL)) 166292740Snp refcount_release(&ic->ic_outstanding_pdus); 167292740Snp#endif 168292740Snp} 169292740Snp 170292740Snpstruct icl_pdu * 171292740Snpicl_cxgbei_new_pdu(int flags) 172292740Snp{ 173292740Snp struct icl_cxgbei_pdu *icp; 174292740Snp struct icl_pdu *ip; 175292740Snp struct mbuf *m; 176292740Snp uintptr_t a; 177292740Snp 178292740Snp m = m_gethdr(flags, MT_DATA); 179292740Snp if (__predict_false(m == NULL)) 180292740Snp return (NULL); 181292740Snp 182292740Snp a = roundup2(mtod(m, uintptr_t), _Alignof(struct icl_cxgbei_pdu)); 183292740Snp icp = (struct icl_cxgbei_pdu *)a; 184292740Snp bzero(icp, sizeof(*icp)); 185292740Snp 186292740Snp icp->icp_signature = CXGBEI_PDU_SIGNATURE; 187292740Snp ip = &icp->ip; 188292740Snp ip->ip_bhs_mbuf = m; 189292740Snp 190292740Snp a = roundup2((uintptr_t)(icp + 1), _Alignof(struct iscsi_bhs *)); 191292740Snp ip->ip_bhs = (struct iscsi_bhs *)a; 192292740Snp#ifdef INVARIANTS 193292740Snp /* Everything must fit entirely in the mbuf. */ 194292740Snp a = (uintptr_t)(ip->ip_bhs + 1); 195292740Snp MPASS(a <= (uintptr_t)m + MSIZE); 196292740Snp#endif 197292740Snp bzero(ip->ip_bhs, sizeof(*ip->ip_bhs)); 198292740Snp 199292740Snp m->m_data = (void *)ip->ip_bhs; 200292740Snp m->m_len = sizeof(struct iscsi_bhs); 201292740Snp m->m_pkthdr.len = m->m_len; 202292740Snp 203292740Snp return (ip); 204292740Snp} 205292740Snp 206292740Snpvoid 207292740Snpicl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic) 208292740Snp{ 209292740Snp 210292740Snp ip->ip_conn = ic; 211292740Snp#ifdef DIAGNOSTIC 212292740Snp refcount_acquire(&ic->ic_outstanding_pdus); 213292740Snp#endif 214292740Snp} 215292740Snp 216292740Snp/* 217292740Snp * Allocate icl_pdu with empty BHS to fill up by the caller. 218292740Snp */ 219292740Snpstatic struct icl_pdu * 220292740Snpicl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags) 221292740Snp{ 222292740Snp struct icl_pdu *ip; 223292740Snp 224292740Snp ip = icl_cxgbei_new_pdu(flags); 225292740Snp if (__predict_false(ip == NULL)) 226292740Snp return (NULL); 227292740Snp icl_cxgbei_new_pdu_set_conn(ip, ic); 228292740Snp 229292740Snp return (ip); 230292740Snp} 231292740Snp 232292740Snpstatic size_t 233292740Snpicl_pdu_data_segment_length(const struct icl_pdu *request) 234292740Snp{ 235292740Snp uint32_t len = 0; 236292740Snp 237292740Snp len += request->ip_bhs->bhs_data_segment_len[0]; 238292740Snp len <<= 8; 239292740Snp len += request->ip_bhs->bhs_data_segment_len[1]; 240292740Snp len <<= 8; 241292740Snp len += request->ip_bhs->bhs_data_segment_len[2]; 242292740Snp 243292740Snp return (len); 244292740Snp} 245292740Snp 246292740Snpsize_t 247292740Snpicl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic, 248292740Snp const struct icl_pdu *request) 249292740Snp{ 250292740Snp 251292740Snp return (icl_pdu_data_segment_length(request)); 252292740Snp} 253292740Snp 254292740Snpstatic uint32_t 255292740Snpicl_conn_build_tasktag(struct icl_conn *ic, uint32_t tag) 256292740Snp{ 257292740Snp return tag; 258292740Snp} 259292740Snp 260292740Snpstatic struct mbuf * 261292740Snpfinalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp) 262292740Snp{ 263292740Snp struct icl_pdu *ip = &icp->ip; 264292740Snp uint8_t ulp_submode, padding; 265292740Snp struct mbuf *m, *last; 266292740Snp struct iscsi_bhs *bhs; 267292740Snp 268292740Snp /* 269292740Snp * Fix up the data segment mbuf first. 270292740Snp */ 271292740Snp m = ip->ip_data_mbuf; 272292740Snp ulp_submode = icc->ulp_submode; 273292740Snp if (m) { 274292740Snp last = m_last(m); 275292740Snp 276292740Snp /* 277292740Snp * Round up the data segment to a 4B boundary. Pad with 0 if 278292740Snp * necessary. There will definitely be room in the mbuf. 279292740Snp */ 280292740Snp padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len; 281292740Snp if (padding) { 282292740Snp bzero(mtod(last, uint8_t *) + last->m_len, padding); 283292740Snp last->m_len += padding; 284292740Snp } 285292740Snp } else { 286292740Snp MPASS(ip->ip_data_len == 0); 287292740Snp ulp_submode &= ~ULP_CRC_DATA; 288292740Snp padding = 0; 289292740Snp } 290292740Snp 291292740Snp /* 292292740Snp * Now the header mbuf that has the BHS. 293292740Snp */ 294292740Snp m = ip->ip_bhs_mbuf; 295292740Snp MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs)); 296292740Snp MPASS(m->m_len == sizeof(struct iscsi_bhs)); 297292740Snp 298292740Snp bhs = ip->ip_bhs; 299292740Snp bhs->bhs_data_segment_len[2] = ip->ip_data_len; 300292740Snp bhs->bhs_data_segment_len[1] = ip->ip_data_len >> 8; 301292740Snp bhs->bhs_data_segment_len[0] = ip->ip_data_len >> 16; 302292740Snp 303292740Snp /* "Convert" PDU to mbuf chain. Do not use icp/ip after this. */ 304292740Snp m->m_pkthdr.len = sizeof(struct iscsi_bhs) + ip->ip_data_len + padding; 305292740Snp m->m_next = ip->ip_data_mbuf; 306292740Snp set_mbuf_ulp_submode(m, ulp_submode); 307292740Snp#ifdef INVARIANTS 308292740Snp bzero(icp, sizeof(*icp)); 309292740Snp#endif 310292740Snp#ifdef DIAGNOSTIC 311292740Snp refcount_release(&icc->ic.ic_outstanding_pdus); 312292740Snp#endif 313292740Snp 314292740Snp return (m); 315292740Snp} 316292740Snp 317292740Snpint 318292740Snpicl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip, 319292740Snp const void *addr, size_t len, int flags) 320292740Snp{ 321292740Snp struct mbuf *m; 322292740Snp#ifdef INVARIANTS 323292740Snp struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 324292740Snp#endif 325292740Snp 326292740Snp MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 327292740Snp MPASS(ic == ip->ip_conn); 328292740Snp KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len)); 329292740Snp 330292740Snp m = ip->ip_data_mbuf; 331292740Snp if (m == NULL) { 332292740Snp m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES); 333292740Snp if (__predict_false(m == NULL)) 334292740Snp return (ENOMEM); 335292740Snp 336292740Snp ip->ip_data_mbuf = m; 337292740Snp } 338292740Snp 339292740Snp if (__predict_true(m_append(m, len, addr) != 0)) { 340292740Snp ip->ip_data_len += len; 341292740Snp MPASS(ip->ip_data_len <= CXGBEI_MAX_DSL); 342292740Snp return (0); 343292740Snp } else { 344292740Snp if (flags & M_WAITOK) { 345292740Snp CXGBE_UNIMPLEMENTED("fail safe append"); 346292740Snp } 347292740Snp ip->ip_data_len = m_length(m, NULL); 348292740Snp return (1); 349292740Snp } 350292740Snp} 351292740Snp 352292740Snpvoid 353292740Snpicl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 354292740Snp size_t off, void *addr, size_t len) 355292740Snp{ 356292740Snp struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 357292740Snp 358292740Snp if (icp->pdu_flags & SBUF_ULP_FLAG_DATA_DDPED) 359292740Snp return; /* data is DDP'ed, no need to copy */ 360292740Snp m_copydata(ip->ip_data_mbuf, off, len, addr); 361292740Snp} 362292740Snp 363292740Snpvoid 364292740Snpicl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 365292740Snp{ 366292740Snp struct icl_cxgbei_conn *icc = ic_to_icc(ic); 367292740Snp struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 368292740Snp struct socket *so = ic->ic_socket; 369292740Snp struct toepcb *toep = icc->toep; 370292740Snp struct inpcb *inp; 371292740Snp struct mbuf *m; 372292740Snp 373292740Snp MPASS(ic == ip->ip_conn); 374292740Snp MPASS(ip->ip_bhs_mbuf != NULL); 375292740Snp /* The kernel doesn't generate PDUs with AHS. */ 376292740Snp MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0); 377292740Snp 378292740Snp ICL_CONN_LOCK_ASSERT(ic); 379292740Snp /* NOTE: sowriteable without so_snd lock is a mostly harmless race. */ 380292740Snp if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) { 381292740Snp icl_cxgbei_conn_pdu_free(ic, ip); 382292740Snp return; 383292740Snp } 384292740Snp 385292740Snp m = finalize_pdu(icc, icp); 386292740Snp M_ASSERTPKTHDR(m); 387292740Snp MPASS((m->m_pkthdr.len & 3) == 0); 388292740Snp MPASS(m->m_pkthdr.len + 8 <= CXGBEI_MAX_PDU); 389292740Snp 390292740Snp /* 391292740Snp * Do not get inp from toep->inp as the toepcb might have detached 392292740Snp * already. 393292740Snp */ 394292740Snp inp = sotoinpcb(so); 395292740Snp INP_WLOCK(inp); 396292740Snp if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) || 397292740Snp __predict_false((toep->flags & TPF_ATTACHED) == 0)) 398292740Snp m_freem(m); 399292740Snp else { 400292740Snp mbufq_enqueue(&toep->ulp_pduq, m); 401292740Snp t4_push_pdus(icc->sc, toep, 0); 402292740Snp } 403292740Snp INP_WUNLOCK(inp); 404292740Snp} 405292740Snp 406292740Snpstatic struct icl_conn * 407292740Snpicl_cxgbei_new_conn(const char *name, struct mtx *lock) 408292740Snp{ 409292740Snp struct icl_cxgbei_conn *icc; 410292740Snp struct icl_conn *ic; 411292740Snp 412292740Snp refcount_acquire(&icl_cxgbei_ncons); 413292740Snp 414292740Snp icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE, 415292740Snp M_WAITOK | M_ZERO); 416292740Snp icc->icc_signature = CXGBEI_CONN_SIGNATURE; 417292740Snp STAILQ_INIT(&icc->rcvd_pdus); 418292740Snp 419292740Snp ic = &icc->ic; 420292740Snp ic->ic_lock = lock; 421292740Snp 422292740Snp /* XXXNP: review. Most of these icl_conn fields aren't really used */ 423292740Snp STAILQ_INIT(&ic->ic_to_send); 424292740Snp cv_init(&ic->ic_send_cv, "icl_cxgbei_tx"); 425292740Snp cv_init(&ic->ic_receive_cv, "icl_cxgbei_rx"); 426292740Snp#ifdef DIAGNOSTIC 427292740Snp refcount_init(&ic->ic_outstanding_pdus, 0); 428292740Snp#endif 429292740Snp ic->ic_max_data_segment_length = CXGBEI_MAX_DSL; 430292740Snp ic->ic_name = name; 431292740Snp ic->ic_offload = "cxgbei"; 432300369Strasz ic->ic_unmapped = false; 433292740Snp 434292740Snp CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); 435292740Snp 436292740Snp return (ic); 437292740Snp} 438292740Snp 439292740Snpvoid 440292740Snpicl_cxgbei_conn_free(struct icl_conn *ic) 441292740Snp{ 442292740Snp struct icl_cxgbei_conn *icc = ic_to_icc(ic); 443292740Snp 444292740Snp MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 445292740Snp 446292740Snp CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); 447292740Snp 448292740Snp cv_destroy(&ic->ic_send_cv); 449292740Snp cv_destroy(&ic->ic_receive_cv); 450292740Snp 451292740Snp kobj_delete((struct kobj *)icc, M_CXGBE); 452292740Snp refcount_release(&icl_cxgbei_ncons); 453292740Snp} 454292740Snp 455292740Snpstatic int 456292740Snpicl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so) 457292740Snp{ 458292740Snp size_t minspace; 459292740Snp struct sockopt opt; 460292740Snp int error, one = 1; 461292740Snp 462292740Snp /* 463292740Snp * For sendspace, this is required because the current code cannot 464292740Snp * send a PDU in pieces; thus, the minimum buffer size is equal 465292740Snp * to the maximum PDU size. "+4" is to account for possible padding. 466292740Snp * 467292740Snp * What we should actually do here is to use autoscaling, but set 468292740Snp * some minimal buffer size to "minspace". I don't know a way to do 469292740Snp * that, though. 470292740Snp */ 471292740Snp minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + 472292740Snp ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 473292740Snp if (sendspace < minspace) 474292740Snp sendspace = minspace; 475292740Snp if (recvspace < minspace) 476292740Snp recvspace = minspace; 477292740Snp 478292740Snp error = soreserve(so, sendspace, recvspace); 479292740Snp if (error != 0) { 480292740Snp icl_cxgbei_conn_close(ic); 481292740Snp return (error); 482292740Snp } 483292740Snp SOCKBUF_LOCK(&so->so_snd); 484292740Snp so->so_snd.sb_flags |= SB_AUTOSIZE; 485292740Snp SOCKBUF_UNLOCK(&so->so_snd); 486292740Snp SOCKBUF_LOCK(&so->so_rcv); 487292740Snp so->so_rcv.sb_flags |= SB_AUTOSIZE; 488292740Snp SOCKBUF_UNLOCK(&so->so_rcv); 489292740Snp 490292740Snp /* 491292740Snp * Disable Nagle. 492292740Snp */ 493292740Snp bzero(&opt, sizeof(opt)); 494292740Snp opt.sopt_dir = SOPT_SET; 495292740Snp opt.sopt_level = IPPROTO_TCP; 496292740Snp opt.sopt_name = TCP_NODELAY; 497292740Snp opt.sopt_val = &one; 498292740Snp opt.sopt_valsize = sizeof(one); 499292740Snp error = sosetopt(so, &opt); 500292740Snp if (error != 0) { 501292740Snp icl_cxgbei_conn_close(ic); 502292740Snp return (error); 503292740Snp } 504292740Snp 505292740Snp return (0); 506292740Snp} 507292740Snp 508292740Snp/* 509292740Snp * Request/response structure used to find out the adapter offloading a socket. 510292740Snp */ 511292740Snpstruct find_ofld_adapter_rr { 512292740Snp struct socket *so; 513292740Snp struct adapter *sc; /* result */ 514292740Snp}; 515292740Snp 516292740Snpstatic void 517292740Snpfind_offload_adapter(struct adapter *sc, void *arg) 518292740Snp{ 519292740Snp struct find_ofld_adapter_rr *fa = arg; 520292740Snp struct socket *so = fa->so; 521292740Snp struct tom_data *td = sc->tom_softc; 522292740Snp struct tcpcb *tp; 523292740Snp struct inpcb *inp; 524292740Snp 525292740Snp /* Non-TCP were filtered out earlier. */ 526292740Snp MPASS(so->so_proto->pr_protocol == IPPROTO_TCP); 527292740Snp 528292740Snp if (fa->sc != NULL) 529292740Snp return; /* Found already. */ 530292740Snp 531292740Snp if (td == NULL) 532292740Snp return; /* TOE not enabled on this adapter. */ 533292740Snp 534292740Snp inp = sotoinpcb(so); 535292740Snp INP_WLOCK(inp); 536292740Snp if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 537292740Snp tp = intotcpcb(inp); 538292740Snp if (tp->t_flags & TF_TOE && tp->tod == &td->tod) 539292740Snp fa->sc = sc; /* Found. */ 540292740Snp } 541292740Snp INP_WUNLOCK(inp); 542292740Snp} 543292740Snp 544292740Snp/* XXXNP: move this to t4_tom. */ 545292740Snpstatic void 546292740Snpsend_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen) 547292740Snp{ 548292740Snp struct wrqe *wr; 549292740Snp struct fw_flowc_wr *flowc; 550292740Snp const u_int nparams = 1; 551292740Snp u_int flowclen; 552292740Snp struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 553292740Snp 554292740Snp flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); 555292740Snp 556292740Snp wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); 557292740Snp if (wr == NULL) { 558292740Snp /* XXX */ 559292740Snp panic("%s: allocation failure.", __func__); 560292740Snp } 561292740Snp flowc = wrtod(wr); 562292740Snp memset(flowc, 0, wr->wr_len); 563292740Snp 564292740Snp flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 565292740Snp V_FW_FLOWC_WR_NPARAMS(nparams)); 566292740Snp flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | 567292740Snp V_FW_WR_FLOWID(toep->tid)); 568292740Snp 569292740Snp flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; 570292740Snp flowc->mnemval[0].val = htobe32(maxlen); 571292740Snp 572292740Snp txsd->tx_credits = howmany(flowclen, 16); 573292740Snp txsd->plen = 0; 574292740Snp KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 575292740Snp ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 576292740Snp toep->tx_credits -= txsd->tx_credits; 577292740Snp if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 578292740Snp toep->txsd_pidx = 0; 579292740Snp toep->txsd_avail--; 580292740Snp 581292740Snp t4_wrq_tx(sc, wr); 582292740Snp} 583292740Snp 584292740Snpstatic void 585292740Snpset_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, int hcrc, int dcrc) 586292740Snp{ 587292740Snp uint64_t val = 0; 588292740Snp 589292740Snp if (hcrc) 590292740Snp val |= ULP_CRC_HEADER; 591292740Snp if (dcrc) 592292740Snp val |= ULP_CRC_DATA; 593292740Snp val <<= 4; 594292740Snp val |= ULP_MODE_ISCSI; 595292740Snp 596292740Snp CTR4(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, CRC hdr=%d data=%d", 597292740Snp __func__, toep->tid, hcrc, dcrc); 598292740Snp 599292740Snp t4_set_tcb_field(sc, toep, 1, 0, 0xfff, val); 600292740Snp} 601292740Snp 602292740Snp/* 603292740Snp * XXXNP: Who is responsible for cleaning up the socket if this returns with an 604292740Snp * error? Review all error paths. 605292740Snp * 606292740Snp * XXXNP: What happens to the socket's fd reference if the operation is 607292740Snp * successful, and how does that affect the socket's life cycle? 608292740Snp */ 609292740Snpint 610292740Snpicl_cxgbei_conn_handoff(struct icl_conn *ic, int fd) 611292740Snp{ 612292740Snp struct icl_cxgbei_conn *icc = ic_to_icc(ic); 613292740Snp struct find_ofld_adapter_rr fa; 614292740Snp struct file *fp; 615292740Snp struct socket *so; 616292740Snp struct inpcb *inp; 617292740Snp struct tcpcb *tp; 618292740Snp struct toepcb *toep; 619292740Snp cap_rights_t rights; 620292740Snp int error; 621292740Snp 622292740Snp MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 623292740Snp ICL_CONN_LOCK_ASSERT_NOT(ic); 624292740Snp 625292740Snp /* 626292740Snp * Steal the socket from userland. 627292740Snp */ 628292740Snp error = fget(curthread, fd, 629292740Snp cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); 630292740Snp if (error != 0) 631292740Snp return (error); 632292740Snp if (fp->f_type != DTYPE_SOCKET) { 633292740Snp fdrop(fp, curthread); 634292740Snp return (EINVAL); 635292740Snp } 636292740Snp so = fp->f_data; 637292740Snp if (so->so_type != SOCK_STREAM || 638292740Snp so->so_proto->pr_protocol != IPPROTO_TCP) { 639292740Snp fdrop(fp, curthread); 640292740Snp return (EINVAL); 641292740Snp } 642292740Snp 643292740Snp ICL_CONN_LOCK(ic); 644292740Snp if (ic->ic_socket != NULL) { 645292740Snp ICL_CONN_UNLOCK(ic); 646292740Snp fdrop(fp, curthread); 647292740Snp return (EBUSY); 648292740Snp } 649292740Snp ic->ic_disconnecting = false; 650292740Snp ic->ic_socket = so; 651292740Snp fp->f_ops = &badfileops; 652292740Snp fp->f_data = NULL; 653292740Snp fdrop(fp, curthread); 654292740Snp ICL_CONN_UNLOCK(ic); 655292740Snp 656292740Snp /* Find the adapter offloading this socket. */ 657292740Snp fa.sc = NULL; 658292740Snp fa.so = so; 659292740Snp t4_iterate(find_offload_adapter, &fa); 660292740Snp if (fa.sc == NULL) 661292740Snp return (EINVAL); 662292740Snp icc->sc = fa.sc; 663292740Snp 664292740Snp error = icl_cxgbei_setsockopt(ic, so); 665292740Snp if (error) 666292740Snp return (error); 667292740Snp 668292740Snp inp = sotoinpcb(so); 669292740Snp INP_WLOCK(inp); 670292740Snp tp = intotcpcb(inp); 671292740Snp if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) 672292740Snp error = EBUSY; 673292740Snp else { 674292740Snp /* 675292740Snp * socket could not have been "unoffloaded" if here. 676292740Snp */ 677292740Snp MPASS(tp->t_flags & TF_TOE); 678292740Snp MPASS(tp->tod != NULL); 679292740Snp MPASS(tp->t_toe != NULL); 680292740Snp toep = tp->t_toe; 681292740Snp MPASS(toep->vi->pi->adapter == icc->sc); 682292740Snp icc->toep = toep; 683292740Snp icc->cwt = cxgbei_select_worker_thread(icc); 684292740Snp icc->ulp_submode = 0; 685292740Snp if (ic->ic_header_crc32c) 686292740Snp icc->ulp_submode |= ULP_CRC_HEADER; 687292740Snp if (ic->ic_data_crc32c) 688292740Snp icc->ulp_submode |= ULP_CRC_DATA; 689292740Snp so->so_options |= SO_NO_DDP; 690292740Snp toep->ulp_mode = ULP_MODE_ISCSI; 691292740Snp toep->ulpcb = icc; 692292740Snp 693292740Snp send_iscsi_flowc_wr(icc->sc, toep, CXGBEI_MAX_PDU); 694292740Snp set_ulp_mode_iscsi(icc->sc, toep, ic->ic_header_crc32c, 695292740Snp ic->ic_data_crc32c); 696292740Snp error = 0; 697292740Snp } 698292740Snp INP_WUNLOCK(inp); 699292740Snp 700292740Snp return (error); 701292740Snp} 702292740Snp 703292740Snpvoid 704292740Snpicl_cxgbei_conn_close(struct icl_conn *ic) 705292740Snp{ 706292740Snp struct icl_cxgbei_conn *icc = ic_to_icc(ic); 707292740Snp struct icl_pdu *ip; 708292740Snp struct socket *so; 709292740Snp struct sockbuf *sb; 710292740Snp struct inpcb *inp; 711292740Snp struct toepcb *toep = icc->toep; 712292740Snp 713292740Snp MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 714292740Snp ICL_CONN_LOCK_ASSERT_NOT(ic); 715292740Snp 716292740Snp ICL_CONN_LOCK(ic); 717292740Snp so = ic->ic_socket; 718292740Snp if (ic->ic_disconnecting || so == NULL) { 719292740Snp CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p", 720292740Snp __func__, icc, ic->ic_disconnecting, so); 721292740Snp ICL_CONN_UNLOCK(ic); 722292740Snp return; 723292740Snp } 724292740Snp ic->ic_disconnecting = true; 725292740Snp 726292740Snp /* These are unused in this driver right now. */ 727292740Snp MPASS(STAILQ_EMPTY(&ic->ic_to_send)); 728292740Snp MPASS(ic->ic_receive_pdu == NULL); 729292740Snp 730292740Snp#ifdef DIAGNOSTIC 731292740Snp KASSERT(ic->ic_outstanding_pdus == 0, 732292740Snp ("destroying session with %d outstanding PDUs", 733292740Snp ic->ic_outstanding_pdus)); 734292740Snp#endif 735292740Snp ICL_CONN_UNLOCK(ic); 736292740Snp 737292740Snp CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1, 738292740Snp icc); 739292740Snp inp = sotoinpcb(so); 740292740Snp sb = &so->so_rcv; 741292740Snp INP_WLOCK(inp); 742292740Snp if (toep != NULL) { /* NULL if connection was never offloaded. */ 743292740Snp toep->ulpcb = NULL; 744292740Snp mbufq_drain(&toep->ulp_pduq); 745292740Snp SOCKBUF_LOCK(sb); 746292740Snp if (icc->rx_flags & RXF_ACTIVE) { 747292740Snp volatile u_int *p = &icc->rx_flags; 748292740Snp 749292740Snp SOCKBUF_UNLOCK(sb); 750292740Snp INP_WUNLOCK(inp); 751292740Snp 752292740Snp while (*p & RXF_ACTIVE) 753292740Snp pause("conclo", 1); 754292740Snp 755292740Snp INP_WLOCK(inp); 756292740Snp SOCKBUF_LOCK(sb); 757292740Snp } 758292740Snp 759292740Snp while (!STAILQ_EMPTY(&icc->rcvd_pdus)) { 760292740Snp ip = STAILQ_FIRST(&icc->rcvd_pdus); 761292740Snp STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next); 762292740Snp icl_cxgbei_conn_pdu_free(ic, ip); 763292740Snp } 764292740Snp SOCKBUF_UNLOCK(sb); 765292740Snp } 766292740Snp INP_WUNLOCK(inp); 767292740Snp 768292740Snp ICL_CONN_LOCK(ic); 769292740Snp ic->ic_socket = NULL; 770292740Snp ICL_CONN_UNLOCK(ic); 771292740Snp 772292740Snp /* 773292740Snp * XXXNP: we should send RST instead of FIN when PDUs held in various 774292740Snp * queues were purged instead of delivered reliably but soabort isn't 775292740Snp * really general purpose and wouldn't do the right thing here. 776292740Snp */ 777292740Snp soclose(so); 778292740Snp} 779292740Snp 780292740Snpint 781300040Straszicl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 782300040Strasz struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp) 783292740Snp{ 784292740Snp void *prv; 785292740Snp 786292740Snp *task_tagp = icl_conn_build_tasktag(ic, *task_tagp); 787292740Snp 788292740Snp prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO); 789292740Snp if (prv == NULL) 790292740Snp return (ENOMEM); 791292740Snp 792292740Snp *prvp = prv; 793292740Snp 794292740Snp cxgbei_conn_task_reserve_itt(ic, prvp, csio, task_tagp); 795292740Snp 796292740Snp return (0); 797292740Snp} 798292740Snp 799292740Snpvoid 800292740Snpicl_cxgbei_conn_task_done(struct icl_conn *ic, void *prv) 801292740Snp{ 802292740Snp 803292740Snp cxgbei_cleanup_task(ic, prv); 804292740Snp uma_zfree(icl_transfer_zone, prv); 805292740Snp} 806292740Snp 807292740Snpint 808292740Snpicl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, 809292740Snp uint32_t *transfer_tag, void **prvp) 810292740Snp{ 811292740Snp void *prv; 812292740Snp 813292740Snp *transfer_tag = icl_conn_build_tasktag(ic, *transfer_tag); 814292740Snp 815292740Snp prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO); 816292740Snp if (prv == NULL) 817292740Snp return (ENOMEM); 818292740Snp 819292740Snp *prvp = prv; 820292740Snp 821292740Snp cxgbei_conn_transfer_reserve_ttt(ic, prvp, io, transfer_tag); 822292740Snp 823292740Snp return (0); 824292740Snp} 825292740Snp 826292740Snpvoid 827292740Snpicl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *prv) 828292740Snp{ 829292740Snp cxgbei_cleanup_task(ic, prv); 830292740Snp uma_zfree(icl_transfer_zone, prv); 831292740Snp} 832292740Snp 833292740Snpstatic int 834292740Snpicl_cxgbei_limits(size_t *limitp) 835292740Snp{ 836292740Snp 837292740Snp *limitp = CXGBEI_MAX_DSL; 838292740Snp 839292740Snp return (0); 840292740Snp} 841292740Snp 842292740Snpstatic int 843292740Snpicl_cxgbei_load(void) 844292740Snp{ 845292740Snp int error; 846292740Snp 847292740Snp icl_transfer_zone = uma_zcreate("icl_transfer", 848292740Snp 16 * 1024, NULL, NULL, NULL, NULL, 849292740Snp UMA_ALIGN_PTR, 0); 850292740Snp 851292740Snp refcount_init(&icl_cxgbei_ncons, 0); 852292740Snp 853301119Strasz error = icl_register("cxgbei", false, -100, icl_cxgbei_limits, 854292740Snp icl_cxgbei_new_conn); 855292740Snp KASSERT(error == 0, ("failed to register")); 856292740Snp 857292740Snp return (error); 858292740Snp} 859292740Snp 860292740Snpstatic int 861292740Snpicl_cxgbei_unload(void) 862292740Snp{ 863292740Snp 864292740Snp if (icl_cxgbei_ncons != 0) 865292740Snp return (EBUSY); 866292740Snp 867300592Strasz icl_unregister("cxgbei", false); 868292740Snp 869292740Snp uma_zdestroy(icl_transfer_zone); 870292740Snp 871292740Snp return (0); 872292740Snp} 873292740Snp 874292740Snpstatic int 875292740Snpicl_cxgbei_modevent(module_t mod, int what, void *arg) 876292740Snp{ 877292740Snp 878292740Snp switch (what) { 879292740Snp case MOD_LOAD: 880292740Snp return (icl_cxgbei_load()); 881292740Snp case MOD_UNLOAD: 882292740Snp return (icl_cxgbei_unload()); 883292740Snp default: 884292740Snp return (EINVAL); 885292740Snp } 886292740Snp} 887292740Snp 888292740Snpmoduledata_t icl_cxgbei_data = { 889292740Snp "icl_cxgbei", 890292740Snp icl_cxgbei_modevent, 891292740Snp 0 892292740Snp}; 893292740Snp 894292740SnpDECLARE_MODULE(icl_cxgbei, icl_cxgbei_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 895292740SnpMODULE_DEPEND(icl_cxgbei, icl, 1, 1, 1); 896292740SnpMODULE_VERSION(icl_cxgbei, 1); 897292740Snp#endif 898