1237263Snp/*- 2292736Snp * Copyright (c) 2012, 2015 Chelsio Communications, Inc. 3237263Snp * All rights reserved. 4237263Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5237263Snp * 6237263Snp * Redistribution and use in source and binary forms, with or without 7237263Snp * modification, are permitted provided that the following conditions 8237263Snp * are met: 9237263Snp * 1. Redistributions of source code must retain the above copyright 10237263Snp * notice, this list of conditions and the following disclaimer. 11237263Snp * 2. Redistributions in binary form must reproduce the above copyright 12237263Snp * notice, this list of conditions and the following disclaimer in the 13237263Snp * documentation and/or other materials provided with the distribution. 14237263Snp * 15237263Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16237263Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17237263Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18237263Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19237263Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20237263Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21237263Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22237263Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23237263Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24237263Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25237263Snp * SUCH DAMAGE. 26237263Snp */ 27237263Snp 28237263Snp#include <sys/cdefs.h> 29237263Snp__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c 355249 2019-11-30 20:22:03Z np $"); 30237263Snp 31237263Snp#include "opt_inet.h" 32346805Snp#include "opt_inet6.h" 33237263Snp 34237263Snp#ifdef TCP_OFFLOAD 35237263Snp#include <sys/param.h> 36306661Sjhb#include <sys/aio.h> 37306661Sjhb#include <sys/file.h> 38237263Snp#include <sys/kernel.h> 39237263Snp#include <sys/ktr.h> 40237263Snp#include <sys/module.h> 41306661Sjhb#include <sys/proc.h> 42237263Snp#include <sys/protosw.h> 43237263Snp#include <sys/domain.h> 44237263Snp#include <sys/socket.h> 45237263Snp#include <sys/socketvar.h> 46237263Snp#include <sys/sglist.h> 47306661Sjhb#include <sys/taskqueue.h> 48237263Snp#include <netinet/in.h> 49237263Snp#include <netinet/in_pcb.h> 50237263Snp#include <netinet/ip.h> 51276574Snp#include <netinet/ip6.h> 52237263Snp#define TCPSTATES 53237263Snp#include <netinet/tcp_fsm.h> 54237263Snp#include <netinet/tcp_seq.h> 55294869Sglebius#include <netinet/tcp_var.h> 56237263Snp#include <netinet/toecore.h> 57237263Snp 58306661Sjhb#include <security/mac/mac_framework.h> 59306661Sjhb 60306661Sjhb#include <vm/vm.h> 61306661Sjhb#include <vm/vm_extern.h> 62306661Sjhb#include <vm/pmap.h> 63306661Sjhb#include <vm/vm_map.h> 64306661Sjhb#include <vm/vm_page.h> 65306661Sjhb 66237263Snp#include "common/common.h" 67237263Snp#include "common/t4_msg.h" 68237263Snp#include "common/t4_regs.h" 69239344Snp#include "common/t4_tcb.h" 70237263Snp#include "tom/t4_tom_l2t.h" 71237263Snp#include "tom/t4_tom.h" 72237263Snp 73237263SnpVNET_DECLARE(int, tcp_do_autosndbuf); 74237263Snp#define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf) 75237263SnpVNET_DECLARE(int, tcp_autosndbuf_inc); 76237263Snp#define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc) 77237263SnpVNET_DECLARE(int, tcp_autosndbuf_max); 78237263Snp#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) 79237263SnpVNET_DECLARE(int, tcp_do_autorcvbuf); 80237263Snp#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) 81237263SnpVNET_DECLARE(int, tcp_autorcvbuf_inc); 82237263Snp#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) 83237263SnpVNET_DECLARE(int, tcp_autorcvbuf_max); 84237263Snp#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) 85237263Snp 86306661Sjhbstatic void t4_aiotx_cancel(struct kaiocb *job); 87351236Sjhbstatic void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep); 88306661Sjhb 89306661Sjhbstatic size_t 90306661Sjhbaiotx_mbuf_pgoff(struct mbuf *m) 91306661Sjhb{ 92306661Sjhb struct aiotx_buffer *ab; 93306661Sjhb 94306661Sjhb MPASS(IS_AIOTX_MBUF(m)); 95306661Sjhb ab = m->m_ext.ext_arg1; 96306661Sjhb return ((ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) % PAGE_SIZE); 97306661Sjhb} 98306661Sjhb 99306661Sjhbstatic vm_page_t * 100306661Sjhbaiotx_mbuf_pages(struct mbuf *m) 101306661Sjhb{ 102306661Sjhb struct aiotx_buffer *ab; 103306661Sjhb int npages; 104306661Sjhb 105306661Sjhb MPASS(IS_AIOTX_MBUF(m)); 106306661Sjhb ab = m->m_ext.ext_arg1; 107306661Sjhb npages = (ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) / PAGE_SIZE; 108306661Sjhb return (ab->ps.pages + npages); 109306661Sjhb} 110306661Sjhb 111237263Snpvoid 112237263Snpsend_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) 113237263Snp{ 114241626Snp struct wrqe *wr; 115241626Snp struct fw_flowc_wr *flowc; 116345664Sjhb unsigned int nparams, flowclen, paramidx; 117291665Sjhb struct vi_info *vi = toep->vi; 118291665Sjhb struct port_info *pi = vi->pi; 119237263Snp struct adapter *sc = pi->adapter; 120346967Snp unsigned int pfvf = sc->pf << S_FW_VIID_PFN; 121237263Snp struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 122237263Snp 123239514Snp KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT), 124237263Snp ("%s: flowc for tid %u sent already", __func__, toep->tid)); 125237263Snp 126345664Sjhb if (ftxp != NULL) 127345664Sjhb nparams = 8; 128345664Sjhb else 129345664Sjhb nparams = 6; 130345664Sjhb if (toep->ulp_mode == ULP_MODE_TLS) 131345664Sjhb nparams++; 132345664Sjhb if (toep->tls.fcplenmax != 0) 133345664Sjhb nparams++; 134346805Snp if (toep->tc_idx != -1) { 135346805Snp MPASS(toep->tc_idx >= 0 && 136346805Snp toep->tc_idx < sc->chip_params->nsched_cls); 137346805Snp nparams++; 138346805Snp } 139345664Sjhb 140237263Snp flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); 141237263Snp 142248925Snp wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); 143237263Snp if (wr == NULL) { 144237263Snp /* XXX */ 145237263Snp panic("%s: allocation failure.", __func__); 146237263Snp } 147237263Snp flowc = wrtod(wr); 148237263Snp memset(flowc, 0, wr->wr_len); 149237263Snp 150237263Snp flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 151237263Snp V_FW_FLOWC_WR_NPARAMS(nparams)); 152237263Snp flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | 153237263Snp V_FW_WR_FLOWID(toep->tid)); 154237263Snp 155345664Sjhb#define FLOWC_PARAM(__m, __v) \ 156345664Sjhb do { \ 157345664Sjhb flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \ 158345664Sjhb flowc->mnemval[paramidx].val = htobe32(__v); \ 159345664Sjhb paramidx++; \ 160345664Sjhb } while (0) 161345664Sjhb 162345664Sjhb paramidx = 0; 163345664Sjhb 164345664Sjhb FLOWC_PARAM(PFNVFN, pfvf); 165345664Sjhb FLOWC_PARAM(CH, pi->tx_chan); 166345664Sjhb FLOWC_PARAM(PORT, pi->tx_chan); 167345664Sjhb FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id); 168237263Snp if (ftxp) { 169237263Snp uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf); 170237263Snp 171345664Sjhb FLOWC_PARAM(SNDNXT, ftxp->snd_nxt); 172345664Sjhb FLOWC_PARAM(RCVNXT, ftxp->rcv_nxt); 173345664Sjhb FLOWC_PARAM(SNDBUF, sndbuf); 174345664Sjhb FLOWC_PARAM(MSS, ftxp->mss); 175276570Snp 176276570Snp CTR6(KTR_CXGBE, 177276570Snp "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x", 178276570Snp __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt, 179276570Snp ftxp->rcv_nxt); 180241642Snp } else { 181345664Sjhb FLOWC_PARAM(SNDBUF, 512); 182345664Sjhb FLOWC_PARAM(MSS, 512); 183276570Snp 184276570Snp CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); 185237263Snp } 186345664Sjhb if (toep->ulp_mode == ULP_MODE_TLS) 187345664Sjhb FLOWC_PARAM(ULP_MODE, toep->ulp_mode); 188345664Sjhb if (toep->tls.fcplenmax != 0) 189345664Sjhb FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax); 190346805Snp if (toep->tc_idx != -1) 191346805Snp FLOWC_PARAM(SCHEDCLASS, toep->tc_idx); 192345664Sjhb#undef FLOWC_PARAM 193237263Snp 194345664Sjhb KASSERT(paramidx == nparams, ("nparams mismatch")); 195345664Sjhb 196237263Snp txsd->tx_credits = howmany(flowclen, 16); 197237263Snp txsd->plen = 0; 198237263Snp KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 199237263Snp ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 200237263Snp toep->tx_credits -= txsd->tx_credits; 201237263Snp if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 202237263Snp toep->txsd_pidx = 0; 203237263Snp toep->txsd_avail--; 204237263Snp 205239514Snp toep->flags |= TPF_FLOWC_WR_SENT; 206237263Snp t4_wrq_tx(sc, wr); 207237263Snp} 208237263Snp 209346805Snp#ifdef RATELIMIT 210346805Snp/* 211346805Snp * Input is Bytes/second (so_max_pacing-rate), chip counts in Kilobits/second. 212346805Snp */ 213346805Snpstatic int 214346805Snpupdate_tx_rate_limit(struct adapter *sc, struct toepcb *toep, u_int Bps) 215346805Snp{ 216346805Snp int tc_idx, rc; 217346805Snp const u_int kbps = (u_int) (uint64_t)Bps * 8ULL / 1000; 218346805Snp const int port_id = toep->vi->pi->port_id; 219346805Snp 220346805Snp CTR3(KTR_CXGBE, "%s: tid %u, rate %uKbps", __func__, toep->tid, kbps); 221346805Snp 222346805Snp if (kbps == 0) { 223346805Snp /* unbind */ 224346805Snp tc_idx = -1; 225346805Snp } else { 226346805Snp rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx); 227346805Snp if (rc != 0) 228346805Snp return (rc); 229346805Snp MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls); 230346805Snp } 231346805Snp 232346805Snp if (toep->tc_idx != tc_idx) { 233346805Snp struct wrqe *wr; 234346805Snp struct fw_flowc_wr *flowc; 235346805Snp int nparams = 1, flowclen, flowclen16; 236346805Snp struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 237346805Snp 238346805Snp flowclen = sizeof(*flowc) + nparams * sizeof(struct 239346805Snp fw_flowc_mnemval); 240346805Snp flowclen16 = howmany(flowclen, 16); 241346805Snp if (toep->tx_credits < flowclen16 || toep->txsd_avail == 0 || 242346805Snp (wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq)) == NULL) { 243346805Snp if (tc_idx >= 0) 244346871Snp t4_release_cl_rl(sc, port_id, tc_idx); 245346805Snp return (ENOMEM); 246346805Snp } 247346805Snp 248346805Snp flowc = wrtod(wr); 249346805Snp memset(flowc, 0, wr->wr_len); 250346805Snp 251346805Snp flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 252346805Snp V_FW_FLOWC_WR_NPARAMS(nparams)); 253346805Snp flowc->flowid_len16 = htonl(V_FW_WR_LEN16(flowclen16) | 254346805Snp V_FW_WR_FLOWID(toep->tid)); 255346805Snp 256346805Snp flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; 257346805Snp if (tc_idx == -1) 258346805Snp flowc->mnemval[0].val = htobe32(0xff); 259346805Snp else 260346805Snp flowc->mnemval[0].val = htobe32(tc_idx); 261346805Snp 262346805Snp txsd->tx_credits = flowclen16; 263346805Snp txsd->plen = 0; 264346805Snp toep->tx_credits -= txsd->tx_credits; 265346805Snp if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 266346805Snp toep->txsd_pidx = 0; 267346805Snp toep->txsd_avail--; 268346805Snp t4_wrq_tx(sc, wr); 269346805Snp } 270346805Snp 271346805Snp if (toep->tc_idx >= 0) 272346871Snp t4_release_cl_rl(sc, port_id, toep->tc_idx); 273346805Snp toep->tc_idx = tc_idx; 274346805Snp 275346805Snp return (0); 276346805Snp} 277346805Snp#endif 278346805Snp 279237263Snpvoid 280237263Snpsend_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) 281237263Snp{ 282237263Snp struct wrqe *wr; 283237263Snp struct cpl_abort_req *req; 284237263Snp int tid = toep->tid; 285237263Snp struct inpcb *inp = toep->inp; 286237263Snp struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */ 287237263Snp 288237263Snp INP_WLOCK_ASSERT(inp); 289237263Snp 290237263Snp CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s", 291237263Snp __func__, toep->tid, 292237263Snp inp->inp_flags & INP_DROPPED ? "inp dropped" : 293237263Snp tcpstates[tp->t_state], 294237263Snp toep->flags, inp->inp_flags, 295239514Snp toep->flags & TPF_ABORT_SHUTDOWN ? 296237263Snp " (abort already in progress)" : ""); 297237263Snp 298239514Snp if (toep->flags & TPF_ABORT_SHUTDOWN) 299237263Snp return; /* abort already in progress */ 300237263Snp 301239514Snp toep->flags |= TPF_ABORT_SHUTDOWN; 302237263Snp 303239514Snp KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 304237263Snp ("%s: flowc_wr not sent for tid %d.", __func__, tid)); 305237263Snp 306237263Snp wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 307237263Snp if (wr == NULL) { 308237263Snp /* XXX */ 309237263Snp panic("%s: allocation failure.", __func__); 310237263Snp } 311237263Snp req = wrtod(wr); 312237263Snp 313237263Snp INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid); 314237263Snp if (inp->inp_flags & INP_DROPPED) 315237263Snp req->rsvd0 = htobe32(snd_nxt); 316237263Snp else 317237263Snp req->rsvd0 = htobe32(tp->snd_nxt); 318239514Snp req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT); 319237263Snp req->cmd = CPL_ABORT_SEND_RST; 320237263Snp 321237263Snp /* 322237263Snp * XXX: What's the correct way to tell that the inp hasn't been detached 323237263Snp * from its socket? Should I even be flushing the snd buffer here? 324237263Snp */ 325237263Snp if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 326237263Snp struct socket *so = inp->inp_socket; 327237263Snp 328237263Snp if (so != NULL) /* because I'm not sure. See comment above */ 329237263Snp sbflush(&so->so_snd); 330237263Snp } 331237263Snp 332237263Snp t4_l2t_send(sc, wr, toep->l2te); 333237263Snp} 334237263Snp 335237263Snp/* 336237263Snp * Called when a connection is established to translate the TCP options 337237263Snp * reported by HW to FreeBSD's native format. 338237263Snp */ 339237263Snpstatic void 340355242Snpassign_rxopt(struct tcpcb *tp, uint16_t opt) 341237263Snp{ 342237263Snp struct toepcb *toep = tp->t_toe; 343276574Snp struct inpcb *inp = tp->t_inpcb; 344237263Snp struct adapter *sc = td_adapter(toep->td); 345237263Snp 346276574Snp INP_LOCK_ASSERT(inp); 347237263Snp 348355242Snp toep->tcp_opt = opt; 349355242Snp toep->mtu_idx = G_TCPOPT_MSS(opt); 350355242Snp tp->t_maxseg = sc->params.mtus[toep->mtu_idx]; 351276574Snp if (inp->inp_inc.inc_flags & INC_ISIPV6) 352355242Snp tp->t_maxseg -= sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 353276574Snp else 354355242Snp tp->t_maxseg -= sizeof(struct ip) + sizeof(struct tcphdr); 355237263Snp 356355242Snp toep->emss = tp->t_maxseg; 357237263Snp if (G_TCPOPT_TSTAMP(opt)) { 358237263Snp tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */ 359237263Snp tp->ts_recent = 0; /* hmmm */ 360237263Snp tp->ts_recent_age = tcp_ts_getticks(); 361355242Snp toep->emss -= TCPOLEN_TSTAMP_APPA; 362237263Snp } 363237263Snp 364355242Snp CTR6(KTR_CXGBE, "%s: tid %d, mtu_idx %u (%u), t_maxseg %u, emss %u", 365355242Snp __func__, toep->tid, toep->mtu_idx, 366355242Snp sc->params.mtus[G_TCPOPT_MSS(opt)], tp->t_maxseg, toep->emss); 367346805Snp 368237263Snp if (G_TCPOPT_SACK(opt)) 369237263Snp tp->t_flags |= TF_SACK_PERMIT; /* should already be set */ 370237263Snp else 371237263Snp tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */ 372237263Snp 373237263Snp if (G_TCPOPT_WSCALE_OK(opt)) 374237263Snp tp->t_flags |= TF_RCVD_SCALE; 375237263Snp 376237263Snp /* Doing window scaling? */ 377237263Snp if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 378237263Snp (TF_RCVD_SCALE | TF_REQ_SCALE)) { 379237263Snp tp->rcv_scale = tp->request_r_scale; 380237263Snp tp->snd_scale = G_TCPOPT_SND_WSCALE(opt); 381237263Snp } 382237263Snp} 383237263Snp 384237263Snp/* 385237263Snp * Completes some final bits of initialization for just established connections 386237263Snp * and changes their state to TCPS_ESTABLISHED. 387237263Snp * 388346970Snp * The ISNs are from the exchange of SYNs. 389237263Snp */ 390237263Snpvoid 391346970Snpmake_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt) 392237263Snp{ 393237263Snp struct inpcb *inp = toep->inp; 394237263Snp struct socket *so = inp->inp_socket; 395237263Snp struct tcpcb *tp = intotcpcb(inp); 396237263Snp long bufsize; 397237263Snp uint16_t tcpopt = be16toh(opt); 398237263Snp struct flowc_tx_params ftxp; 399237263Snp 400237263Snp INP_WLOCK_ASSERT(inp); 401237263Snp KASSERT(tp->t_state == TCPS_SYN_SENT || 402237263Snp tp->t_state == TCPS_SYN_RECEIVED, 403237263Snp ("%s: TCP state %s", __func__, tcpstates[tp->t_state])); 404237263Snp 405330307Snp CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p", 406330307Snp __func__, toep->tid, so, inp, tp, toep); 407237263Snp 408342583Sjhb tcp_state_change(tp, TCPS_ESTABLISHED); 409237263Snp tp->t_starttime = ticks; 410237263Snp TCPSTAT_INC(tcps_connects); 411237263Snp 412237263Snp tp->irs = irs; 413237263Snp tcp_rcvseqinit(tp); 414355242Snp tp->rcv_wnd = (u_int)toep->opt0_rcv_bufsize << 10; 415237263Snp tp->rcv_adv += tp->rcv_wnd; 416237263Snp tp->last_ack_sent = tp->rcv_nxt; 417237263Snp 418237263Snp tp->iss = iss; 419237263Snp tcp_sendseqinit(tp); 420237263Snp tp->snd_una = iss + 1; 421237263Snp tp->snd_nxt = iss + 1; 422237263Snp tp->snd_max = iss + 1; 423237263Snp 424237263Snp assign_rxopt(tp, tcpopt); 425237263Snp 426237263Snp SOCKBUF_LOCK(&so->so_snd); 427237263Snp if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf) 428237263Snp bufsize = V_tcp_autosndbuf_max; 429237263Snp else 430237263Snp bufsize = sbspace(&so->so_snd); 431237263Snp SOCKBUF_UNLOCK(&so->so_snd); 432237263Snp 433237263Snp ftxp.snd_nxt = tp->snd_nxt; 434237263Snp ftxp.rcv_nxt = tp->rcv_nxt; 435237263Snp ftxp.snd_space = bufsize; 436355242Snp ftxp.mss = toep->emss; 437237263Snp send_flowc_wr(toep, &ftxp); 438237263Snp 439237263Snp soisconnected(so); 440237263Snp} 441237263Snp 442345664Sjhbint 443239344Snpsend_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) 444237263Snp{ 445237263Snp struct wrqe *wr; 446237263Snp struct cpl_rx_data_ack *req; 447237263Snp uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 448237263Snp 449239344Snp KASSERT(credits >= 0, ("%s: %d credits", __func__, credits)); 450239344Snp 451237263Snp wr = alloc_wrqe(sizeof(*req), toep->ctrlq); 452237263Snp if (wr == NULL) 453237263Snp return (0); 454237263Snp req = wrtod(wr); 455237263Snp 456237263Snp INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); 457237263Snp req->credit_dack = htobe32(dack | V_RX_CREDITS(credits)); 458237263Snp 459237263Snp t4_wrq_tx(sc, wr); 460237263Snp return (credits); 461237263Snp} 462237263Snp 463237263Snpvoid 464345664Sjhbsend_rx_modulate(struct adapter *sc, struct toepcb *toep) 465345664Sjhb{ 466345664Sjhb struct wrqe *wr; 467345664Sjhb struct cpl_rx_data_ack *req; 468345664Sjhb 469345664Sjhb wr = alloc_wrqe(sizeof(*req), toep->ctrlq); 470345664Sjhb if (wr == NULL) 471345664Sjhb return; 472345664Sjhb req = wrtod(wr); 473345664Sjhb 474345664Sjhb INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); 475345664Sjhb req->credit_dack = htobe32(F_RX_MODULATE_RX); 476345664Sjhb 477345664Sjhb t4_wrq_tx(sc, wr); 478345664Sjhb} 479345664Sjhb 480345664Sjhbvoid 481299210Sjhbt4_rcvd_locked(struct toedev *tod, struct tcpcb *tp) 482237263Snp{ 483237263Snp struct adapter *sc = tod->tod_softc; 484237263Snp struct inpcb *inp = tp->t_inpcb; 485237263Snp struct socket *so = inp->inp_socket; 486239344Snp struct sockbuf *sb = &so->so_rcv; 487237263Snp struct toepcb *toep = tp->t_toe; 488348704Snp int rx_credits; 489237263Snp 490237263Snp INP_WLOCK_ASSERT(inp); 491299210Sjhb SOCKBUF_LOCK_ASSERT(sb); 492292736Snp 493348704Snp rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 494345664Sjhb if (toep->ulp_mode == ULP_MODE_TLS) { 495348704Snp if (toep->tls.rcv_over >= rx_credits) { 496348704Snp toep->tls.rcv_over -= rx_credits; 497348704Snp rx_credits = 0; 498345664Sjhb } else { 499348704Snp rx_credits -= toep->tls.rcv_over; 500345664Sjhb toep->tls.rcv_over = 0; 501345664Sjhb } 502345664Sjhb } 503292736Snp 504348704Snp if (rx_credits > 0 && 505348704Snp (tp->rcv_wnd <= 32 * 1024 || rx_credits >= 64 * 1024 || 506348704Snp (rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || 507348704Snp sbused(sb) + tp->rcv_wnd < sb->sb_lowat)) { 508348704Snp rx_credits = send_rx_credits(sc, toep, rx_credits); 509348704Snp tp->rcv_wnd += rx_credits; 510348704Snp tp->rcv_adv += rx_credits; 511345664Sjhb } else if (toep->flags & TPF_FORCE_CREDITS) 512345664Sjhb send_rx_modulate(sc, toep); 513299210Sjhb} 514299210Sjhb 515299210Sjhbvoid 516299210Sjhbt4_rcvd(struct toedev *tod, struct tcpcb *tp) 517299210Sjhb{ 518299210Sjhb struct inpcb *inp = tp->t_inpcb; 519299210Sjhb struct socket *so = inp->inp_socket; 520299210Sjhb struct sockbuf *sb = &so->so_rcv; 521299210Sjhb 522299210Sjhb SOCKBUF_LOCK(sb); 523299210Sjhb t4_rcvd_locked(tod, tp); 524280878Snp SOCKBUF_UNLOCK(sb); 525237263Snp} 526237263Snp 527237263Snp/* 528237263Snp * Close a connection by sending a CPL_CLOSE_CON_REQ message. 529237263Snp */ 530345664Sjhbint 531345664Sjhbt4_close_conn(struct adapter *sc, struct toepcb *toep) 532237263Snp{ 533237263Snp struct wrqe *wr; 534237263Snp struct cpl_close_con_req *req; 535237263Snp unsigned int tid = toep->tid; 536237263Snp 537237263Snp CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid, 538239514Snp toep->flags & TPF_FIN_SENT ? ", IGNORED" : ""); 539237263Snp 540239514Snp if (toep->flags & TPF_FIN_SENT) 541237263Snp return (0); 542237263Snp 543239514Snp KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 544237263Snp ("%s: flowc_wr not sent for tid %u.", __func__, tid)); 545237263Snp 546237263Snp wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 547237263Snp if (wr == NULL) { 548237263Snp /* XXX */ 549237263Snp panic("%s: allocation failure.", __func__); 550237263Snp } 551237263Snp req = wrtod(wr); 552237263Snp 553237263Snp req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | 554237263Snp V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr))); 555237263Snp req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) | 556237263Snp V_FW_WR_FLOWID(tid)); 557237263Snp req->wr.wr_lo = cpu_to_be64(0); 558237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 559237263Snp req->rsvd = 0; 560237263Snp 561239514Snp toep->flags |= TPF_FIN_SENT; 562239514Snp toep->flags &= ~TPF_SEND_FIN; 563237263Snp t4_l2t_send(sc, wr, toep->l2te); 564237263Snp 565237263Snp return (0); 566237263Snp} 567237263Snp 568237263Snp#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 569237263Snp#define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16)) 570237263Snp 571237263Snp/* Maximum amount of immediate data we could stuff in a WR */ 572237263Snpstatic inline int 573237263Snpmax_imm_payload(int tx_credits) 574237263Snp{ 575355249Snp const int n = 1; /* Use no more than one desc for imm. data WR */ 576237263Snp 577237263Snp KASSERT(tx_credits >= 0 && 578237263Snp tx_credits <= MAX_OFLD_TX_CREDITS, 579237263Snp ("%s: %d credits", __func__, tx_credits)); 580237263Snp 581237263Snp if (tx_credits < MIN_OFLD_TX_CREDITS) 582237263Snp return (0); 583237263Snp 584237263Snp if (tx_credits >= (n * EQ_ESIZE) / 16) 585237263Snp return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr)); 586237263Snp else 587237263Snp return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr)); 588237263Snp} 589237263Snp 590237263Snp/* Maximum number of SGL entries we could stuff in a WR */ 591237263Snpstatic inline int 592237263Snpmax_dsgl_nsegs(int tx_credits) 593237263Snp{ 594237263Snp int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */ 595237263Snp int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS; 596237263Snp 597237263Snp KASSERT(tx_credits >= 0 && 598237263Snp tx_credits <= MAX_OFLD_TX_CREDITS, 599237263Snp ("%s: %d credits", __func__, tx_credits)); 600237263Snp 601237263Snp if (tx_credits < MIN_OFLD_TX_CREDITS) 602237263Snp return (0); 603237263Snp 604237263Snp nseg += 2 * (sge_pair_credits * 16 / 24); 605237263Snp if ((sge_pair_credits * 16) % 24 == 16) 606237263Snp nseg++; 607237263Snp 608237263Snp return (nseg); 609237263Snp} 610237263Snp 611237263Snpstatic inline void 612237263Snpwrite_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, 613292736Snp unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign) 614237263Snp{ 615237263Snp struct fw_ofld_tx_data_wr *txwr = dst; 616237263Snp 617237263Snp txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | 618255411Snp V_FW_WR_IMMDLEN(immdlen)); 619237263Snp txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | 620237263Snp V_FW_WR_LEN16(credits)); 621292736Snp txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) | 622292736Snp V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove)); 623237263Snp txwr->plen = htobe32(plen); 624276597Snp 625276597Snp if (txalign > 0) { 626276597Snp struct tcpcb *tp = intotcpcb(toep->inp); 627276597Snp 628355242Snp if (plen < 2 * toep->emss) 629285527Snp txwr->lsodisable_to_flags |= 630276597Snp htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE); 631276597Snp else 632285527Snp txwr->lsodisable_to_flags |= 633276597Snp htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD | 634276597Snp (tp->t_flags & TF_NODELAY ? 0 : 635276597Snp F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE)); 636276597Snp } 637237263Snp} 638237263Snp 639237263Snp/* 640237263Snp * Generate a DSGL from a starting mbuf. The total number of segments and the 641237263Snp * maximum segments in any one mbuf are provided. 642237263Snp */ 643237263Snpstatic void 644237263Snpwrite_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) 645237263Snp{ 646237263Snp struct mbuf *m; 647237263Snp struct ulptx_sgl *usgl = dst; 648237263Snp int i, j, rc; 649237263Snp struct sglist sg; 650237263Snp struct sglist_seg segs[n]; 651237263Snp 652237263Snp KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 653237263Snp 654237263Snp sglist_init(&sg, n, segs); 655237263Snp usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 656237263Snp V_ULPTX_NSGE(nsegs)); 657237263Snp 658237263Snp i = -1; 659237263Snp for (m = start; m != stop; m = m->m_next) { 660306661Sjhb if (IS_AIOTX_MBUF(m)) 661306661Sjhb rc = sglist_append_vmpages(&sg, aiotx_mbuf_pages(m), 662306661Sjhb aiotx_mbuf_pgoff(m), m->m_len); 663306661Sjhb else 664306661Sjhb rc = sglist_append(&sg, mtod(m, void *), m->m_len); 665237263Snp if (__predict_false(rc != 0)) 666237263Snp panic("%s: sglist_append %d", __func__, rc); 667237263Snp 668237263Snp for (j = 0; j < sg.sg_nseg; i++, j++) { 669237263Snp if (i < 0) { 670237263Snp usgl->len0 = htobe32(segs[j].ss_len); 671237263Snp usgl->addr0 = htobe64(segs[j].ss_paddr); 672237263Snp } else { 673237263Snp usgl->sge[i / 2].len[i & 1] = 674237263Snp htobe32(segs[j].ss_len); 675237263Snp usgl->sge[i / 2].addr[i & 1] = 676237263Snp htobe64(segs[j].ss_paddr); 677237263Snp } 678237263Snp#ifdef INVARIANTS 679237263Snp nsegs--; 680237263Snp#endif 681237263Snp } 682237263Snp sglist_reset(&sg); 683237263Snp } 684237263Snp if (i & 1) 685237263Snp usgl->sge[i / 2].len[1] = htobe32(0); 686237263Snp KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", 687237263Snp __func__, nsegs, start, stop)); 688237263Snp} 689237263Snp 690237263Snp/* 691237263Snp * Max number of SGL entries an offload tx work request can have. This is 41 692237263Snp * (1 + 40) for a full 512B work request. 693237263Snp * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) 694237263Snp */ 695237263Snp#define OFLD_SGL_LEN (41) 696237263Snp 697237263Snp/* 698237263Snp * Send data and/or a FIN to the peer. 699237263Snp * 700237263Snp * The socket's so_snd buffer consists of a stream of data starting with sb_mb 701237263Snp * and linked together with m_next. sb_sndptr, if set, is the last mbuf that 702237263Snp * was transmitted. 703255411Snp * 704255411Snp * drop indicates the number of bytes that should be dropped from the head of 705255411Snp * the send buffer. It is an optimization that lets do_fw4_ack avoid creating 706255411Snp * contention on the send buffer lock (before this change it used to do 707255411Snp * sowwakeup and then t4_push_frames right after that when recovering from tx 708255411Snp * stalls). When drop is set this function MUST drop the bytes and wake up any 709255411Snp * writers. 710237263Snp */ 711269076Snpvoid 712255411Snpt4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 713237263Snp{ 714237263Snp struct mbuf *sndptr, *m, *sb_sndptr; 715237263Snp struct fw_ofld_tx_data_wr *txwr; 716237263Snp struct wrqe *wr; 717255411Snp u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 718237263Snp struct inpcb *inp = toep->inp; 719237263Snp struct tcpcb *tp = intotcpcb(inp); 720237263Snp struct socket *so = inp->inp_socket; 721237263Snp struct sockbuf *sb = &so->so_snd; 722301932Sjhb int tx_credits, shove, compl, sowwakeup; 723346805Snp struct ofld_tx_sdesc *txsd; 724306661Sjhb bool aiotx_mbuf_seen; 725237263Snp 726237263Snp INP_WLOCK_ASSERT(inp); 727239514Snp KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 728237263Snp ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 729237263Snp 730255005Snp KASSERT(toep->ulp_mode == ULP_MODE_NONE || 731255005Snp toep->ulp_mode == ULP_MODE_TCPDDP || 732345664Sjhb toep->ulp_mode == ULP_MODE_TLS || 733255005Snp toep->ulp_mode == ULP_MODE_RDMA, 734255005Snp ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 735237263Snp 736306661Sjhb#ifdef VERBOSE_TRACES 737306661Sjhb CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d", 738306661Sjhb __func__, toep->tid, toep->flags, tp->t_flags); 739306661Sjhb#endif 740292736Snp if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 741292736Snp return; 742292736Snp 743346805Snp#ifdef RATELIMIT 744346805Snp if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) && 745346805Snp (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) { 746346805Snp inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED; 747346805Snp } 748346805Snp#endif 749346805Snp 750237263Snp /* 751237263Snp * This function doesn't resume by itself. Someone else must clear the 752237263Snp * flag and call this function. 753237263Snp */ 754255411Snp if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 755255411Snp KASSERT(drop == 0, 756255411Snp ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 757237263Snp return; 758255411Snp } 759237263Snp 760346805Snp txsd = &toep->txsd[toep->txsd_pidx]; 761237263Snp do { 762237263Snp tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 763237263Snp max_imm = max_imm_payload(tx_credits); 764237263Snp max_nsegs = max_dsgl_nsegs(tx_credits); 765237263Snp 766237263Snp SOCKBUF_LOCK(sb); 767255411Snp sowwakeup = drop; 768255411Snp if (drop) { 769255411Snp sbdrop_locked(sb, drop); 770255411Snp drop = 0; 771255411Snp } 772237263Snp sb_sndptr = sb->sb_sndptr; 773237263Snp sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; 774237263Snp plen = 0; 775237263Snp nsegs = 0; 776237263Snp max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 777306661Sjhb aiotx_mbuf_seen = false; 778237263Snp for (m = sndptr; m != NULL; m = m->m_next) { 779306661Sjhb int n; 780237263Snp 781306661Sjhb if (IS_AIOTX_MBUF(m)) 782306661Sjhb n = sglist_count_vmpages(aiotx_mbuf_pages(m), 783306661Sjhb aiotx_mbuf_pgoff(m), m->m_len); 784306661Sjhb else 785306661Sjhb n = sglist_count(mtod(m, void *), m->m_len); 786306661Sjhb 787237263Snp nsegs += n; 788237263Snp plen += m->m_len; 789237263Snp 790237263Snp /* This mbuf sent us _over_ the nsegs limit, back out */ 791237263Snp if (plen > max_imm && nsegs > max_nsegs) { 792237263Snp nsegs -= n; 793237263Snp plen -= m->m_len; 794237263Snp if (plen == 0) { 795237263Snp /* Too few credits */ 796239514Snp toep->flags |= TPF_TX_SUSPENDED; 797306661Sjhb if (sowwakeup) { 798306661Sjhb if (!TAILQ_EMPTY( 799306661Sjhb &toep->aiotx_jobq)) 800351236Sjhb t4_aiotx_queue_toep(so, 801306661Sjhb toep); 802255411Snp sowwakeup_locked(so); 803306661Sjhb } else 804255411Snp SOCKBUF_UNLOCK(sb); 805255411Snp SOCKBUF_UNLOCK_ASSERT(sb); 806237263Snp return; 807237263Snp } 808237263Snp break; 809237263Snp } 810237263Snp 811306661Sjhb if (IS_AIOTX_MBUF(m)) 812306661Sjhb aiotx_mbuf_seen = true; 813237263Snp if (max_nsegs_1mbuf < n) 814237263Snp max_nsegs_1mbuf = n; 815237263Snp sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ 816237263Snp 817237263Snp /* This mbuf put us right at the max_nsegs limit */ 818237263Snp if (plen > max_imm && nsegs == max_nsegs) { 819237263Snp m = m->m_next; 820237263Snp break; 821237263Snp } 822237263Snp } 823237263Snp 824301932Sjhb if (sbused(sb) > sb->sb_hiwat * 5 / 8 && 825255411Snp toep->plen_nocompl + plen >= sb->sb_hiwat / 4) 826255411Snp compl = 1; 827255411Snp else 828255411Snp compl = 0; 829255411Snp 830237263Snp if (sb->sb_flags & SB_AUTOSIZE && 831237263Snp V_tcp_do_autosndbuf && 832237263Snp sb->sb_hiwat < V_tcp_autosndbuf_max && 833301932Sjhb sbused(sb) >= sb->sb_hiwat * 7 / 8) { 834237263Snp int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 835237263Snp V_tcp_autosndbuf_max); 836237263Snp 837237263Snp if (!sbreserve_locked(sb, newsize, so, NULL)) 838237263Snp sb->sb_flags &= ~SB_AUTOSIZE; 839255411Snp else 840255411Snp sowwakeup = 1; /* room available */ 841237263Snp } 842306661Sjhb if (sowwakeup) { 843306661Sjhb if (!TAILQ_EMPTY(&toep->aiotx_jobq)) 844351236Sjhb t4_aiotx_queue_toep(so, toep); 845255411Snp sowwakeup_locked(so); 846306661Sjhb } else 847255411Snp SOCKBUF_UNLOCK(sb); 848255411Snp SOCKBUF_UNLOCK_ASSERT(sb); 849237263Snp 850237263Snp /* nothing to send */ 851237263Snp if (plen == 0) { 852237263Snp KASSERT(m == NULL, 853237263Snp ("%s: nothing to send, but m != NULL", __func__)); 854237263Snp break; 855237263Snp } 856237263Snp 857239514Snp if (__predict_false(toep->flags & TPF_FIN_SENT)) 858237263Snp panic("%s: excess tx.", __func__); 859237263Snp 860290175Snp shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); 861306661Sjhb if (plen <= max_imm && !aiotx_mbuf_seen) { 862237263Snp 863237263Snp /* Immediate data tx */ 864237263Snp 865248925Snp wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), 866237263Snp toep->ofld_txq); 867237263Snp if (wr == NULL) { 868237263Snp /* XXX: how will we recover from this? */ 869239514Snp toep->flags |= TPF_TX_SUSPENDED; 870237263Snp return; 871237263Snp } 872237263Snp txwr = wrtod(wr); 873237263Snp credits = howmany(wr->wr_len, 16); 874276597Snp write_tx_wr(txwr, toep, plen, plen, credits, shove, 0, 875276597Snp sc->tt.tx_align); 876237263Snp m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 877255411Snp nsegs = 0; 878237263Snp } else { 879237263Snp int wr_len; 880237263Snp 881237263Snp /* DSGL tx */ 882237263Snp 883237263Snp wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 884237263Snp ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 885248925Snp wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 886237263Snp if (wr == NULL) { 887237263Snp /* XXX: how will we recover from this? */ 888239514Snp toep->flags |= TPF_TX_SUSPENDED; 889237263Snp return; 890237263Snp } 891237263Snp txwr = wrtod(wr); 892237263Snp credits = howmany(wr_len, 16); 893276597Snp write_tx_wr(txwr, toep, 0, plen, credits, shove, 0, 894276597Snp sc->tt.tx_align); 895237263Snp write_tx_sgl(txwr + 1, sndptr, m, nsegs, 896237263Snp max_nsegs_1mbuf); 897237263Snp if (wr_len & 0xf) { 898237263Snp uint64_t *pad = (uint64_t *) 899237263Snp ((uintptr_t)txwr + wr_len); 900237263Snp *pad = 0; 901237263Snp } 902237263Snp } 903237263Snp 904237263Snp KASSERT(toep->tx_credits >= credits, 905237263Snp ("%s: not enough credits", __func__)); 906237263Snp 907237263Snp toep->tx_credits -= credits; 908255411Snp toep->tx_nocompl += credits; 909255411Snp toep->plen_nocompl += plen; 910255411Snp if (toep->tx_credits <= toep->tx_total * 3 / 8 && 911255411Snp toep->tx_nocompl >= toep->tx_total / 4) 912255411Snp compl = 1; 913237263Snp 914273797Snp if (compl || toep->ulp_mode == ULP_MODE_RDMA) { 915255411Snp txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 916255411Snp toep->tx_nocompl = 0; 917255411Snp toep->plen_nocompl = 0; 918255411Snp } 919255411Snp 920237263Snp tp->snd_nxt += plen; 921237263Snp tp->snd_max += plen; 922237263Snp 923237263Snp SOCKBUF_LOCK(sb); 924237263Snp KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__)); 925237263Snp sb->sb_sndptr = sb_sndptr; 926237263Snp SOCKBUF_UNLOCK(sb); 927237263Snp 928239514Snp toep->flags |= TPF_TX_DATA_SENT; 929255411Snp if (toep->tx_credits < MIN_OFLD_TX_CREDITS) 930255411Snp toep->flags |= TPF_TX_SUSPENDED; 931237263Snp 932237263Snp KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 933237263Snp txsd->plen = plen; 934237263Snp txsd->tx_credits = credits; 935237263Snp txsd++; 936237263Snp if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 937237263Snp toep->txsd_pidx = 0; 938237263Snp txsd = &toep->txsd[0]; 939237263Snp } 940237263Snp toep->txsd_avail--; 941237263Snp 942237263Snp t4_l2t_send(sc, wr, toep->l2te); 943237263Snp } while (m != NULL); 944237263Snp 945237263Snp /* Send a FIN if requested, but only if there's no more data to send */ 946239514Snp if (m == NULL && toep->flags & TPF_SEND_FIN) 947345664Sjhb t4_close_conn(sc, toep); 948237263Snp} 949237263Snp 950292736Snpstatic inline void 951292736Snprqdrop_locked(struct mbufq *q, int plen) 952292736Snp{ 953292736Snp struct mbuf *m; 954292736Snp 955292736Snp while (plen > 0) { 956292736Snp m = mbufq_dequeue(q); 957292736Snp 958292736Snp /* Too many credits. */ 959292736Snp MPASS(m != NULL); 960292736Snp M_ASSERTPKTHDR(m); 961292736Snp 962292736Snp /* Partial credits. */ 963292736Snp MPASS(plen >= m->m_pkthdr.len); 964292736Snp 965292736Snp plen -= m->m_pkthdr.len; 966292736Snp m_freem(m); 967292736Snp } 968292736Snp} 969292736Snp 970269076Snpvoid 971292736Snpt4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) 972269076Snp{ 973292736Snp struct mbuf *sndptr, *m; 974269076Snp struct fw_ofld_tx_data_wr *txwr; 975269076Snp struct wrqe *wr; 976292736Snp u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 977292736Snp u_int adjusted_plen, ulp_submode; 978269076Snp struct inpcb *inp = toep->inp; 979292736Snp struct tcpcb *tp = intotcpcb(inp); 980292736Snp int tx_credits, shove; 981292736Snp struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 982292736Snp struct mbufq *pduq = &toep->ulp_pduq; 983292736Snp static const u_int ulp_extra_len[] = {0, 4, 4, 8}; 984269076Snp 985269076Snp INP_WLOCK_ASSERT(inp); 986269076Snp KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 987269076Snp ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 988292736Snp KASSERT(toep->ulp_mode == ULP_MODE_ISCSI, 989292736Snp ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 990269076Snp 991292736Snp if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 992292736Snp return; 993292736Snp 994269076Snp /* 995269076Snp * This function doesn't resume by itself. Someone else must clear the 996269076Snp * flag and call this function. 997269076Snp */ 998292736Snp if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 999292736Snp KASSERT(drop == 0, 1000292736Snp ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 1001269076Snp return; 1002292736Snp } 1003269076Snp 1004292736Snp if (drop) 1005292736Snp rqdrop_locked(&toep->ulp_pdu_reclaimq, drop); 1006269076Snp 1007292736Snp while ((sndptr = mbufq_first(pduq)) != NULL) { 1008292736Snp M_ASSERTPKTHDR(sndptr); 1009292736Snp 1010269076Snp tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 1011269076Snp max_imm = max_imm_payload(tx_credits); 1012269076Snp max_nsegs = max_dsgl_nsegs(tx_credits); 1013269076Snp 1014269076Snp plen = 0; 1015269076Snp nsegs = 0; 1016269076Snp max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 1017269076Snp for (m = sndptr; m != NULL; m = m->m_next) { 1018269076Snp int n = sglist_count(mtod(m, void *), m->m_len); 1019269076Snp 1020269076Snp nsegs += n; 1021269076Snp plen += m->m_len; 1022269076Snp 1023292736Snp /* 1024292736Snp * This mbuf would send us _over_ the nsegs limit. 1025292736Snp * Suspend tx because the PDU can't be sent out. 1026292736Snp */ 1027269076Snp if (plen > max_imm && nsegs > max_nsegs) { 1028269076Snp toep->flags |= TPF_TX_SUSPENDED; 1029269076Snp return; 1030269076Snp } 1031269076Snp 1032269076Snp if (max_nsegs_1mbuf < n) 1033269076Snp max_nsegs_1mbuf = n; 1034269076Snp } 1035269076Snp 1036269076Snp if (__predict_false(toep->flags & TPF_FIN_SENT)) 1037269076Snp panic("%s: excess tx.", __func__); 1038269076Snp 1039292736Snp /* 1040292736Snp * We have a PDU to send. All of it goes out in one WR so 'm' 1041292736Snp * is NULL. A PDU's length is always a multiple of 4. 1042292736Snp */ 1043292736Snp MPASS(m == NULL); 1044292736Snp MPASS((plen & 3) == 0); 1045292736Snp MPASS(sndptr->m_pkthdr.len == plen); 1046292736Snp 1047292736Snp shove = !(tp->t_flags & TF_MORETOCOME); 1048292736Snp ulp_submode = mbuf_ulp_submode(sndptr); 1049292736Snp MPASS(ulp_submode < nitems(ulp_extra_len)); 1050292736Snp 1051292736Snp /* 1052292736Snp * plen doesn't include header and data digests, which are 1053292736Snp * generated and inserted in the right places by the TOE, but 1054292736Snp * they do occupy TCP sequence space and need to be accounted 1055292736Snp * for. 1056292736Snp */ 1057292736Snp adjusted_plen = plen + ulp_extra_len[ulp_submode]; 1058269076Snp if (plen <= max_imm) { 1059269076Snp 1060269076Snp /* Immediate data tx */ 1061292736Snp 1062292736Snp wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), 1063269076Snp toep->ofld_txq); 1064269076Snp if (wr == NULL) { 1065269076Snp /* XXX: how will we recover from this? */ 1066269076Snp toep->flags |= TPF_TX_SUSPENDED; 1067269076Snp return; 1068269076Snp } 1069269076Snp txwr = wrtod(wr); 1070269076Snp credits = howmany(wr->wr_len, 16); 1071292736Snp write_tx_wr(txwr, toep, plen, adjusted_plen, credits, 1072292736Snp shove, ulp_submode, sc->tt.tx_align); 1073269076Snp m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 1074292736Snp nsegs = 0; 1075269076Snp } else { 1076269076Snp int wr_len; 1077269076Snp 1078269076Snp /* DSGL tx */ 1079269076Snp wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 1080269076Snp ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 1081292736Snp wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 1082269076Snp if (wr == NULL) { 1083269076Snp /* XXX: how will we recover from this? */ 1084269076Snp toep->flags |= TPF_TX_SUSPENDED; 1085269076Snp return; 1086269076Snp } 1087269076Snp txwr = wrtod(wr); 1088269076Snp credits = howmany(wr_len, 16); 1089292736Snp write_tx_wr(txwr, toep, 0, adjusted_plen, credits, 1090292736Snp shove, ulp_submode, sc->tt.tx_align); 1091269076Snp write_tx_sgl(txwr + 1, sndptr, m, nsegs, 1092269076Snp max_nsegs_1mbuf); 1093269076Snp if (wr_len & 0xf) { 1094269076Snp uint64_t *pad = (uint64_t *) 1095269076Snp ((uintptr_t)txwr + wr_len); 1096269076Snp *pad = 0; 1097269076Snp } 1098269076Snp } 1099269076Snp 1100269076Snp KASSERT(toep->tx_credits >= credits, 1101269076Snp ("%s: not enough credits", __func__)); 1102269076Snp 1103292736Snp m = mbufq_dequeue(pduq); 1104292736Snp MPASS(m == sndptr); 1105292736Snp mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); 1106292736Snp 1107269076Snp toep->tx_credits -= credits; 1108269076Snp toep->tx_nocompl += credits; 1109269076Snp toep->plen_nocompl += plen; 1110269076Snp if (toep->tx_credits <= toep->tx_total * 3 / 8 && 1111292736Snp toep->tx_nocompl >= toep->tx_total / 4) { 1112269076Snp txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 1113269076Snp toep->tx_nocompl = 0; 1114269076Snp toep->plen_nocompl = 0; 1115269076Snp } 1116269076Snp 1117292736Snp tp->snd_nxt += adjusted_plen; 1118292736Snp tp->snd_max += adjusted_plen; 1119269076Snp 1120269076Snp toep->flags |= TPF_TX_DATA_SENT; 1121292736Snp if (toep->tx_credits < MIN_OFLD_TX_CREDITS) 1122269076Snp toep->flags |= TPF_TX_SUSPENDED; 1123269076Snp 1124269076Snp KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 1125269076Snp txsd->plen = plen; 1126269076Snp txsd->tx_credits = credits; 1127269076Snp txsd++; 1128269076Snp if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 1129269076Snp toep->txsd_pidx = 0; 1130269076Snp txsd = &toep->txsd[0]; 1131269076Snp } 1132269076Snp toep->txsd_avail--; 1133269076Snp 1134269076Snp t4_l2t_send(sc, wr, toep->l2te); 1135292736Snp } 1136269076Snp 1137292736Snp /* Send a FIN if requested, but only if there are no more PDUs to send */ 1138292736Snp if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) 1139345664Sjhb t4_close_conn(sc, toep); 1140269076Snp} 1141269076Snp 1142237263Snpint 1143237263Snpt4_tod_output(struct toedev *tod, struct tcpcb *tp) 1144237263Snp{ 1145237263Snp struct adapter *sc = tod->tod_softc; 1146237263Snp#ifdef INVARIANTS 1147237263Snp struct inpcb *inp = tp->t_inpcb; 1148237263Snp#endif 1149237263Snp struct toepcb *toep = tp->t_toe; 1150237263Snp 1151237263Snp INP_WLOCK_ASSERT(inp); 1152237263Snp KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1153237263Snp ("%s: inp %p dropped.", __func__, inp)); 1154237263Snp KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1155237263Snp 1156292736Snp if (toep->ulp_mode == ULP_MODE_ISCSI) 1157292736Snp t4_push_pdus(sc, toep, 0); 1158345664Sjhb else if (tls_tx_key(toep)) 1159345664Sjhb t4_push_tls_records(sc, toep, 0); 1160292736Snp else 1161292736Snp t4_push_frames(sc, toep, 0); 1162237263Snp 1163237263Snp return (0); 1164237263Snp} 1165237263Snp 1166237263Snpint 1167237263Snpt4_send_fin(struct toedev *tod, struct tcpcb *tp) 1168237263Snp{ 1169237263Snp struct adapter *sc = tod->tod_softc; 1170237263Snp#ifdef INVARIANTS 1171237263Snp struct inpcb *inp = tp->t_inpcb; 1172237263Snp#endif 1173237263Snp struct toepcb *toep = tp->t_toe; 1174237263Snp 1175237263Snp INP_WLOCK_ASSERT(inp); 1176237263Snp KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1177237263Snp ("%s: inp %p dropped.", __func__, inp)); 1178237263Snp KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1179237263Snp 1180239514Snp toep->flags |= TPF_SEND_FIN; 1181269076Snp if (tp->t_state >= TCPS_ESTABLISHED) { 1182269076Snp if (toep->ulp_mode == ULP_MODE_ISCSI) 1183292736Snp t4_push_pdus(sc, toep, 0); 1184345664Sjhb else if (tls_tx_key(toep)) 1185345664Sjhb t4_push_tls_records(sc, toep, 0); 1186269076Snp else 1187269076Snp t4_push_frames(sc, toep, 0); 1188269076Snp } 1189237263Snp 1190237263Snp return (0); 1191237263Snp} 1192237263Snp 1193237263Snpint 1194237263Snpt4_send_rst(struct toedev *tod, struct tcpcb *tp) 1195237263Snp{ 1196237263Snp struct adapter *sc = tod->tod_softc; 1197237263Snp#if defined(INVARIANTS) 1198237263Snp struct inpcb *inp = tp->t_inpcb; 1199237263Snp#endif 1200237263Snp struct toepcb *toep = tp->t_toe; 1201237263Snp 1202237263Snp INP_WLOCK_ASSERT(inp); 1203237263Snp KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1204237263Snp ("%s: inp %p dropped.", __func__, inp)); 1205237263Snp KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1206237263Snp 1207237263Snp /* hmmmm */ 1208239514Snp KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1209237263Snp ("%s: flowc for tid %u [%s] not sent already", 1210237263Snp __func__, toep->tid, tcpstates[tp->t_state])); 1211237263Snp 1212237263Snp send_reset(sc, toep, 0); 1213237263Snp return (0); 1214237263Snp} 1215237263Snp 1216237263Snp/* 1217237263Snp * Peer has sent us a FIN. 1218237263Snp */ 1219237263Snpstatic int 1220237263Snpdo_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1221237263Snp{ 1222237263Snp struct adapter *sc = iq->adapter; 1223237263Snp const struct cpl_peer_close *cpl = (const void *)(rss + 1); 1224237263Snp unsigned int tid = GET_TID(cpl); 1225237263Snp struct toepcb *toep = lookup_tid(sc, tid); 1226237263Snp struct inpcb *inp = toep->inp; 1227237263Snp struct tcpcb *tp = NULL; 1228239344Snp struct socket *so; 1229237263Snp#ifdef INVARIANTS 1230237263Snp unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1231237263Snp#endif 1232237263Snp 1233237263Snp KASSERT(opcode == CPL_PEER_CLOSE, 1234237263Snp ("%s: unexpected opcode 0x%x", __func__, opcode)); 1235237263Snp KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1236243680Snp 1237243680Snp if (__predict_false(toep->flags & TPF_SYNQE)) { 1238346970Snp /* 1239346970Snp * do_pass_establish must have run before do_peer_close and if 1240346970Snp * this is still a synqe instead of a toepcb then the connection 1241346970Snp * must be getting aborted. 1242346970Snp */ 1243346970Snp MPASS(toep->flags & TPF_ABORT_SHUTDOWN); 1244243680Snp CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1245243680Snp toep, toep->flags); 1246243680Snp return (0); 1247243680Snp } 1248243680Snp 1249237263Snp KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1250237263Snp 1251312116Snp CURVNET_SET(toep->vnet); 1252286227Sjch INP_INFO_RLOCK(&V_tcbinfo); 1253237263Snp INP_WLOCK(inp); 1254237263Snp tp = intotcpcb(inp); 1255237263Snp 1256237263Snp CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__, 1257237263Snp tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp); 1258237263Snp 1259239514Snp if (toep->flags & TPF_ABORT_SHUTDOWN) 1260237263Snp goto done; 1261237263Snp 1262239344Snp tp->rcv_nxt++; /* FIN */ 1263239344Snp 1264237263Snp so = inp->inp_socket; 1265299210Sjhb if (toep->ulp_mode == ULP_MODE_TCPDDP) { 1266299210Sjhb DDP_LOCK(toep); 1267331645Sjhb if (__predict_false(toep->ddp.flags & 1268299210Sjhb (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) 1269299210Sjhb handle_ddp_close(toep, tp, cpl->rcv_nxt); 1270299210Sjhb DDP_UNLOCK(toep); 1271239344Snp } 1272299210Sjhb socantrcvmore(so); 1273239344Snp 1274255005Snp if (toep->ulp_mode != ULP_MODE_RDMA) { 1275255005Snp KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt), 1276255005Snp ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt, 1277255005Snp be32toh(cpl->rcv_nxt))); 1278255005Snp } 1279237263Snp 1280237263Snp switch (tp->t_state) { 1281237263Snp case TCPS_SYN_RECEIVED: 1282237263Snp tp->t_starttime = ticks; 1283237263Snp /* FALLTHROUGH */ 1284237263Snp 1285237263Snp case TCPS_ESTABLISHED: 1286342583Sjhb tcp_state_change(tp, TCPS_CLOSE_WAIT); 1287237263Snp break; 1288237263Snp 1289237263Snp case TCPS_FIN_WAIT_1: 1290342583Sjhb tcp_state_change(tp, TCPS_CLOSING); 1291237263Snp break; 1292237263Snp 1293237263Snp case TCPS_FIN_WAIT_2: 1294237263Snp tcp_twstart(tp); 1295237263Snp INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1296286227Sjch INP_INFO_RUNLOCK(&V_tcbinfo); 1297312116Snp CURVNET_RESTORE(); 1298237263Snp 1299237263Snp INP_WLOCK(inp); 1300237263Snp final_cpl_received(toep); 1301237263Snp return (0); 1302237263Snp 1303237263Snp default: 1304237263Snp log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n", 1305237263Snp __func__, tid, tp->t_state); 1306237263Snp } 1307237263Snpdone: 1308237263Snp INP_WUNLOCK(inp); 1309286227Sjch INP_INFO_RUNLOCK(&V_tcbinfo); 1310312116Snp CURVNET_RESTORE(); 1311237263Snp return (0); 1312237263Snp} 1313237263Snp 1314237263Snp/* 1315237263Snp * Peer has ACK'd our FIN. 1316237263Snp */ 1317237263Snpstatic int 1318237263Snpdo_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, 1319237263Snp struct mbuf *m) 1320237263Snp{ 1321237263Snp struct adapter *sc = iq->adapter; 1322237263Snp const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1); 1323237263Snp unsigned int tid = GET_TID(cpl); 1324237263Snp struct toepcb *toep = lookup_tid(sc, tid); 1325237263Snp struct inpcb *inp = toep->inp; 1326237263Snp struct tcpcb *tp = NULL; 1327237263Snp struct socket *so = NULL; 1328237263Snp#ifdef INVARIANTS 1329237263Snp unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1330237263Snp#endif 1331237263Snp 1332237263Snp KASSERT(opcode == CPL_CLOSE_CON_RPL, 1333237263Snp ("%s: unexpected opcode 0x%x", __func__, opcode)); 1334237263Snp KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1335237263Snp KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1336237263Snp 1337312116Snp CURVNET_SET(toep->vnet); 1338286227Sjch INP_INFO_RLOCK(&V_tcbinfo); 1339237263Snp INP_WLOCK(inp); 1340237263Snp tp = intotcpcb(inp); 1341237263Snp 1342237263Snp CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x", 1343237263Snp __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags); 1344237263Snp 1345239514Snp if (toep->flags & TPF_ABORT_SHUTDOWN) 1346237263Snp goto done; 1347237263Snp 1348237263Snp so = inp->inp_socket; 1349237263Snp tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */ 1350237263Snp 1351237263Snp switch (tp->t_state) { 1352237263Snp case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */ 1353237263Snp tcp_twstart(tp); 1354237263Snprelease: 1355237263Snp INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1356286227Sjch INP_INFO_RUNLOCK(&V_tcbinfo); 1357312116Snp CURVNET_RESTORE(); 1358237263Snp 1359237263Snp INP_WLOCK(inp); 1360237263Snp final_cpl_received(toep); /* no more CPLs expected */ 1361237263Snp 1362237263Snp return (0); 1363237263Snp case TCPS_LAST_ACK: 1364237263Snp if (tcp_close(tp)) 1365237263Snp INP_WUNLOCK(inp); 1366237263Snp goto release; 1367237263Snp 1368237263Snp case TCPS_FIN_WAIT_1: 1369237263Snp if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1370237263Snp soisdisconnected(so); 1371342583Sjhb tcp_state_change(tp, TCPS_FIN_WAIT_2); 1372237263Snp break; 1373237263Snp 1374237263Snp default: 1375237263Snp log(LOG_ERR, 1376237263Snp "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n", 1377237263Snp __func__, tid, tcpstates[tp->t_state]); 1378237263Snp } 1379237263Snpdone: 1380237263Snp INP_WUNLOCK(inp); 1381286227Sjch INP_INFO_RUNLOCK(&V_tcbinfo); 1382312116Snp CURVNET_RESTORE(); 1383237263Snp return (0); 1384237263Snp} 1385237263Snp 1386237263Snpvoid 1387237263Snpsend_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid, 1388237263Snp int rst_status) 1389237263Snp{ 1390237263Snp struct wrqe *wr; 1391237263Snp struct cpl_abort_rpl *cpl; 1392237263Snp 1393237263Snp wr = alloc_wrqe(sizeof(*cpl), ofld_txq); 1394237263Snp if (wr == NULL) { 1395237263Snp /* XXX */ 1396237263Snp panic("%s: allocation failure.", __func__); 1397237263Snp } 1398237263Snp cpl = wrtod(wr); 1399237263Snp 1400237263Snp INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid); 1401237263Snp cpl->cmd = rst_status; 1402237263Snp 1403237263Snp t4_wrq_tx(sc, wr); 1404237263Snp} 1405237263Snp 1406237263Snpstatic int 1407237263Snpabort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason) 1408237263Snp{ 1409237263Snp switch (abort_reason) { 1410237263Snp case CPL_ERR_BAD_SYN: 1411237263Snp case CPL_ERR_CONN_RESET: 1412237263Snp return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); 1413237263Snp case CPL_ERR_XMIT_TIMEDOUT: 1414237263Snp case CPL_ERR_PERSIST_TIMEDOUT: 1415237263Snp case CPL_ERR_FINWAIT2_TIMEDOUT: 1416237263Snp case CPL_ERR_KEEPALIVE_TIMEDOUT: 1417237263Snp return (ETIMEDOUT); 1418237263Snp default: 1419237263Snp return (EIO); 1420237263Snp } 1421237263Snp} 1422237263Snp 1423237263Snp/* 1424237263Snp * TCP RST from the peer, timeout, or some other such critical error. 1425237263Snp */ 1426237263Snpstatic int 1427237263Snpdo_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1428237263Snp{ 1429237263Snp struct adapter *sc = iq->adapter; 1430237263Snp const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); 1431237263Snp unsigned int tid = GET_TID(cpl); 1432237263Snp struct toepcb *toep = lookup_tid(sc, tid); 1433237263Snp struct sge_wrq *ofld_txq = toep->ofld_txq; 1434237263Snp struct inpcb *inp; 1435237263Snp struct tcpcb *tp; 1436237263Snp#ifdef INVARIANTS 1437237263Snp unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1438237263Snp#endif 1439237263Snp 1440237263Snp KASSERT(opcode == CPL_ABORT_REQ_RSS, 1441237263Snp ("%s: unexpected opcode 0x%x", __func__, opcode)); 1442237263Snp KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1443237263Snp 1444239514Snp if (toep->flags & TPF_SYNQE) 1445237263Snp return (do_abort_req_synqe(iq, rss, m)); 1446237263Snp 1447237263Snp KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1448237263Snp 1449245935Snp if (negative_advice(cpl->status)) { 1450237263Snp CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)", 1451237263Snp __func__, cpl->status, tid, toep->flags); 1452237263Snp return (0); /* Ignore negative advice */ 1453237263Snp } 1454237263Snp 1455237263Snp inp = toep->inp; 1456312116Snp CURVNET_SET(toep->vnet); 1457286227Sjch INP_INFO_RLOCK(&V_tcbinfo); /* for tcp_close */ 1458237263Snp INP_WLOCK(inp); 1459237263Snp 1460237263Snp tp = intotcpcb(inp); 1461237263Snp 1462237263Snp CTR6(KTR_CXGBE, 1463237263Snp "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d", 1464239528Snp __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, 1465239528Snp inp->inp_flags, cpl->status); 1466237263Snp 1467237263Snp /* 1468237263Snp * If we'd initiated an abort earlier the reply to it is responsible for 1469237263Snp * cleaning up resources. Otherwise we tear everything down right here 1470237263Snp * right now. We owe the T4 a CPL_ABORT_RPL no matter what. 1471237263Snp */ 1472239514Snp if (toep->flags & TPF_ABORT_SHUTDOWN) { 1473237263Snp INP_WUNLOCK(inp); 1474237263Snp goto done; 1475237263Snp } 1476239514Snp toep->flags |= TPF_ABORT_SHUTDOWN; 1477237263Snp 1478242671Snp if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 1479242671Snp struct socket *so = inp->inp_socket; 1480237263Snp 1481242671Snp if (so != NULL) 1482242671Snp so_error_set(so, abort_status_to_errno(tp, 1483242671Snp cpl->status)); 1484242671Snp tp = tcp_close(tp); 1485242671Snp if (tp == NULL) 1486242671Snp INP_WLOCK(inp); /* re-acquire */ 1487242671Snp } 1488242671Snp 1489237263Snp final_cpl_received(toep); 1490237263Snpdone: 1491286227Sjch INP_INFO_RUNLOCK(&V_tcbinfo); 1492312116Snp CURVNET_RESTORE(); 1493237263Snp send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST); 1494237263Snp return (0); 1495237263Snp} 1496237263Snp 1497237263Snp/* 1498237263Snp * Reply to the CPL_ABORT_REQ (send_reset) 1499237263Snp */ 1500237263Snpstatic int 1501237263Snpdo_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1502237263Snp{ 1503237263Snp struct adapter *sc = iq->adapter; 1504237263Snp const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); 1505237263Snp unsigned int tid = GET_TID(cpl); 1506237263Snp struct toepcb *toep = lookup_tid(sc, tid); 1507237263Snp struct inpcb *inp = toep->inp; 1508237263Snp#ifdef INVARIANTS 1509237263Snp unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1510237263Snp#endif 1511237263Snp 1512237263Snp KASSERT(opcode == CPL_ABORT_RPL_RSS, 1513237263Snp ("%s: unexpected opcode 0x%x", __func__, opcode)); 1514237263Snp KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1515237263Snp 1516239514Snp if (toep->flags & TPF_SYNQE) 1517237263Snp return (do_abort_rpl_synqe(iq, rss, m)); 1518237263Snp 1519237263Snp KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1520237263Snp 1521237263Snp CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d", 1522237263Snp __func__, tid, toep, inp, cpl->status); 1523237263Snp 1524239514Snp KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1525237263Snp ("%s: wasn't expecting abort reply", __func__)); 1526237263Snp 1527237263Snp INP_WLOCK(inp); 1528237263Snp final_cpl_received(toep); 1529237263Snp 1530237263Snp return (0); 1531237263Snp} 1532237263Snp 1533237263Snpstatic int 1534237263Snpdo_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1535237263Snp{ 1536237263Snp struct adapter *sc = iq->adapter; 1537237263Snp const struct cpl_rx_data *cpl = mtod(m, const void *); 1538237263Snp unsigned int tid = GET_TID(cpl); 1539237263Snp struct toepcb *toep = lookup_tid(sc, tid); 1540237263Snp struct inpcb *inp = toep->inp; 1541237263Snp struct tcpcb *tp; 1542237263Snp struct socket *so; 1543239344Snp struct sockbuf *sb; 1544348704Snp int len, rx_credits; 1545243681Snp uint32_t ddp_placed = 0; 1546237263Snp 1547239514Snp if (__predict_false(toep->flags & TPF_SYNQE)) { 1548346970Snp /* 1549346970Snp * do_pass_establish must have run before do_rx_data and if this 1550346970Snp * is still a synqe instead of a toepcb then the connection must 1551346970Snp * be getting aborted. 1552346970Snp */ 1553346970Snp MPASS(toep->flags & TPF_ABORT_SHUTDOWN); 1554243680Snp CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1555243680Snp toep, toep->flags); 1556237263Snp m_freem(m); 1557237263Snp return (0); 1558237263Snp } 1559237263Snp 1560237263Snp KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1561237263Snp 1562237263Snp /* strip off CPL header */ 1563237263Snp m_adj(m, sizeof(*cpl)); 1564239344Snp len = m->m_pkthdr.len; 1565237263Snp 1566237263Snp INP_WLOCK(inp); 1567237263Snp if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1568237263Snp CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1569239344Snp __func__, tid, len, inp->inp_flags); 1570237263Snp INP_WUNLOCK(inp); 1571237263Snp m_freem(m); 1572237263Snp return (0); 1573237263Snp } 1574237263Snp 1575237263Snp tp = intotcpcb(inp); 1576237263Snp 1577243681Snp if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq))) 1578243681Snp ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt; 1579237263Snp 1580239344Snp tp->rcv_nxt += len; 1581300895Snp if (tp->rcv_wnd < len) { 1582301898Snp KASSERT(toep->ulp_mode == ULP_MODE_RDMA, 1583300895Snp ("%s: negative window size", __func__)); 1584300895Snp } 1585300895Snp 1586239344Snp tp->rcv_wnd -= len; 1587237263Snp tp->t_rcvtime = ticks; 1588237263Snp 1589299210Sjhb if (toep->ulp_mode == ULP_MODE_TCPDDP) 1590299210Sjhb DDP_LOCK(toep); 1591237263Snp so = inp_inpcbtosocket(inp); 1592239344Snp sb = &so->so_rcv; 1593239344Snp SOCKBUF_LOCK(sb); 1594237263Snp 1595239344Snp if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1596237263Snp CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1597239344Snp __func__, tid, len); 1598237263Snp m_freem(m); 1599239344Snp SOCKBUF_UNLOCK(sb); 1600299210Sjhb if (toep->ulp_mode == ULP_MODE_TCPDDP) 1601299210Sjhb DDP_UNLOCK(toep); 1602237263Snp INP_WUNLOCK(inp); 1603237263Snp 1604312116Snp CURVNET_SET(toep->vnet); 1605286227Sjch INP_INFO_RLOCK(&V_tcbinfo); 1606237263Snp INP_WLOCK(inp); 1607237263Snp tp = tcp_drop(tp, ECONNRESET); 1608237263Snp if (tp) 1609237263Snp INP_WUNLOCK(inp); 1610286227Sjch INP_INFO_RUNLOCK(&V_tcbinfo); 1611312116Snp CURVNET_RESTORE(); 1612237263Snp 1613237263Snp return (0); 1614237263Snp } 1615237263Snp 1616237263Snp /* receive buffer autosize */ 1617312116Snp MPASS(toep->vnet == so->so_vnet); 1618312116Snp CURVNET_SET(toep->vnet); 1619239344Snp if (sb->sb_flags & SB_AUTOSIZE && 1620237263Snp V_tcp_do_autorcvbuf && 1621239344Snp sb->sb_hiwat < V_tcp_autorcvbuf_max && 1622239344Snp len > (sbspace(sb) / 8 * 7)) { 1623239344Snp unsigned int hiwat = sb->sb_hiwat; 1624237263Snp unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, 1625237263Snp V_tcp_autorcvbuf_max); 1626237263Snp 1627239344Snp if (!sbreserve_locked(sb, newsize, so, NULL)) 1628239344Snp sb->sb_flags &= ~SB_AUTOSIZE; 1629237263Snp } 1630239344Snp 1631239344Snp if (toep->ulp_mode == ULP_MODE_TCPDDP) { 1632331645Sjhb int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off; 1633239344Snp 1634331645Sjhb if (toep->ddp.waiting_count != 0 || toep->ddp.active_count != 0) 1635331645Sjhb CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)", 1636331645Sjhb __func__, tid, len); 1637331645Sjhb 1638239344Snp if (changed) { 1639331645Sjhb if (toep->ddp.flags & DDP_SC_REQ) 1640331645Sjhb toep->ddp.flags ^= DDP_ON | DDP_SC_REQ; 1641243681Snp else { 1642243681Snp KASSERT(cpl->ddp_off == 1, 1643243681Snp ("%s: DDP switched on by itself.", 1644243681Snp __func__)); 1645243681Snp 1646243681Snp /* Fell out of DDP mode */ 1647331645Sjhb toep->ddp.flags &= ~DDP_ON; 1648299210Sjhb CTR1(KTR_CXGBE, "%s: fell out of DDP mode", 1649299210Sjhb __func__); 1650243681Snp 1651299210Sjhb insert_ddp_data(toep, ddp_placed); 1652239344Snp } 1653239344Snp } 1654239344Snp 1655331645Sjhb if (toep->ddp.flags & DDP_ON) { 1656239344Snp /* 1657299210Sjhb * CPL_RX_DATA with DDP on can only be an indicate. 1658299210Sjhb * Start posting queued AIO requests via DDP. The 1659299210Sjhb * payload that arrived in this indicate is appended 1660299210Sjhb * to the socket buffer as usual. 1661239344Snp */ 1662299210Sjhb handle_ddp_indicate(toep); 1663239344Snp } 1664239344Snp } 1665239344Snp 1666275329Sglebius sbappendstream_locked(sb, m, 0); 1667348704Snp rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; 1668348704Snp if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) { 1669348704Snp rx_credits = send_rx_credits(sc, toep, rx_credits); 1670348704Snp tp->rcv_wnd += rx_credits; 1671348704Snp tp->rcv_adv += rx_credits; 1672280878Snp } 1673299210Sjhb 1674331645Sjhb if (toep->ulp_mode == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 && 1675331645Sjhb sbavail(sb) != 0) { 1676299210Sjhb CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__, 1677299210Sjhb tid); 1678299210Sjhb ddp_queue_toep(toep); 1679299210Sjhb } 1680237263Snp sorwakeup_locked(so); 1681239344Snp SOCKBUF_UNLOCK_ASSERT(sb); 1682299210Sjhb if (toep->ulp_mode == ULP_MODE_TCPDDP) 1683299210Sjhb DDP_UNLOCK(toep); 1684237263Snp 1685237263Snp INP_WUNLOCK(inp); 1686299206Sjhb CURVNET_RESTORE(); 1687237263Snp return (0); 1688237263Snp} 1689237263Snp 1690237263Snpstatic int 1691237263Snpdo_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1692237263Snp{ 1693237263Snp struct adapter *sc = iq->adapter; 1694237263Snp const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 1695237263Snp unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 1696237263Snp struct toepcb *toep = lookup_tid(sc, tid); 1697237263Snp struct inpcb *inp; 1698237263Snp struct tcpcb *tp; 1699237263Snp struct socket *so; 1700237263Snp uint8_t credits = cpl->credits; 1701237263Snp struct ofld_tx_sdesc *txsd; 1702237263Snp int plen; 1703237263Snp#ifdef INVARIANTS 1704237263Snp unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl))); 1705237263Snp#endif 1706237263Snp 1707237263Snp /* 1708237263Snp * Very unusual case: we'd sent a flowc + abort_req for a synq entry and 1709237263Snp * now this comes back carrying the credits for the flowc. 1710237263Snp */ 1711239514Snp if (__predict_false(toep->flags & TPF_SYNQE)) { 1712239514Snp KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1713237263Snp ("%s: credits for a synq entry %p", __func__, toep)); 1714237263Snp return (0); 1715237263Snp } 1716237263Snp 1717237263Snp inp = toep->inp; 1718237263Snp 1719237263Snp KASSERT(opcode == CPL_FW4_ACK, 1720237263Snp ("%s: unexpected opcode 0x%x", __func__, opcode)); 1721237263Snp KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1722237263Snp KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1723237263Snp 1724237263Snp INP_WLOCK(inp); 1725237263Snp 1726239514Snp if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) { 1727237263Snp INP_WUNLOCK(inp); 1728237263Snp return (0); 1729237263Snp } 1730237263Snp 1731237263Snp KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0, 1732237263Snp ("%s: inp_flags 0x%x", __func__, inp->inp_flags)); 1733237263Snp 1734237263Snp tp = intotcpcb(inp); 1735237263Snp 1736237436Snp if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) { 1737237263Snp tcp_seq snd_una = be32toh(cpl->snd_una); 1738237263Snp 1739237263Snp#ifdef INVARIANTS 1740237263Snp if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { 1741237263Snp log(LOG_ERR, 1742237263Snp "%s: unexpected seq# %x for TID %u, snd_una %x\n", 1743237263Snp __func__, snd_una, toep->tid, tp->snd_una); 1744237263Snp } 1745237263Snp#endif 1746237263Snp 1747237263Snp if (tp->snd_una != snd_una) { 1748237263Snp tp->snd_una = snd_una; 1749237263Snp tp->ts_recent_age = tcp_ts_getticks(); 1750237263Snp } 1751237263Snp } 1752237263Snp 1753306661Sjhb#ifdef VERBOSE_TRACES 1754306661Sjhb CTR3(KTR_CXGBE, "%s: tid %d credits %u", __func__, tid, credits); 1755306661Sjhb#endif 1756237263Snp so = inp->inp_socket; 1757237263Snp txsd = &toep->txsd[toep->txsd_cidx]; 1758237263Snp plen = 0; 1759237263Snp while (credits) { 1760237263Snp KASSERT(credits >= txsd->tx_credits, 1761237263Snp ("%s: too many (or partial) credits", __func__)); 1762237263Snp credits -= txsd->tx_credits; 1763237263Snp toep->tx_credits += txsd->tx_credits; 1764237263Snp plen += txsd->plen; 1765345664Sjhb if (txsd->iv_buffer) { 1766345664Sjhb free(txsd->iv_buffer, M_CXGBE); 1767345664Sjhb txsd->iv_buffer = NULL; 1768345664Sjhb } 1769237263Snp txsd++; 1770237263Snp toep->txsd_avail++; 1771237263Snp KASSERT(toep->txsd_avail <= toep->txsd_total, 1772237263Snp ("%s: txsd avail > total", __func__)); 1773237263Snp if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) { 1774237263Snp txsd = &toep->txsd[0]; 1775237263Snp toep->txsd_cidx = 0; 1776237263Snp } 1777237263Snp } 1778237263Snp 1779255411Snp if (toep->tx_credits == toep->tx_total) { 1780255411Snp toep->tx_nocompl = 0; 1781255411Snp toep->plen_nocompl = 0; 1782255411Snp } 1783255411Snp 1784255411Snp if (toep->flags & TPF_TX_SUSPENDED && 1785255411Snp toep->tx_credits >= toep->tx_total / 4) { 1786306661Sjhb#ifdef VERBOSE_TRACES 1787306661Sjhb CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__, 1788306661Sjhb tid); 1789306661Sjhb#endif 1790255411Snp toep->flags &= ~TPF_TX_SUSPENDED; 1791312116Snp CURVNET_SET(toep->vnet); 1792269076Snp if (toep->ulp_mode == ULP_MODE_ISCSI) 1793292736Snp t4_push_pdus(sc, toep, plen); 1794345664Sjhb else if (tls_tx_key(toep)) 1795345664Sjhb t4_push_tls_records(sc, toep, plen); 1796269076Snp else 1797269076Snp t4_push_frames(sc, toep, plen); 1798312116Snp CURVNET_RESTORE(); 1799255411Snp } else if (plen > 0) { 1800237263Snp struct sockbuf *sb = &so->so_snd; 1801292736Snp int sbu; 1802237263Snp 1803292736Snp SOCKBUF_LOCK(sb); 1804292736Snp sbu = sbused(sb); 1805292736Snp if (toep->ulp_mode == ULP_MODE_ISCSI) { 1806292736Snp 1807292736Snp if (__predict_false(sbu > 0)) { 1808292736Snp /* 1809292736Snp * The data trasmitted before the tid's ULP mode 1810292736Snp * changed to ISCSI is still in so_snd. 1811292736Snp * Incoming credits should account for so_snd 1812292736Snp * first. 1813292736Snp */ 1814292736Snp sbdrop_locked(sb, min(sbu, plen)); 1815292736Snp plen -= min(sbu, plen); 1816292736Snp } 1817292736Snp sowwakeup_locked(so); /* unlocks so_snd */ 1818292736Snp rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); 1819292736Snp } else { 1820306661Sjhb#ifdef VERBOSE_TRACES 1821306661Sjhb CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__, 1822306661Sjhb tid, plen); 1823306661Sjhb#endif 1824269076Snp sbdrop_locked(sb, plen); 1825345664Sjhb if (tls_tx_key(toep)) { 1826345664Sjhb struct tls_ofld_info *tls_ofld = &toep->tls; 1827345664Sjhb 1828345664Sjhb MPASS(tls_ofld->sb_off >= plen); 1829345664Sjhb tls_ofld->sb_off -= plen; 1830345664Sjhb } 1831306661Sjhb if (!TAILQ_EMPTY(&toep->aiotx_jobq)) 1832351236Sjhb t4_aiotx_queue_toep(so, toep); 1833292736Snp sowwakeup_locked(so); /* unlocks so_snd */ 1834269076Snp } 1835292736Snp SOCKBUF_UNLOCK_ASSERT(sb); 1836237263Snp } 1837237263Snp 1838237263Snp INP_WUNLOCK(inp); 1839237263Snp 1840237263Snp return (0); 1841237263Snp} 1842237263Snp 1843237263Snpvoid 1844345664Sjhbt4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep, 1845345664Sjhb uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie) 1846239338Snp{ 1847239338Snp struct wrqe *wr; 1848239338Snp struct cpl_set_tcb_field *req; 1849345664Sjhb struct ofld_tx_sdesc *txsd; 1850239338Snp 1851302339Snp MPASS((cookie & ~M_COOKIE) == 0); 1852346852Snp if (reply) { 1853346852Snp MPASS(cookie != CPL_COOKIE_RESERVED); 1854346852Snp } 1855239338Snp 1856302339Snp wr = alloc_wrqe(sizeof(*req), wrq); 1857299210Sjhb if (wr == NULL) { 1858299210Sjhb /* XXX */ 1859299210Sjhb panic("%s: allocation failure.", __func__); 1860299210Sjhb } 1861299210Sjhb req = wrtod(wr); 1862299210Sjhb 1863345664Sjhb INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); 1864345664Sjhb req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id)); 1865302339Snp if (reply == 0) 1866302339Snp req->reply_ctrl |= htobe16(F_NO_REPLY); 1867299210Sjhb req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie)); 1868299210Sjhb req->mask = htobe64(mask); 1869299210Sjhb req->val = htobe64(val); 1870345664Sjhb if ((wrq->eq.flags & EQ_TYPEMASK) == EQ_OFLD) { 1871345664Sjhb txsd = &toep->txsd[toep->txsd_pidx]; 1872345664Sjhb txsd->tx_credits = howmany(sizeof(*req), 16); 1873345664Sjhb txsd->plen = 0; 1874345664Sjhb KASSERT(toep->tx_credits >= txsd->tx_credits && 1875345664Sjhb toep->txsd_avail > 0, 1876345664Sjhb ("%s: not enough credits (%d)", __func__, 1877345664Sjhb toep->tx_credits)); 1878345664Sjhb toep->tx_credits -= txsd->tx_credits; 1879345664Sjhb if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 1880345664Sjhb toep->txsd_pidx = 0; 1881345664Sjhb toep->txsd_avail--; 1882345664Sjhb } 1883299210Sjhb 1884299210Sjhb t4_wrq_tx(sc, wr); 1885299210Sjhb} 1886299210Sjhb 1887299210Sjhbvoid 1888302339Snpt4_init_cpl_io_handlers(void) 1889237263Snp{ 1890237263Snp 1891302339Snp t4_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close); 1892302339Snp t4_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl); 1893302339Snp t4_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req); 1894346855Snp t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl, 1895346855Snp CPL_COOKIE_TOM); 1896302339Snp t4_register_cpl_handler(CPL_RX_DATA, do_rx_data); 1897346863Snp t4_register_shared_cpl_handler(CPL_FW4_ACK, do_fw4_ack, CPL_COOKIE_TOM); 1898237263Snp} 1899239338Snp 1900239338Snpvoid 1901302339Snpt4_uninit_cpl_io_handlers(void) 1902239338Snp{ 1903239338Snp 1904313178Sjhb t4_register_cpl_handler(CPL_PEER_CLOSE, NULL); 1905313178Sjhb t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL); 1906313178Sjhb t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL); 1907346864Snp t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM); 1908313178Sjhb t4_register_cpl_handler(CPL_RX_DATA, NULL); 1909346863Snp t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM); 1910239338Snp} 1911306661Sjhb 1912306661Sjhb/* 1913306661Sjhb * Use the 'backend3' field in AIO jobs to store the amount of data 1914306661Sjhb * sent by the AIO job so far and the 'backend4' field to hold an 1915306661Sjhb * error that should be reported when the job is completed. 1916306661Sjhb */ 1917306661Sjhb#define aio_sent backend3 1918306661Sjhb#define aio_error backend4 1919306661Sjhb 1920306661Sjhb#define jobtotid(job) \ 1921306661Sjhb (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid) 1922306661Sjhb 1923306661Sjhbstatic void 1924306661Sjhbfree_aiotx_buffer(struct aiotx_buffer *ab) 1925306661Sjhb{ 1926306661Sjhb struct kaiocb *job; 1927306661Sjhb long status; 1928306661Sjhb int error; 1929306661Sjhb 1930306661Sjhb if (refcount_release(&ab->refcount) == 0) 1931306661Sjhb return; 1932306661Sjhb 1933306661Sjhb job = ab->job; 1934306661Sjhb error = job->aio_error; 1935306661Sjhb status = job->aio_sent; 1936306661Sjhb vm_page_unhold_pages(ab->ps.pages, ab->ps.npages); 1937306661Sjhb free(ab, M_CXGBE); 1938306661Sjhb#ifdef VERBOSE_TRACES 1939306661Sjhb CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__, 1940306661Sjhb jobtotid(job), job, status, error); 1941237263Snp#endif 1942306661Sjhb if (error == ECANCELED && status != 0) 1943306661Sjhb error = 0; 1944306661Sjhb if (error == ECANCELED) 1945306661Sjhb aio_cancel(job); 1946306661Sjhb else if (error) 1947306661Sjhb aio_complete(job, -1, error); 1948306661Sjhb else 1949306661Sjhb aio_complete(job, status, 0); 1950306661Sjhb} 1951306661Sjhb 1952306661Sjhbstatic void 1953306661Sjhbt4_aiotx_mbuf_free(struct mbuf *m, void *buffer, void *arg) 1954306661Sjhb{ 1955306661Sjhb struct aiotx_buffer *ab = buffer; 1956306661Sjhb 1957306661Sjhb#ifdef VERBOSE_TRACES 1958306661Sjhb CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, 1959306661Sjhb m->m_len, jobtotid(ab->job)); 1960306661Sjhb#endif 1961306661Sjhb free_aiotx_buffer(ab); 1962306661Sjhb} 1963306661Sjhb 1964306661Sjhb/* 1965306661Sjhb * Hold the buffer backing an AIO request and return an AIO transmit 1966306661Sjhb * buffer. 1967306661Sjhb */ 1968306661Sjhbstatic int 1969306661Sjhbhold_aio(struct kaiocb *job) 1970306661Sjhb{ 1971306661Sjhb struct aiotx_buffer *ab; 1972306661Sjhb struct vmspace *vm; 1973306661Sjhb vm_map_t map; 1974306661Sjhb vm_offset_t start, end, pgoff; 1975306661Sjhb int n; 1976306661Sjhb 1977306661Sjhb MPASS(job->backend1 == NULL); 1978306661Sjhb 1979306661Sjhb /* 1980306661Sjhb * The AIO subsystem will cancel and drain all requests before 1981306661Sjhb * permitting a process to exit or exec, so p_vmspace should 1982306661Sjhb * be stable here. 1983306661Sjhb */ 1984306661Sjhb vm = job->userproc->p_vmspace; 1985306661Sjhb map = &vm->vm_map; 1986306661Sjhb start = (uintptr_t)job->uaiocb.aio_buf; 1987306661Sjhb pgoff = start & PAGE_MASK; 1988306661Sjhb end = round_page(start + job->uaiocb.aio_nbytes); 1989306661Sjhb start = trunc_page(start); 1990306661Sjhb n = atop(end - start); 1991306661Sjhb 1992306661Sjhb ab = malloc(sizeof(*ab) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK | 1993306661Sjhb M_ZERO); 1994306661Sjhb refcount_init(&ab->refcount, 1); 1995306661Sjhb ab->ps.pages = (vm_page_t *)(ab + 1); 1996306661Sjhb ab->ps.npages = vm_fault_quick_hold_pages(map, start, end - start, 1997306661Sjhb VM_PROT_WRITE, ab->ps.pages, n); 1998306661Sjhb if (ab->ps.npages < 0) { 1999306661Sjhb free(ab, M_CXGBE); 2000306661Sjhb return (EFAULT); 2001306661Sjhb } 2002306661Sjhb 2003306661Sjhb KASSERT(ab->ps.npages == n, 2004306661Sjhb ("hold_aio: page count mismatch: %d vs %d", ab->ps.npages, n)); 2005306661Sjhb 2006306661Sjhb ab->ps.offset = pgoff; 2007306661Sjhb ab->ps.len = job->uaiocb.aio_nbytes; 2008306661Sjhb ab->job = job; 2009306661Sjhb job->backend1 = ab; 2010306661Sjhb#ifdef VERBOSE_TRACES 2011306661Sjhb CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d", 2012306661Sjhb __func__, jobtotid(job), &ab->ps, job, ab->ps.npages); 2013306661Sjhb#endif 2014306661Sjhb return (0); 2015306661Sjhb} 2016306661Sjhb 2017306661Sjhbstatic void 2018306661Sjhbt4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job) 2019306661Sjhb{ 2020306661Sjhb struct adapter *sc; 2021306661Sjhb struct sockbuf *sb; 2022306661Sjhb struct file *fp; 2023306661Sjhb struct aiotx_buffer *ab; 2024306661Sjhb struct inpcb *inp; 2025306661Sjhb struct tcpcb *tp; 2026306661Sjhb struct mbuf *m; 2027306661Sjhb int error; 2028306661Sjhb bool moretocome, sendmore; 2029306661Sjhb 2030306661Sjhb sc = td_adapter(toep->td); 2031306661Sjhb sb = &so->so_snd; 2032306661Sjhb SOCKBUF_UNLOCK(sb); 2033306661Sjhb fp = job->fd_file; 2034306661Sjhb ab = job->backend1; 2035306661Sjhb m = NULL; 2036306661Sjhb 2037306661Sjhb#ifdef MAC 2038306661Sjhb error = mac_socket_check_send(fp->f_cred, so); 2039306661Sjhb if (error != 0) 2040306661Sjhb goto out; 2041306661Sjhb#endif 2042306661Sjhb 2043306661Sjhb if (ab == NULL) { 2044306661Sjhb error = hold_aio(job); 2045306661Sjhb if (error != 0) 2046306661Sjhb goto out; 2047306661Sjhb ab = job->backend1; 2048306661Sjhb } 2049306661Sjhb 2050306661Sjhb /* Inline sosend_generic(). */ 2051306661Sjhb 2052306661Sjhb job->msgsnd = 1; 2053306661Sjhb 2054306661Sjhb error = sblock(sb, SBL_WAIT); 2055306661Sjhb MPASS(error == 0); 2056306661Sjhb 2057306661Sjhbsendanother: 2058306661Sjhb m = m_get(M_WAITOK, MT_DATA); 2059306661Sjhb 2060306661Sjhb SOCKBUF_LOCK(sb); 2061306661Sjhb if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2062306661Sjhb SOCKBUF_UNLOCK(sb); 2063306661Sjhb sbunlock(sb); 2064306661Sjhb if ((so->so_options & SO_NOSIGPIPE) == 0) { 2065306661Sjhb PROC_LOCK(job->userproc); 2066306661Sjhb kern_psignal(job->userproc, SIGPIPE); 2067306661Sjhb PROC_UNLOCK(job->userproc); 2068306661Sjhb } 2069306661Sjhb error = EPIPE; 2070306661Sjhb goto out; 2071306661Sjhb } 2072306661Sjhb if (so->so_error) { 2073306661Sjhb error = so->so_error; 2074306661Sjhb so->so_error = 0; 2075306661Sjhb SOCKBUF_UNLOCK(sb); 2076306661Sjhb sbunlock(sb); 2077306661Sjhb goto out; 2078306661Sjhb } 2079306661Sjhb if ((so->so_state & SS_ISCONNECTED) == 0) { 2080306661Sjhb SOCKBUF_UNLOCK(sb); 2081306661Sjhb sbunlock(sb); 2082306661Sjhb error = ENOTCONN; 2083306661Sjhb goto out; 2084306661Sjhb } 2085306661Sjhb if (sbspace(sb) < sb->sb_lowat) { 2086306661Sjhb MPASS(job->aio_sent == 0 || !(so->so_state & SS_NBIO)); 2087306661Sjhb 2088306661Sjhb /* 2089306661Sjhb * Don't block if there is too little room in the socket 2090306661Sjhb * buffer. Instead, requeue the request. 2091306661Sjhb */ 2092306661Sjhb if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { 2093306661Sjhb SOCKBUF_UNLOCK(sb); 2094306661Sjhb sbunlock(sb); 2095306661Sjhb error = ECANCELED; 2096306661Sjhb goto out; 2097306661Sjhb } 2098306661Sjhb TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); 2099306661Sjhb SOCKBUF_UNLOCK(sb); 2100306661Sjhb sbunlock(sb); 2101306661Sjhb goto out; 2102306661Sjhb } 2103306661Sjhb 2104306661Sjhb /* 2105306661Sjhb * Write as much data as the socket permits, but no more than a 2106306661Sjhb * a single sndbuf at a time. 2107306661Sjhb */ 2108306661Sjhb m->m_len = sbspace(sb); 2109306661Sjhb if (m->m_len > ab->ps.len - job->aio_sent) { 2110306661Sjhb m->m_len = ab->ps.len - job->aio_sent; 2111306661Sjhb moretocome = false; 2112306661Sjhb } else 2113306661Sjhb moretocome = true; 2114306661Sjhb if (m->m_len > sc->tt.sndbuf) { 2115306661Sjhb m->m_len = sc->tt.sndbuf; 2116306661Sjhb sendmore = true; 2117306661Sjhb } else 2118306661Sjhb sendmore = false; 2119306661Sjhb 2120306661Sjhb if (!TAILQ_EMPTY(&toep->aiotx_jobq)) 2121306661Sjhb moretocome = true; 2122306661Sjhb SOCKBUF_UNLOCK(sb); 2123306661Sjhb MPASS(m->m_len != 0); 2124306661Sjhb 2125306661Sjhb /* Inlined tcp_usr_send(). */ 2126306661Sjhb 2127306661Sjhb inp = toep->inp; 2128306661Sjhb INP_WLOCK(inp); 2129306661Sjhb if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 2130306661Sjhb INP_WUNLOCK(inp); 2131306661Sjhb sbunlock(sb); 2132306661Sjhb error = ECONNRESET; 2133306661Sjhb goto out; 2134306661Sjhb } 2135306661Sjhb 2136306661Sjhb refcount_acquire(&ab->refcount); 2137306661Sjhb m_extadd(m, NULL, ab->ps.len, t4_aiotx_mbuf_free, ab, 2138306661Sjhb (void *)(uintptr_t)job->aio_sent, 0, EXT_NET_DRV); 2139306661Sjhb m->m_ext.ext_flags |= EXT_FLAG_AIOTX; 2140306661Sjhb job->aio_sent += m->m_len; 2141306661Sjhb 2142306661Sjhb sbappendstream(sb, m, 0); 2143306661Sjhb m = NULL; 2144306661Sjhb 2145306661Sjhb if (!(inp->inp_flags & INP_DROPPED)) { 2146306661Sjhb tp = intotcpcb(inp); 2147306661Sjhb if (moretocome) 2148306661Sjhb tp->t_flags |= TF_MORETOCOME; 2149306661Sjhb error = tp->t_fb->tfb_tcp_output(tp); 2150306661Sjhb if (moretocome) 2151306661Sjhb tp->t_flags &= ~TF_MORETOCOME; 2152306661Sjhb } 2153306661Sjhb 2154306661Sjhb INP_WUNLOCK(inp); 2155306661Sjhb if (sendmore) 2156306661Sjhb goto sendanother; 2157306661Sjhb sbunlock(sb); 2158306661Sjhb 2159306661Sjhb if (error) 2160306661Sjhb goto out; 2161306661Sjhb 2162306661Sjhb /* 2163306661Sjhb * If this is a non-blocking socket and the request has not 2164306661Sjhb * been fully completed, requeue it until the socket is ready 2165306661Sjhb * again. 2166306661Sjhb */ 2167306661Sjhb if (job->aio_sent < job->uaiocb.aio_nbytes && 2168306661Sjhb !(so->so_state & SS_NBIO)) { 2169306661Sjhb SOCKBUF_LOCK(sb); 2170306661Sjhb if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { 2171306661Sjhb SOCKBUF_UNLOCK(sb); 2172306661Sjhb error = ECANCELED; 2173306661Sjhb goto out; 2174306661Sjhb } 2175306661Sjhb TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); 2176306661Sjhb return; 2177306661Sjhb } 2178306661Sjhb 2179306661Sjhb /* 2180306661Sjhb * If the request will not be requeued, drop a reference on 2181306661Sjhb * the the aiotx buffer. Any mbufs in flight should still 2182306661Sjhb * contain a reference, but this drops the reference that the 2183306661Sjhb * job owns while it is waiting to queue mbufs to the socket. 2184306661Sjhb */ 2185306661Sjhb free_aiotx_buffer(ab); 2186306661Sjhb 2187306661Sjhbout: 2188306661Sjhb if (error) { 2189306661Sjhb if (ab != NULL) { 2190306661Sjhb job->aio_error = error; 2191306661Sjhb free_aiotx_buffer(ab); 2192306661Sjhb } else { 2193306661Sjhb MPASS(job->aio_sent == 0); 2194306661Sjhb aio_complete(job, -1, error); 2195306661Sjhb } 2196306661Sjhb } 2197306661Sjhb if (m != NULL) 2198306661Sjhb m_free(m); 2199306661Sjhb SOCKBUF_LOCK(sb); 2200306661Sjhb} 2201306661Sjhb 2202306661Sjhbstatic void 2203306661Sjhbt4_aiotx_task(void *context, int pending) 2204306661Sjhb{ 2205306661Sjhb struct toepcb *toep = context; 2206351236Sjhb struct socket *so; 2207306661Sjhb struct kaiocb *job; 2208306661Sjhb 2209351236Sjhb so = toep->aiotx_so; 2210312116Snp CURVNET_SET(toep->vnet); 2211306661Sjhb SOCKBUF_LOCK(&so->so_snd); 2212306661Sjhb while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) { 2213306661Sjhb job = TAILQ_FIRST(&toep->aiotx_jobq); 2214306661Sjhb TAILQ_REMOVE(&toep->aiotx_jobq, job, list); 2215306661Sjhb if (!aio_clear_cancel_function(job)) 2216306661Sjhb continue; 2217306661Sjhb 2218306661Sjhb t4_aiotx_process_job(toep, so, job); 2219306661Sjhb } 2220351236Sjhb toep->aiotx_so = NULL; 2221306661Sjhb SOCKBUF_UNLOCK(&so->so_snd); 2222306661Sjhb CURVNET_RESTORE(); 2223306661Sjhb 2224306661Sjhb free_toepcb(toep); 2225351236Sjhb SOCK_LOCK(so); 2226351236Sjhb sorele(so); 2227306661Sjhb} 2228306661Sjhb 2229306661Sjhbstatic void 2230351236Sjhbt4_aiotx_queue_toep(struct socket *so, struct toepcb *toep) 2231306661Sjhb{ 2232306661Sjhb 2233306661Sjhb SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd); 2234306661Sjhb#ifdef VERBOSE_TRACES 2235306661Sjhb CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s", 2236306661Sjhb __func__, toep->tid, toep->aiotx_task_active ? "true" : "false"); 2237306661Sjhb#endif 2238351236Sjhb if (toep->aiotx_so != NULL) 2239306661Sjhb return; 2240351236Sjhb soref(so); 2241351236Sjhb toep->aiotx_so = so; 2242306661Sjhb hold_toepcb(toep); 2243306661Sjhb soaio_enqueue(&toep->aiotx_task); 2244306661Sjhb} 2245306661Sjhb 2246306661Sjhbstatic void 2247306661Sjhbt4_aiotx_cancel(struct kaiocb *job) 2248306661Sjhb{ 2249306661Sjhb struct aiotx_buffer *ab; 2250306661Sjhb struct socket *so; 2251306661Sjhb struct sockbuf *sb; 2252306661Sjhb struct tcpcb *tp; 2253306661Sjhb struct toepcb *toep; 2254306661Sjhb 2255306661Sjhb so = job->fd_file->f_data; 2256306661Sjhb tp = so_sototcpcb(so); 2257306661Sjhb toep = tp->t_toe; 2258306661Sjhb MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); 2259306661Sjhb sb = &so->so_snd; 2260306661Sjhb 2261306661Sjhb SOCKBUF_LOCK(sb); 2262306661Sjhb if (!aio_cancel_cleared(job)) 2263306661Sjhb TAILQ_REMOVE(&toep->aiotx_jobq, job, list); 2264306661Sjhb SOCKBUF_UNLOCK(sb); 2265306661Sjhb 2266306661Sjhb ab = job->backend1; 2267306661Sjhb if (ab != NULL) 2268306661Sjhb free_aiotx_buffer(ab); 2269306661Sjhb else 2270306661Sjhb aio_cancel(job); 2271306661Sjhb} 2272306661Sjhb 2273306661Sjhbint 2274306661Sjhbt4_aio_queue_aiotx(struct socket *so, struct kaiocb *job) 2275306661Sjhb{ 2276306661Sjhb struct tcpcb *tp = so_sototcpcb(so); 2277306661Sjhb struct toepcb *toep = tp->t_toe; 2278306661Sjhb struct adapter *sc = td_adapter(toep->td); 2279306661Sjhb 2280306661Sjhb /* This only handles writes. */ 2281306661Sjhb if (job->uaiocb.aio_lio_opcode != LIO_WRITE) 2282306661Sjhb return (EOPNOTSUPP); 2283306661Sjhb 2284306661Sjhb if (!sc->tt.tx_zcopy) 2285306661Sjhb return (EOPNOTSUPP); 2286306661Sjhb 2287345664Sjhb if (tls_tx_key(toep)) 2288345664Sjhb return (EOPNOTSUPP); 2289345664Sjhb 2290306661Sjhb SOCKBUF_LOCK(&so->so_snd); 2291306661Sjhb#ifdef VERBOSE_TRACES 2292306661Sjhb CTR2(KTR_CXGBE, "%s: queueing %p", __func__, job); 2293306661Sjhb#endif 2294306661Sjhb if (!aio_set_cancel_function(job, t4_aiotx_cancel)) 2295306661Sjhb panic("new job was cancelled"); 2296306661Sjhb TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); 2297306661Sjhb if (sowriteable(so)) 2298351236Sjhb t4_aiotx_queue_toep(so, toep); 2299306661Sjhb SOCKBUF_UNLOCK(&so->so_snd); 2300306661Sjhb return (0); 2301306661Sjhb} 2302306661Sjhb 2303306661Sjhbvoid 2304306661Sjhbaiotx_init_toep(struct toepcb *toep) 2305306661Sjhb{ 2306306661Sjhb 2307306661Sjhb TAILQ_INIT(&toep->aiotx_jobq); 2308306661Sjhb TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep); 2309306661Sjhb} 2310306661Sjhb#endif 2311