t4_cpl_io.c revision 299206
1/*- 2 * Copyright (c) 2012, 2015 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/tom/t4_cpl_io.c 299206 2016-05-06 23:49:10Z jhb $"); 30 31#include "opt_inet.h" 32 33#ifdef TCP_OFFLOAD 34#include <sys/param.h> 35#include <sys/types.h> 36#include <sys/kernel.h> 37#include <sys/ktr.h> 38#include <sys/module.h> 39#include <sys/protosw.h> 40#include <sys/domain.h> 41#include <sys/socket.h> 42#include <sys/socketvar.h> 43#include <sys/sglist.h> 44#include <netinet/in.h> 45#include <netinet/in_pcb.h> 46#include <netinet/ip.h> 47#include <netinet/ip6.h> 48#define TCPSTATES 49#include <netinet/tcp_fsm.h> 50#include <netinet/tcp_seq.h> 51#include <netinet/tcp_var.h> 52#include <netinet/toecore.h> 53 54#include "common/common.h" 55#include "common/t4_msg.h" 56#include "common/t4_regs.h" 57#include "common/t4_tcb.h" 58#include "tom/t4_tom_l2t.h" 59#include "tom/t4_tom.h" 60 61VNET_DECLARE(int, tcp_do_autosndbuf); 62#define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf) 63VNET_DECLARE(int, tcp_autosndbuf_inc); 64#define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc) 65VNET_DECLARE(int, tcp_autosndbuf_max); 66#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) 67VNET_DECLARE(int, tcp_do_autorcvbuf); 68#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) 69VNET_DECLARE(int, tcp_autorcvbuf_inc); 70#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) 71VNET_DECLARE(int, tcp_autorcvbuf_max); 72#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) 73 74void 75send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) 76{ 77 struct wrqe *wr; 78 struct fw_flowc_wr *flowc; 79 unsigned int nparams = ftxp ? 8 : 6, flowclen; 80 struct vi_info *vi = toep->vi; 81 struct port_info *pi = vi->pi; 82 struct adapter *sc = pi->adapter; 83 unsigned int pfvf = G_FW_VIID_PFN(vi->viid) << S_FW_VIID_PFN; 84 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 85 86 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT), 87 ("%s: flowc for tid %u sent already", __func__, toep->tid)); 88 89 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); 90 91 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); 92 if (wr == NULL) { 93 /* XXX */ 94 panic("%s: allocation failure.", __func__); 95 } 96 flowc = wrtod(wr); 97 memset(flowc, 0, wr->wr_len); 98 99 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 100 V_FW_FLOWC_WR_NPARAMS(nparams)); 101 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | 102 V_FW_WR_FLOWID(toep->tid)); 103 104 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 105 flowc->mnemval[0].val = htobe32(pfvf); 106 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 107 flowc->mnemval[1].val = htobe32(pi->tx_chan); 108 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 109 flowc->mnemval[2].val = htobe32(pi->tx_chan); 110 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 111 flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id); 112 if (ftxp) { 113 uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf); 114 115 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 116 flowc->mnemval[4].val = htobe32(ftxp->snd_nxt); 117 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 118 flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt); 119 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 120 flowc->mnemval[6].val = htobe32(sndbuf); 121 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 122 flowc->mnemval[7].val = htobe32(ftxp->mss); 123 124 CTR6(KTR_CXGBE, 125 "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x", 126 __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt, 127 ftxp->rcv_nxt); 128 } else { 129 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF; 130 flowc->mnemval[4].val = htobe32(512); 131 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS; 132 flowc->mnemval[5].val = htobe32(512); 133 134 CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); 135 } 136 137 txsd->tx_credits = howmany(flowclen, 16); 138 txsd->plen = 0; 139 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 140 ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 141 toep->tx_credits -= txsd->tx_credits; 142 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 143 toep->txsd_pidx = 0; 144 toep->txsd_avail--; 145 146 toep->flags |= TPF_FLOWC_WR_SENT; 147 t4_wrq_tx(sc, wr); 148} 149 150void 151send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) 152{ 153 struct wrqe *wr; 154 struct cpl_abort_req *req; 155 int tid = toep->tid; 156 struct inpcb *inp = toep->inp; 157 struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */ 158 159 INP_WLOCK_ASSERT(inp); 160 161 CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s", 162 __func__, toep->tid, 163 inp->inp_flags & INP_DROPPED ? "inp dropped" : 164 tcpstates[tp->t_state], 165 toep->flags, inp->inp_flags, 166 toep->flags & TPF_ABORT_SHUTDOWN ? 167 " (abort already in progress)" : ""); 168 169 if (toep->flags & TPF_ABORT_SHUTDOWN) 170 return; /* abort already in progress */ 171 172 toep->flags |= TPF_ABORT_SHUTDOWN; 173 174 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 175 ("%s: flowc_wr not sent for tid %d.", __func__, tid)); 176 177 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 178 if (wr == NULL) { 179 /* XXX */ 180 panic("%s: allocation failure.", __func__); 181 } 182 req = wrtod(wr); 183 184 INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid); 185 if (inp->inp_flags & INP_DROPPED) 186 req->rsvd0 = htobe32(snd_nxt); 187 else 188 req->rsvd0 = htobe32(tp->snd_nxt); 189 req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT); 190 req->cmd = CPL_ABORT_SEND_RST; 191 192 /* 193 * XXX: What's the correct way to tell that the inp hasn't been detached 194 * from its socket? Should I even be flushing the snd buffer here? 195 */ 196 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 197 struct socket *so = inp->inp_socket; 198 199 if (so != NULL) /* because I'm not sure. See comment above */ 200 sbflush(&so->so_snd); 201 } 202 203 t4_l2t_send(sc, wr, toep->l2te); 204} 205 206/* 207 * Called when a connection is established to translate the TCP options 208 * reported by HW to FreeBSD's native format. 209 */ 210static void 211assign_rxopt(struct tcpcb *tp, unsigned int opt) 212{ 213 struct toepcb *toep = tp->t_toe; 214 struct inpcb *inp = tp->t_inpcb; 215 struct adapter *sc = td_adapter(toep->td); 216 int n; 217 218 INP_LOCK_ASSERT(inp); 219 220 if (inp->inp_inc.inc_flags & INC_ISIPV6) 221 n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 222 else 223 n = sizeof(struct ip) + sizeof(struct tcphdr); 224 tp->t_maxseg = sc->params.mtus[G_TCPOPT_MSS(opt)] - n; 225 226 CTR4(KTR_CXGBE, "%s: tid %d, mtu_idx %u (%u)", __func__, toep->tid, 227 G_TCPOPT_MSS(opt), sc->params.mtus[G_TCPOPT_MSS(opt)]); 228 229 if (G_TCPOPT_TSTAMP(opt)) { 230 tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */ 231 tp->ts_recent = 0; /* hmmm */ 232 tp->ts_recent_age = tcp_ts_getticks(); 233 } 234 235 if (G_TCPOPT_SACK(opt)) 236 tp->t_flags |= TF_SACK_PERMIT; /* should already be set */ 237 else 238 tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */ 239 240 if (G_TCPOPT_WSCALE_OK(opt)) 241 tp->t_flags |= TF_RCVD_SCALE; 242 243 /* Doing window scaling? */ 244 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 245 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 246 tp->rcv_scale = tp->request_r_scale; 247 tp->snd_scale = G_TCPOPT_SND_WSCALE(opt); 248 } 249} 250 251/* 252 * Completes some final bits of initialization for just established connections 253 * and changes their state to TCPS_ESTABLISHED. 254 * 255 * The ISNs are from after the exchange of SYNs. i.e., the true ISN + 1. 256 */ 257void 258make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn, 259 uint16_t opt) 260{ 261 struct inpcb *inp = toep->inp; 262 struct socket *so = inp->inp_socket; 263 struct tcpcb *tp = intotcpcb(inp); 264 long bufsize; 265 uint32_t iss = be32toh(snd_isn) - 1; /* true ISS */ 266 uint32_t irs = be32toh(rcv_isn) - 1; /* true IRS */ 267 uint16_t tcpopt = be16toh(opt); 268 struct flowc_tx_params ftxp; 269 270 CURVNET_SET(so->so_vnet); 271 INP_WLOCK_ASSERT(inp); 272 KASSERT(tp->t_state == TCPS_SYN_SENT || 273 tp->t_state == TCPS_SYN_RECEIVED, 274 ("%s: TCP state %s", __func__, tcpstates[tp->t_state])); 275 276 CTR4(KTR_CXGBE, "%s: tid %d, toep %p, inp %p", 277 __func__, toep->tid, toep, inp); 278 279 tp->t_state = TCPS_ESTABLISHED; 280 tp->t_starttime = ticks; 281 TCPSTAT_INC(tcps_connects); 282 283 tp->irs = irs; 284 tcp_rcvseqinit(tp); 285 tp->rcv_wnd = toep->rx_credits << 10; 286 tp->rcv_adv += tp->rcv_wnd; 287 tp->last_ack_sent = tp->rcv_nxt; 288 289 /* 290 * If we were unable to send all rx credits via opt0, save the remainder 291 * in rx_credits so that they can be handed over with the next credit 292 * update. 293 */ 294 SOCKBUF_LOCK(&so->so_rcv); 295 bufsize = select_rcv_wnd(so); 296 SOCKBUF_UNLOCK(&so->so_rcv); 297 toep->rx_credits = bufsize - tp->rcv_wnd; 298 299 tp->iss = iss; 300 tcp_sendseqinit(tp); 301 tp->snd_una = iss + 1; 302 tp->snd_nxt = iss + 1; 303 tp->snd_max = iss + 1; 304 305 assign_rxopt(tp, tcpopt); 306 307 SOCKBUF_LOCK(&so->so_snd); 308 if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf) 309 bufsize = V_tcp_autosndbuf_max; 310 else 311 bufsize = sbspace(&so->so_snd); 312 SOCKBUF_UNLOCK(&so->so_snd); 313 314 ftxp.snd_nxt = tp->snd_nxt; 315 ftxp.rcv_nxt = tp->rcv_nxt; 316 ftxp.snd_space = bufsize; 317 ftxp.mss = tp->t_maxseg; 318 send_flowc_wr(toep, &ftxp); 319 320 soisconnected(so); 321 CURVNET_RESTORE(); 322} 323 324static int 325send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) 326{ 327 struct wrqe *wr; 328 struct cpl_rx_data_ack *req; 329 uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 330 331 KASSERT(credits >= 0, ("%s: %d credits", __func__, credits)); 332 333 wr = alloc_wrqe(sizeof(*req), toep->ctrlq); 334 if (wr == NULL) 335 return (0); 336 req = wrtod(wr); 337 338 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); 339 req->credit_dack = htobe32(dack | V_RX_CREDITS(credits)); 340 341 t4_wrq_tx(sc, wr); 342 return (credits); 343} 344 345void 346t4_rcvd(struct toedev *tod, struct tcpcb *tp) 347{ 348 struct adapter *sc = tod->tod_softc; 349 struct inpcb *inp = tp->t_inpcb; 350 struct socket *so = inp->inp_socket; 351 struct sockbuf *sb = &so->so_rcv; 352 struct toepcb *toep = tp->t_toe; 353 int credits; 354 355 INP_WLOCK_ASSERT(inp); 356 357 SOCKBUF_LOCK(sb); 358 KASSERT(toep->sb_cc >= sbused(sb), 359 ("%s: sb %p has more data (%d) than last time (%d).", 360 __func__, sb, sbused(sb), toep->sb_cc)); 361 362 toep->rx_credits += toep->sb_cc - sbused(sb); 363 toep->sb_cc = sbused(sb); 364 365 if (toep->rx_credits > 0 && 366 (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || 367 (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || 368 toep->sb_cc + tp->rcv_wnd < sb->sb_lowat)) { 369 370 credits = send_rx_credits(sc, toep, toep->rx_credits); 371 toep->rx_credits -= credits; 372 tp->rcv_wnd += credits; 373 tp->rcv_adv += credits; 374 } 375 SOCKBUF_UNLOCK(sb); 376} 377 378/* 379 * Close a connection by sending a CPL_CLOSE_CON_REQ message. 380 */ 381static int 382close_conn(struct adapter *sc, struct toepcb *toep) 383{ 384 struct wrqe *wr; 385 struct cpl_close_con_req *req; 386 unsigned int tid = toep->tid; 387 388 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid, 389 toep->flags & TPF_FIN_SENT ? ", IGNORED" : ""); 390 391 if (toep->flags & TPF_FIN_SENT) 392 return (0); 393 394 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 395 ("%s: flowc_wr not sent for tid %u.", __func__, tid)); 396 397 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 398 if (wr == NULL) { 399 /* XXX */ 400 panic("%s: allocation failure.", __func__); 401 } 402 req = wrtod(wr); 403 404 req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | 405 V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr))); 406 req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) | 407 V_FW_WR_FLOWID(tid)); 408 req->wr.wr_lo = cpu_to_be64(0); 409 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 410 req->rsvd = 0; 411 412 toep->flags |= TPF_FIN_SENT; 413 toep->flags &= ~TPF_SEND_FIN; 414 t4_l2t_send(sc, wr, toep->l2te); 415 416 return (0); 417} 418 419#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 420#define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16)) 421 422/* Maximum amount of immediate data we could stuff in a WR */ 423static inline int 424max_imm_payload(int tx_credits) 425{ 426 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 427 428 KASSERT(tx_credits >= 0 && 429 tx_credits <= MAX_OFLD_TX_CREDITS, 430 ("%s: %d credits", __func__, tx_credits)); 431 432 if (tx_credits < MIN_OFLD_TX_CREDITS) 433 return (0); 434 435 if (tx_credits >= (n * EQ_ESIZE) / 16) 436 return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr)); 437 else 438 return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr)); 439} 440 441/* Maximum number of SGL entries we could stuff in a WR */ 442static inline int 443max_dsgl_nsegs(int tx_credits) 444{ 445 int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */ 446 int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS; 447 448 KASSERT(tx_credits >= 0 && 449 tx_credits <= MAX_OFLD_TX_CREDITS, 450 ("%s: %d credits", __func__, tx_credits)); 451 452 if (tx_credits < MIN_OFLD_TX_CREDITS) 453 return (0); 454 455 nseg += 2 * (sge_pair_credits * 16 / 24); 456 if ((sge_pair_credits * 16) % 24 == 16) 457 nseg++; 458 459 return (nseg); 460} 461 462static inline void 463write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, 464 unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign) 465{ 466 struct fw_ofld_tx_data_wr *txwr = dst; 467 468 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | 469 V_FW_WR_IMMDLEN(immdlen)); 470 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | 471 V_FW_WR_LEN16(credits)); 472 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) | 473 V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove)); 474 txwr->plen = htobe32(plen); 475 476 if (txalign > 0) { 477 struct tcpcb *tp = intotcpcb(toep->inp); 478 479 if (plen < 2 * tp->t_maxseg || is_10G_port(toep->vi->pi)) 480 txwr->lsodisable_to_flags |= 481 htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE); 482 else 483 txwr->lsodisable_to_flags |= 484 htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD | 485 (tp->t_flags & TF_NODELAY ? 0 : 486 F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE)); 487 } 488} 489 490/* 491 * Generate a DSGL from a starting mbuf. The total number of segments and the 492 * maximum segments in any one mbuf are provided. 493 */ 494static void 495write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) 496{ 497 struct mbuf *m; 498 struct ulptx_sgl *usgl = dst; 499 int i, j, rc; 500 struct sglist sg; 501 struct sglist_seg segs[n]; 502 503 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 504 505 sglist_init(&sg, n, segs); 506 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 507 V_ULPTX_NSGE(nsegs)); 508 509 i = -1; 510 for (m = start; m != stop; m = m->m_next) { 511 rc = sglist_append(&sg, mtod(m, void *), m->m_len); 512 if (__predict_false(rc != 0)) 513 panic("%s: sglist_append %d", __func__, rc); 514 515 for (j = 0; j < sg.sg_nseg; i++, j++) { 516 if (i < 0) { 517 usgl->len0 = htobe32(segs[j].ss_len); 518 usgl->addr0 = htobe64(segs[j].ss_paddr); 519 } else { 520 usgl->sge[i / 2].len[i & 1] = 521 htobe32(segs[j].ss_len); 522 usgl->sge[i / 2].addr[i & 1] = 523 htobe64(segs[j].ss_paddr); 524 } 525#ifdef INVARIANTS 526 nsegs--; 527#endif 528 } 529 sglist_reset(&sg); 530 } 531 if (i & 1) 532 usgl->sge[i / 2].len[1] = htobe32(0); 533 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", 534 __func__, nsegs, start, stop)); 535} 536 537/* 538 * Max number of SGL entries an offload tx work request can have. This is 41 539 * (1 + 40) for a full 512B work request. 540 * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) 541 */ 542#define OFLD_SGL_LEN (41) 543 544/* 545 * Send data and/or a FIN to the peer. 546 * 547 * The socket's so_snd buffer consists of a stream of data starting with sb_mb 548 * and linked together with m_next. sb_sndptr, if set, is the last mbuf that 549 * was transmitted. 550 * 551 * drop indicates the number of bytes that should be dropped from the head of 552 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating 553 * contention on the send buffer lock (before this change it used to do 554 * sowwakeup and then t4_push_frames right after that when recovering from tx 555 * stalls). When drop is set this function MUST drop the bytes and wake up any 556 * writers. 557 */ 558void 559t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 560{ 561 struct mbuf *sndptr, *m, *sb_sndptr; 562 struct fw_ofld_tx_data_wr *txwr; 563 struct wrqe *wr; 564 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 565 struct inpcb *inp = toep->inp; 566 struct tcpcb *tp = intotcpcb(inp); 567 struct socket *so = inp->inp_socket; 568 struct sockbuf *sb = &so->so_snd; 569 int tx_credits, shove, compl, space, sowwakeup; 570 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 571 572 INP_WLOCK_ASSERT(inp); 573 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 574 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 575 576 KASSERT(toep->ulp_mode == ULP_MODE_NONE || 577 toep->ulp_mode == ULP_MODE_TCPDDP || 578 toep->ulp_mode == ULP_MODE_RDMA, 579 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 580 581 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 582 return; 583 584 /* 585 * This function doesn't resume by itself. Someone else must clear the 586 * flag and call this function. 587 */ 588 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 589 KASSERT(drop == 0, 590 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 591 return; 592 } 593 594 do { 595 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 596 max_imm = max_imm_payload(tx_credits); 597 max_nsegs = max_dsgl_nsegs(tx_credits); 598 599 SOCKBUF_LOCK(sb); 600 sowwakeup = drop; 601 if (drop) { 602 sbdrop_locked(sb, drop); 603 drop = 0; 604 } 605 sb_sndptr = sb->sb_sndptr; 606 sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; 607 plen = 0; 608 nsegs = 0; 609 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 610 for (m = sndptr; m != NULL; m = m->m_next) { 611 int n = sglist_count(mtod(m, void *), m->m_len); 612 613 nsegs += n; 614 plen += m->m_len; 615 616 /* This mbuf sent us _over_ the nsegs limit, back out */ 617 if (plen > max_imm && nsegs > max_nsegs) { 618 nsegs -= n; 619 plen -= m->m_len; 620 if (plen == 0) { 621 /* Too few credits */ 622 toep->flags |= TPF_TX_SUSPENDED; 623 if (sowwakeup) 624 sowwakeup_locked(so); 625 else 626 SOCKBUF_UNLOCK(sb); 627 SOCKBUF_UNLOCK_ASSERT(sb); 628 return; 629 } 630 break; 631 } 632 633 if (max_nsegs_1mbuf < n) 634 max_nsegs_1mbuf = n; 635 sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ 636 637 /* This mbuf put us right at the max_nsegs limit */ 638 if (plen > max_imm && nsegs == max_nsegs) { 639 m = m->m_next; 640 break; 641 } 642 } 643 644 space = sbspace(sb); 645 646 if (space <= sb->sb_hiwat * 3 / 8 && 647 toep->plen_nocompl + plen >= sb->sb_hiwat / 4) 648 compl = 1; 649 else 650 compl = 0; 651 652 if (sb->sb_flags & SB_AUTOSIZE && 653 V_tcp_do_autosndbuf && 654 sb->sb_hiwat < V_tcp_autosndbuf_max && 655 space < sb->sb_hiwat / 8) { 656 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 657 V_tcp_autosndbuf_max); 658 659 if (!sbreserve_locked(sb, newsize, so, NULL)) 660 sb->sb_flags &= ~SB_AUTOSIZE; 661 else 662 sowwakeup = 1; /* room available */ 663 } 664 if (sowwakeup) 665 sowwakeup_locked(so); 666 else 667 SOCKBUF_UNLOCK(sb); 668 SOCKBUF_UNLOCK_ASSERT(sb); 669 670 /* nothing to send */ 671 if (plen == 0) { 672 KASSERT(m == NULL, 673 ("%s: nothing to send, but m != NULL", __func__)); 674 break; 675 } 676 677 if (__predict_false(toep->flags & TPF_FIN_SENT)) 678 panic("%s: excess tx.", __func__); 679 680 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); 681 if (plen <= max_imm) { 682 683 /* Immediate data tx */ 684 685 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), 686 toep->ofld_txq); 687 if (wr == NULL) { 688 /* XXX: how will we recover from this? */ 689 toep->flags |= TPF_TX_SUSPENDED; 690 return; 691 } 692 txwr = wrtod(wr); 693 credits = howmany(wr->wr_len, 16); 694 write_tx_wr(txwr, toep, plen, plen, credits, shove, 0, 695 sc->tt.tx_align); 696 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 697 nsegs = 0; 698 } else { 699 int wr_len; 700 701 /* DSGL tx */ 702 703 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 704 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 705 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 706 if (wr == NULL) { 707 /* XXX: how will we recover from this? */ 708 toep->flags |= TPF_TX_SUSPENDED; 709 return; 710 } 711 txwr = wrtod(wr); 712 credits = howmany(wr_len, 16); 713 write_tx_wr(txwr, toep, 0, plen, credits, shove, 0, 714 sc->tt.tx_align); 715 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 716 max_nsegs_1mbuf); 717 if (wr_len & 0xf) { 718 uint64_t *pad = (uint64_t *) 719 ((uintptr_t)txwr + wr_len); 720 *pad = 0; 721 } 722 } 723 724 KASSERT(toep->tx_credits >= credits, 725 ("%s: not enough credits", __func__)); 726 727 toep->tx_credits -= credits; 728 toep->tx_nocompl += credits; 729 toep->plen_nocompl += plen; 730 if (toep->tx_credits <= toep->tx_total * 3 / 8 && 731 toep->tx_nocompl >= toep->tx_total / 4) 732 compl = 1; 733 734 if (compl || toep->ulp_mode == ULP_MODE_RDMA) { 735 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 736 toep->tx_nocompl = 0; 737 toep->plen_nocompl = 0; 738 } 739 740 tp->snd_nxt += plen; 741 tp->snd_max += plen; 742 743 SOCKBUF_LOCK(sb); 744 KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__)); 745 sb->sb_sndptr = sb_sndptr; 746 SOCKBUF_UNLOCK(sb); 747 748 toep->flags |= TPF_TX_DATA_SENT; 749 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) 750 toep->flags |= TPF_TX_SUSPENDED; 751 752 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 753 txsd->plen = plen; 754 txsd->tx_credits = credits; 755 txsd++; 756 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 757 toep->txsd_pidx = 0; 758 txsd = &toep->txsd[0]; 759 } 760 toep->txsd_avail--; 761 762 t4_l2t_send(sc, wr, toep->l2te); 763 } while (m != NULL); 764 765 /* Send a FIN if requested, but only if there's no more data to send */ 766 if (m == NULL && toep->flags & TPF_SEND_FIN) 767 close_conn(sc, toep); 768} 769 770static inline void 771rqdrop_locked(struct mbufq *q, int plen) 772{ 773 struct mbuf *m; 774 775 while (plen > 0) { 776 m = mbufq_dequeue(q); 777 778 /* Too many credits. */ 779 MPASS(m != NULL); 780 M_ASSERTPKTHDR(m); 781 782 /* Partial credits. */ 783 MPASS(plen >= m->m_pkthdr.len); 784 785 plen -= m->m_pkthdr.len; 786 m_freem(m); 787 } 788} 789 790void 791t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) 792{ 793 struct mbuf *sndptr, *m; 794 struct fw_ofld_tx_data_wr *txwr; 795 struct wrqe *wr; 796 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 797 u_int adjusted_plen, ulp_submode; 798 struct inpcb *inp = toep->inp; 799 struct tcpcb *tp = intotcpcb(inp); 800 int tx_credits, shove; 801 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 802 struct mbufq *pduq = &toep->ulp_pduq; 803 static const u_int ulp_extra_len[] = {0, 4, 4, 8}; 804 805 INP_WLOCK_ASSERT(inp); 806 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 807 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 808 KASSERT(toep->ulp_mode == ULP_MODE_ISCSI, 809 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 810 811 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) 812 return; 813 814 /* 815 * This function doesn't resume by itself. Someone else must clear the 816 * flag and call this function. 817 */ 818 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 819 KASSERT(drop == 0, 820 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 821 return; 822 } 823 824 if (drop) 825 rqdrop_locked(&toep->ulp_pdu_reclaimq, drop); 826 827 while ((sndptr = mbufq_first(pduq)) != NULL) { 828 M_ASSERTPKTHDR(sndptr); 829 830 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 831 max_imm = max_imm_payload(tx_credits); 832 max_nsegs = max_dsgl_nsegs(tx_credits); 833 834 plen = 0; 835 nsegs = 0; 836 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 837 for (m = sndptr; m != NULL; m = m->m_next) { 838 int n = sglist_count(mtod(m, void *), m->m_len); 839 840 nsegs += n; 841 plen += m->m_len; 842 843 /* 844 * This mbuf would send us _over_ the nsegs limit. 845 * Suspend tx because the PDU can't be sent out. 846 */ 847 if (plen > max_imm && nsegs > max_nsegs) { 848 toep->flags |= TPF_TX_SUSPENDED; 849 return; 850 } 851 852 if (max_nsegs_1mbuf < n) 853 max_nsegs_1mbuf = n; 854 } 855 856 if (__predict_false(toep->flags & TPF_FIN_SENT)) 857 panic("%s: excess tx.", __func__); 858 859 /* 860 * We have a PDU to send. All of it goes out in one WR so 'm' 861 * is NULL. A PDU's length is always a multiple of 4. 862 */ 863 MPASS(m == NULL); 864 MPASS((plen & 3) == 0); 865 MPASS(sndptr->m_pkthdr.len == plen); 866 867 shove = !(tp->t_flags & TF_MORETOCOME); 868 ulp_submode = mbuf_ulp_submode(sndptr); 869 MPASS(ulp_submode < nitems(ulp_extra_len)); 870 871 /* 872 * plen doesn't include header and data digests, which are 873 * generated and inserted in the right places by the TOE, but 874 * they do occupy TCP sequence space and need to be accounted 875 * for. 876 */ 877 adjusted_plen = plen + ulp_extra_len[ulp_submode]; 878 if (plen <= max_imm) { 879 880 /* Immediate data tx */ 881 882 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), 883 toep->ofld_txq); 884 if (wr == NULL) { 885 /* XXX: how will we recover from this? */ 886 toep->flags |= TPF_TX_SUSPENDED; 887 return; 888 } 889 txwr = wrtod(wr); 890 credits = howmany(wr->wr_len, 16); 891 write_tx_wr(txwr, toep, plen, adjusted_plen, credits, 892 shove, ulp_submode, sc->tt.tx_align); 893 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 894 nsegs = 0; 895 } else { 896 int wr_len; 897 898 /* DSGL tx */ 899 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 900 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 901 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 902 if (wr == NULL) { 903 /* XXX: how will we recover from this? */ 904 toep->flags |= TPF_TX_SUSPENDED; 905 return; 906 } 907 txwr = wrtod(wr); 908 credits = howmany(wr_len, 16); 909 write_tx_wr(txwr, toep, 0, adjusted_plen, credits, 910 shove, ulp_submode, sc->tt.tx_align); 911 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 912 max_nsegs_1mbuf); 913 if (wr_len & 0xf) { 914 uint64_t *pad = (uint64_t *) 915 ((uintptr_t)txwr + wr_len); 916 *pad = 0; 917 } 918 } 919 920 KASSERT(toep->tx_credits >= credits, 921 ("%s: not enough credits", __func__)); 922 923 m = mbufq_dequeue(pduq); 924 MPASS(m == sndptr); 925 mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); 926 927 toep->tx_credits -= credits; 928 toep->tx_nocompl += credits; 929 toep->plen_nocompl += plen; 930 if (toep->tx_credits <= toep->tx_total * 3 / 8 && 931 toep->tx_nocompl >= toep->tx_total / 4) { 932 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 933 toep->tx_nocompl = 0; 934 toep->plen_nocompl = 0; 935 } 936 937 tp->snd_nxt += adjusted_plen; 938 tp->snd_max += adjusted_plen; 939 940 toep->flags |= TPF_TX_DATA_SENT; 941 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) 942 toep->flags |= TPF_TX_SUSPENDED; 943 944 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 945 txsd->plen = plen; 946 txsd->tx_credits = credits; 947 txsd++; 948 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 949 toep->txsd_pidx = 0; 950 txsd = &toep->txsd[0]; 951 } 952 toep->txsd_avail--; 953 954 t4_l2t_send(sc, wr, toep->l2te); 955 } 956 957 /* Send a FIN if requested, but only if there are no more PDUs to send */ 958 if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) 959 close_conn(sc, toep); 960} 961 962int 963t4_tod_output(struct toedev *tod, struct tcpcb *tp) 964{ 965 struct adapter *sc = tod->tod_softc; 966#ifdef INVARIANTS 967 struct inpcb *inp = tp->t_inpcb; 968#endif 969 struct toepcb *toep = tp->t_toe; 970 971 INP_WLOCK_ASSERT(inp); 972 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 973 ("%s: inp %p dropped.", __func__, inp)); 974 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 975 976 if (toep->ulp_mode == ULP_MODE_ISCSI) 977 t4_push_pdus(sc, toep, 0); 978 else 979 t4_push_frames(sc, toep, 0); 980 981 return (0); 982} 983 984int 985t4_send_fin(struct toedev *tod, struct tcpcb *tp) 986{ 987 struct adapter *sc = tod->tod_softc; 988#ifdef INVARIANTS 989 struct inpcb *inp = tp->t_inpcb; 990#endif 991 struct toepcb *toep = tp->t_toe; 992 993 INP_WLOCK_ASSERT(inp); 994 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 995 ("%s: inp %p dropped.", __func__, inp)); 996 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 997 998 toep->flags |= TPF_SEND_FIN; 999 if (tp->t_state >= TCPS_ESTABLISHED) { 1000 if (toep->ulp_mode == ULP_MODE_ISCSI) 1001 t4_push_pdus(sc, toep, 0); 1002 else 1003 t4_push_frames(sc, toep, 0); 1004 } 1005 1006 return (0); 1007} 1008 1009int 1010t4_send_rst(struct toedev *tod, struct tcpcb *tp) 1011{ 1012 struct adapter *sc = tod->tod_softc; 1013#if defined(INVARIANTS) 1014 struct inpcb *inp = tp->t_inpcb; 1015#endif 1016 struct toepcb *toep = tp->t_toe; 1017 1018 INP_WLOCK_ASSERT(inp); 1019 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1020 ("%s: inp %p dropped.", __func__, inp)); 1021 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1022 1023 /* hmmmm */ 1024 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1025 ("%s: flowc for tid %u [%s] not sent already", 1026 __func__, toep->tid, tcpstates[tp->t_state])); 1027 1028 send_reset(sc, toep, 0); 1029 return (0); 1030} 1031 1032/* 1033 * Peer has sent us a FIN. 1034 */ 1035static int 1036do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1037{ 1038 struct adapter *sc = iq->adapter; 1039 const struct cpl_peer_close *cpl = (const void *)(rss + 1); 1040 unsigned int tid = GET_TID(cpl); 1041 struct toepcb *toep = lookup_tid(sc, tid); 1042 struct inpcb *inp = toep->inp; 1043 struct tcpcb *tp = NULL; 1044 struct socket *so; 1045 struct sockbuf *sb; 1046#ifdef INVARIANTS 1047 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1048#endif 1049 1050 KASSERT(opcode == CPL_PEER_CLOSE, 1051 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1052 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1053 1054 if (__predict_false(toep->flags & TPF_SYNQE)) { 1055#ifdef INVARIANTS 1056 struct synq_entry *synqe = (void *)toep; 1057 1058 INP_WLOCK(synqe->lctx->inp); 1059 if (synqe->flags & TPF_SYNQE_HAS_L2TE) { 1060 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, 1061 ("%s: listen socket closed but tid %u not aborted.", 1062 __func__, tid)); 1063 } else { 1064 /* 1065 * do_pass_accept_req is still running and will 1066 * eventually take care of this tid. 1067 */ 1068 } 1069 INP_WUNLOCK(synqe->lctx->inp); 1070#endif 1071 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1072 toep, toep->flags); 1073 return (0); 1074 } 1075 1076 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1077 1078 INP_INFO_RLOCK(&V_tcbinfo); 1079 INP_WLOCK(inp); 1080 tp = intotcpcb(inp); 1081 1082 CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__, 1083 tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp); 1084 1085 if (toep->flags & TPF_ABORT_SHUTDOWN) 1086 goto done; 1087 1088 tp->rcv_nxt++; /* FIN */ 1089 1090 so = inp->inp_socket; 1091 sb = &so->so_rcv; 1092 SOCKBUF_LOCK(sb); 1093 if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) { 1094 handle_ddp_close(toep, tp, sb, cpl->rcv_nxt); 1095 } 1096 socantrcvmore_locked(so); /* unlocks the sockbuf */ 1097 1098 if (toep->ulp_mode != ULP_MODE_RDMA) { 1099 KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt), 1100 ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt, 1101 be32toh(cpl->rcv_nxt))); 1102 } 1103 1104 switch (tp->t_state) { 1105 case TCPS_SYN_RECEIVED: 1106 tp->t_starttime = ticks; 1107 /* FALLTHROUGH */ 1108 1109 case TCPS_ESTABLISHED: 1110 tp->t_state = TCPS_CLOSE_WAIT; 1111 break; 1112 1113 case TCPS_FIN_WAIT_1: 1114 tp->t_state = TCPS_CLOSING; 1115 break; 1116 1117 case TCPS_FIN_WAIT_2: 1118 tcp_twstart(tp); 1119 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1120 INP_INFO_RUNLOCK(&V_tcbinfo); 1121 1122 INP_WLOCK(inp); 1123 final_cpl_received(toep); 1124 return (0); 1125 1126 default: 1127 log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n", 1128 __func__, tid, tp->t_state); 1129 } 1130done: 1131 INP_WUNLOCK(inp); 1132 INP_INFO_RUNLOCK(&V_tcbinfo); 1133 return (0); 1134} 1135 1136/* 1137 * Peer has ACK'd our FIN. 1138 */ 1139static int 1140do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, 1141 struct mbuf *m) 1142{ 1143 struct adapter *sc = iq->adapter; 1144 const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1); 1145 unsigned int tid = GET_TID(cpl); 1146 struct toepcb *toep = lookup_tid(sc, tid); 1147 struct inpcb *inp = toep->inp; 1148 struct tcpcb *tp = NULL; 1149 struct socket *so = NULL; 1150#ifdef INVARIANTS 1151 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1152#endif 1153 1154 KASSERT(opcode == CPL_CLOSE_CON_RPL, 1155 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1156 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1157 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1158 1159 INP_INFO_RLOCK(&V_tcbinfo); 1160 INP_WLOCK(inp); 1161 tp = intotcpcb(inp); 1162 1163 CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x", 1164 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags); 1165 1166 if (toep->flags & TPF_ABORT_SHUTDOWN) 1167 goto done; 1168 1169 so = inp->inp_socket; 1170 tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */ 1171 1172 switch (tp->t_state) { 1173 case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */ 1174 tcp_twstart(tp); 1175release: 1176 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1177 INP_INFO_RUNLOCK(&V_tcbinfo); 1178 1179 INP_WLOCK(inp); 1180 final_cpl_received(toep); /* no more CPLs expected */ 1181 1182 return (0); 1183 case TCPS_LAST_ACK: 1184 if (tcp_close(tp)) 1185 INP_WUNLOCK(inp); 1186 goto release; 1187 1188 case TCPS_FIN_WAIT_1: 1189 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1190 soisdisconnected(so); 1191 tp->t_state = TCPS_FIN_WAIT_2; 1192 break; 1193 1194 default: 1195 log(LOG_ERR, 1196 "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n", 1197 __func__, tid, tcpstates[tp->t_state]); 1198 } 1199done: 1200 INP_WUNLOCK(inp); 1201 INP_INFO_RUNLOCK(&V_tcbinfo); 1202 return (0); 1203} 1204 1205void 1206send_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid, 1207 int rst_status) 1208{ 1209 struct wrqe *wr; 1210 struct cpl_abort_rpl *cpl; 1211 1212 wr = alloc_wrqe(sizeof(*cpl), ofld_txq); 1213 if (wr == NULL) { 1214 /* XXX */ 1215 panic("%s: allocation failure.", __func__); 1216 } 1217 cpl = wrtod(wr); 1218 1219 INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid); 1220 cpl->cmd = rst_status; 1221 1222 t4_wrq_tx(sc, wr); 1223} 1224 1225static int 1226abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason) 1227{ 1228 switch (abort_reason) { 1229 case CPL_ERR_BAD_SYN: 1230 case CPL_ERR_CONN_RESET: 1231 return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); 1232 case CPL_ERR_XMIT_TIMEDOUT: 1233 case CPL_ERR_PERSIST_TIMEDOUT: 1234 case CPL_ERR_FINWAIT2_TIMEDOUT: 1235 case CPL_ERR_KEEPALIVE_TIMEDOUT: 1236 return (ETIMEDOUT); 1237 default: 1238 return (EIO); 1239 } 1240} 1241 1242/* 1243 * TCP RST from the peer, timeout, or some other such critical error. 1244 */ 1245static int 1246do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1247{ 1248 struct adapter *sc = iq->adapter; 1249 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); 1250 unsigned int tid = GET_TID(cpl); 1251 struct toepcb *toep = lookup_tid(sc, tid); 1252 struct sge_wrq *ofld_txq = toep->ofld_txq; 1253 struct inpcb *inp; 1254 struct tcpcb *tp; 1255#ifdef INVARIANTS 1256 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1257#endif 1258 1259 KASSERT(opcode == CPL_ABORT_REQ_RSS, 1260 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1261 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1262 1263 if (toep->flags & TPF_SYNQE) 1264 return (do_abort_req_synqe(iq, rss, m)); 1265 1266 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1267 1268 if (negative_advice(cpl->status)) { 1269 CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)", 1270 __func__, cpl->status, tid, toep->flags); 1271 return (0); /* Ignore negative advice */ 1272 } 1273 1274 inp = toep->inp; 1275 INP_INFO_RLOCK(&V_tcbinfo); /* for tcp_close */ 1276 INP_WLOCK(inp); 1277 1278 tp = intotcpcb(inp); 1279 1280 CTR6(KTR_CXGBE, 1281 "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d", 1282 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, 1283 inp->inp_flags, cpl->status); 1284 1285 /* 1286 * If we'd initiated an abort earlier the reply to it is responsible for 1287 * cleaning up resources. Otherwise we tear everything down right here 1288 * right now. We owe the T4 a CPL_ABORT_RPL no matter what. 1289 */ 1290 if (toep->flags & TPF_ABORT_SHUTDOWN) { 1291 INP_WUNLOCK(inp); 1292 goto done; 1293 } 1294 toep->flags |= TPF_ABORT_SHUTDOWN; 1295 1296 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 1297 struct socket *so = inp->inp_socket; 1298 1299 if (so != NULL) 1300 so_error_set(so, abort_status_to_errno(tp, 1301 cpl->status)); 1302 tp = tcp_close(tp); 1303 if (tp == NULL) 1304 INP_WLOCK(inp); /* re-acquire */ 1305 } 1306 1307 final_cpl_received(toep); 1308done: 1309 INP_INFO_RUNLOCK(&V_tcbinfo); 1310 send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST); 1311 return (0); 1312} 1313 1314/* 1315 * Reply to the CPL_ABORT_REQ (send_reset) 1316 */ 1317static int 1318do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1319{ 1320 struct adapter *sc = iq->adapter; 1321 const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); 1322 unsigned int tid = GET_TID(cpl); 1323 struct toepcb *toep = lookup_tid(sc, tid); 1324 struct inpcb *inp = toep->inp; 1325#ifdef INVARIANTS 1326 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1327#endif 1328 1329 KASSERT(opcode == CPL_ABORT_RPL_RSS, 1330 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1331 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1332 1333 if (toep->flags & TPF_SYNQE) 1334 return (do_abort_rpl_synqe(iq, rss, m)); 1335 1336 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1337 1338 CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d", 1339 __func__, tid, toep, inp, cpl->status); 1340 1341 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1342 ("%s: wasn't expecting abort reply", __func__)); 1343 1344 INP_WLOCK(inp); 1345 final_cpl_received(toep); 1346 1347 return (0); 1348} 1349 1350static int 1351do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1352{ 1353 struct adapter *sc = iq->adapter; 1354 const struct cpl_rx_data *cpl = mtod(m, const void *); 1355 unsigned int tid = GET_TID(cpl); 1356 struct toepcb *toep = lookup_tid(sc, tid); 1357 struct inpcb *inp = toep->inp; 1358 struct tcpcb *tp; 1359 struct socket *so; 1360 struct sockbuf *sb; 1361 int len; 1362 uint32_t ddp_placed = 0; 1363 1364 if (__predict_false(toep->flags & TPF_SYNQE)) { 1365#ifdef INVARIANTS 1366 struct synq_entry *synqe = (void *)toep; 1367 1368 INP_WLOCK(synqe->lctx->inp); 1369 if (synqe->flags & TPF_SYNQE_HAS_L2TE) { 1370 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, 1371 ("%s: listen socket closed but tid %u not aborted.", 1372 __func__, tid)); 1373 } else { 1374 /* 1375 * do_pass_accept_req is still running and will 1376 * eventually take care of this tid. 1377 */ 1378 } 1379 INP_WUNLOCK(synqe->lctx->inp); 1380#endif 1381 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1382 toep, toep->flags); 1383 m_freem(m); 1384 return (0); 1385 } 1386 1387 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1388 1389 /* strip off CPL header */ 1390 m_adj(m, sizeof(*cpl)); 1391 len = m->m_pkthdr.len; 1392 1393 INP_WLOCK(inp); 1394 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1395 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1396 __func__, tid, len, inp->inp_flags); 1397 INP_WUNLOCK(inp); 1398 m_freem(m); 1399 return (0); 1400 } 1401 1402 tp = intotcpcb(inp); 1403 1404 if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq))) 1405 ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt; 1406 1407 tp->rcv_nxt += len; 1408 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); 1409 tp->rcv_wnd -= len; 1410 tp->t_rcvtime = ticks; 1411 1412 so = inp_inpcbtosocket(inp); 1413 sb = &so->so_rcv; 1414 SOCKBUF_LOCK(sb); 1415 1416 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1417 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1418 __func__, tid, len); 1419 m_freem(m); 1420 SOCKBUF_UNLOCK(sb); 1421 INP_WUNLOCK(inp); 1422 1423 INP_INFO_RLOCK(&V_tcbinfo); 1424 INP_WLOCK(inp); 1425 tp = tcp_drop(tp, ECONNRESET); 1426 if (tp) 1427 INP_WUNLOCK(inp); 1428 INP_INFO_RUNLOCK(&V_tcbinfo); 1429 1430 return (0); 1431 } 1432 1433 /* receive buffer autosize */ 1434 CURVNET_SET(so->so_vnet); 1435 if (sb->sb_flags & SB_AUTOSIZE && 1436 V_tcp_do_autorcvbuf && 1437 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1438 len > (sbspace(sb) / 8 * 7)) { 1439 unsigned int hiwat = sb->sb_hiwat; 1440 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, 1441 V_tcp_autorcvbuf_max); 1442 1443 if (!sbreserve_locked(sb, newsize, so, NULL)) 1444 sb->sb_flags &= ~SB_AUTOSIZE; 1445 else 1446 toep->rx_credits += newsize - hiwat; 1447 } 1448 1449 if (toep->ulp_mode == ULP_MODE_TCPDDP) { 1450 int changed = !(toep->ddp_flags & DDP_ON) ^ cpl->ddp_off; 1451 1452 if (changed) { 1453 if (toep->ddp_flags & DDP_SC_REQ) 1454 toep->ddp_flags ^= DDP_ON | DDP_SC_REQ; 1455 else { 1456 KASSERT(cpl->ddp_off == 1, 1457 ("%s: DDP switched on by itself.", 1458 __func__)); 1459 1460 /* Fell out of DDP mode */ 1461 toep->ddp_flags &= ~(DDP_ON | DDP_BUF0_ACTIVE | 1462 DDP_BUF1_ACTIVE); 1463 1464 if (ddp_placed) 1465 insert_ddp_data(toep, ddp_placed); 1466 } 1467 } 1468 1469 if ((toep->ddp_flags & DDP_OK) == 0 && 1470 time_uptime >= toep->ddp_disabled + DDP_RETRY_WAIT) { 1471 toep->ddp_score = DDP_LOW_SCORE; 1472 toep->ddp_flags |= DDP_OK; 1473 CTR3(KTR_CXGBE, "%s: tid %u DDP_OK @ %u", 1474 __func__, tid, time_uptime); 1475 } 1476 1477 if (toep->ddp_flags & DDP_ON) { 1478 1479 /* 1480 * CPL_RX_DATA with DDP on can only be an indicate. Ask 1481 * soreceive to post a buffer or disable DDP. The 1482 * payload that arrived in this indicate is appended to 1483 * the socket buffer as usual. 1484 */ 1485 1486#if 0 1487 CTR5(KTR_CXGBE, 1488 "%s: tid %u (0x%x) DDP indicate (seq 0x%x, len %d)", 1489 __func__, tid, toep->flags, be32toh(cpl->seq), len); 1490#endif 1491 sb->sb_flags |= SB_DDP_INDICATE; 1492 } else if ((toep->ddp_flags & (DDP_OK|DDP_SC_REQ)) == DDP_OK && 1493 tp->rcv_wnd > DDP_RSVD_WIN && len >= sc->tt.ddp_thres) { 1494 1495 /* 1496 * DDP allowed but isn't on (and a request to switch it 1497 * on isn't pending either), and conditions are ripe for 1498 * it to work. Switch it on. 1499 */ 1500 1501 enable_ddp(sc, toep); 1502 } 1503 } 1504 1505 KASSERT(toep->sb_cc >= sbused(sb), 1506 ("%s: sb %p has more data (%d) than last time (%d).", 1507 __func__, sb, sbused(sb), toep->sb_cc)); 1508 toep->rx_credits += toep->sb_cc - sbused(sb); 1509 sbappendstream_locked(sb, m, 0); 1510 toep->sb_cc = sbused(sb); 1511 if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) { 1512 int credits; 1513 1514 credits = send_rx_credits(sc, toep, toep->rx_credits); 1515 toep->rx_credits -= credits; 1516 tp->rcv_wnd += credits; 1517 tp->rcv_adv += credits; 1518 } 1519 sorwakeup_locked(so); 1520 SOCKBUF_UNLOCK_ASSERT(sb); 1521 1522 INP_WUNLOCK(inp); 1523 CURVNET_RESTORE(); 1524 return (0); 1525} 1526 1527#define S_CPL_FW4_ACK_OPCODE 24 1528#define M_CPL_FW4_ACK_OPCODE 0xff 1529#define V_CPL_FW4_ACK_OPCODE(x) ((x) << S_CPL_FW4_ACK_OPCODE) 1530#define G_CPL_FW4_ACK_OPCODE(x) \ 1531 (((x) >> S_CPL_FW4_ACK_OPCODE) & M_CPL_FW4_ACK_OPCODE) 1532 1533#define S_CPL_FW4_ACK_FLOWID 0 1534#define M_CPL_FW4_ACK_FLOWID 0xffffff 1535#define V_CPL_FW4_ACK_FLOWID(x) ((x) << S_CPL_FW4_ACK_FLOWID) 1536#define G_CPL_FW4_ACK_FLOWID(x) \ 1537 (((x) >> S_CPL_FW4_ACK_FLOWID) & M_CPL_FW4_ACK_FLOWID) 1538 1539#define S_CPL_FW4_ACK_CR 24 1540#define M_CPL_FW4_ACK_CR 0xff 1541#define V_CPL_FW4_ACK_CR(x) ((x) << S_CPL_FW4_ACK_CR) 1542#define G_CPL_FW4_ACK_CR(x) (((x) >> S_CPL_FW4_ACK_CR) & M_CPL_FW4_ACK_CR) 1543 1544#define S_CPL_FW4_ACK_SEQVAL 0 1545#define M_CPL_FW4_ACK_SEQVAL 0x1 1546#define V_CPL_FW4_ACK_SEQVAL(x) ((x) << S_CPL_FW4_ACK_SEQVAL) 1547#define G_CPL_FW4_ACK_SEQVAL(x) \ 1548 (((x) >> S_CPL_FW4_ACK_SEQVAL) & M_CPL_FW4_ACK_SEQVAL) 1549#define F_CPL_FW4_ACK_SEQVAL V_CPL_FW4_ACK_SEQVAL(1U) 1550 1551static int 1552do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1553{ 1554 struct adapter *sc = iq->adapter; 1555 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 1556 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 1557 struct toepcb *toep = lookup_tid(sc, tid); 1558 struct inpcb *inp; 1559 struct tcpcb *tp; 1560 struct socket *so; 1561 uint8_t credits = cpl->credits; 1562 struct ofld_tx_sdesc *txsd; 1563 int plen; 1564#ifdef INVARIANTS 1565 unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl))); 1566#endif 1567 1568 /* 1569 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and 1570 * now this comes back carrying the credits for the flowc. 1571 */ 1572 if (__predict_false(toep->flags & TPF_SYNQE)) { 1573 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1574 ("%s: credits for a synq entry %p", __func__, toep)); 1575 return (0); 1576 } 1577 1578 inp = toep->inp; 1579 1580 KASSERT(opcode == CPL_FW4_ACK, 1581 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1582 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1583 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1584 1585 INP_WLOCK(inp); 1586 1587 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) { 1588 INP_WUNLOCK(inp); 1589 return (0); 1590 } 1591 1592 KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0, 1593 ("%s: inp_flags 0x%x", __func__, inp->inp_flags)); 1594 1595 tp = intotcpcb(inp); 1596 1597 if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) { 1598 tcp_seq snd_una = be32toh(cpl->snd_una); 1599 1600#ifdef INVARIANTS 1601 if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { 1602 log(LOG_ERR, 1603 "%s: unexpected seq# %x for TID %u, snd_una %x\n", 1604 __func__, snd_una, toep->tid, tp->snd_una); 1605 } 1606#endif 1607 1608 if (tp->snd_una != snd_una) { 1609 tp->snd_una = snd_una; 1610 tp->ts_recent_age = tcp_ts_getticks(); 1611 } 1612 } 1613 1614 so = inp->inp_socket; 1615 txsd = &toep->txsd[toep->txsd_cidx]; 1616 plen = 0; 1617 while (credits) { 1618 KASSERT(credits >= txsd->tx_credits, 1619 ("%s: too many (or partial) credits", __func__)); 1620 credits -= txsd->tx_credits; 1621 toep->tx_credits += txsd->tx_credits; 1622 plen += txsd->plen; 1623 txsd++; 1624 toep->txsd_avail++; 1625 KASSERT(toep->txsd_avail <= toep->txsd_total, 1626 ("%s: txsd avail > total", __func__)); 1627 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) { 1628 txsd = &toep->txsd[0]; 1629 toep->txsd_cidx = 0; 1630 } 1631 } 1632 1633 if (toep->tx_credits == toep->tx_total) { 1634 toep->tx_nocompl = 0; 1635 toep->plen_nocompl = 0; 1636 } 1637 1638 if (toep->flags & TPF_TX_SUSPENDED && 1639 toep->tx_credits >= toep->tx_total / 4) { 1640 toep->flags &= ~TPF_TX_SUSPENDED; 1641 if (toep->ulp_mode == ULP_MODE_ISCSI) 1642 t4_push_pdus(sc, toep, plen); 1643 else 1644 t4_push_frames(sc, toep, plen); 1645 } else if (plen > 0) { 1646 struct sockbuf *sb = &so->so_snd; 1647 int sbu; 1648 1649 SOCKBUF_LOCK(sb); 1650 sbu = sbused(sb); 1651 if (toep->ulp_mode == ULP_MODE_ISCSI) { 1652 1653 if (__predict_false(sbu > 0)) { 1654 /* 1655 * The data trasmitted before the tid's ULP mode 1656 * changed to ISCSI is still in so_snd. 1657 * Incoming credits should account for so_snd 1658 * first. 1659 */ 1660 sbdrop_locked(sb, min(sbu, plen)); 1661 plen -= min(sbu, plen); 1662 } 1663 sowwakeup_locked(so); /* unlocks so_snd */ 1664 rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); 1665 } else { 1666 sbdrop_locked(sb, plen); 1667 sowwakeup_locked(so); /* unlocks so_snd */ 1668 } 1669 SOCKBUF_UNLOCK_ASSERT(sb); 1670 } 1671 1672 INP_WUNLOCK(inp); 1673 1674 return (0); 1675} 1676 1677static int 1678do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1679{ 1680 struct adapter *sc = iq->adapter; 1681 const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); 1682 unsigned int tid = GET_TID(cpl); 1683#ifdef INVARIANTS 1684 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1685#endif 1686 1687 KASSERT(opcode == CPL_SET_TCB_RPL, 1688 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1689 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1690 1691 if (is_ftid(sc, tid)) 1692 return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ 1693 1694 /* 1695 * TOM and/or other ULPs don't request replies for CPL_SET_TCB or 1696 * CPL_SET_TCB_FIELD requests. This can easily change and when it does 1697 * the dispatch code will go here. 1698 */ 1699#ifdef INVARIANTS 1700 panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__, 1701 tid, iq); 1702#else 1703 log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n", 1704 __func__, tid, iq); 1705#endif 1706 1707 return (0); 1708} 1709 1710void 1711t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl, 1712 uint16_t word, uint64_t mask, uint64_t val) 1713{ 1714 struct wrqe *wr; 1715 struct cpl_set_tcb_field *req; 1716 1717 wr = alloc_wrqe(sizeof(*req), ctrl ? toep->ctrlq : toep->ofld_txq); 1718 if (wr == NULL) { 1719 /* XXX */ 1720 panic("%s: allocation failure.", __func__); 1721 } 1722 req = wrtod(wr); 1723 1724 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); 1725 req->reply_ctrl = htobe16(V_NO_REPLY(1) | 1726 V_QUEUENO(toep->ofld_rxq->iq.abs_id)); 1727 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0)); 1728 req->mask = htobe64(mask); 1729 req->val = htobe64(val); 1730 1731 t4_wrq_tx(sc, wr); 1732} 1733 1734void 1735t4_init_cpl_io_handlers(struct adapter *sc) 1736{ 1737 1738 t4_register_cpl_handler(sc, CPL_PEER_CLOSE, do_peer_close); 1739 t4_register_cpl_handler(sc, CPL_CLOSE_CON_RPL, do_close_con_rpl); 1740 t4_register_cpl_handler(sc, CPL_ABORT_REQ_RSS, do_abort_req); 1741 t4_register_cpl_handler(sc, CPL_ABORT_RPL_RSS, do_abort_rpl); 1742 t4_register_cpl_handler(sc, CPL_RX_DATA, do_rx_data); 1743 t4_register_cpl_handler(sc, CPL_FW4_ACK, do_fw4_ack); 1744 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, do_set_tcb_rpl); 1745} 1746 1747void 1748t4_uninit_cpl_io_handlers(struct adapter *sc) 1749{ 1750 1751 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 1752} 1753#endif 1754