cxgb_listen.c revision 276563
1237263Snp/*- 2237263Snp * Copyright (c) 2012 Chelsio Communications, Inc. 3237263Snp * All rights reserved. 4237263Snp * 5237263Snp * Redistribution and use in source and binary forms, with or without 6237263Snp * modification, are permitted provided that the following conditions 7237263Snp * are met: 8237263Snp * 1. Redistributions of source code must retain the above copyright 9237263Snp * notice, this list of conditions and the following disclaimer. 10237263Snp * 2. Redistributions in binary form must reproduce the above copyright 11237263Snp * notice, this list of conditions and the following disclaimer in the 12237263Snp * documentation and/or other materials provided with the distribution. 13237263Snp * 14237263Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15237263Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16237263Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17237263Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18237263Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19237263Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20237263Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21237263Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22237263Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23237263Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24237263Snp * SUCH DAMAGE. 25237263Snp */ 26174641Skmacy 27237263Snp#include <sys/cdefs.h> 28237263Snp__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_listen.c 276563 2015-01-02 19:06:27Z rwatson $"); 29174641Skmacy 30237263Snp#include "opt_inet.h" 31174641Skmacy 32237263Snp#ifdef TCP_OFFLOAD 33237263Snp#include <sys/param.h> 34237263Snp#include <sys/refcount.h> 35237263Snp#include <sys/socket.h> 36237263Snp#include <sys/socketvar.h> 37237263Snp#include <sys/sysctl.h> 38237263Snp#include <net/if.h> 39257241Sglebius#include <net/if_var.h> 40237263Snp#include <net/route.h> 41237263Snp#include <netinet/in.h> 42237263Snp#include <netinet/ip.h> 43237263Snp#include <netinet/in_pcb.h> 44237263Snp#include <netinet/in_var.h> 45239544Snp#include <netinet/tcp_timer.h> 46237263Snp#include <netinet/tcp_var.h> 47237263Snp#define TCPSTATES 48237263Snp#include <netinet/tcp_fsm.h> 49237263Snp#include <netinet/toecore.h> 50174641Skmacy 51237263Snp#include "cxgb_include.h" 52237263Snp#include "ulp/tom/cxgb_tom.h" 53237263Snp#include "ulp/tom/cxgb_l2t.h" 54237263Snp#include "ulp/tom/cxgb_toepcb.h" 55174641Skmacy 56237263Snpstatic void t3_send_reset_synqe(struct toedev *, struct synq_entry *); 57174641Skmacy 58237263Snpstatic int 59237263Snpalloc_stid(struct tid_info *t, void *ctx) 60237263Snp{ 61237263Snp int stid = -1; 62174641Skmacy 63237263Snp mtx_lock(&t->stid_lock); 64237263Snp if (t->sfree) { 65237263Snp union listen_entry *p = t->sfree; 66174641Skmacy 67237263Snp stid = (p - t->stid_tab) + t->stid_base; 68237263Snp t->sfree = p->next; 69237263Snp p->ctx = ctx; 70237263Snp t->stids_in_use++; 71237263Snp } 72237263Snp mtx_unlock(&t->stid_lock); 73237263Snp return (stid); 74237263Snp} 75181067Skmacy 76237263Snpstatic void 77237263Snpfree_stid(struct tid_info *t, int stid) 78237263Snp{ 79237263Snp union listen_entry *p = stid2entry(t, stid); 80181067Skmacy 81237263Snp mtx_lock(&t->stid_lock); 82237263Snp p->next = t->sfree; 83237263Snp t->sfree = p; 84237263Snp t->stids_in_use--; 85237263Snp mtx_unlock(&t->stid_lock); 86237263Snp} 87174641Skmacy 88237263Snpstatic struct listen_ctx * 89237263Snpalloc_lctx(struct tom_data *td, struct inpcb *inp, int qset) 90237263Snp{ 91237263Snp struct listen_ctx *lctx; 92174641Skmacy 93237263Snp INP_WLOCK_ASSERT(inp); 94174641Skmacy 95237263Snp lctx = malloc(sizeof(struct listen_ctx), M_CXGB, M_NOWAIT | M_ZERO); 96237263Snp if (lctx == NULL) 97237263Snp return (NULL); 98174641Skmacy 99237263Snp lctx->stid = alloc_stid(&td->tid_maps, lctx); 100237263Snp if (lctx->stid < 0) { 101237263Snp free(lctx, M_CXGB); 102237263Snp return (NULL); 103237263Snp } 104174641Skmacy 105237263Snp lctx->inp = inp; 106237263Snp in_pcbref(inp); 107174641Skmacy 108237263Snp lctx->qset = qset; 109237263Snp refcount_init(&lctx->refcnt, 1); 110237263Snp TAILQ_INIT(&lctx->synq); 111174641Skmacy 112237263Snp return (lctx); 113237263Snp} 114174641Skmacy 115237263Snp/* Don't call this directly, use release_lctx instead */ 116237263Snpstatic int 117237263Snpfree_lctx(struct tom_data *td, struct listen_ctx *lctx) 118237263Snp{ 119237263Snp struct inpcb *inp = lctx->inp; 120174641Skmacy 121237263Snp INP_WLOCK_ASSERT(inp); 122237263Snp KASSERT(lctx->refcnt == 0, 123237263Snp ("%s: refcnt %d", __func__, lctx->refcnt)); 124237263Snp KASSERT(TAILQ_EMPTY(&lctx->synq), 125237263Snp ("%s: synq not empty.", __func__)); 126237263Snp KASSERT(lctx->stid >= 0, ("%s: bad stid %d.", __func__, lctx->stid)); 127174641Skmacy 128237263Snp CTR4(KTR_CXGB, "%s: stid %u, lctx %p, inp %p", 129237263Snp __func__, lctx->stid, lctx, lctx->inp); 130237263Snp 131237263Snp free_stid(&td->tid_maps, lctx->stid); 132237263Snp free(lctx, M_CXGB); 133237263Snp 134237263Snp return in_pcbrele_wlocked(inp); 135237263Snp} 136237263Snp 137237263Snpstatic void 138237263Snphold_lctx(struct listen_ctx *lctx) 139237263Snp{ 140237263Snp 141237263Snp refcount_acquire(&lctx->refcnt); 142237263Snp} 143237263Snp 144237263Snpstatic inline uint32_t 145237263Snplisten_hashfn(void *key, u_long mask) 146237263Snp{ 147237263Snp 148237263Snp return (fnv_32_buf(&key, sizeof(key), FNV1_32_INIT) & mask); 149237263Snp} 150237263Snp 151174641Skmacy/* 152237263Snp * Add a listen_ctx entry to the listen hash table. 153237263Snp */ 154237263Snpstatic void 155237263Snplisten_hash_add(struct tom_data *td, struct listen_ctx *lctx) 156237263Snp{ 157237263Snp int bucket = listen_hashfn(lctx->inp, td->listen_mask); 158237263Snp 159237263Snp mtx_lock(&td->lctx_hash_lock); 160237263Snp LIST_INSERT_HEAD(&td->listen_hash[bucket], lctx, link); 161237263Snp td->lctx_count++; 162237263Snp mtx_unlock(&td->lctx_hash_lock); 163237263Snp} 164237263Snp 165237263Snp/* 166237263Snp * Look for the listening socket's context entry in the hash and return it. 167237263Snp */ 168237263Snpstatic struct listen_ctx * 169237263Snplisten_hash_find(struct tom_data *td, struct inpcb *inp) 170237263Snp{ 171237263Snp int bucket = listen_hashfn(inp, td->listen_mask); 172237263Snp struct listen_ctx *lctx; 173237263Snp 174237263Snp mtx_lock(&td->lctx_hash_lock); 175237263Snp LIST_FOREACH(lctx, &td->listen_hash[bucket], link) { 176237263Snp if (lctx->inp == inp) 177237263Snp break; 178237263Snp } 179237263Snp mtx_unlock(&td->lctx_hash_lock); 180237263Snp 181237263Snp return (lctx); 182237263Snp} 183237263Snp 184237263Snp/* 185237263Snp * Removes the listen_ctx structure for inp from the hash and returns it. 186237263Snp */ 187237263Snpstatic struct listen_ctx * 188237263Snplisten_hash_del(struct tom_data *td, struct inpcb *inp) 189237263Snp{ 190237263Snp int bucket = listen_hashfn(inp, td->listen_mask); 191237263Snp struct listen_ctx *lctx, *l; 192237263Snp 193237263Snp mtx_lock(&td->lctx_hash_lock); 194237263Snp LIST_FOREACH_SAFE(lctx, &td->listen_hash[bucket], link, l) { 195237263Snp if (lctx->inp == inp) { 196237263Snp LIST_REMOVE(lctx, link); 197237263Snp td->lctx_count--; 198237263Snp break; 199237263Snp } 200237263Snp } 201237263Snp mtx_unlock(&td->lctx_hash_lock); 202237263Snp 203237263Snp return (lctx); 204237263Snp} 205237263Snp 206237263Snp/* 207237263Snp * Releases a hold on the lctx. Must be called with the listening socket's inp 208237263Snp * locked. The inp may be freed by this function and it returns NULL to 209237263Snp * indicate this. 210237263Snp */ 211237263Snpstatic struct inpcb * 212237263Snprelease_lctx(struct tom_data *td, struct listen_ctx *lctx) 213237263Snp{ 214237263Snp struct inpcb *inp = lctx->inp; 215237263Snp int inp_freed = 0; 216237263Snp 217237263Snp INP_WLOCK_ASSERT(inp); 218237263Snp if (refcount_release(&lctx->refcnt)) 219237263Snp inp_freed = free_lctx(td, lctx); 220237263Snp 221237263Snp return (inp_freed ? NULL : inp); 222237263Snp} 223237263Snp 224237263Snpstatic int 225237263Snpcreate_server(struct adapter *sc, struct listen_ctx *lctx) 226237263Snp{ 227237263Snp struct mbuf *m; 228237263Snp struct cpl_pass_open_req *req; 229237263Snp struct inpcb *inp = lctx->inp; 230237263Snp 231237263Snp m = M_GETHDR_OFLD(lctx->qset, CPL_PRIORITY_CONTROL, req); 232237263Snp if (m == NULL) 233237263Snp return (ENOMEM); 234237263Snp 235237263Snp req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 236237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, lctx->stid)); 237237263Snp req->local_port = inp->inp_lport; 238237263Snp memcpy(&req->local_ip, &inp->inp_laddr, 4); 239237263Snp req->peer_port = 0; 240237263Snp req->peer_ip = 0; 241237263Snp req->peer_netmask = 0; 242237263Snp req->opt0h = htonl(F_DELACK | F_TCAM_BYPASS); 243237263Snp req->opt0l = htonl(V_RCV_BUFSIZ(16)); 244237263Snp req->opt1 = htonl(V_CONN_POLICY(CPL_CONN_POLICY_ASK)); 245237263Snp 246237263Snp t3_offload_tx(sc, m); 247237263Snp 248237263Snp return (0); 249237263Snp} 250237263Snp 251237263Snpstatic int 252237263Snpdestroy_server(struct adapter *sc, struct listen_ctx *lctx) 253237263Snp{ 254237263Snp struct mbuf *m; 255237263Snp struct cpl_close_listserv_req *req; 256237263Snp 257237263Snp m = M_GETHDR_OFLD(lctx->qset, CPL_PRIORITY_CONTROL, req); 258237263Snp if (m == NULL) 259237263Snp return (ENOMEM); 260237263Snp 261237263Snp req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 262237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, 263237263Snp lctx->stid)); 264237263Snp req->cpu_idx = 0; 265237263Snp 266237263Snp t3_offload_tx(sc, m); 267237263Snp 268237263Snp return (0); 269237263Snp} 270237263Snp 271237263Snp/* 272174641Skmacy * Process a CPL_CLOSE_LISTSRV_RPL message. If the status is good we release 273174641Skmacy * the STID. 274174641Skmacy */ 275174641Skmacystatic int 276237263Snpdo_close_server_rpl(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 277174641Skmacy{ 278237263Snp struct adapter *sc = qs->adap; 279237263Snp struct tom_data *td = sc->tom_softc; 280237263Snp struct cpl_close_listserv_rpl *rpl = mtod(m, void *); 281174641Skmacy unsigned int stid = GET_TID(rpl); 282237263Snp struct listen_ctx *lctx = lookup_stid(&td->tid_maps, stid); 283237263Snp struct inpcb *inp = lctx->inp; 284174641Skmacy 285237263Snp CTR3(KTR_CXGB, "%s: stid %u, status %u", __func__, stid, rpl->status); 286174641Skmacy 287237263Snp if (rpl->status != CPL_ERR_NONE) { 288237263Snp log(LOG_ERR, "%s: failed (%u) to close listener for stid %u", 289237263Snp __func__, rpl->status, stid); 290237263Snp } else { 291237263Snp INP_WLOCK(inp); 292237263Snp KASSERT(listen_hash_del(td, lctx->inp) == NULL, 293237263Snp ("%s: inp %p still in listen hash", __func__, inp)); 294237263Snp if (release_lctx(td, lctx) != NULL) 295237263Snp INP_WUNLOCK(inp); 296174641Skmacy } 297174641Skmacy 298237263Snp m_freem(m); 299237263Snp return (0); 300174641Skmacy} 301174641Skmacy 302174641Skmacy/* 303237263Snp * Process a CPL_PASS_OPEN_RPL message. Remove the lctx from the listen hash 304237263Snp * table and free it if there was any error, otherwise nothing to do. 305174641Skmacy */ 306174641Skmacystatic int 307237263Snpdo_pass_open_rpl(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 308174641Skmacy{ 309237263Snp struct adapter *sc = qs->adap; 310237263Snp struct tom_data *td = sc->tom_softc; 311237263Snp struct cpl_pass_open_rpl *rpl = mtod(m, void *); 312237263Snp int stid = GET_TID(rpl); 313237263Snp struct listen_ctx *lctx; 314237263Snp struct inpcb *inp; 315174641Skmacy 316237263Snp /* 317237263Snp * We get these replies also when setting up HW filters. Just throw 318237263Snp * those away. 319237263Snp */ 320237263Snp if (stid >= td->tid_maps.stid_base + td->tid_maps.nstids) 321237263Snp goto done; 322237263Snp 323237263Snp lctx = lookup_stid(&td->tid_maps, stid); 324237263Snp inp = lctx->inp; 325237263Snp 326237263Snp INP_WLOCK(inp); 327237263Snp 328237263Snp CTR4(KTR_CXGB, "%s: stid %u, status %u, flags 0x%x", 329237263Snp __func__, stid, rpl->status, lctx->flags); 330237263Snp 331237263Snp lctx->flags &= ~LCTX_RPL_PENDING; 332237263Snp 333174641Skmacy if (rpl->status != CPL_ERR_NONE) { 334237263Snp log(LOG_ERR, "%s: %s: hw listen (stid %d) failed: %d\n", 335237263Snp __func__, device_get_nameunit(sc->dev), stid, rpl->status); 336237263Snp } 337174641Skmacy 338237263Snp#ifdef INVARIANTS 339237263Snp /* 340237263Snp * If the inp has been dropped (listening socket closed) then 341237263Snp * listen_stop must have run and taken the inp out of the hash. 342237263Snp */ 343237263Snp if (inp->inp_flags & INP_DROPPED) { 344237263Snp KASSERT(listen_hash_del(td, inp) == NULL, 345237263Snp ("%s: inp %p still in listen hash", __func__, inp)); 346237263Snp } 347174641Skmacy#endif 348237263Snp 349237263Snp if (inp->inp_flags & INP_DROPPED && rpl->status != CPL_ERR_NONE) { 350237263Snp if (release_lctx(td, lctx) != NULL) 351237263Snp INP_WUNLOCK(inp); 352237263Snp goto done; 353174641Skmacy } 354237263Snp 355237263Snp /* 356237263Snp * Listening socket stopped listening earlier and now the chip tells us 357237263Snp * it has started the hardware listener. Stop it; the lctx will be 358237263Snp * released in do_close_server_rpl. 359237263Snp */ 360237263Snp if (inp->inp_flags & INP_DROPPED) { 361237263Snp destroy_server(sc, lctx); 362237263Snp INP_WUNLOCK(inp); 363237263Snp goto done; 364237263Snp } 365237263Snp 366237263Snp /* 367237263Snp * Failed to start hardware listener. Take inp out of the hash and 368237263Snp * release our reference on it. An error message has been logged 369237263Snp * already. 370237263Snp */ 371237263Snp if (rpl->status != CPL_ERR_NONE) { 372237263Snp listen_hash_del(td, inp); 373237263Snp if (release_lctx(td, lctx) != NULL) 374237263Snp INP_WUNLOCK(inp); 375237263Snp goto done; 376237263Snp } 377237263Snp 378237263Snp /* hardware listener open for business */ 379237263Snp 380237263Snp INP_WUNLOCK(inp); 381237263Snpdone: 382237263Snp m_freem(m); 383237263Snp return (0); 384174641Skmacy} 385174641Skmacy 386237263Snpstatic void 387237263Snppass_accept_req_to_protohdrs(const struct cpl_pass_accept_req *cpl, 388237263Snp struct in_conninfo *inc, struct tcphdr *th, struct tcpopt *to) 389174641Skmacy{ 390237263Snp const struct tcp_options *t3opt = &cpl->tcp_options; 391237263Snp 392237263Snp bzero(inc, sizeof(*inc)); 393237263Snp inc->inc_faddr.s_addr = cpl->peer_ip; 394237263Snp inc->inc_laddr.s_addr = cpl->local_ip; 395237263Snp inc->inc_fport = cpl->peer_port; 396237263Snp inc->inc_lport = cpl->local_port; 397237263Snp 398237263Snp bzero(th, sizeof(*th)); 399237263Snp th->th_sport = cpl->peer_port; 400237263Snp th->th_dport = cpl->local_port; 401237263Snp th->th_seq = be32toh(cpl->rcv_isn); /* as in tcp_fields_to_host */ 402237263Snp th->th_flags = TH_SYN; 403237263Snp 404237263Snp bzero(to, sizeof(*to)); 405237263Snp if (t3opt->mss) { 406237263Snp to->to_flags |= TOF_MSS; 407237263Snp to->to_mss = be16toh(t3opt->mss); 408237263Snp } 409237263Snp if (t3opt->wsf) { 410237263Snp to->to_flags |= TOF_SCALE; 411237263Snp to->to_wscale = t3opt->wsf; 412237263Snp } 413237263Snp if (t3opt->tstamp) 414237263Snp to->to_flags |= TOF_TS; 415237263Snp if (t3opt->sack) 416237263Snp to->to_flags |= TOF_SACKPERM; 417174641Skmacy} 418174641Skmacy 419237263Snpstatic inline void 420237263Snphold_synqe(struct synq_entry *synqe) 421174641Skmacy{ 422237263Snp 423237263Snp refcount_acquire(&synqe->refcnt); 424174641Skmacy} 425174641Skmacy 426237263Snpstatic inline void 427237263Snprelease_synqe(struct synq_entry *synqe) 428237263Snp{ 429237263Snp 430237263Snp if (refcount_release(&synqe->refcnt)) 431237263Snp m_freem(synqe->m); 432237263Snp} 433237263Snp 434174641Skmacy/* 435237263Snp * Use the trailing space in the mbuf in which the PASS_ACCEPT_REQ arrived to 436237263Snp * store some state temporarily. There will be enough room in the mbuf's 437237263Snp * trailing space as the CPL is not that large. 438237263Snp * 439237263Snp * XXX: bad hack. 440174641Skmacy */ 441237263Snpstatic struct synq_entry * 442237263Snpmbuf_to_synq_entry(struct mbuf *m) 443174641Skmacy{ 444237263Snp int len = roundup(sizeof (struct synq_entry), 8); 445174641Skmacy 446237263Snp if (__predict_false(M_TRAILINGSPACE(m) < len)) { 447237263Snp panic("%s: no room for synq_entry (%td, %d)\n", __func__, 448237263Snp M_TRAILINGSPACE(m), len); 449237263Snp } 450174641Skmacy 451276563Srwatson return ((void *)(M_START(m) + M_SIZE(m) - len)); 452174641Skmacy} 453174641Skmacy 454237263Snp#ifdef KTR 455237263Snp#define REJECT_PASS_ACCEPT() do { \ 456237263Snp reject_reason = __LINE__; \ 457237263Snp goto reject; \ 458237263Snp} while (0) 459237263Snp#else 460237263Snp#define REJECT_PASS_ACCEPT() do { goto reject; } while (0) 461237263Snp#endif 462237263Snp 463174641Skmacy/* 464237263Snp * The context associated with a tid entry via insert_tid could be a synq_entry 465237263Snp * or a toepcb. The only way CPL handlers can tell is via a bit in these flags. 466174641Skmacy */ 467237263SnpCTASSERT(offsetof(struct toepcb, tp_flags) == offsetof(struct synq_entry, flags)); 468237263Snp 469237263Snp/* 470237263Snp * Handle a CPL_PASS_ACCEPT_REQ message. 471237263Snp */ 472174641Skmacystatic int 473237263Snpdo_pass_accept_req(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 474174641Skmacy{ 475237263Snp struct adapter *sc = qs->adap; 476237263Snp struct tom_data *td = sc->tom_softc; 477237263Snp struct toedev *tod = &td->tod; 478237263Snp const struct cpl_pass_accept_req *req = mtod(m, void *); 479237263Snp unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 480237263Snp unsigned int tid = GET_TID(req); 481237263Snp struct listen_ctx *lctx = lookup_stid(&td->tid_maps, stid); 482237263Snp struct l2t_entry *e = NULL; 483237263Snp struct sockaddr_in nam; 484237263Snp struct rtentry *rt; 485237263Snp struct inpcb *inp; 486237263Snp struct socket *so; 487237263Snp struct port_info *pi; 488237263Snp struct ifnet *ifp; 489237263Snp struct in_conninfo inc; 490237263Snp struct tcphdr th; 491237263Snp struct tcpopt to; 492237263Snp struct synq_entry *synqe = NULL; 493237263Snp int i; 494237263Snp#ifdef KTR 495237263Snp int reject_reason; 496237263Snp#endif 497174641Skmacy 498237263Snp CTR4(KTR_CXGB, "%s: stid %u, tid %u, lctx %p", __func__, stid, tid, 499237263Snp lctx); 500237263Snp 501237263Snp pass_accept_req_to_protohdrs(req, &inc, &th, &to); 502237263Snp 503237263Snp /* 504237263Snp * Don't offload if the interface that received the SYN doesn't have 505237263Snp * IFCAP_TOE enabled. 506237263Snp */ 507237263Snp pi = NULL; 508237263Snp for_each_port(sc, i) { 509237263Snp if (memcmp(sc->port[i].hw_addr, req->dst_mac, ETHER_ADDR_LEN)) 510237263Snp continue; 511237263Snp pi = &sc->port[i]; 512237263Snp break; 513237263Snp } 514237263Snp if (pi == NULL) 515237263Snp REJECT_PASS_ACCEPT(); 516237263Snp ifp = pi->ifp; 517237263Snp if ((ifp->if_capenable & IFCAP_TOE4) == 0) 518237263Snp REJECT_PASS_ACCEPT(); 519237263Snp 520237263Snp /* 521237263Snp * Don't offload if the outgoing interface for the route back to the 522237263Snp * peer is not the same as the interface that received the SYN. 523237263Snp */ 524237263Snp bzero(&nam, sizeof(nam)); 525237263Snp nam.sin_len = sizeof(nam); 526237263Snp nam.sin_family = AF_INET; 527237263Snp nam.sin_addr = inc.inc_faddr; 528237263Snp rt = rtalloc1((struct sockaddr *)&nam, 0, 0); 529237263Snp if (rt == NULL) 530237263Snp REJECT_PASS_ACCEPT(); 531237263Snp else { 532237263Snp struct sockaddr *nexthop; 533237263Snp 534237263Snp RT_UNLOCK(rt); 535237263Snp nexthop = rt->rt_flags & RTF_GATEWAY ? rt->rt_gateway : 536237263Snp (struct sockaddr *)&nam; 537237263Snp if (rt->rt_ifp == ifp) 538237263Snp e = t3_l2t_get(pi, rt->rt_ifp, nexthop); 539237263Snp RTFREE(rt); 540237263Snp if (e == NULL) 541237263Snp REJECT_PASS_ACCEPT(); /* no l2te, or ifp mismatch */ 542237263Snp } 543237263Snp 544237263Snp INP_INFO_WLOCK(&V_tcbinfo); 545237263Snp 546237263Snp /* Don't offload if the 4-tuple is already in use */ 547237263Snp if (toe_4tuple_check(&inc, &th, ifp) != 0) { 548237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 549237263Snp REJECT_PASS_ACCEPT(); 550237263Snp } 551237263Snp 552237263Snp inp = lctx->inp; /* listening socket (not owned by the TOE) */ 553237263Snp INP_WLOCK(inp); 554237263Snp if (__predict_false(inp->inp_flags & INP_DROPPED)) { 555237263Snp /* 556237263Snp * The listening socket has closed. The reply from the TOE to 557237263Snp * our CPL_CLOSE_LISTSRV_REQ will ultimately release all 558237263Snp * resources tied to this listen context. 559237263Snp */ 560237263Snp INP_WUNLOCK(inp); 561237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 562237263Snp REJECT_PASS_ACCEPT(); 563237263Snp } 564237263Snp so = inp->inp_socket; 565237263Snp 566237263Snp /* Reuse the mbuf that delivered the CPL to us */ 567237263Snp synqe = mbuf_to_synq_entry(m); 568237263Snp synqe->flags = TP_IS_A_SYNQ_ENTRY; 569237263Snp synqe->m = m; 570237263Snp synqe->lctx = lctx; 571237263Snp synqe->tid = tid; 572237263Snp synqe->e = e; 573237263Snp synqe->opt0h = calc_opt0h(so, 0, 0, e); 574237263Snp synqe->qset = pi->first_qset + (arc4random() % pi->nqsets); 575237263Snp SOCKBUF_LOCK(&so->so_rcv); 576237263Snp synqe->rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ); 577237263Snp SOCKBUF_UNLOCK(&so->so_rcv); 578237263Snp refcount_init(&synqe->refcnt, 1); 579237263Snp atomic_store_rel_int(&synqe->reply, RPL_OK); 580237263Snp 581237263Snp insert_tid(td, synqe, tid); 582237263Snp TAILQ_INSERT_TAIL(&lctx->synq, synqe, link); 583237263Snp hold_synqe(synqe); 584237263Snp hold_lctx(lctx); 585237263Snp 586237263Snp /* syncache_add releases both pcbinfo and pcb locks */ 587237263Snp toe_syncache_add(&inc, &to, &th, inp, tod, synqe); 588237263Snp INP_UNLOCK_ASSERT(inp); 589237263Snp INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); 590237263Snp 591237263Snp /* 592237263Snp * If we replied during syncache_add (reply is RPL_DONE), good. 593237263Snp * Otherwise (reply is unchanged - RPL_OK) it's no longer ok to reply. 594237263Snp * The mbuf will stick around as long as the entry is in the syncache. 595237263Snp * The kernel is free to retry syncache_respond but we'll ignore it due 596237263Snp * to RPL_DONT. 597237263Snp */ 598237263Snp if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONT)) { 599237263Snp 600237263Snp INP_WLOCK(inp); 601237263Snp if (__predict_false(inp->inp_flags & INP_DROPPED)) { 602237263Snp /* listener closed. synqe must have been aborted. */ 603237263Snp KASSERT(synqe->flags & TP_ABORT_SHUTDOWN, 604237263Snp ("%s: listener %p closed but synqe %p not aborted", 605237263Snp __func__, inp, synqe)); 606237263Snp 607237263Snp CTR5(KTR_CXGB, 608237263Snp "%s: stid %u, tid %u, lctx %p, synqe %p, ABORTED", 609237263Snp __func__, stid, tid, lctx, synqe); 610237263Snp INP_WUNLOCK(inp); 611237263Snp release_synqe(synqe); 612237263Snp return (__LINE__); 613174641Skmacy } 614237263Snp 615237263Snp KASSERT(!(synqe->flags & TP_ABORT_SHUTDOWN), 616237263Snp ("%s: synqe %p aborted, but listener %p not dropped.", 617237263Snp __func__, synqe, inp)); 618237263Snp 619237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 620237263Snp release_synqe(synqe); /* removed from synq list */ 621237263Snp inp = release_lctx(td, lctx); 622237263Snp if (inp) 623237263Snp INP_WUNLOCK(inp); 624237263Snp 625237263Snp release_synqe(synqe); /* about to exit function */ 626237263Snp REJECT_PASS_ACCEPT(); 627237263Snp } 628237263Snp 629237263Snp KASSERT(synqe->reply == RPL_DONE, 630237263Snp ("%s: reply %d", __func__, synqe->reply)); 631237263Snp 632237263Snp CTR3(KTR_CXGB, "%s: stid %u, tid %u, OK", __func__, stid, tid); 633237263Snp release_synqe(synqe); 634237263Snp return (0); 635237263Snp 636237263Snpreject: 637237263Snp CTR4(KTR_CXGB, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid, 638237263Snp reject_reason); 639237263Snp 640237263Snp if (synqe == NULL) 641237263Snp m_freem(m); 642237263Snp if (e) 643237263Snp l2t_release(td->l2t, e); 644237263Snp queue_tid_release(tod, tid); 645237263Snp 646237263Snp return (0); 647174641Skmacy} 648174641Skmacy 649237263Snpstatic void 650237263Snppass_establish_to_protohdrs(const struct cpl_pass_establish *cpl, 651237263Snp struct in_conninfo *inc, struct tcphdr *th, struct tcpopt *to) 652237263Snp{ 653237263Snp uint16_t tcp_opt = be16toh(cpl->tcp_opt); 654237263Snp 655237263Snp bzero(inc, sizeof(*inc)); 656237263Snp inc->inc_faddr.s_addr = cpl->peer_ip; 657237263Snp inc->inc_laddr.s_addr = cpl->local_ip; 658237263Snp inc->inc_fport = cpl->peer_port; 659237263Snp inc->inc_lport = cpl->local_port; 660237263Snp 661237263Snp bzero(th, sizeof(*th)); 662237263Snp th->th_sport = cpl->peer_port; 663237263Snp th->th_dport = cpl->local_port; 664237263Snp th->th_flags = TH_ACK; 665237263Snp th->th_seq = be32toh(cpl->rcv_isn); /* as in tcp_fields_to_host */ 666237263Snp th->th_ack = be32toh(cpl->snd_isn); /* ditto */ 667237263Snp 668237263Snp bzero(to, sizeof(*to)); 669237263Snp if (G_TCPOPT_TSTAMP(tcp_opt)) 670237263Snp to->to_flags |= TOF_TS; 671237263Snp} 672237263Snp 673174641Skmacy/* 674237263Snp * Process a CPL_PASS_ESTABLISH message. The T3 has already established a 675237263Snp * connection and we need to do the software side setup. 676174641Skmacy */ 677174641Skmacystatic int 678237263Snpdo_pass_establish(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 679174641Skmacy{ 680237263Snp struct adapter *sc = qs->adap; 681237263Snp struct tom_data *td = sc->tom_softc; 682237263Snp struct cpl_pass_establish *cpl = mtod(m, void *); 683237263Snp struct toedev *tod = &td->tod; 684237263Snp unsigned int tid = GET_TID(cpl); 685237263Snp struct synq_entry *synqe = lookup_tid(&td->tid_maps, tid); 686237263Snp struct toepcb *toep; 687237263Snp struct socket *so; 688237263Snp struct listen_ctx *lctx = synqe->lctx; 689237263Snp struct inpcb *inp = lctx->inp; 690237263Snp struct tcpopt to; 691237263Snp struct tcphdr th; 692237263Snp struct in_conninfo inc; 693237263Snp#ifdef KTR 694237263Snp int stid = G_PASS_OPEN_TID(ntohl(cpl->tos_tid)); 695237263Snp#endif 696174641Skmacy 697237263Snp CTR5(KTR_CXGB, "%s: stid %u, tid %u, lctx %p, inp_flags 0x%x", 698237263Snp __func__, stid, tid, lctx, inp->inp_flags); 699174641Skmacy 700237263Snp KASSERT(qs->idx == synqe->qset, 701237263Snp ("%s qset mismatch %d %d", __func__, qs->idx, synqe->qset)); 702237263Snp 703237263Snp INP_INFO_WLOCK(&V_tcbinfo); /* for syncache_expand */ 704237263Snp INP_WLOCK(inp); 705237263Snp 706237263Snp if (__predict_false(inp->inp_flags & INP_DROPPED)) { 707237263Snp /* 708237263Snp * The listening socket has closed. The TOM must have aborted 709237263Snp * all the embryonic connections (including this one) that were 710237263Snp * on the lctx's synq. do_abort_rpl for the tid is responsible 711237263Snp * for cleaning up. 712237263Snp */ 713237263Snp KASSERT(synqe->flags & TP_ABORT_SHUTDOWN, 714237263Snp ("%s: listen socket dropped but tid %u not aborted.", 715237263Snp __func__, tid)); 716237263Snp INP_WUNLOCK(inp); 717237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 718237263Snp m_freem(m); 719237263Snp return (0); 720237263Snp } 721237263Snp 722237263Snp pass_establish_to_protohdrs(cpl, &inc, &th, &to); 723237263Snp 724237263Snp /* Lie in order to pass the checks in syncache_expand */ 725237263Snp to.to_tsecr = synqe->ts; 726237263Snp th.th_ack = synqe->iss + 1; 727237263Snp 728237263Snp toep = toepcb_alloc(tod); 729237263Snp if (toep == NULL) { 730237263Snpreset: 731237263Snp t3_send_reset_synqe(tod, synqe); 732237263Snp INP_WUNLOCK(inp); 733237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 734237263Snp m_freem(m); 735237263Snp return (0); 736237263Snp } 737237263Snp toep->tp_qset = qs->idx; 738237263Snp toep->tp_l2t = synqe->e; 739237263Snp toep->tp_tid = tid; 740237263Snp toep->tp_rx_credits = synqe->rx_credits; 741237263Snp 742237263Snp synqe->toep = toep; 743237263Snp synqe->cpl = cpl; 744237263Snp 745237263Snp so = inp->inp_socket; 746237263Snp if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) { 747237263Snp toepcb_free(toep); 748237263Snp goto reset; 749237263Snp } 750237263Snp 751239544Snp if (__predict_false(!(synqe->flags & TP_SYNQE_EXPANDED))) { 752239544Snp struct inpcb *new_inp = sotoinpcb(so); 753239544Snp 754239544Snp INP_WLOCK(new_inp); 755239544Snp tcp_timer_activate(intotcpcb(new_inp), TT_KEEP, 0); 756239544Snp t3_offload_socket(tod, synqe, so); 757239544Snp INP_WUNLOCK(new_inp); 758239544Snp } 759239544Snp 760237263Snp /* Remove the synq entry and release its reference on the lctx */ 761237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 762237263Snp inp = release_lctx(td, lctx); 763237263Snp if (inp) 764237263Snp INP_WUNLOCK(inp); 765237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 766237263Snp release_synqe(synqe); 767237263Snp 768237263Snp m_freem(m); 769237263Snp return (0); 770174641Skmacy} 771174641Skmacy 772237263Snpvoid 773237263Snpt3_init_listen_cpl_handlers(struct adapter *sc) 774237263Snp{ 775237263Snp t3_register_cpl_handler(sc, CPL_PASS_OPEN_RPL, do_pass_open_rpl); 776237263Snp t3_register_cpl_handler(sc, CPL_CLOSE_LISTSRV_RPL, do_close_server_rpl); 777237263Snp t3_register_cpl_handler(sc, CPL_PASS_ACCEPT_REQ, do_pass_accept_req); 778237263Snp t3_register_cpl_handler(sc, CPL_PASS_ESTABLISH, do_pass_establish); 779237263Snp} 780237263Snp 781174641Skmacy/* 782174641Skmacy * Start a listening server by sending a passive open request to HW. 783237263Snp * 784237263Snp * Can't take adapter lock here and access to sc->flags, sc->open_device_map, 785237263Snp * sc->offload_map, if_capenable are all race prone. 786174641Skmacy */ 787237263Snpint 788237263Snpt3_listen_start(struct toedev *tod, struct tcpcb *tp) 789174641Skmacy{ 790237263Snp struct tom_data *td = t3_tomdata(tod); 791237263Snp struct adapter *sc = tod->tod_softc; 792237263Snp struct port_info *pi; 793237263Snp struct inpcb *inp = tp->t_inpcb; 794237263Snp struct listen_ctx *lctx; 795237263Snp int i; 796174641Skmacy 797237263Snp INP_WLOCK_ASSERT(inp); 798174641Skmacy 799237263Snp if ((inp->inp_vflag & INP_IPV4) == 0) 800237263Snp return (0); 801174641Skmacy 802237263Snp#ifdef notyet 803237263Snp ADAPTER_LOCK(sc); 804237263Snp if (IS_BUSY(sc)) { 805237263Snp log(LOG_ERR, "%s: listen request ignored, %s is busy", 806237263Snp __func__, device_get_nameunit(sc->dev)); 807237263Snp goto done; 808237263Snp } 809174641Skmacy 810237263Snp KASSERT(sc->flags & TOM_INIT_DONE, 811237263Snp ("%s: TOM not initialized", __func__)); 812237263Snp#endif 813174641Skmacy 814237263Snp if ((sc->open_device_map & sc->offload_map) == 0) 815237263Snp goto done; /* no port that's UP with IFCAP_TOE enabled */ 816174641Skmacy 817237263Snp /* 818237263Snp * Find a running port with IFCAP_TOE4. We'll use the first such port's 819237263Snp * queues to send the passive open and receive the reply to it. 820237263Snp * 821237263Snp * XXX: need a way to mark an port in use by offload. if_cxgbe should 822237263Snp * then reject any attempt to bring down such a port (and maybe reject 823237263Snp * attempts to disable IFCAP_TOE on that port too?). 824237263Snp */ 825237263Snp for_each_port(sc, i) { 826237263Snp if (isset(&sc->open_device_map, i) && 827237263Snp sc->port[i].ifp->if_capenable & IFCAP_TOE4) 828237263Snp break; 829237263Snp } 830237263Snp KASSERT(i < sc->params.nports, 831237263Snp ("%s: no running port with TOE capability enabled.", __func__)); 832237263Snp pi = &sc->port[i]; 833174641Skmacy 834237263Snp if (listen_hash_find(td, inp) != NULL) 835237263Snp goto done; /* already setup */ 836174641Skmacy 837237263Snp lctx = alloc_lctx(td, inp, pi->first_qset); 838237263Snp if (lctx == NULL) { 839237263Snp log(LOG_ERR, 840237263Snp "%s: listen request ignored, %s couldn't allocate lctx\n", 841237263Snp __func__, device_get_nameunit(sc->dev)); 842237263Snp goto done; 843237263Snp } 844237263Snp listen_hash_add(td, lctx); 845237263Snp 846237263Snp CTR5(KTR_CXGB, "%s: stid %u (%s), lctx %p, inp %p", __func__, 847237263Snp lctx->stid, tcpstates[tp->t_state], lctx, inp); 848237263Snp 849237263Snp if (create_server(sc, lctx) != 0) { 850237263Snp log(LOG_ERR, "%s: %s failed to create hw listener.\n", __func__, 851237263Snp device_get_nameunit(sc->dev)); 852237263Snp (void) listen_hash_del(td, inp); 853237263Snp inp = release_lctx(td, lctx); 854237263Snp /* can't be freed, host stack has a reference */ 855237263Snp KASSERT(inp != NULL, ("%s: inp freed", __func__)); 856237263Snp goto done; 857237263Snp } 858237263Snp lctx->flags |= LCTX_RPL_PENDING; 859237263Snpdone: 860237263Snp#ifdef notyet 861237263Snp ADAPTER_UNLOCK(sc); 862237263Snp#endif 863237263Snp return (0); 864174641Skmacy} 865174641Skmacy 866174641Skmacy/* 867174641Skmacy * Stop a listening server by sending a close_listsvr request to HW. 868174641Skmacy * The server TID is freed when we get the reply. 869174641Skmacy */ 870237263Snpint 871237263Snpt3_listen_stop(struct toedev *tod, struct tcpcb *tp) 872174641Skmacy{ 873174641Skmacy struct listen_ctx *lctx; 874237263Snp struct adapter *sc = tod->tod_softc; 875237263Snp struct tom_data *td = t3_tomdata(tod); 876237263Snp struct inpcb *inp = tp->t_inpcb; 877237263Snp struct synq_entry *synqe; 878174641Skmacy 879237263Snp INP_WLOCK_ASSERT(inp); 880237263Snp 881237263Snp lctx = listen_hash_del(td, inp); 882237263Snp if (lctx == NULL) 883237263Snp return (ENOENT); /* no hardware listener for this inp */ 884237263Snp 885237263Snp CTR4(KTR_CXGB, "%s: stid %u, lctx %p, flags %x", __func__, lctx->stid, 886237263Snp lctx, lctx->flags); 887237263Snp 888174641Skmacy /* 889237263Snp * If the reply to the PASS_OPEN is still pending we'll wait for it to 890237263Snp * arrive and clean up when it does. 891174641Skmacy */ 892237263Snp if (lctx->flags & LCTX_RPL_PENDING) { 893237263Snp KASSERT(TAILQ_EMPTY(&lctx->synq), 894237263Snp ("%s: synq not empty.", __func__)); 895237263Snp return (EINPROGRESS); 896237263Snp } 897174641Skmacy 898237263Snp /* 899237263Snp * The host stack will abort all the connections on the listening 900237263Snp * socket's so_comp. It doesn't know about the connections on the synq 901237263Snp * so we need to take care of those. 902237263Snp */ 903237263Snp TAILQ_FOREACH(synqe, &lctx->synq, link) { 904237263Snp KASSERT(synqe->lctx == lctx, ("%s: synq corrupt", __func__)); 905237263Snp t3_send_reset_synqe(tod, synqe); 906174641Skmacy } 907174641Skmacy 908237263Snp destroy_server(sc, lctx); 909237263Snp return (0); 910237263Snp} 911174641Skmacy 912237263Snpvoid 913237263Snpt3_syncache_added(struct toedev *tod __unused, void *arg) 914237263Snp{ 915237263Snp struct synq_entry *synqe = arg; 916237263Snp 917237263Snp hold_synqe(synqe); 918174641Skmacy} 919237263Snp 920237263Snpvoid 921237263Snpt3_syncache_removed(struct toedev *tod __unused, void *arg) 922237263Snp{ 923237263Snp struct synq_entry *synqe = arg; 924237263Snp 925237263Snp release_synqe(synqe); 926237263Snp} 927237263Snp 928237263Snp/* XXX */ 929237263Snpextern void tcp_dooptions(struct tcpopt *, u_char *, int, int); 930237263Snp 931237263Snpint 932237263Snpt3_syncache_respond(struct toedev *tod, void *arg, struct mbuf *m) 933237263Snp{ 934237263Snp struct adapter *sc = tod->tod_softc; 935237263Snp struct synq_entry *synqe = arg; 936237263Snp struct l2t_entry *e = synqe->e; 937237263Snp struct ip *ip = mtod(m, struct ip *); 938237263Snp struct tcphdr *th = (void *)(ip + 1); 939237263Snp struct cpl_pass_accept_rpl *rpl; 940237263Snp struct mbuf *r; 941237263Snp struct listen_ctx *lctx = synqe->lctx; 942237263Snp struct tcpopt to; 943237263Snp int mtu_idx, cpu_idx; 944237263Snp 945237263Snp /* 946237263Snp * The first time we run it's during the call to syncache_add. That's 947237263Snp * the only one we care about. 948237263Snp */ 949237263Snp if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONE) == 0) 950237263Snp goto done; /* reply to the CPL only if it's ok to do so */ 951237263Snp 952237263Snp r = M_GETHDR_OFLD(lctx->qset, CPL_PRIORITY_CONTROL, rpl); 953237263Snp if (r == NULL) 954237263Snp goto done; 955237263Snp 956237263Snp /* 957237263Snp * Use only the provided mbuf (with ip and tcp headers) and what's in 958237263Snp * synqe. Avoid looking at the listening socket (lctx->inp) here. 959237263Snp * 960237263Snp * XXX: if the incoming SYN had the TCP timestamp option but the kernel 961237263Snp * decides it doesn't want to use TCP timestamps we have no way of 962237263Snp * relaying this info to the chip on a per-tid basis (all we have is a 963237263Snp * global knob). 964237263Snp */ 965237263Snp bzero(&to, sizeof(to)); 966237263Snp tcp_dooptions(&to, (void *)(th + 1), (th->th_off << 2) - sizeof(*th), 967237263Snp TO_SYN); 968237263Snp 969237263Snp /* stash them for later */ 970237263Snp synqe->iss = be32toh(th->th_seq); 971237263Snp synqe->ts = to.to_tsval; 972237263Snp 973237263Snp mtu_idx = find_best_mtu_idx(sc, NULL, to.to_mss); 974237263Snp cpu_idx = sc->rrss_map[synqe->qset]; 975237263Snp 976237263Snp rpl->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 977237263Snp rpl->wr.wrh_lo = 0; 978237263Snp OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, synqe->tid)); 979237263Snp rpl->opt2 = calc_opt2(cpu_idx); 980237263Snp rpl->rsvd = rpl->opt2; /* workaround for HW bug */ 981237263Snp rpl->peer_ip = ip->ip_dst.s_addr; 982237263Snp rpl->opt0h = synqe->opt0h | 983237263Snp calc_opt0h(NULL, mtu_idx, to.to_wscale, NULL); 984237263Snp rpl->opt0l_status = htobe32(CPL_PASS_OPEN_ACCEPT) | 985237263Snp calc_opt0l(NULL, synqe->rx_credits); 986237263Snp 987237263Snp l2t_send(sc, r, e); 988237263Snpdone: 989237263Snp m_freem(m); 990237263Snp return (0); 991237263Snp} 992237263Snp 993237263Snpint 994237263Snpdo_abort_req_synqe(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 995237263Snp{ 996237263Snp struct adapter *sc = qs->adap; 997237263Snp struct tom_data *td = sc->tom_softc; 998237263Snp struct toedev *tod = &td->tod; 999237263Snp const struct cpl_abort_req_rss *req = mtod(m, void *); 1000237263Snp unsigned int tid = GET_TID(req); 1001237263Snp struct synq_entry *synqe = lookup_tid(&td->tid_maps, tid); 1002237263Snp struct listen_ctx *lctx = synqe->lctx; 1003237263Snp struct inpcb *inp = lctx->inp; 1004237263Snp 1005237263Snp KASSERT(synqe->flags & TP_IS_A_SYNQ_ENTRY, 1006237263Snp ("%s: !SYNQ_ENTRY", __func__)); 1007237263Snp 1008237263Snp CTR6(KTR_CXGB, "%s: tid %u, synqe %p (%x), lctx %p, status %d", 1009237263Snp __func__, tid, synqe, synqe->flags, synqe->lctx, req->status); 1010237263Snp 1011237263Snp INP_WLOCK(inp); 1012237263Snp 1013237263Snp if (!(synqe->flags & TP_ABORT_REQ_RCVD)) { 1014237263Snp synqe->flags |= TP_ABORT_REQ_RCVD; 1015237263Snp synqe->flags |= TP_ABORT_SHUTDOWN; 1016237263Snp INP_WUNLOCK(inp); 1017237263Snp m_freem(m); 1018237263Snp return (0); 1019237263Snp } 1020237263Snp synqe->flags &= ~TP_ABORT_REQ_RCVD; 1021237263Snp 1022237263Snp /* 1023237263Snp * If we'd sent a reset on this synqe, we'll ignore this and clean up in 1024237263Snp * the T3's reply to our reset instead. 1025237263Snp */ 1026237263Snp if (synqe->flags & TP_ABORT_RPL_PENDING) { 1027237263Snp synqe->flags |= TP_ABORT_RPL_SENT; 1028237263Snp INP_WUNLOCK(inp); 1029237263Snp } else { 1030237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 1031237263Snp inp = release_lctx(td, lctx); 1032237263Snp if (inp) 1033237263Snp INP_WUNLOCK(inp); 1034237263Snp release_tid(tod, tid, qs->idx); 1035237263Snp l2t_release(td->l2t, synqe->e); 1036237263Snp release_synqe(synqe); 1037237263Snp } 1038237263Snp 1039237263Snp send_abort_rpl(tod, tid, qs->idx); 1040237263Snp m_freem(m); 1041237263Snp return (0); 1042237263Snp} 1043237263Snp 1044237263Snpint 1045237263Snpdo_abort_rpl_synqe(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 1046237263Snp{ 1047237263Snp struct adapter *sc = qs->adap; 1048237263Snp struct tom_data *td = sc->tom_softc; 1049237263Snp struct toedev *tod = &td->tod; 1050237263Snp const struct cpl_abort_rpl_rss *rpl = mtod(m, void *); 1051237263Snp unsigned int tid = GET_TID(rpl); 1052237263Snp struct synq_entry *synqe = lookup_tid(&td->tid_maps, tid); 1053237263Snp struct listen_ctx *lctx = synqe->lctx; 1054237263Snp struct inpcb *inp = lctx->inp; 1055237263Snp 1056237263Snp CTR3(KTR_CXGB, "%s: tid %d, synqe %p, status %d", tid, synqe, 1057237263Snp rpl->status); 1058237263Snp 1059237263Snp INP_WLOCK(inp); 1060237263Snp 1061237263Snp if (synqe->flags & TP_ABORT_RPL_PENDING) { 1062237263Snp if (!(synqe->flags & TP_ABORT_RPL_RCVD)) { 1063237263Snp synqe->flags |= TP_ABORT_RPL_RCVD; 1064237263Snp INP_WUNLOCK(inp); 1065237263Snp } else { 1066237263Snp synqe->flags &= ~TP_ABORT_RPL_RCVD; 1067237263Snp synqe->flags &= TP_ABORT_RPL_PENDING; 1068237263Snp 1069237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 1070237263Snp inp = release_lctx(td, lctx); 1071237263Snp if (inp) 1072237263Snp INP_WUNLOCK(inp); 1073237263Snp release_tid(tod, tid, qs->idx); 1074237263Snp l2t_release(td->l2t, synqe->e); 1075237263Snp release_synqe(synqe); 1076237263Snp } 1077237263Snp } 1078237263Snp 1079237263Snp m_freem(m); 1080237263Snp return (0); 1081237263Snp} 1082237263Snp 1083237263Snpstatic void 1084237263Snpt3_send_reset_synqe(struct toedev *tod, struct synq_entry *synqe) 1085237263Snp{ 1086237263Snp struct cpl_abort_req *req; 1087237263Snp unsigned int tid = synqe->tid; 1088237263Snp struct adapter *sc = tod->tod_softc; 1089237263Snp struct mbuf *m; 1090237263Snp#ifdef INVARIANTS 1091237263Snp struct listen_ctx *lctx = synqe->lctx; 1092237263Snp struct inpcb *inp = lctx->inp; 1093237263Snp#endif 1094237263Snp 1095237263Snp INP_WLOCK_ASSERT(inp); 1096237263Snp 1097237263Snp CTR4(KTR_CXGB, "%s: tid %d, synqe %p (%x)", __func__, tid, synqe, 1098237263Snp synqe->flags); 1099237263Snp 1100237263Snp if (synqe->flags & TP_ABORT_SHUTDOWN) 1101237263Snp return; 1102237263Snp 1103237263Snp synqe->flags |= (TP_ABORT_RPL_PENDING | TP_ABORT_SHUTDOWN); 1104237263Snp 1105237263Snp m = M_GETHDR_OFLD(synqe->qset, CPL_PRIORITY_DATA, req); 1106237263Snp if (m == NULL) 1107237263Snp CXGB_UNIMPLEMENTED(); 1108237263Snp 1109237263Snp req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 1110237263Snp req->wr.wrh_lo = htonl(V_WR_TID(tid)); 1111237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); 1112237263Snp req->rsvd0 = 0; 1113237263Snp req->rsvd1 = !(synqe->flags & TP_DATASENT); 1114237263Snp req->cmd = CPL_ABORT_SEND_RST; 1115237263Snp 1116237263Snp l2t_send(sc, m, synqe->e); 1117237263Snp} 1118237263Snp 1119237263Snpvoid 1120237263Snpt3_offload_socket(struct toedev *tod, void *arg, struct socket *so) 1121237263Snp{ 1122237263Snp struct adapter *sc = tod->tod_softc; 1123237263Snp struct tom_data *td = sc->tom_softc; 1124237263Snp struct synq_entry *synqe = arg; 1125237263Snp#ifdef INVARIANTS 1126237263Snp struct inpcb *inp = sotoinpcb(so); 1127237263Snp#endif 1128237263Snp struct cpl_pass_establish *cpl = synqe->cpl; 1129237263Snp struct toepcb *toep = synqe->toep; 1130237263Snp 1131237263Snp INP_INFO_LOCK_ASSERT(&V_tcbinfo); /* prevents bad race with accept() */ 1132237263Snp INP_WLOCK_ASSERT(inp); 1133237263Snp 1134237263Snp offload_socket(so, toep); 1135237263Snp make_established(so, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt); 1136237263Snp update_tid(td, toep, synqe->tid); 1137239544Snp synqe->flags |= TP_SYNQE_EXPANDED; 1138237263Snp} 1139237263Snp#endif 1140