1237263Snp/*- 2237263Snp * Copyright (c) 2012 Chelsio Communications, Inc. 3237263Snp * All rights reserved. 4237263Snp * 5237263Snp * Redistribution and use in source and binary forms, with or without 6237263Snp * modification, are permitted provided that the following conditions 7237263Snp * are met: 8237263Snp * 1. Redistributions of source code must retain the above copyright 9237263Snp * notice, this list of conditions and the following disclaimer. 10237263Snp * 2. Redistributions in binary form must reproduce the above copyright 11237263Snp * notice, this list of conditions and the following disclaimer in the 12237263Snp * documentation and/or other materials provided with the distribution. 13237263Snp * 14237263Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15237263Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16237263Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17237263Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18237263Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19237263Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20237263Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21237263Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22237263Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23237263Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24237263Snp * SUCH DAMAGE. 25237263Snp */ 26174641Skmacy 27237263Snp#include <sys/cdefs.h> 28237263Snp__FBSDID("$FreeBSD: releng/10.3/sys/dev/cxgb/ulp/tom/cxgb_listen.c 239544 2012-08-21 22:23:17Z np $"); 29174641Skmacy 30237263Snp#include "opt_inet.h" 31174641Skmacy 32237263Snp#ifdef TCP_OFFLOAD 33237263Snp#include <sys/param.h> 34237263Snp#include <sys/refcount.h> 35237263Snp#include <sys/socket.h> 36237263Snp#include <sys/socketvar.h> 37237263Snp#include <sys/sysctl.h> 38237263Snp#include <net/if.h> 39237263Snp#include <net/route.h> 40237263Snp#include <netinet/in.h> 41237263Snp#include <netinet/ip.h> 42237263Snp#include <netinet/in_pcb.h> 43237263Snp#include <netinet/in_var.h> 44239544Snp#include <netinet/tcp_timer.h> 45237263Snp#include <netinet/tcp_var.h> 46237263Snp#define TCPSTATES 47237263Snp#include <netinet/tcp_fsm.h> 48237263Snp#include <netinet/toecore.h> 49174641Skmacy 50237263Snp#include "cxgb_include.h" 51237263Snp#include "ulp/tom/cxgb_tom.h" 52237263Snp#include "ulp/tom/cxgb_l2t.h" 53237263Snp#include "ulp/tom/cxgb_toepcb.h" 54174641Skmacy 55237263Snpstatic void t3_send_reset_synqe(struct toedev *, struct synq_entry *); 56174641Skmacy 57237263Snpstatic int 58237263Snpalloc_stid(struct tid_info *t, void *ctx) 59237263Snp{ 60237263Snp int stid = -1; 61174641Skmacy 62237263Snp mtx_lock(&t->stid_lock); 63237263Snp if (t->sfree) { 64237263Snp union listen_entry *p = t->sfree; 65174641Skmacy 66237263Snp stid = (p - t->stid_tab) + t->stid_base; 67237263Snp t->sfree = p->next; 68237263Snp p->ctx = ctx; 69237263Snp t->stids_in_use++; 70237263Snp } 71237263Snp mtx_unlock(&t->stid_lock); 72237263Snp return (stid); 73237263Snp} 74181067Skmacy 75237263Snpstatic void 76237263Snpfree_stid(struct tid_info *t, int stid) 77237263Snp{ 78237263Snp union listen_entry *p = stid2entry(t, stid); 79181067Skmacy 80237263Snp mtx_lock(&t->stid_lock); 81237263Snp p->next = t->sfree; 82237263Snp t->sfree = p; 83237263Snp t->stids_in_use--; 84237263Snp mtx_unlock(&t->stid_lock); 85237263Snp} 86174641Skmacy 87237263Snpstatic struct listen_ctx * 88237263Snpalloc_lctx(struct tom_data *td, struct inpcb *inp, int qset) 89237263Snp{ 90237263Snp struct listen_ctx *lctx; 91174641Skmacy 92237263Snp INP_WLOCK_ASSERT(inp); 93174641Skmacy 94237263Snp lctx = malloc(sizeof(struct listen_ctx), M_CXGB, M_NOWAIT | M_ZERO); 95237263Snp if (lctx == NULL) 96237263Snp return (NULL); 97174641Skmacy 98237263Snp lctx->stid = alloc_stid(&td->tid_maps, lctx); 99237263Snp if (lctx->stid < 0) { 100237263Snp free(lctx, M_CXGB); 101237263Snp return (NULL); 102237263Snp } 103174641Skmacy 104237263Snp lctx->inp = inp; 105237263Snp in_pcbref(inp); 106174641Skmacy 107237263Snp lctx->qset = qset; 108237263Snp refcount_init(&lctx->refcnt, 1); 109237263Snp TAILQ_INIT(&lctx->synq); 110174641Skmacy 111237263Snp return (lctx); 112237263Snp} 113174641Skmacy 114237263Snp/* Don't call this directly, use release_lctx instead */ 115237263Snpstatic int 116237263Snpfree_lctx(struct tom_data *td, struct listen_ctx *lctx) 117237263Snp{ 118237263Snp struct inpcb *inp = lctx->inp; 119174641Skmacy 120237263Snp INP_WLOCK_ASSERT(inp); 121237263Snp KASSERT(lctx->refcnt == 0, 122237263Snp ("%s: refcnt %d", __func__, lctx->refcnt)); 123237263Snp KASSERT(TAILQ_EMPTY(&lctx->synq), 124237263Snp ("%s: synq not empty.", __func__)); 125237263Snp KASSERT(lctx->stid >= 0, ("%s: bad stid %d.", __func__, lctx->stid)); 126174641Skmacy 127237263Snp CTR4(KTR_CXGB, "%s: stid %u, lctx %p, inp %p", 128237263Snp __func__, lctx->stid, lctx, lctx->inp); 129237263Snp 130237263Snp free_stid(&td->tid_maps, lctx->stid); 131237263Snp free(lctx, M_CXGB); 132237263Snp 133237263Snp return in_pcbrele_wlocked(inp); 134237263Snp} 135237263Snp 136237263Snpstatic void 137237263Snphold_lctx(struct listen_ctx *lctx) 138237263Snp{ 139237263Snp 140237263Snp refcount_acquire(&lctx->refcnt); 141237263Snp} 142237263Snp 143237263Snpstatic inline uint32_t 144237263Snplisten_hashfn(void *key, u_long mask) 145237263Snp{ 146237263Snp 147237263Snp return (fnv_32_buf(&key, sizeof(key), FNV1_32_INIT) & mask); 148237263Snp} 149237263Snp 150174641Skmacy/* 151237263Snp * Add a listen_ctx entry to the listen hash table. 152237263Snp */ 153237263Snpstatic void 154237263Snplisten_hash_add(struct tom_data *td, struct listen_ctx *lctx) 155237263Snp{ 156237263Snp int bucket = listen_hashfn(lctx->inp, td->listen_mask); 157237263Snp 158237263Snp mtx_lock(&td->lctx_hash_lock); 159237263Snp LIST_INSERT_HEAD(&td->listen_hash[bucket], lctx, link); 160237263Snp td->lctx_count++; 161237263Snp mtx_unlock(&td->lctx_hash_lock); 162237263Snp} 163237263Snp 164237263Snp/* 165237263Snp * Look for the listening socket's context entry in the hash and return it. 166237263Snp */ 167237263Snpstatic struct listen_ctx * 168237263Snplisten_hash_find(struct tom_data *td, struct inpcb *inp) 169237263Snp{ 170237263Snp int bucket = listen_hashfn(inp, td->listen_mask); 171237263Snp struct listen_ctx *lctx; 172237263Snp 173237263Snp mtx_lock(&td->lctx_hash_lock); 174237263Snp LIST_FOREACH(lctx, &td->listen_hash[bucket], link) { 175237263Snp if (lctx->inp == inp) 176237263Snp break; 177237263Snp } 178237263Snp mtx_unlock(&td->lctx_hash_lock); 179237263Snp 180237263Snp return (lctx); 181237263Snp} 182237263Snp 183237263Snp/* 184237263Snp * Removes the listen_ctx structure for inp from the hash and returns it. 185237263Snp */ 186237263Snpstatic struct listen_ctx * 187237263Snplisten_hash_del(struct tom_data *td, struct inpcb *inp) 188237263Snp{ 189237263Snp int bucket = listen_hashfn(inp, td->listen_mask); 190237263Snp struct listen_ctx *lctx, *l; 191237263Snp 192237263Snp mtx_lock(&td->lctx_hash_lock); 193237263Snp LIST_FOREACH_SAFE(lctx, &td->listen_hash[bucket], link, l) { 194237263Snp if (lctx->inp == inp) { 195237263Snp LIST_REMOVE(lctx, link); 196237263Snp td->lctx_count--; 197237263Snp break; 198237263Snp } 199237263Snp } 200237263Snp mtx_unlock(&td->lctx_hash_lock); 201237263Snp 202237263Snp return (lctx); 203237263Snp} 204237263Snp 205237263Snp/* 206237263Snp * Releases a hold on the lctx. Must be called with the listening socket's inp 207237263Snp * locked. The inp may be freed by this function and it returns NULL to 208237263Snp * indicate this. 209237263Snp */ 210237263Snpstatic struct inpcb * 211237263Snprelease_lctx(struct tom_data *td, struct listen_ctx *lctx) 212237263Snp{ 213237263Snp struct inpcb *inp = lctx->inp; 214237263Snp int inp_freed = 0; 215237263Snp 216237263Snp INP_WLOCK_ASSERT(inp); 217237263Snp if (refcount_release(&lctx->refcnt)) 218237263Snp inp_freed = free_lctx(td, lctx); 219237263Snp 220237263Snp return (inp_freed ? NULL : inp); 221237263Snp} 222237263Snp 223237263Snpstatic int 224237263Snpcreate_server(struct adapter *sc, struct listen_ctx *lctx) 225237263Snp{ 226237263Snp struct mbuf *m; 227237263Snp struct cpl_pass_open_req *req; 228237263Snp struct inpcb *inp = lctx->inp; 229237263Snp 230237263Snp m = M_GETHDR_OFLD(lctx->qset, CPL_PRIORITY_CONTROL, req); 231237263Snp if (m == NULL) 232237263Snp return (ENOMEM); 233237263Snp 234237263Snp req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 235237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, lctx->stid)); 236237263Snp req->local_port = inp->inp_lport; 237237263Snp memcpy(&req->local_ip, &inp->inp_laddr, 4); 238237263Snp req->peer_port = 0; 239237263Snp req->peer_ip = 0; 240237263Snp req->peer_netmask = 0; 241237263Snp req->opt0h = htonl(F_DELACK | F_TCAM_BYPASS); 242237263Snp req->opt0l = htonl(V_RCV_BUFSIZ(16)); 243237263Snp req->opt1 = htonl(V_CONN_POLICY(CPL_CONN_POLICY_ASK)); 244237263Snp 245237263Snp t3_offload_tx(sc, m); 246237263Snp 247237263Snp return (0); 248237263Snp} 249237263Snp 250237263Snpstatic int 251237263Snpdestroy_server(struct adapter *sc, struct listen_ctx *lctx) 252237263Snp{ 253237263Snp struct mbuf *m; 254237263Snp struct cpl_close_listserv_req *req; 255237263Snp 256237263Snp m = M_GETHDR_OFLD(lctx->qset, CPL_PRIORITY_CONTROL, req); 257237263Snp if (m == NULL) 258237263Snp return (ENOMEM); 259237263Snp 260237263Snp req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 261237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, 262237263Snp lctx->stid)); 263237263Snp req->cpu_idx = 0; 264237263Snp 265237263Snp t3_offload_tx(sc, m); 266237263Snp 267237263Snp return (0); 268237263Snp} 269237263Snp 270237263Snp/* 271174641Skmacy * Process a CPL_CLOSE_LISTSRV_RPL message. If the status is good we release 272174641Skmacy * the STID. 273174641Skmacy */ 274174641Skmacystatic int 275237263Snpdo_close_server_rpl(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 276174641Skmacy{ 277237263Snp struct adapter *sc = qs->adap; 278237263Snp struct tom_data *td = sc->tom_softc; 279237263Snp struct cpl_close_listserv_rpl *rpl = mtod(m, void *); 280174641Skmacy unsigned int stid = GET_TID(rpl); 281237263Snp struct listen_ctx *lctx = lookup_stid(&td->tid_maps, stid); 282237263Snp struct inpcb *inp = lctx->inp; 283174641Skmacy 284237263Snp CTR3(KTR_CXGB, "%s: stid %u, status %u", __func__, stid, rpl->status); 285174641Skmacy 286237263Snp if (rpl->status != CPL_ERR_NONE) { 287237263Snp log(LOG_ERR, "%s: failed (%u) to close listener for stid %u", 288237263Snp __func__, rpl->status, stid); 289237263Snp } else { 290237263Snp INP_WLOCK(inp); 291237263Snp KASSERT(listen_hash_del(td, lctx->inp) == NULL, 292237263Snp ("%s: inp %p still in listen hash", __func__, inp)); 293237263Snp if (release_lctx(td, lctx) != NULL) 294237263Snp INP_WUNLOCK(inp); 295174641Skmacy } 296174641Skmacy 297237263Snp m_freem(m); 298237263Snp return (0); 299174641Skmacy} 300174641Skmacy 301174641Skmacy/* 302237263Snp * Process a CPL_PASS_OPEN_RPL message. Remove the lctx from the listen hash 303237263Snp * table and free it if there was any error, otherwise nothing to do. 304174641Skmacy */ 305174641Skmacystatic int 306237263Snpdo_pass_open_rpl(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 307174641Skmacy{ 308237263Snp struct adapter *sc = qs->adap; 309237263Snp struct tom_data *td = sc->tom_softc; 310237263Snp struct cpl_pass_open_rpl *rpl = mtod(m, void *); 311237263Snp int stid = GET_TID(rpl); 312237263Snp struct listen_ctx *lctx; 313237263Snp struct inpcb *inp; 314174641Skmacy 315237263Snp /* 316237263Snp * We get these replies also when setting up HW filters. Just throw 317237263Snp * those away. 318237263Snp */ 319237263Snp if (stid >= td->tid_maps.stid_base + td->tid_maps.nstids) 320237263Snp goto done; 321237263Snp 322237263Snp lctx = lookup_stid(&td->tid_maps, stid); 323237263Snp inp = lctx->inp; 324237263Snp 325237263Snp INP_WLOCK(inp); 326237263Snp 327237263Snp CTR4(KTR_CXGB, "%s: stid %u, status %u, flags 0x%x", 328237263Snp __func__, stid, rpl->status, lctx->flags); 329237263Snp 330237263Snp lctx->flags &= ~LCTX_RPL_PENDING; 331237263Snp 332174641Skmacy if (rpl->status != CPL_ERR_NONE) { 333237263Snp log(LOG_ERR, "%s: %s: hw listen (stid %d) failed: %d\n", 334237263Snp __func__, device_get_nameunit(sc->dev), stid, rpl->status); 335237263Snp } 336174641Skmacy 337237263Snp#ifdef INVARIANTS 338237263Snp /* 339237263Snp * If the inp has been dropped (listening socket closed) then 340237263Snp * listen_stop must have run and taken the inp out of the hash. 341237263Snp */ 342237263Snp if (inp->inp_flags & INP_DROPPED) { 343237263Snp KASSERT(listen_hash_del(td, inp) == NULL, 344237263Snp ("%s: inp %p still in listen hash", __func__, inp)); 345237263Snp } 346174641Skmacy#endif 347237263Snp 348237263Snp if (inp->inp_flags & INP_DROPPED && rpl->status != CPL_ERR_NONE) { 349237263Snp if (release_lctx(td, lctx) != NULL) 350237263Snp INP_WUNLOCK(inp); 351237263Snp goto done; 352174641Skmacy } 353237263Snp 354237263Snp /* 355237263Snp * Listening socket stopped listening earlier and now the chip tells us 356237263Snp * it has started the hardware listener. Stop it; the lctx will be 357237263Snp * released in do_close_server_rpl. 358237263Snp */ 359237263Snp if (inp->inp_flags & INP_DROPPED) { 360237263Snp destroy_server(sc, lctx); 361237263Snp INP_WUNLOCK(inp); 362237263Snp goto done; 363237263Snp } 364237263Snp 365237263Snp /* 366237263Snp * Failed to start hardware listener. Take inp out of the hash and 367237263Snp * release our reference on it. An error message has been logged 368237263Snp * already. 369237263Snp */ 370237263Snp if (rpl->status != CPL_ERR_NONE) { 371237263Snp listen_hash_del(td, inp); 372237263Snp if (release_lctx(td, lctx) != NULL) 373237263Snp INP_WUNLOCK(inp); 374237263Snp goto done; 375237263Snp } 376237263Snp 377237263Snp /* hardware listener open for business */ 378237263Snp 379237263Snp INP_WUNLOCK(inp); 380237263Snpdone: 381237263Snp m_freem(m); 382237263Snp return (0); 383174641Skmacy} 384174641Skmacy 385237263Snpstatic void 386237263Snppass_accept_req_to_protohdrs(const struct cpl_pass_accept_req *cpl, 387237263Snp struct in_conninfo *inc, struct tcphdr *th, struct tcpopt *to) 388174641Skmacy{ 389237263Snp const struct tcp_options *t3opt = &cpl->tcp_options; 390237263Snp 391237263Snp bzero(inc, sizeof(*inc)); 392237263Snp inc->inc_faddr.s_addr = cpl->peer_ip; 393237263Snp inc->inc_laddr.s_addr = cpl->local_ip; 394237263Snp inc->inc_fport = cpl->peer_port; 395237263Snp inc->inc_lport = cpl->local_port; 396237263Snp 397237263Snp bzero(th, sizeof(*th)); 398237263Snp th->th_sport = cpl->peer_port; 399237263Snp th->th_dport = cpl->local_port; 400237263Snp th->th_seq = be32toh(cpl->rcv_isn); /* as in tcp_fields_to_host */ 401237263Snp th->th_flags = TH_SYN; 402237263Snp 403237263Snp bzero(to, sizeof(*to)); 404237263Snp if (t3opt->mss) { 405237263Snp to->to_flags |= TOF_MSS; 406237263Snp to->to_mss = be16toh(t3opt->mss); 407237263Snp } 408237263Snp if (t3opt->wsf) { 409237263Snp to->to_flags |= TOF_SCALE; 410237263Snp to->to_wscale = t3opt->wsf; 411237263Snp } 412237263Snp if (t3opt->tstamp) 413237263Snp to->to_flags |= TOF_TS; 414237263Snp if (t3opt->sack) 415237263Snp to->to_flags |= TOF_SACKPERM; 416174641Skmacy} 417174641Skmacy 418237263Snpstatic inline void 419237263Snphold_synqe(struct synq_entry *synqe) 420174641Skmacy{ 421237263Snp 422237263Snp refcount_acquire(&synqe->refcnt); 423174641Skmacy} 424174641Skmacy 425237263Snpstatic inline void 426237263Snprelease_synqe(struct synq_entry *synqe) 427237263Snp{ 428237263Snp 429237263Snp if (refcount_release(&synqe->refcnt)) 430237263Snp m_freem(synqe->m); 431237263Snp} 432237263Snp 433174641Skmacy/* 434237263Snp * Use the trailing space in the mbuf in which the PASS_ACCEPT_REQ arrived to 435237263Snp * store some state temporarily. There will be enough room in the mbuf's 436237263Snp * trailing space as the CPL is not that large. 437237263Snp * 438237263Snp * XXX: bad hack. 439174641Skmacy */ 440237263Snpstatic struct synq_entry * 441237263Snpmbuf_to_synq_entry(struct mbuf *m) 442174641Skmacy{ 443237263Snp int len = roundup(sizeof (struct synq_entry), 8); 444237263Snp uint8_t *buf; 445237263Snp int buflen; 446174641Skmacy 447237263Snp if (__predict_false(M_TRAILINGSPACE(m) < len)) { 448237263Snp panic("%s: no room for synq_entry (%td, %d)\n", __func__, 449237263Snp M_TRAILINGSPACE(m), len); 450237263Snp } 451174641Skmacy 452237263Snp if (m->m_flags & M_EXT) { 453237263Snp buf = m->m_ext.ext_buf; 454237263Snp buflen = m->m_ext.ext_size; 455237263Snp } else if (m->m_flags & M_PKTHDR) { 456237263Snp buf = &m->m_pktdat[0]; 457237263Snp buflen = MHLEN; 458237263Snp } else { 459237263Snp buf = &m->m_dat[0]; 460237263Snp buflen = MLEN; 461174641Skmacy } 462237263Snp 463237263Snp return ((void *)(buf + buflen - len)); 464174641Skmacy} 465174641Skmacy 466237263Snp#ifdef KTR 467237263Snp#define REJECT_PASS_ACCEPT() do { \ 468237263Snp reject_reason = __LINE__; \ 469237263Snp goto reject; \ 470237263Snp} while (0) 471237263Snp#else 472237263Snp#define REJECT_PASS_ACCEPT() do { goto reject; } while (0) 473237263Snp#endif 474237263Snp 475174641Skmacy/* 476237263Snp * The context associated with a tid entry via insert_tid could be a synq_entry 477237263Snp * or a toepcb. The only way CPL handlers can tell is via a bit in these flags. 478174641Skmacy */ 479237263SnpCTASSERT(offsetof(struct toepcb, tp_flags) == offsetof(struct synq_entry, flags)); 480237263Snp 481237263Snp/* 482237263Snp * Handle a CPL_PASS_ACCEPT_REQ message. 483237263Snp */ 484174641Skmacystatic int 485237263Snpdo_pass_accept_req(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 486174641Skmacy{ 487237263Snp struct adapter *sc = qs->adap; 488237263Snp struct tom_data *td = sc->tom_softc; 489237263Snp struct toedev *tod = &td->tod; 490237263Snp const struct cpl_pass_accept_req *req = mtod(m, void *); 491237263Snp unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 492237263Snp unsigned int tid = GET_TID(req); 493237263Snp struct listen_ctx *lctx = lookup_stid(&td->tid_maps, stid); 494237263Snp struct l2t_entry *e = NULL; 495237263Snp struct sockaddr_in nam; 496237263Snp struct rtentry *rt; 497237263Snp struct inpcb *inp; 498237263Snp struct socket *so; 499237263Snp struct port_info *pi; 500237263Snp struct ifnet *ifp; 501237263Snp struct in_conninfo inc; 502237263Snp struct tcphdr th; 503237263Snp struct tcpopt to; 504237263Snp struct synq_entry *synqe = NULL; 505237263Snp int i; 506237263Snp#ifdef KTR 507237263Snp int reject_reason; 508237263Snp#endif 509174641Skmacy 510237263Snp CTR4(KTR_CXGB, "%s: stid %u, tid %u, lctx %p", __func__, stid, tid, 511237263Snp lctx); 512237263Snp 513237263Snp pass_accept_req_to_protohdrs(req, &inc, &th, &to); 514237263Snp 515237263Snp /* 516237263Snp * Don't offload if the interface that received the SYN doesn't have 517237263Snp * IFCAP_TOE enabled. 518237263Snp */ 519237263Snp pi = NULL; 520237263Snp for_each_port(sc, i) { 521237263Snp if (memcmp(sc->port[i].hw_addr, req->dst_mac, ETHER_ADDR_LEN)) 522237263Snp continue; 523237263Snp pi = &sc->port[i]; 524237263Snp break; 525237263Snp } 526237263Snp if (pi == NULL) 527237263Snp REJECT_PASS_ACCEPT(); 528237263Snp ifp = pi->ifp; 529237263Snp if ((ifp->if_capenable & IFCAP_TOE4) == 0) 530237263Snp REJECT_PASS_ACCEPT(); 531237263Snp 532237263Snp /* 533237263Snp * Don't offload if the outgoing interface for the route back to the 534237263Snp * peer is not the same as the interface that received the SYN. 535237263Snp */ 536237263Snp bzero(&nam, sizeof(nam)); 537237263Snp nam.sin_len = sizeof(nam); 538237263Snp nam.sin_family = AF_INET; 539237263Snp nam.sin_addr = inc.inc_faddr; 540237263Snp rt = rtalloc1((struct sockaddr *)&nam, 0, 0); 541237263Snp if (rt == NULL) 542237263Snp REJECT_PASS_ACCEPT(); 543237263Snp else { 544237263Snp struct sockaddr *nexthop; 545237263Snp 546237263Snp RT_UNLOCK(rt); 547237263Snp nexthop = rt->rt_flags & RTF_GATEWAY ? rt->rt_gateway : 548237263Snp (struct sockaddr *)&nam; 549237263Snp if (rt->rt_ifp == ifp) 550237263Snp e = t3_l2t_get(pi, rt->rt_ifp, nexthop); 551237263Snp RTFREE(rt); 552237263Snp if (e == NULL) 553237263Snp REJECT_PASS_ACCEPT(); /* no l2te, or ifp mismatch */ 554237263Snp } 555237263Snp 556237263Snp INP_INFO_WLOCK(&V_tcbinfo); 557237263Snp 558237263Snp /* Don't offload if the 4-tuple is already in use */ 559237263Snp if (toe_4tuple_check(&inc, &th, ifp) != 0) { 560237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 561237263Snp REJECT_PASS_ACCEPT(); 562237263Snp } 563237263Snp 564237263Snp inp = lctx->inp; /* listening socket (not owned by the TOE) */ 565237263Snp INP_WLOCK(inp); 566237263Snp if (__predict_false(inp->inp_flags & INP_DROPPED)) { 567237263Snp /* 568237263Snp * The listening socket has closed. The reply from the TOE to 569237263Snp * our CPL_CLOSE_LISTSRV_REQ will ultimately release all 570237263Snp * resources tied to this listen context. 571237263Snp */ 572237263Snp INP_WUNLOCK(inp); 573237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 574237263Snp REJECT_PASS_ACCEPT(); 575237263Snp } 576237263Snp so = inp->inp_socket; 577237263Snp 578237263Snp /* Reuse the mbuf that delivered the CPL to us */ 579237263Snp synqe = mbuf_to_synq_entry(m); 580237263Snp synqe->flags = TP_IS_A_SYNQ_ENTRY; 581237263Snp synqe->m = m; 582237263Snp synqe->lctx = lctx; 583237263Snp synqe->tid = tid; 584237263Snp synqe->e = e; 585237263Snp synqe->opt0h = calc_opt0h(so, 0, 0, e); 586237263Snp synqe->qset = pi->first_qset + (arc4random() % pi->nqsets); 587237263Snp SOCKBUF_LOCK(&so->so_rcv); 588237263Snp synqe->rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ); 589237263Snp SOCKBUF_UNLOCK(&so->so_rcv); 590237263Snp refcount_init(&synqe->refcnt, 1); 591237263Snp atomic_store_rel_int(&synqe->reply, RPL_OK); 592237263Snp 593237263Snp insert_tid(td, synqe, tid); 594237263Snp TAILQ_INSERT_TAIL(&lctx->synq, synqe, link); 595237263Snp hold_synqe(synqe); 596237263Snp hold_lctx(lctx); 597237263Snp 598237263Snp /* syncache_add releases both pcbinfo and pcb locks */ 599237263Snp toe_syncache_add(&inc, &to, &th, inp, tod, synqe); 600237263Snp INP_UNLOCK_ASSERT(inp); 601237263Snp INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); 602237263Snp 603237263Snp /* 604237263Snp * If we replied during syncache_add (reply is RPL_DONE), good. 605237263Snp * Otherwise (reply is unchanged - RPL_OK) it's no longer ok to reply. 606237263Snp * The mbuf will stick around as long as the entry is in the syncache. 607237263Snp * The kernel is free to retry syncache_respond but we'll ignore it due 608237263Snp * to RPL_DONT. 609237263Snp */ 610237263Snp if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONT)) { 611237263Snp 612237263Snp INP_WLOCK(inp); 613237263Snp if (__predict_false(inp->inp_flags & INP_DROPPED)) { 614237263Snp /* listener closed. synqe must have been aborted. */ 615237263Snp KASSERT(synqe->flags & TP_ABORT_SHUTDOWN, 616237263Snp ("%s: listener %p closed but synqe %p not aborted", 617237263Snp __func__, inp, synqe)); 618237263Snp 619237263Snp CTR5(KTR_CXGB, 620237263Snp "%s: stid %u, tid %u, lctx %p, synqe %p, ABORTED", 621237263Snp __func__, stid, tid, lctx, synqe); 622237263Snp INP_WUNLOCK(inp); 623237263Snp release_synqe(synqe); 624237263Snp return (__LINE__); 625174641Skmacy } 626237263Snp 627237263Snp KASSERT(!(synqe->flags & TP_ABORT_SHUTDOWN), 628237263Snp ("%s: synqe %p aborted, but listener %p not dropped.", 629237263Snp __func__, synqe, inp)); 630237263Snp 631237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 632237263Snp release_synqe(synqe); /* removed from synq list */ 633237263Snp inp = release_lctx(td, lctx); 634237263Snp if (inp) 635237263Snp INP_WUNLOCK(inp); 636237263Snp 637237263Snp release_synqe(synqe); /* about to exit function */ 638237263Snp REJECT_PASS_ACCEPT(); 639237263Snp } 640237263Snp 641237263Snp KASSERT(synqe->reply == RPL_DONE, 642237263Snp ("%s: reply %d", __func__, synqe->reply)); 643237263Snp 644237263Snp CTR3(KTR_CXGB, "%s: stid %u, tid %u, OK", __func__, stid, tid); 645237263Snp release_synqe(synqe); 646237263Snp return (0); 647237263Snp 648237263Snpreject: 649237263Snp CTR4(KTR_CXGB, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid, 650237263Snp reject_reason); 651237263Snp 652237263Snp if (synqe == NULL) 653237263Snp m_freem(m); 654237263Snp if (e) 655237263Snp l2t_release(td->l2t, e); 656237263Snp queue_tid_release(tod, tid); 657237263Snp 658237263Snp return (0); 659174641Skmacy} 660174641Skmacy 661237263Snpstatic void 662237263Snppass_establish_to_protohdrs(const struct cpl_pass_establish *cpl, 663237263Snp struct in_conninfo *inc, struct tcphdr *th, struct tcpopt *to) 664237263Snp{ 665237263Snp uint16_t tcp_opt = be16toh(cpl->tcp_opt); 666237263Snp 667237263Snp bzero(inc, sizeof(*inc)); 668237263Snp inc->inc_faddr.s_addr = cpl->peer_ip; 669237263Snp inc->inc_laddr.s_addr = cpl->local_ip; 670237263Snp inc->inc_fport = cpl->peer_port; 671237263Snp inc->inc_lport = cpl->local_port; 672237263Snp 673237263Snp bzero(th, sizeof(*th)); 674237263Snp th->th_sport = cpl->peer_port; 675237263Snp th->th_dport = cpl->local_port; 676237263Snp th->th_flags = TH_ACK; 677237263Snp th->th_seq = be32toh(cpl->rcv_isn); /* as in tcp_fields_to_host */ 678237263Snp th->th_ack = be32toh(cpl->snd_isn); /* ditto */ 679237263Snp 680237263Snp bzero(to, sizeof(*to)); 681237263Snp if (G_TCPOPT_TSTAMP(tcp_opt)) 682237263Snp to->to_flags |= TOF_TS; 683237263Snp} 684237263Snp 685174641Skmacy/* 686237263Snp * Process a CPL_PASS_ESTABLISH message. The T3 has already established a 687237263Snp * connection and we need to do the software side setup. 688174641Skmacy */ 689174641Skmacystatic int 690237263Snpdo_pass_establish(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 691174641Skmacy{ 692237263Snp struct adapter *sc = qs->adap; 693237263Snp struct tom_data *td = sc->tom_softc; 694237263Snp struct cpl_pass_establish *cpl = mtod(m, void *); 695237263Snp struct toedev *tod = &td->tod; 696237263Snp unsigned int tid = GET_TID(cpl); 697237263Snp struct synq_entry *synqe = lookup_tid(&td->tid_maps, tid); 698237263Snp struct toepcb *toep; 699237263Snp struct socket *so; 700237263Snp struct listen_ctx *lctx = synqe->lctx; 701237263Snp struct inpcb *inp = lctx->inp; 702237263Snp struct tcpopt to; 703237263Snp struct tcphdr th; 704237263Snp struct in_conninfo inc; 705237263Snp#ifdef KTR 706237263Snp int stid = G_PASS_OPEN_TID(ntohl(cpl->tos_tid)); 707237263Snp#endif 708174641Skmacy 709237263Snp CTR5(KTR_CXGB, "%s: stid %u, tid %u, lctx %p, inp_flags 0x%x", 710237263Snp __func__, stid, tid, lctx, inp->inp_flags); 711174641Skmacy 712237263Snp KASSERT(qs->idx == synqe->qset, 713237263Snp ("%s qset mismatch %d %d", __func__, qs->idx, synqe->qset)); 714237263Snp 715237263Snp INP_INFO_WLOCK(&V_tcbinfo); /* for syncache_expand */ 716237263Snp INP_WLOCK(inp); 717237263Snp 718237263Snp if (__predict_false(inp->inp_flags & INP_DROPPED)) { 719237263Snp /* 720237263Snp * The listening socket has closed. The TOM must have aborted 721237263Snp * all the embryonic connections (including this one) that were 722237263Snp * on the lctx's synq. do_abort_rpl for the tid is responsible 723237263Snp * for cleaning up. 724237263Snp */ 725237263Snp KASSERT(synqe->flags & TP_ABORT_SHUTDOWN, 726237263Snp ("%s: listen socket dropped but tid %u not aborted.", 727237263Snp __func__, tid)); 728237263Snp INP_WUNLOCK(inp); 729237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 730237263Snp m_freem(m); 731237263Snp return (0); 732237263Snp } 733237263Snp 734237263Snp pass_establish_to_protohdrs(cpl, &inc, &th, &to); 735237263Snp 736237263Snp /* Lie in order to pass the checks in syncache_expand */ 737237263Snp to.to_tsecr = synqe->ts; 738237263Snp th.th_ack = synqe->iss + 1; 739237263Snp 740237263Snp toep = toepcb_alloc(tod); 741237263Snp if (toep == NULL) { 742237263Snpreset: 743237263Snp t3_send_reset_synqe(tod, synqe); 744237263Snp INP_WUNLOCK(inp); 745237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 746237263Snp m_freem(m); 747237263Snp return (0); 748237263Snp } 749237263Snp toep->tp_qset = qs->idx; 750237263Snp toep->tp_l2t = synqe->e; 751237263Snp toep->tp_tid = tid; 752237263Snp toep->tp_rx_credits = synqe->rx_credits; 753237263Snp 754237263Snp synqe->toep = toep; 755237263Snp synqe->cpl = cpl; 756237263Snp 757237263Snp so = inp->inp_socket; 758237263Snp if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) { 759237263Snp toepcb_free(toep); 760237263Snp goto reset; 761237263Snp } 762237263Snp 763239544Snp if (__predict_false(!(synqe->flags & TP_SYNQE_EXPANDED))) { 764239544Snp struct inpcb *new_inp = sotoinpcb(so); 765239544Snp 766239544Snp INP_WLOCK(new_inp); 767239544Snp tcp_timer_activate(intotcpcb(new_inp), TT_KEEP, 0); 768239544Snp t3_offload_socket(tod, synqe, so); 769239544Snp INP_WUNLOCK(new_inp); 770239544Snp } 771239544Snp 772237263Snp /* Remove the synq entry and release its reference on the lctx */ 773237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 774237263Snp inp = release_lctx(td, lctx); 775237263Snp if (inp) 776237263Snp INP_WUNLOCK(inp); 777237263Snp INP_INFO_WUNLOCK(&V_tcbinfo); 778237263Snp release_synqe(synqe); 779237263Snp 780237263Snp m_freem(m); 781237263Snp return (0); 782174641Skmacy} 783174641Skmacy 784237263Snpvoid 785237263Snpt3_init_listen_cpl_handlers(struct adapter *sc) 786237263Snp{ 787237263Snp t3_register_cpl_handler(sc, CPL_PASS_OPEN_RPL, do_pass_open_rpl); 788237263Snp t3_register_cpl_handler(sc, CPL_CLOSE_LISTSRV_RPL, do_close_server_rpl); 789237263Snp t3_register_cpl_handler(sc, CPL_PASS_ACCEPT_REQ, do_pass_accept_req); 790237263Snp t3_register_cpl_handler(sc, CPL_PASS_ESTABLISH, do_pass_establish); 791237263Snp} 792237263Snp 793174641Skmacy/* 794174641Skmacy * Start a listening server by sending a passive open request to HW. 795237263Snp * 796237263Snp * Can't take adapter lock here and access to sc->flags, sc->open_device_map, 797237263Snp * sc->offload_map, if_capenable are all race prone. 798174641Skmacy */ 799237263Snpint 800237263Snpt3_listen_start(struct toedev *tod, struct tcpcb *tp) 801174641Skmacy{ 802237263Snp struct tom_data *td = t3_tomdata(tod); 803237263Snp struct adapter *sc = tod->tod_softc; 804237263Snp struct port_info *pi; 805237263Snp struct inpcb *inp = tp->t_inpcb; 806237263Snp struct listen_ctx *lctx; 807237263Snp int i; 808174641Skmacy 809237263Snp INP_WLOCK_ASSERT(inp); 810174641Skmacy 811237263Snp if ((inp->inp_vflag & INP_IPV4) == 0) 812237263Snp return (0); 813174641Skmacy 814237263Snp#ifdef notyet 815237263Snp ADAPTER_LOCK(sc); 816237263Snp if (IS_BUSY(sc)) { 817237263Snp log(LOG_ERR, "%s: listen request ignored, %s is busy", 818237263Snp __func__, device_get_nameunit(sc->dev)); 819237263Snp goto done; 820237263Snp } 821174641Skmacy 822237263Snp KASSERT(sc->flags & TOM_INIT_DONE, 823237263Snp ("%s: TOM not initialized", __func__)); 824237263Snp#endif 825174641Skmacy 826237263Snp if ((sc->open_device_map & sc->offload_map) == 0) 827237263Snp goto done; /* no port that's UP with IFCAP_TOE enabled */ 828174641Skmacy 829237263Snp /* 830237263Snp * Find a running port with IFCAP_TOE4. We'll use the first such port's 831237263Snp * queues to send the passive open and receive the reply to it. 832237263Snp * 833237263Snp * XXX: need a way to mark an port in use by offload. if_cxgbe should 834237263Snp * then reject any attempt to bring down such a port (and maybe reject 835237263Snp * attempts to disable IFCAP_TOE on that port too?). 836237263Snp */ 837237263Snp for_each_port(sc, i) { 838237263Snp if (isset(&sc->open_device_map, i) && 839237263Snp sc->port[i].ifp->if_capenable & IFCAP_TOE4) 840237263Snp break; 841237263Snp } 842237263Snp KASSERT(i < sc->params.nports, 843237263Snp ("%s: no running port with TOE capability enabled.", __func__)); 844237263Snp pi = &sc->port[i]; 845174641Skmacy 846237263Snp if (listen_hash_find(td, inp) != NULL) 847237263Snp goto done; /* already setup */ 848174641Skmacy 849237263Snp lctx = alloc_lctx(td, inp, pi->first_qset); 850237263Snp if (lctx == NULL) { 851237263Snp log(LOG_ERR, 852237263Snp "%s: listen request ignored, %s couldn't allocate lctx\n", 853237263Snp __func__, device_get_nameunit(sc->dev)); 854237263Snp goto done; 855237263Snp } 856237263Snp listen_hash_add(td, lctx); 857237263Snp 858237263Snp CTR5(KTR_CXGB, "%s: stid %u (%s), lctx %p, inp %p", __func__, 859237263Snp lctx->stid, tcpstates[tp->t_state], lctx, inp); 860237263Snp 861237263Snp if (create_server(sc, lctx) != 0) { 862237263Snp log(LOG_ERR, "%s: %s failed to create hw listener.\n", __func__, 863237263Snp device_get_nameunit(sc->dev)); 864237263Snp (void) listen_hash_del(td, inp); 865237263Snp inp = release_lctx(td, lctx); 866237263Snp /* can't be freed, host stack has a reference */ 867237263Snp KASSERT(inp != NULL, ("%s: inp freed", __func__)); 868237263Snp goto done; 869237263Snp } 870237263Snp lctx->flags |= LCTX_RPL_PENDING; 871237263Snpdone: 872237263Snp#ifdef notyet 873237263Snp ADAPTER_UNLOCK(sc); 874237263Snp#endif 875237263Snp return (0); 876174641Skmacy} 877174641Skmacy 878174641Skmacy/* 879174641Skmacy * Stop a listening server by sending a close_listsvr request to HW. 880174641Skmacy * The server TID is freed when we get the reply. 881174641Skmacy */ 882237263Snpint 883237263Snpt3_listen_stop(struct toedev *tod, struct tcpcb *tp) 884174641Skmacy{ 885174641Skmacy struct listen_ctx *lctx; 886237263Snp struct adapter *sc = tod->tod_softc; 887237263Snp struct tom_data *td = t3_tomdata(tod); 888237263Snp struct inpcb *inp = tp->t_inpcb; 889237263Snp struct synq_entry *synqe; 890174641Skmacy 891237263Snp INP_WLOCK_ASSERT(inp); 892237263Snp 893237263Snp lctx = listen_hash_del(td, inp); 894237263Snp if (lctx == NULL) 895237263Snp return (ENOENT); /* no hardware listener for this inp */ 896237263Snp 897237263Snp CTR4(KTR_CXGB, "%s: stid %u, lctx %p, flags %x", __func__, lctx->stid, 898237263Snp lctx, lctx->flags); 899237263Snp 900174641Skmacy /* 901237263Snp * If the reply to the PASS_OPEN is still pending we'll wait for it to 902237263Snp * arrive and clean up when it does. 903174641Skmacy */ 904237263Snp if (lctx->flags & LCTX_RPL_PENDING) { 905237263Snp KASSERT(TAILQ_EMPTY(&lctx->synq), 906237263Snp ("%s: synq not empty.", __func__)); 907237263Snp return (EINPROGRESS); 908237263Snp } 909174641Skmacy 910237263Snp /* 911237263Snp * The host stack will abort all the connections on the listening 912237263Snp * socket's so_comp. It doesn't know about the connections on the synq 913237263Snp * so we need to take care of those. 914237263Snp */ 915237263Snp TAILQ_FOREACH(synqe, &lctx->synq, link) { 916237263Snp KASSERT(synqe->lctx == lctx, ("%s: synq corrupt", __func__)); 917237263Snp t3_send_reset_synqe(tod, synqe); 918174641Skmacy } 919174641Skmacy 920237263Snp destroy_server(sc, lctx); 921237263Snp return (0); 922237263Snp} 923174641Skmacy 924237263Snpvoid 925237263Snpt3_syncache_added(struct toedev *tod __unused, void *arg) 926237263Snp{ 927237263Snp struct synq_entry *synqe = arg; 928237263Snp 929237263Snp hold_synqe(synqe); 930174641Skmacy} 931237263Snp 932237263Snpvoid 933237263Snpt3_syncache_removed(struct toedev *tod __unused, void *arg) 934237263Snp{ 935237263Snp struct synq_entry *synqe = arg; 936237263Snp 937237263Snp release_synqe(synqe); 938237263Snp} 939237263Snp 940237263Snp/* XXX */ 941237263Snpextern void tcp_dooptions(struct tcpopt *, u_char *, int, int); 942237263Snp 943237263Snpint 944237263Snpt3_syncache_respond(struct toedev *tod, void *arg, struct mbuf *m) 945237263Snp{ 946237263Snp struct adapter *sc = tod->tod_softc; 947237263Snp struct synq_entry *synqe = arg; 948237263Snp struct l2t_entry *e = synqe->e; 949237263Snp struct ip *ip = mtod(m, struct ip *); 950237263Snp struct tcphdr *th = (void *)(ip + 1); 951237263Snp struct cpl_pass_accept_rpl *rpl; 952237263Snp struct mbuf *r; 953237263Snp struct listen_ctx *lctx = synqe->lctx; 954237263Snp struct tcpopt to; 955237263Snp int mtu_idx, cpu_idx; 956237263Snp 957237263Snp /* 958237263Snp * The first time we run it's during the call to syncache_add. That's 959237263Snp * the only one we care about. 960237263Snp */ 961237263Snp if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONE) == 0) 962237263Snp goto done; /* reply to the CPL only if it's ok to do so */ 963237263Snp 964237263Snp r = M_GETHDR_OFLD(lctx->qset, CPL_PRIORITY_CONTROL, rpl); 965237263Snp if (r == NULL) 966237263Snp goto done; 967237263Snp 968237263Snp /* 969237263Snp * Use only the provided mbuf (with ip and tcp headers) and what's in 970237263Snp * synqe. Avoid looking at the listening socket (lctx->inp) here. 971237263Snp * 972237263Snp * XXX: if the incoming SYN had the TCP timestamp option but the kernel 973237263Snp * decides it doesn't want to use TCP timestamps we have no way of 974237263Snp * relaying this info to the chip on a per-tid basis (all we have is a 975237263Snp * global knob). 976237263Snp */ 977237263Snp bzero(&to, sizeof(to)); 978237263Snp tcp_dooptions(&to, (void *)(th + 1), (th->th_off << 2) - sizeof(*th), 979237263Snp TO_SYN); 980237263Snp 981237263Snp /* stash them for later */ 982237263Snp synqe->iss = be32toh(th->th_seq); 983237263Snp synqe->ts = to.to_tsval; 984237263Snp 985237263Snp mtu_idx = find_best_mtu_idx(sc, NULL, to.to_mss); 986237263Snp cpu_idx = sc->rrss_map[synqe->qset]; 987237263Snp 988237263Snp rpl->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 989237263Snp rpl->wr.wrh_lo = 0; 990237263Snp OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, synqe->tid)); 991237263Snp rpl->opt2 = calc_opt2(cpu_idx); 992237263Snp rpl->rsvd = rpl->opt2; /* workaround for HW bug */ 993237263Snp rpl->peer_ip = ip->ip_dst.s_addr; 994237263Snp rpl->opt0h = synqe->opt0h | 995237263Snp calc_opt0h(NULL, mtu_idx, to.to_wscale, NULL); 996237263Snp rpl->opt0l_status = htobe32(CPL_PASS_OPEN_ACCEPT) | 997237263Snp calc_opt0l(NULL, synqe->rx_credits); 998237263Snp 999237263Snp l2t_send(sc, r, e); 1000237263Snpdone: 1001237263Snp m_freem(m); 1002237263Snp return (0); 1003237263Snp} 1004237263Snp 1005237263Snpint 1006237263Snpdo_abort_req_synqe(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 1007237263Snp{ 1008237263Snp struct adapter *sc = qs->adap; 1009237263Snp struct tom_data *td = sc->tom_softc; 1010237263Snp struct toedev *tod = &td->tod; 1011237263Snp const struct cpl_abort_req_rss *req = mtod(m, void *); 1012237263Snp unsigned int tid = GET_TID(req); 1013237263Snp struct synq_entry *synqe = lookup_tid(&td->tid_maps, tid); 1014237263Snp struct listen_ctx *lctx = synqe->lctx; 1015237263Snp struct inpcb *inp = lctx->inp; 1016237263Snp 1017237263Snp KASSERT(synqe->flags & TP_IS_A_SYNQ_ENTRY, 1018237263Snp ("%s: !SYNQ_ENTRY", __func__)); 1019237263Snp 1020237263Snp CTR6(KTR_CXGB, "%s: tid %u, synqe %p (%x), lctx %p, status %d", 1021237263Snp __func__, tid, synqe, synqe->flags, synqe->lctx, req->status); 1022237263Snp 1023237263Snp INP_WLOCK(inp); 1024237263Snp 1025237263Snp if (!(synqe->flags & TP_ABORT_REQ_RCVD)) { 1026237263Snp synqe->flags |= TP_ABORT_REQ_RCVD; 1027237263Snp synqe->flags |= TP_ABORT_SHUTDOWN; 1028237263Snp INP_WUNLOCK(inp); 1029237263Snp m_freem(m); 1030237263Snp return (0); 1031237263Snp } 1032237263Snp synqe->flags &= ~TP_ABORT_REQ_RCVD; 1033237263Snp 1034237263Snp /* 1035237263Snp * If we'd sent a reset on this synqe, we'll ignore this and clean up in 1036237263Snp * the T3's reply to our reset instead. 1037237263Snp */ 1038237263Snp if (synqe->flags & TP_ABORT_RPL_PENDING) { 1039237263Snp synqe->flags |= TP_ABORT_RPL_SENT; 1040237263Snp INP_WUNLOCK(inp); 1041237263Snp } else { 1042237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 1043237263Snp inp = release_lctx(td, lctx); 1044237263Snp if (inp) 1045237263Snp INP_WUNLOCK(inp); 1046237263Snp release_tid(tod, tid, qs->idx); 1047237263Snp l2t_release(td->l2t, synqe->e); 1048237263Snp release_synqe(synqe); 1049237263Snp } 1050237263Snp 1051237263Snp send_abort_rpl(tod, tid, qs->idx); 1052237263Snp m_freem(m); 1053237263Snp return (0); 1054237263Snp} 1055237263Snp 1056237263Snpint 1057237263Snpdo_abort_rpl_synqe(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 1058237263Snp{ 1059237263Snp struct adapter *sc = qs->adap; 1060237263Snp struct tom_data *td = sc->tom_softc; 1061237263Snp struct toedev *tod = &td->tod; 1062237263Snp const struct cpl_abort_rpl_rss *rpl = mtod(m, void *); 1063237263Snp unsigned int tid = GET_TID(rpl); 1064237263Snp struct synq_entry *synqe = lookup_tid(&td->tid_maps, tid); 1065237263Snp struct listen_ctx *lctx = synqe->lctx; 1066237263Snp struct inpcb *inp = lctx->inp; 1067237263Snp 1068237263Snp CTR3(KTR_CXGB, "%s: tid %d, synqe %p, status %d", tid, synqe, 1069237263Snp rpl->status); 1070237263Snp 1071237263Snp INP_WLOCK(inp); 1072237263Snp 1073237263Snp if (synqe->flags & TP_ABORT_RPL_PENDING) { 1074237263Snp if (!(synqe->flags & TP_ABORT_RPL_RCVD)) { 1075237263Snp synqe->flags |= TP_ABORT_RPL_RCVD; 1076237263Snp INP_WUNLOCK(inp); 1077237263Snp } else { 1078237263Snp synqe->flags &= ~TP_ABORT_RPL_RCVD; 1079237263Snp synqe->flags &= TP_ABORT_RPL_PENDING; 1080237263Snp 1081237263Snp TAILQ_REMOVE(&lctx->synq, synqe, link); 1082237263Snp inp = release_lctx(td, lctx); 1083237263Snp if (inp) 1084237263Snp INP_WUNLOCK(inp); 1085237263Snp release_tid(tod, tid, qs->idx); 1086237263Snp l2t_release(td->l2t, synqe->e); 1087237263Snp release_synqe(synqe); 1088237263Snp } 1089237263Snp } 1090237263Snp 1091237263Snp m_freem(m); 1092237263Snp return (0); 1093237263Snp} 1094237263Snp 1095237263Snpstatic void 1096237263Snpt3_send_reset_synqe(struct toedev *tod, struct synq_entry *synqe) 1097237263Snp{ 1098237263Snp struct cpl_abort_req *req; 1099237263Snp unsigned int tid = synqe->tid; 1100237263Snp struct adapter *sc = tod->tod_softc; 1101237263Snp struct mbuf *m; 1102237263Snp#ifdef INVARIANTS 1103237263Snp struct listen_ctx *lctx = synqe->lctx; 1104237263Snp struct inpcb *inp = lctx->inp; 1105237263Snp#endif 1106237263Snp 1107237263Snp INP_WLOCK_ASSERT(inp); 1108237263Snp 1109237263Snp CTR4(KTR_CXGB, "%s: tid %d, synqe %p (%x)", __func__, tid, synqe, 1110237263Snp synqe->flags); 1111237263Snp 1112237263Snp if (synqe->flags & TP_ABORT_SHUTDOWN) 1113237263Snp return; 1114237263Snp 1115237263Snp synqe->flags |= (TP_ABORT_RPL_PENDING | TP_ABORT_SHUTDOWN); 1116237263Snp 1117237263Snp m = M_GETHDR_OFLD(synqe->qset, CPL_PRIORITY_DATA, req); 1118237263Snp if (m == NULL) 1119237263Snp CXGB_UNIMPLEMENTED(); 1120237263Snp 1121237263Snp req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 1122237263Snp req->wr.wrh_lo = htonl(V_WR_TID(tid)); 1123237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); 1124237263Snp req->rsvd0 = 0; 1125237263Snp req->rsvd1 = !(synqe->flags & TP_DATASENT); 1126237263Snp req->cmd = CPL_ABORT_SEND_RST; 1127237263Snp 1128237263Snp l2t_send(sc, m, synqe->e); 1129237263Snp} 1130237263Snp 1131237263Snpvoid 1132237263Snpt3_offload_socket(struct toedev *tod, void *arg, struct socket *so) 1133237263Snp{ 1134237263Snp struct adapter *sc = tod->tod_softc; 1135237263Snp struct tom_data *td = sc->tom_softc; 1136237263Snp struct synq_entry *synqe = arg; 1137237263Snp#ifdef INVARIANTS 1138237263Snp struct inpcb *inp = sotoinpcb(so); 1139237263Snp#endif 1140237263Snp struct cpl_pass_establish *cpl = synqe->cpl; 1141237263Snp struct toepcb *toep = synqe->toep; 1142237263Snp 1143237263Snp INP_INFO_LOCK_ASSERT(&V_tcbinfo); /* prevents bad race with accept() */ 1144237263Snp INP_WLOCK_ASSERT(inp); 1145237263Snp 1146237263Snp offload_socket(so, toep); 1147237263Snp make_established(so, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt); 1148237263Snp update_tid(td, toep, synqe->tid); 1149239544Snp synqe->flags |= TP_SYNQE_EXPANDED; 1150237263Snp} 1151237263Snp#endif 1152