t4_tom.c revision 249627
1237263Snp/*- 2237263Snp * Copyright (c) 2012 Chelsio Communications, Inc. 3237263Snp * All rights reserved. 4237263Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5237263Snp * 6237263Snp * Redistribution and use in source and binary forms, with or without 7237263Snp * modification, are permitted provided that the following conditions 8237263Snp * are met: 9237263Snp * 1. Redistributions of source code must retain the above copyright 10237263Snp * notice, this list of conditions and the following disclaimer. 11237263Snp * 2. Redistributions in binary form must reproduce the above copyright 12237263Snp * notice, this list of conditions and the following disclaimer in the 13237263Snp * documentation and/or other materials provided with the distribution. 14237263Snp * 15237263Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16237263Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17237263Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18237263Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19237263Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20237263Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21237263Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22237263Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23237263Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24237263Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25237263Snp * SUCH DAMAGE. 26237263Snp */ 27237263Snp 28237263Snp#include <sys/cdefs.h> 29237263Snp__FBSDID("$FreeBSD: head/sys/dev/cxgbe/tom/t4_tom.c 249627 2013-04-18 19:52:11Z np $"); 30237263Snp 31237263Snp#include "opt_inet.h" 32245441Snp#include "opt_inet6.h" 33237263Snp 34237263Snp#include <sys/param.h> 35237263Snp#include <sys/types.h> 36237263Snp#include <sys/systm.h> 37237263Snp#include <sys/kernel.h> 38237263Snp#include <sys/ktr.h> 39237263Snp#include <sys/module.h> 40237263Snp#include <sys/protosw.h> 41237263Snp#include <sys/domain.h> 42237263Snp#include <sys/socket.h> 43237263Snp#include <sys/socketvar.h> 44249627Snp#include <sys/taskqueue.h> 45245448Snp#include <net/if.h> 46237263Snp#include <netinet/in.h> 47237263Snp#include <netinet/in_pcb.h> 48245448Snp#include <netinet/in_var.h> 49237263Snp#include <netinet/ip.h> 50245441Snp#include <netinet/ip6.h> 51237263Snp#include <netinet/tcp_var.h> 52245448Snp#include <netinet6/scope6_var.h> 53237263Snp#define TCPSTATES 54237263Snp#include <netinet/tcp_fsm.h> 55237263Snp#include <netinet/toecore.h> 56237263Snp 57237263Snp#ifdef TCP_OFFLOAD 58237263Snp#include "common/common.h" 59237263Snp#include "common/t4_msg.h" 60237263Snp#include "common/t4_regs.h" 61237263Snp#include "tom/t4_tom_l2t.h" 62237263Snp#include "tom/t4_tom.h" 63237263Snp 64239344Snpstatic struct protosw ddp_protosw; 65239344Snpstatic struct pr_usrreqs ddp_usrreqs; 66239344Snp 67245441Snpstatic struct protosw ddp6_protosw; 68245441Snpstatic struct pr_usrreqs ddp6_usrreqs; 69245441Snp 70237263Snp/* Module ops */ 71237263Snpstatic int t4_tom_mod_load(void); 72237263Snpstatic int t4_tom_mod_unload(void); 73237263Snpstatic int t4_tom_modevent(module_t, int, void *); 74237263Snp 75237263Snp/* ULD ops and helpers */ 76237263Snpstatic int t4_tom_activate(struct adapter *); 77237263Snpstatic int t4_tom_deactivate(struct adapter *); 78237263Snp 79237263Snpstatic struct uld_info tom_uld_info = { 80237263Snp .uld_id = ULD_TOM, 81237263Snp .activate = t4_tom_activate, 82237263Snp .deactivate = t4_tom_deactivate, 83237263Snp}; 84237263Snp 85237263Snpstatic void queue_tid_release(struct adapter *, int); 86237263Snpstatic void release_offload_resources(struct toepcb *); 87237263Snpstatic int alloc_tid_tabs(struct tid_info *); 88237263Snpstatic void free_tid_tabs(struct tid_info *); 89245448Snpstatic int add_lip(struct adapter *, struct in6_addr *); 90245448Snpstatic int delete_lip(struct adapter *, struct in6_addr *); 91245448Snpstatic struct clip_entry *search_lip(struct tom_data *, struct in6_addr *); 92245448Snpstatic void init_clip_table(struct adapter *, struct tom_data *); 93249627Snpstatic void update_clip(struct adapter *, void *); 94249627Snpstatic void t4_clip_task(void *, int); 95249627Snpstatic void update_clip_table(struct adapter *, struct tom_data *); 96245448Snpstatic void destroy_clip_table(struct adapter *, struct tom_data *); 97237263Snpstatic void free_tom_data(struct adapter *, struct tom_data *); 98237263Snp 99249627Snpstatic int in6_ifaddr_gen; 100249627Snpstatic eventhandler_tag ifaddr_evhandler; 101249627Snpstatic struct timeout_task clip_task; 102249627Snp 103237263Snpstruct toepcb * 104237263Snpalloc_toepcb(struct port_info *pi, int txqid, int rxqid, int flags) 105237263Snp{ 106237263Snp struct adapter *sc = pi->adapter; 107237263Snp struct toepcb *toep; 108237263Snp int tx_credits, txsd_total, len; 109237263Snp 110237263Snp /* 111237263Snp * The firmware counts tx work request credits in units of 16 bytes 112237263Snp * each. Reserve room for an ABORT_REQ so the driver never has to worry 113237263Snp * about tx credits if it wants to abort a connection. 114237263Snp */ 115237263Snp tx_credits = sc->params.ofldq_wr_cred; 116237263Snp tx_credits -= howmany(sizeof(struct cpl_abort_req), 16); 117237263Snp 118237263Snp /* 119237263Snp * Shortest possible tx work request is a fw_ofld_tx_data_wr + 1 byte 120237263Snp * immediate payload, and firmware counts tx work request credits in 121237263Snp * units of 16 byte. Calculate the maximum work requests possible. 122237263Snp */ 123237263Snp txsd_total = tx_credits / 124237263Snp howmany((sizeof(struct fw_ofld_tx_data_wr) + 1), 16); 125237263Snp 126237263Snp if (txqid < 0) 127237263Snp txqid = (arc4random() % pi->nofldtxq) + pi->first_ofld_txq; 128237263Snp KASSERT(txqid >= pi->first_ofld_txq && 129237263Snp txqid < pi->first_ofld_txq + pi->nofldtxq, 130237263Snp ("%s: txqid %d for port %p (first %d, n %d)", __func__, txqid, pi, 131237263Snp pi->first_ofld_txq, pi->nofldtxq)); 132237263Snp 133237263Snp if (rxqid < 0) 134237263Snp rxqid = (arc4random() % pi->nofldrxq) + pi->first_ofld_rxq; 135237263Snp KASSERT(rxqid >= pi->first_ofld_rxq && 136237263Snp rxqid < pi->first_ofld_rxq + pi->nofldrxq, 137237263Snp ("%s: rxqid %d for port %p (first %d, n %d)", __func__, rxqid, pi, 138237263Snp pi->first_ofld_rxq, pi->nofldrxq)); 139237263Snp 140237263Snp len = offsetof(struct toepcb, txsd) + 141237263Snp txsd_total * sizeof(struct ofld_tx_sdesc); 142237263Snp 143237263Snp toep = malloc(len, M_CXGBE, M_ZERO | flags); 144237263Snp if (toep == NULL) 145237263Snp return (NULL); 146237263Snp 147237263Snp toep->td = sc->tom_softc; 148237263Snp toep->port = pi; 149237263Snp toep->tx_credits = tx_credits; 150237263Snp toep->ofld_txq = &sc->sge.ofld_txq[txqid]; 151237263Snp toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid]; 152237263Snp toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; 153237263Snp toep->txsd_total = txsd_total; 154237263Snp toep->txsd_avail = txsd_total; 155237263Snp toep->txsd_pidx = 0; 156237263Snp toep->txsd_cidx = 0; 157237263Snp 158237263Snp return (toep); 159237263Snp} 160237263Snp 161237263Snpvoid 162237263Snpfree_toepcb(struct toepcb *toep) 163237263Snp{ 164237263Snp 165239514Snp KASSERT(!(toep->flags & TPF_ATTACHED), 166237263Snp ("%s: attached to an inpcb", __func__)); 167239514Snp KASSERT(!(toep->flags & TPF_CPL_PENDING), 168237263Snp ("%s: CPL pending", __func__)); 169237263Snp 170237263Snp free(toep, M_CXGBE); 171237263Snp} 172237263Snp 173237263Snp/* 174237263Snp * Set up the socket for TCP offload. 175237263Snp */ 176237263Snpvoid 177237263Snpoffload_socket(struct socket *so, struct toepcb *toep) 178237263Snp{ 179237263Snp struct tom_data *td = toep->td; 180237263Snp struct inpcb *inp = sotoinpcb(so); 181237263Snp struct tcpcb *tp = intotcpcb(inp); 182237263Snp struct sockbuf *sb; 183237263Snp 184237263Snp INP_WLOCK_ASSERT(inp); 185237263Snp 186237263Snp /* Update socket */ 187237263Snp sb = &so->so_snd; 188237263Snp SOCKBUF_LOCK(sb); 189237263Snp sb->sb_flags |= SB_NOCOALESCE; 190237263Snp SOCKBUF_UNLOCK(sb); 191237263Snp sb = &so->so_rcv; 192237263Snp SOCKBUF_LOCK(sb); 193237263Snp sb->sb_flags |= SB_NOCOALESCE; 194245441Snp if (toep->ulp_mode == ULP_MODE_TCPDDP) { 195245441Snp if (inp->inp_vflag & INP_IPV6) 196245441Snp so->so_proto = &ddp6_protosw; 197245441Snp else 198245441Snp so->so_proto = &ddp_protosw; 199245441Snp } 200237263Snp SOCKBUF_UNLOCK(sb); 201237263Snp 202237263Snp /* Update TCP PCB */ 203237263Snp tp->tod = &td->tod; 204237263Snp tp->t_toe = toep; 205237263Snp tp->t_flags |= TF_TOE; 206237263Snp 207237263Snp /* Install an extra hold on inp */ 208237263Snp toep->inp = inp; 209239514Snp toep->flags |= TPF_ATTACHED; 210237263Snp in_pcbref(inp); 211237263Snp 212237263Snp /* Add the TOE PCB to the active list */ 213237263Snp mtx_lock(&td->toep_list_lock); 214237263Snp TAILQ_INSERT_HEAD(&td->toep_list, toep, link); 215237263Snp mtx_unlock(&td->toep_list_lock); 216237263Snp} 217237263Snp 218237263Snp/* This is _not_ the normal way to "unoffload" a socket. */ 219237263Snpvoid 220237263Snpundo_offload_socket(struct socket *so) 221237263Snp{ 222237263Snp struct inpcb *inp = sotoinpcb(so); 223237263Snp struct tcpcb *tp = intotcpcb(inp); 224237263Snp struct toepcb *toep = tp->t_toe; 225237263Snp struct tom_data *td = toep->td; 226237263Snp struct sockbuf *sb; 227237263Snp 228237263Snp INP_WLOCK_ASSERT(inp); 229237263Snp 230237263Snp sb = &so->so_snd; 231237263Snp SOCKBUF_LOCK(sb); 232237263Snp sb->sb_flags &= ~SB_NOCOALESCE; 233237263Snp SOCKBUF_UNLOCK(sb); 234237263Snp sb = &so->so_rcv; 235237263Snp SOCKBUF_LOCK(sb); 236237263Snp sb->sb_flags &= ~SB_NOCOALESCE; 237237263Snp SOCKBUF_UNLOCK(sb); 238237263Snp 239237263Snp tp->tod = NULL; 240237263Snp tp->t_toe = NULL; 241237263Snp tp->t_flags &= ~TF_TOE; 242237263Snp 243237263Snp toep->inp = NULL; 244239514Snp toep->flags &= ~TPF_ATTACHED; 245237263Snp if (in_pcbrele_wlocked(inp)) 246237263Snp panic("%s: inp freed.", __func__); 247237263Snp 248237263Snp mtx_lock(&td->toep_list_lock); 249237263Snp TAILQ_REMOVE(&td->toep_list, toep, link); 250237263Snp mtx_unlock(&td->toep_list_lock); 251237263Snp} 252237263Snp 253237263Snpstatic void 254237263Snprelease_offload_resources(struct toepcb *toep) 255237263Snp{ 256237263Snp struct tom_data *td = toep->td; 257237263Snp struct adapter *sc = td_adapter(td); 258237263Snp int tid = toep->tid; 259237263Snp 260239514Snp KASSERT(!(toep->flags & TPF_CPL_PENDING), 261237263Snp ("%s: %p has CPL pending.", __func__, toep)); 262239514Snp KASSERT(!(toep->flags & TPF_ATTACHED), 263237263Snp ("%s: %p is still attached.", __func__, toep)); 264237263Snp 265245448Snp CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)", 266245448Snp __func__, toep, tid, toep->l2te, toep->ce); 267237263Snp 268239344Snp if (toep->ulp_mode == ULP_MODE_TCPDDP) 269239344Snp release_ddp_resources(toep); 270239344Snp 271237263Snp if (toep->l2te) 272237263Snp t4_l2t_release(toep->l2te); 273237263Snp 274237263Snp if (tid >= 0) { 275237263Snp remove_tid(sc, tid); 276237263Snp release_tid(sc, tid, toep->ctrlq); 277237263Snp } 278237263Snp 279245448Snp if (toep->ce) 280245448Snp release_lip(td, toep->ce); 281245448Snp 282237263Snp mtx_lock(&td->toep_list_lock); 283237263Snp TAILQ_REMOVE(&td->toep_list, toep, link); 284237263Snp mtx_unlock(&td->toep_list_lock); 285237263Snp 286237263Snp free_toepcb(toep); 287237263Snp} 288237263Snp 289237263Snp/* 290237263Snp * The kernel is done with the TCP PCB and this is our opportunity to unhook the 291237263Snp * toepcb hanging off of it. If the TOE driver is also done with the toepcb (no 292237263Snp * pending CPL) then it is time to release all resources tied to the toepcb. 293237263Snp * 294237263Snp * Also gets called when an offloaded active open fails and the TOM wants the 295237263Snp * kernel to take the TCP PCB back. 296237263Snp */ 297237263Snpstatic void 298237263Snpt4_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp) 299237263Snp{ 300237263Snp#if defined(KTR) || defined(INVARIANTS) 301237263Snp struct inpcb *inp = tp->t_inpcb; 302237263Snp#endif 303237263Snp struct toepcb *toep = tp->t_toe; 304237263Snp 305237263Snp INP_WLOCK_ASSERT(inp); 306237263Snp 307237263Snp KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 308239514Snp KASSERT(toep->flags & TPF_ATTACHED, 309237263Snp ("%s: not attached", __func__)); 310237263Snp 311237263Snp#ifdef KTR 312237263Snp if (tp->t_state == TCPS_SYN_SENT) { 313237263Snp CTR6(KTR_CXGBE, "%s: atid %d, toep %p (0x%x), inp %p (0x%x)", 314237263Snp __func__, toep->tid, toep, toep->flags, inp, 315237263Snp inp->inp_flags); 316237263Snp } else { 317237263Snp CTR6(KTR_CXGBE, 318237263Snp "t4_pcb_detach: tid %d (%s), toep %p (0x%x), inp %p (0x%x)", 319237263Snp toep->tid, tcpstates[tp->t_state], toep, toep->flags, inp, 320237263Snp inp->inp_flags); 321237263Snp } 322237263Snp#endif 323237263Snp 324237263Snp tp->t_toe = NULL; 325237263Snp tp->t_flags &= ~TF_TOE; 326239514Snp toep->flags &= ~TPF_ATTACHED; 327237263Snp 328239514Snp if (!(toep->flags & TPF_CPL_PENDING)) 329237263Snp release_offload_resources(toep); 330237263Snp} 331237263Snp 332237263Snp/* 333237263Snp * The TOE driver will not receive any more CPLs for the tid associated with the 334237263Snp * toepcb; release the hold on the inpcb. 335237263Snp */ 336237263Snpvoid 337237263Snpfinal_cpl_received(struct toepcb *toep) 338237263Snp{ 339237263Snp struct inpcb *inp = toep->inp; 340237263Snp 341237263Snp KASSERT(inp != NULL, ("%s: inp is NULL", __func__)); 342237263Snp INP_WLOCK_ASSERT(inp); 343239514Snp KASSERT(toep->flags & TPF_CPL_PENDING, 344237263Snp ("%s: CPL not pending already?", __func__)); 345237263Snp 346237263Snp CTR6(KTR_CXGBE, "%s: tid %d, toep %p (0x%x), inp %p (0x%x)", 347237263Snp __func__, toep->tid, toep, toep->flags, inp, inp->inp_flags); 348237263Snp 349237263Snp toep->inp = NULL; 350239514Snp toep->flags &= ~TPF_CPL_PENDING; 351237263Snp 352239514Snp if (!(toep->flags & TPF_ATTACHED)) 353237263Snp release_offload_resources(toep); 354237263Snp 355237263Snp if (!in_pcbrele_wlocked(inp)) 356237263Snp INP_WUNLOCK(inp); 357237263Snp} 358237263Snp 359237263Snpvoid 360237263Snpinsert_tid(struct adapter *sc, int tid, void *ctx) 361237263Snp{ 362237263Snp struct tid_info *t = &sc->tids; 363237263Snp 364237263Snp t->tid_tab[tid] = ctx; 365237263Snp atomic_add_int(&t->tids_in_use, 1); 366237263Snp} 367237263Snp 368237263Snpvoid * 369237263Snplookup_tid(struct adapter *sc, int tid) 370237263Snp{ 371237263Snp struct tid_info *t = &sc->tids; 372237263Snp 373237263Snp return (t->tid_tab[tid]); 374237263Snp} 375237263Snp 376237263Snpvoid 377237263Snpupdate_tid(struct adapter *sc, int tid, void *ctx) 378237263Snp{ 379237263Snp struct tid_info *t = &sc->tids; 380237263Snp 381237263Snp t->tid_tab[tid] = ctx; 382237263Snp} 383237263Snp 384237263Snpvoid 385237263Snpremove_tid(struct adapter *sc, int tid) 386237263Snp{ 387237263Snp struct tid_info *t = &sc->tids; 388237263Snp 389237263Snp t->tid_tab[tid] = NULL; 390237263Snp atomic_subtract_int(&t->tids_in_use, 1); 391237263Snp} 392237263Snp 393237263Snpvoid 394237263Snprelease_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq) 395237263Snp{ 396237263Snp struct wrqe *wr; 397237263Snp struct cpl_tid_release *req; 398237263Snp 399237263Snp wr = alloc_wrqe(sizeof(*req), ctrlq); 400237263Snp if (wr == NULL) { 401237263Snp queue_tid_release(sc, tid); /* defer */ 402237263Snp return; 403237263Snp } 404237263Snp req = wrtod(wr); 405237263Snp 406237263Snp INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid); 407237263Snp 408237263Snp t4_wrq_tx(sc, wr); 409237263Snp} 410237263Snp 411237263Snpstatic void 412237263Snpqueue_tid_release(struct adapter *sc, int tid) 413237263Snp{ 414237263Snp 415237263Snp CXGBE_UNIMPLEMENTED("deferred tid release"); 416237263Snp} 417237263Snp 418237263Snp/* 419237263Snp * What mtu_idx to use, given a 4-tuple and/or an MSS cap 420237263Snp */ 421237263Snpint 422237263Snpfind_best_mtu_idx(struct adapter *sc, struct in_conninfo *inc, int pmss) 423237263Snp{ 424237263Snp unsigned short *mtus = &sc->params.mtus[0]; 425245441Snp int i, mss, n; 426237263Snp 427237263Snp KASSERT(inc != NULL || pmss > 0, 428237263Snp ("%s: at least one of inc/pmss must be specified", __func__)); 429237263Snp 430237263Snp mss = inc ? tcp_mssopt(inc) : pmss; 431237263Snp if (pmss > 0 && mss > pmss) 432237263Snp mss = pmss; 433237263Snp 434245441Snp if (inc->inc_flags & INC_ISIPV6) 435245441Snp n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 436245441Snp else 437245441Snp n = sizeof(struct ip) + sizeof(struct tcphdr); 438237263Snp 439245441Snp for (i = 0; i < NMTUS - 1 && mtus[i + 1] <= mss + n; i++) 440245441Snp continue; 441245441Snp 442237263Snp return (i); 443237263Snp} 444237263Snp 445237263Snp/* 446237263Snp * Determine the receive window size for a socket. 447237263Snp */ 448237263Snpu_long 449237263Snpselect_rcv_wnd(struct socket *so) 450237263Snp{ 451237263Snp unsigned long wnd; 452237263Snp 453237263Snp SOCKBUF_LOCK_ASSERT(&so->so_rcv); 454237263Snp 455237263Snp wnd = sbspace(&so->so_rcv); 456237263Snp if (wnd < MIN_RCV_WND) 457237263Snp wnd = MIN_RCV_WND; 458237263Snp 459237263Snp return min(wnd, MAX_RCV_WND); 460237263Snp} 461237263Snp 462237263Snpint 463237263Snpselect_rcv_wscale(void) 464237263Snp{ 465237263Snp int wscale = 0; 466237263Snp unsigned long space = sb_max; 467237263Snp 468237263Snp if (space > MAX_RCV_WND) 469237263Snp space = MAX_RCV_WND; 470237263Snp 471237263Snp while (wscale < TCP_MAX_WINSHIFT && (TCP_MAXWIN << wscale) < space) 472237263Snp wscale++; 473237263Snp 474237263Snp return (wscale); 475237263Snp} 476237263Snp 477237263Snpextern int always_keepalive; 478237263Snp#define VIID_SMACIDX(v) (((unsigned int)(v) & 0x7f) << 1) 479237263Snp 480237263Snp/* 481237263Snp * socket so could be a listening socket too. 482237263Snp */ 483237263Snpuint64_t 484237263Snpcalc_opt0(struct socket *so, struct port_info *pi, struct l2t_entry *e, 485237263Snp int mtu_idx, int rscale, int rx_credits, int ulp_mode) 486237263Snp{ 487237263Snp uint64_t opt0; 488237263Snp 489237263Snp KASSERT(rx_credits <= M_RCV_BUFSIZ, 490237263Snp ("%s: rcv_bufsiz too high", __func__)); 491237263Snp 492237263Snp opt0 = F_TCAM_BYPASS | V_WND_SCALE(rscale) | V_MSS_IDX(mtu_idx) | 493237263Snp V_ULP_MODE(ulp_mode) | V_RCV_BUFSIZ(rx_credits); 494237263Snp 495237263Snp if (so != NULL) { 496237263Snp struct inpcb *inp = sotoinpcb(so); 497237263Snp struct tcpcb *tp = intotcpcb(inp); 498237263Snp int keepalive = always_keepalive || 499237263Snp so_options_get(so) & SO_KEEPALIVE; 500237263Snp 501237263Snp opt0 |= V_NAGLE((tp->t_flags & TF_NODELAY) == 0); 502237263Snp opt0 |= V_KEEP_ALIVE(keepalive != 0); 503237263Snp } 504237263Snp 505237263Snp if (e != NULL) 506237263Snp opt0 |= V_L2T_IDX(e->idx); 507237263Snp 508237263Snp if (pi != NULL) { 509237263Snp opt0 |= V_SMAC_SEL(VIID_SMACIDX(pi->viid)); 510237263Snp opt0 |= V_TX_CHAN(pi->tx_chan); 511237263Snp } 512237263Snp 513237263Snp return htobe64(opt0); 514237263Snp} 515237263Snp 516237263Snp#define FILTER_SEL_WIDTH_P_FC (3 + 1) 517237263Snp#define FILTER_SEL_WIDTH_VIN_P_FC (6 + 7 + FILTER_SEL_WIDTH_P_FC) 518237263Snp#define FILTER_SEL_WIDTH_TAG_P_FC (3 + FILTER_SEL_WIDTH_VIN_P_FC) 519237263Snp#define FILTER_SEL_WIDTH_VLD_TAG_P_FC (1 + FILTER_SEL_WIDTH_TAG_P_FC) 520237263Snp#define VLAN_NONE 0xfff 521237263Snp#define FILTER_SEL_VLAN_NONE 0xffff 522237263Snp 523248925Snpuint64_t 524237263Snpselect_ntuple(struct port_info *pi, struct l2t_entry *e, uint32_t filter_mode) 525237263Snp{ 526237263Snp uint16_t viid = pi->viid; 527237263Snp uint32_t ntuple = 0; 528237263Snp 529237263Snp if (filter_mode == HW_TPL_FR_MT_PR_IV_P_FC) { 530237263Snp if (e->vlan == VLAN_NONE) 531237263Snp ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC; 532237263Snp else { 533237263Snp ntuple |= e->vlan << FILTER_SEL_WIDTH_P_FC; 534237263Snp ntuple |= 1 << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 535237263Snp } 536237263Snp ntuple |= e->lport << S_PORT; 537237263Snp ntuple |= IPPROTO_TCP << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 538237263Snp } else if (filter_mode == HW_TPL_FR_MT_PR_OV_P_FC) { 539237263Snp ntuple |= G_FW_VIID_VIN(viid) << FILTER_SEL_WIDTH_P_FC; 540237263Snp ntuple |= G_FW_VIID_PFN(viid) << FILTER_SEL_WIDTH_VIN_P_FC; 541237263Snp ntuple |= G_FW_VIID_VIVLD(viid) << FILTER_SEL_WIDTH_TAG_P_FC; 542237263Snp ntuple |= e->lport << S_PORT; 543237263Snp ntuple |= IPPROTO_TCP << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 544237263Snp } 545237263Snp 546248925Snp if (is_t4(pi->adapter)) 547248925Snp return (htobe32(ntuple)); 548248925Snp else 549248925Snp return (htobe64(V_FILTER_TUPLE(ntuple))); 550237263Snp} 551237263Snp 552245441Snpvoid 553245441Snpset_tcpddp_ulp_mode(struct toepcb *toep) 554245441Snp{ 555245441Snp 556245441Snp toep->ulp_mode = ULP_MODE_TCPDDP; 557245441Snp toep->ddp_flags = DDP_OK; 558245441Snp toep->ddp_score = DDP_LOW_SCORE; 559245441Snp} 560245441Snp 561245935Snpint 562245935Snpnegative_advice(int status) 563245935Snp{ 564245935Snp 565245935Snp return (status == CPL_ERR_RTX_NEG_ADVICE || 566245935Snp status == CPL_ERR_PERSIST_NEG_ADVICE || 567245935Snp status == CPL_ERR_KEEPALV_NEG_ADVICE); 568245935Snp} 569245935Snp 570237263Snpstatic int 571237263Snpalloc_tid_tabs(struct tid_info *t) 572237263Snp{ 573237263Snp size_t size; 574237263Snp unsigned int i; 575237263Snp 576237263Snp size = t->ntids * sizeof(*t->tid_tab) + 577237263Snp t->natids * sizeof(*t->atid_tab) + 578237263Snp t->nstids * sizeof(*t->stid_tab); 579237263Snp 580237263Snp t->tid_tab = malloc(size, M_CXGBE, M_ZERO | M_NOWAIT); 581237263Snp if (t->tid_tab == NULL) 582237263Snp return (ENOMEM); 583237263Snp 584237263Snp mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF); 585237263Snp t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 586237263Snp t->afree = t->atid_tab; 587237263Snp t->atids_in_use = 0; 588237263Snp for (i = 1; i < t->natids; i++) 589237263Snp t->atid_tab[i - 1].next = &t->atid_tab[i]; 590237263Snp t->atid_tab[t->natids - 1].next = NULL; 591237263Snp 592237263Snp mtx_init(&t->stid_lock, "stid lock", NULL, MTX_DEF); 593245276Snp t->stid_tab = (struct listen_ctx **)&t->atid_tab[t->natids]; 594237263Snp t->stids_in_use = 0; 595245276Snp TAILQ_INIT(&t->stids); 596245276Snp t->nstids_free_head = t->nstids; 597237263Snp 598237263Snp atomic_store_rel_int(&t->tids_in_use, 0); 599237263Snp 600237263Snp return (0); 601237263Snp} 602237263Snp 603237263Snpstatic void 604237263Snpfree_tid_tabs(struct tid_info *t) 605237263Snp{ 606237263Snp KASSERT(t->tids_in_use == 0, 607237263Snp ("%s: %d tids still in use.", __func__, t->tids_in_use)); 608237263Snp KASSERT(t->atids_in_use == 0, 609237263Snp ("%s: %d atids still in use.", __func__, t->atids_in_use)); 610237263Snp KASSERT(t->stids_in_use == 0, 611237263Snp ("%s: %d tids still in use.", __func__, t->stids_in_use)); 612237263Snp 613237263Snp free(t->tid_tab, M_CXGBE); 614237263Snp t->tid_tab = NULL; 615237263Snp 616237263Snp if (mtx_initialized(&t->atid_lock)) 617237263Snp mtx_destroy(&t->atid_lock); 618237263Snp if (mtx_initialized(&t->stid_lock)) 619237263Snp mtx_destroy(&t->stid_lock); 620237263Snp} 621237263Snp 622245448Snpstatic int 623245448Snpadd_lip(struct adapter *sc, struct in6_addr *lip) 624245448Snp{ 625245448Snp struct fw_clip_cmd c; 626245448Snp 627245448Snp ASSERT_SYNCHRONIZED_OP(sc); 628245448Snp /* mtx_assert(&td->clip_table_lock, MA_OWNED); */ 629245448Snp 630245448Snp memset(&c, 0, sizeof(c)); 631245448Snp c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST | 632245448Snp F_FW_CMD_WRITE); 633245448Snp c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c)); 634245448Snp c.ip_hi = *(uint64_t *)&lip->s6_addr[0]; 635245448Snp c.ip_lo = *(uint64_t *)&lip->s6_addr[8]; 636245448Snp 637249627Snp return (-t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c)); 638245448Snp} 639245448Snp 640245448Snpstatic int 641245448Snpdelete_lip(struct adapter *sc, struct in6_addr *lip) 642245448Snp{ 643245448Snp struct fw_clip_cmd c; 644245448Snp 645245448Snp ASSERT_SYNCHRONIZED_OP(sc); 646245448Snp /* mtx_assert(&td->clip_table_lock, MA_OWNED); */ 647245448Snp 648245448Snp memset(&c, 0, sizeof(c)); 649245448Snp c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST | 650245448Snp F_FW_CMD_READ); 651245448Snp c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c)); 652245448Snp c.ip_hi = *(uint64_t *)&lip->s6_addr[0]; 653245448Snp c.ip_lo = *(uint64_t *)&lip->s6_addr[8]; 654245448Snp 655249627Snp return (-t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c)); 656245448Snp} 657245448Snp 658245448Snpstatic struct clip_entry * 659245448Snpsearch_lip(struct tom_data *td, struct in6_addr *lip) 660245448Snp{ 661245448Snp struct clip_entry *ce; 662245448Snp 663245448Snp mtx_assert(&td->clip_table_lock, MA_OWNED); 664245448Snp 665245448Snp TAILQ_FOREACH(ce, &td->clip_table, link) { 666245448Snp if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip)) 667245448Snp return (ce); 668245448Snp } 669245448Snp 670245448Snp return (NULL); 671245448Snp} 672245448Snp 673245448Snpstruct clip_entry * 674245448Snphold_lip(struct tom_data *td, struct in6_addr *lip) 675245448Snp{ 676245448Snp struct clip_entry *ce; 677245448Snp 678245448Snp mtx_lock(&td->clip_table_lock); 679245448Snp ce = search_lip(td, lip); 680245448Snp if (ce != NULL) 681245448Snp ce->refcount++; 682245448Snp mtx_unlock(&td->clip_table_lock); 683245448Snp 684245448Snp return (ce); 685245448Snp} 686245448Snp 687245448Snpvoid 688245448Snprelease_lip(struct tom_data *td, struct clip_entry *ce) 689245448Snp{ 690245448Snp 691245448Snp mtx_lock(&td->clip_table_lock); 692245448Snp KASSERT(search_lip(td, &ce->lip) == ce, 693245448Snp ("%s: CLIP entry %p p not in CLIP table.", __func__, ce)); 694245448Snp KASSERT(ce->refcount > 0, 695245448Snp ("%s: CLIP entry %p has refcount 0", __func__, ce)); 696245448Snp --ce->refcount; 697245448Snp mtx_unlock(&td->clip_table_lock); 698245448Snp} 699245448Snp 700237263Snpstatic void 701245448Snpinit_clip_table(struct adapter *sc, struct tom_data *td) 702245448Snp{ 703245448Snp 704245448Snp ASSERT_SYNCHRONIZED_OP(sc); 705245448Snp 706245448Snp mtx_init(&td->clip_table_lock, "CLIP table lock", NULL, MTX_DEF); 707245448Snp TAILQ_INIT(&td->clip_table); 708249627Snp td->clip_gen = -1; 709245448Snp 710249627Snp update_clip_table(sc, td); 711249627Snp} 712249627Snp 713249627Snpstatic void 714249627Snpupdate_clip(struct adapter *sc, void *arg __unused) 715249627Snp{ 716249627Snp 717249627Snp if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4tomuc")) 718249627Snp return; 719249627Snp 720249627Snp if (sc->flags & TOM_INIT_DONE) 721249627Snp update_clip_table(sc, sc->tom_softc); 722249627Snp 723249627Snp end_synchronized_op(sc, LOCK_HELD); 724249627Snp} 725249627Snp 726249627Snpstatic void 727249627Snpt4_clip_task(void *arg, int count) 728249627Snp{ 729249627Snp 730249627Snp t4_iterate(update_clip, NULL); 731249627Snp} 732249627Snp 733249627Snpstatic void 734249627Snpupdate_clip_table(struct adapter *sc, struct tom_data *td) 735249627Snp{ 736249627Snp struct in6_ifaddr *ia; 737249627Snp struct in6_addr *lip, tlip; 738249627Snp struct clip_head stale; 739249627Snp struct clip_entry *ce, *ce_temp; 740249627Snp int rc, gen = atomic_load_acq_int(&in6_ifaddr_gen); 741249627Snp 742249627Snp ASSERT_SYNCHRONIZED_OP(sc); 743249627Snp 744245448Snp IN6_IFADDR_RLOCK(); 745249627Snp mtx_lock(&td->clip_table_lock); 746249627Snp 747249627Snp if (gen == td->clip_gen) 748249627Snp goto done; 749249627Snp 750249627Snp TAILQ_INIT(&stale); 751249627Snp TAILQ_CONCAT(&stale, &td->clip_table, link); 752249627Snp 753245448Snp TAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) { 754245448Snp lip = &ia->ia_addr.sin6_addr; 755245448Snp 756245448Snp KASSERT(!IN6_IS_ADDR_MULTICAST(lip), 757245448Snp ("%s: mcast address in in6_ifaddr list", __func__)); 758245448Snp 759245448Snp if (IN6_IS_ADDR_LOOPBACK(lip)) 760245448Snp continue; 761245448Snp if (IN6_IS_SCOPE_EMBED(lip)) { 762245448Snp /* Remove the embedded scope */ 763245448Snp tlip = *lip; 764245448Snp lip = &tlip; 765245448Snp in6_clearscope(lip); 766245448Snp } 767245448Snp /* 768245448Snp * XXX: how to weed out the link local address for the loopback 769245448Snp * interface? It's fe80::1 usually (always?). 770245448Snp */ 771245448Snp 772249627Snp /* 773249627Snp * If it's in the main list then we already know it's not stale. 774249627Snp */ 775249627Snp TAILQ_FOREACH(ce, &td->clip_table, link) { 776249627Snp if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip)) 777249627Snp goto next; 778249627Snp } 779249627Snp 780249627Snp /* 781249627Snp * If it's in the stale list we should move it to the main list. 782249627Snp */ 783249627Snp TAILQ_FOREACH(ce, &stale, link) { 784249627Snp if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip)) { 785249627Snp TAILQ_REMOVE(&stale, ce, link); 786245448Snp TAILQ_INSERT_TAIL(&td->clip_table, ce, link); 787249627Snp goto next; 788249627Snp } 789249627Snp } 790249627Snp 791249627Snp /* A new IP6 address; add it to the CLIP table */ 792249627Snp ce = malloc(sizeof(*ce), M_CXGBE, M_NOWAIT); 793249627Snp memcpy(&ce->lip, lip, sizeof(ce->lip)); 794249627Snp ce->refcount = 0; 795249627Snp rc = add_lip(sc, lip); 796249627Snp if (rc == 0) 797249627Snp TAILQ_INSERT_TAIL(&td->clip_table, ce, link); 798249627Snp else { 799249627Snp char ip[INET6_ADDRSTRLEN]; 800249627Snp 801249627Snp inet_ntop(AF_INET6, &ce->lip, &ip[0], sizeof(ip)); 802249627Snp log(LOG_ERR, "%s: could not add %s (%d)\n", 803249627Snp __func__, ip, rc); 804249627Snp free(ce, M_CXGBE); 805249627Snp } 806249627Snpnext: 807249627Snp continue; 808249627Snp } 809249627Snp 810249627Snp /* 811249627Snp * Remove stale addresses (those no longer in V_in6_ifaddrhead) that are 812249627Snp * no longer referenced by the driver. 813249627Snp */ 814249627Snp TAILQ_FOREACH_SAFE(ce, &stale, link, ce_temp) { 815249627Snp if (ce->refcount == 0) { 816249627Snp rc = delete_lip(sc, &ce->lip); 817249627Snp if (rc == 0) { 818249627Snp TAILQ_REMOVE(&stale, ce, link); 819245448Snp free(ce, M_CXGBE); 820249627Snp } else { 821249627Snp char ip[INET6_ADDRSTRLEN]; 822249627Snp 823249627Snp inet_ntop(AF_INET6, &ce->lip, &ip[0], 824249627Snp sizeof(ip)); 825249627Snp log(LOG_ERR, "%s: could not delete %s (%d)\n", 826249627Snp __func__, ip, rc); 827249627Snp } 828245448Snp } 829245448Snp } 830249627Snp /* The ones that are still referenced need to stay in the CLIP table */ 831249627Snp TAILQ_CONCAT(&td->clip_table, &stale, link); 832249627Snp 833249627Snp td->clip_gen = gen; 834249627Snpdone: 835249627Snp mtx_unlock(&td->clip_table_lock); 836245448Snp IN6_IFADDR_RUNLOCK(); 837245448Snp} 838245448Snp 839245448Snpstatic void 840245448Snpdestroy_clip_table(struct adapter *sc, struct tom_data *td) 841245448Snp{ 842245448Snp struct clip_entry *ce, *ce_temp; 843245448Snp 844245448Snp if (mtx_initialized(&td->clip_table_lock)) { 845245448Snp mtx_lock(&td->clip_table_lock); 846245448Snp TAILQ_FOREACH_SAFE(ce, &td->clip_table, link, ce_temp) { 847245448Snp KASSERT(ce->refcount == 0, 848245448Snp ("%s: CLIP entry %p still in use (%d)", __func__, 849245448Snp ce, ce->refcount)); 850245448Snp TAILQ_REMOVE(&td->clip_table, ce, link); 851245448Snp delete_lip(sc, &ce->lip); 852245448Snp free(ce, M_CXGBE); 853245448Snp } 854245448Snp mtx_unlock(&td->clip_table_lock); 855245448Snp mtx_destroy(&td->clip_table_lock); 856245448Snp } 857245448Snp} 858245448Snp 859245448Snpstatic void 860237263Snpfree_tom_data(struct adapter *sc, struct tom_data *td) 861237263Snp{ 862245448Snp 863245448Snp ASSERT_SYNCHRONIZED_OP(sc); 864245448Snp 865237263Snp KASSERT(TAILQ_EMPTY(&td->toep_list), 866237263Snp ("%s: TOE PCB list is not empty.", __func__)); 867237263Snp KASSERT(td->lctx_count == 0, 868237263Snp ("%s: lctx hash table is not empty.", __func__)); 869237263Snp 870237263Snp t4_uninit_l2t_cpl_handlers(sc); 871239344Snp t4_uninit_cpl_io_handlers(sc); 872239344Snp t4_uninit_ddp(sc, td); 873245448Snp destroy_clip_table(sc, td); 874237263Snp 875237263Snp if (td->listen_mask != 0) 876237263Snp hashdestroy(td->listen_hash, M_CXGBE, td->listen_mask); 877237263Snp 878237263Snp if (mtx_initialized(&td->lctx_hash_lock)) 879237263Snp mtx_destroy(&td->lctx_hash_lock); 880237263Snp if (mtx_initialized(&td->toep_list_lock)) 881237263Snp mtx_destroy(&td->toep_list_lock); 882237263Snp 883237263Snp free_tid_tabs(&sc->tids); 884237263Snp free(td, M_CXGBE); 885237263Snp} 886237263Snp 887237263Snp/* 888237263Snp * Ground control to Major TOM 889237263Snp * Commencing countdown, engines on 890237263Snp */ 891237263Snpstatic int 892237263Snpt4_tom_activate(struct adapter *sc) 893237263Snp{ 894237263Snp struct tom_data *td; 895237263Snp struct toedev *tod; 896237263Snp int i, rc; 897237263Snp 898245274Snp ASSERT_SYNCHRONIZED_OP(sc); 899237263Snp 900237263Snp /* per-adapter softc for TOM */ 901237263Snp td = malloc(sizeof(*td), M_CXGBE, M_ZERO | M_NOWAIT); 902237263Snp if (td == NULL) 903237263Snp return (ENOMEM); 904237263Snp 905237263Snp /* List of TOE PCBs and associated lock */ 906237263Snp mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF); 907237263Snp TAILQ_INIT(&td->toep_list); 908237263Snp 909237263Snp /* Listen context */ 910237263Snp mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF); 911237263Snp td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGBE, 912237263Snp &td->listen_mask, HASH_NOWAIT); 913237263Snp 914237263Snp /* TID tables */ 915237263Snp rc = alloc_tid_tabs(&sc->tids); 916237263Snp if (rc != 0) 917237263Snp goto done; 918237263Snp 919245448Snp /* DDP page pods and CPL handlers */ 920239344Snp t4_init_ddp(sc, td); 921239344Snp 922245448Snp /* CLIP table for IPv6 offload */ 923245448Snp init_clip_table(sc, td); 924245448Snp 925237263Snp /* CPL handlers */ 926237263Snp t4_init_connect_cpl_handlers(sc); 927237263Snp t4_init_l2t_cpl_handlers(sc); 928237263Snp t4_init_listen_cpl_handlers(sc); 929237263Snp t4_init_cpl_io_handlers(sc); 930237263Snp 931237263Snp /* toedev ops */ 932237263Snp tod = &td->tod; 933237263Snp init_toedev(tod); 934237263Snp tod->tod_softc = sc; 935237263Snp tod->tod_connect = t4_connect; 936237263Snp tod->tod_listen_start = t4_listen_start; 937237263Snp tod->tod_listen_stop = t4_listen_stop; 938237263Snp tod->tod_rcvd = t4_rcvd; 939237263Snp tod->tod_output = t4_tod_output; 940237263Snp tod->tod_send_rst = t4_send_rst; 941237263Snp tod->tod_send_fin = t4_send_fin; 942237263Snp tod->tod_pcb_detach = t4_pcb_detach; 943237263Snp tod->tod_l2_update = t4_l2_update; 944237263Snp tod->tod_syncache_added = t4_syncache_added; 945237263Snp tod->tod_syncache_removed = t4_syncache_removed; 946237263Snp tod->tod_syncache_respond = t4_syncache_respond; 947237263Snp tod->tod_offload_socket = t4_offload_socket; 948237263Snp 949237263Snp for_each_port(sc, i) 950237263Snp TOEDEV(sc->port[i]->ifp) = &td->tod; 951237263Snp 952237263Snp sc->tom_softc = td; 953237263Snp sc->flags |= TOM_INIT_DONE; 954237263Snp register_toedev(sc->tom_softc); 955237263Snp 956237263Snpdone: 957237263Snp if (rc != 0) 958237263Snp free_tom_data(sc, td); 959237263Snp return (rc); 960237263Snp} 961237263Snp 962237263Snpstatic int 963237263Snpt4_tom_deactivate(struct adapter *sc) 964237263Snp{ 965237263Snp int rc = 0; 966237263Snp struct tom_data *td = sc->tom_softc; 967237263Snp 968245274Snp ASSERT_SYNCHRONIZED_OP(sc); 969237263Snp 970237263Snp if (td == NULL) 971237263Snp return (0); /* XXX. KASSERT? */ 972237263Snp 973237263Snp if (sc->offload_map != 0) 974237263Snp return (EBUSY); /* at least one port has IFCAP_TOE enabled */ 975237263Snp 976237263Snp mtx_lock(&td->toep_list_lock); 977237263Snp if (!TAILQ_EMPTY(&td->toep_list)) 978237263Snp rc = EBUSY; 979237263Snp mtx_unlock(&td->toep_list_lock); 980237263Snp 981237263Snp mtx_lock(&td->lctx_hash_lock); 982237263Snp if (td->lctx_count > 0) 983237263Snp rc = EBUSY; 984237263Snp mtx_unlock(&td->lctx_hash_lock); 985237263Snp 986237263Snp if (rc == 0) { 987237263Snp unregister_toedev(sc->tom_softc); 988237263Snp free_tom_data(sc, td); 989237263Snp sc->tom_softc = NULL; 990237263Snp sc->flags &= ~TOM_INIT_DONE; 991237263Snp } 992237263Snp 993237263Snp return (rc); 994237263Snp} 995237263Snp 996249627Snpstatic void 997249627Snpt4_tom_ifaddr_event(void *arg __unused, struct ifnet *ifp) 998249627Snp{ 999249627Snp 1000249627Snp atomic_add_rel_int(&in6_ifaddr_gen, 1); 1001249627Snp taskqueue_enqueue_timeout(taskqueue_thread, &clip_task, -hz / 4); 1002249627Snp} 1003249627Snp 1004237263Snpstatic int 1005237263Snpt4_tom_mod_load(void) 1006237263Snp{ 1007237263Snp int rc; 1008245441Snp struct protosw *tcp_protosw, *tcp6_protosw; 1009237263Snp 1010239344Snp tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM); 1011239344Snp if (tcp_protosw == NULL) 1012239344Snp return (ENOPROTOOPT); 1013239344Snp bcopy(tcp_protosw, &ddp_protosw, sizeof(ddp_protosw)); 1014239344Snp bcopy(tcp_protosw->pr_usrreqs, &ddp_usrreqs, sizeof(ddp_usrreqs)); 1015239344Snp ddp_usrreqs.pru_soreceive = t4_soreceive_ddp; 1016239344Snp ddp_protosw.pr_usrreqs = &ddp_usrreqs; 1017239344Snp 1018245441Snp tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM); 1019245441Snp if (tcp6_protosw == NULL) 1020245441Snp return (ENOPROTOOPT); 1021245441Snp bcopy(tcp6_protosw, &ddp6_protosw, sizeof(ddp6_protosw)); 1022245441Snp bcopy(tcp6_protosw->pr_usrreqs, &ddp6_usrreqs, sizeof(ddp6_usrreqs)); 1023245441Snp ddp6_usrreqs.pru_soreceive = t4_soreceive_ddp; 1024245441Snp ddp6_protosw.pr_usrreqs = &ddp6_usrreqs; 1025245441Snp 1026249627Snp TIMEOUT_TASK_INIT(taskqueue_thread, &clip_task, 0, t4_clip_task, NULL); 1027249627Snp ifaddr_evhandler = EVENTHANDLER_REGISTER(ifaddr_event, 1028249627Snp t4_tom_ifaddr_event, NULL, EVENTHANDLER_PRI_ANY); 1029249627Snp 1030237263Snp rc = t4_register_uld(&tom_uld_info); 1031237263Snp if (rc != 0) 1032237263Snp t4_tom_mod_unload(); 1033237263Snp 1034237263Snp return (rc); 1035237263Snp} 1036237263Snp 1037237263Snpstatic void 1038237263Snptom_uninit(struct adapter *sc, void *arg __unused) 1039237263Snp{ 1040245274Snp if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4tomun")) 1041245274Snp return; 1042245274Snp 1043237263Snp /* Try to free resources (works only if no port has IFCAP_TOE) */ 1044237263Snp if (sc->flags & TOM_INIT_DONE) 1045237263Snp t4_deactivate_uld(sc, ULD_TOM); 1046245274Snp 1047245274Snp end_synchronized_op(sc, LOCK_HELD); 1048237263Snp} 1049237263Snp 1050237263Snpstatic int 1051237263Snpt4_tom_mod_unload(void) 1052237263Snp{ 1053237263Snp t4_iterate(tom_uninit, NULL); 1054237263Snp 1055237263Snp if (t4_unregister_uld(&tom_uld_info) == EBUSY) 1056237263Snp return (EBUSY); 1057237263Snp 1058249627Snp if (ifaddr_evhandler) { 1059249627Snp EVENTHANDLER_DEREGISTER(ifaddr_event, ifaddr_evhandler); 1060249627Snp taskqueue_cancel_timeout(taskqueue_thread, &clip_task, NULL); 1061249627Snp } 1062249627Snp 1063237263Snp return (0); 1064237263Snp} 1065237263Snp#endif /* TCP_OFFLOAD */ 1066237263Snp 1067237263Snpstatic int 1068237263Snpt4_tom_modevent(module_t mod, int cmd, void *arg) 1069237263Snp{ 1070237263Snp int rc = 0; 1071237263Snp 1072237263Snp#ifdef TCP_OFFLOAD 1073237263Snp switch (cmd) { 1074237263Snp case MOD_LOAD: 1075237263Snp rc = t4_tom_mod_load(); 1076237263Snp break; 1077237263Snp 1078237263Snp case MOD_UNLOAD: 1079237263Snp rc = t4_tom_mod_unload(); 1080237263Snp break; 1081237263Snp 1082237263Snp default: 1083237263Snp rc = EINVAL; 1084237263Snp } 1085237263Snp#else 1086237263Snp printf("t4_tom: compiled without TCP_OFFLOAD support.\n"); 1087237263Snp rc = EOPNOTSUPP; 1088237263Snp#endif 1089237263Snp return (rc); 1090237263Snp} 1091237263Snp 1092237263Snpstatic moduledata_t t4_tom_moddata= { 1093237263Snp "t4_tom", 1094237263Snp t4_tom_modevent, 1095241394Skevlo 0 1096237263Snp}; 1097237263Snp 1098237263SnpMODULE_VERSION(t4_tom, 1); 1099237263SnpMODULE_DEPEND(t4_tom, toecore, 1, 1, 1); 1100237263SnpMODULE_DEPEND(t4_tom, t4nex, 1, 1, 1); 1101237263SnpDECLARE_MODULE(t4_tom, t4_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY); 1102