1237263Snp/*- 2237263Snp * Copyright (c) 2012 Chelsio Communications, Inc. 3237263Snp * All rights reserved. 4237263Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5237263Snp * 6237263Snp * Redistribution and use in source and binary forms, with or without 7237263Snp * modification, are permitted provided that the following conditions 8237263Snp * are met: 9237263Snp * 1. Redistributions of source code must retain the above copyright 10237263Snp * notice, this list of conditions and the following disclaimer. 11237263Snp * 2. Redistributions in binary form must reproduce the above copyright 12237263Snp * notice, this list of conditions and the following disclaimer in the 13237263Snp * documentation and/or other materials provided with the distribution. 14237263Snp * 15237263Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16237263Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17237263Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18237263Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19237263Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20237263Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21237263Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22237263Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23237263Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24237263Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25237263Snp * SUCH DAMAGE. 26237263Snp */ 27174641Skmacy 28174641Skmacy#include <sys/cdefs.h> 29174641Skmacy__FBSDID("$FreeBSD$"); 30174641Skmacy 31237263Snp#include "opt_inet.h" 32237263Snp 33174641Skmacy#include <sys/param.h> 34237263Snp#include <sys/types.h> 35174641Skmacy#include <sys/kernel.h> 36237263Snp#include <sys/queue.h> 37237263Snp#include <sys/malloc.h> 38174641Skmacy#include <sys/module.h> 39174641Skmacy#include <sys/socket.h> 40174641Skmacy#include <sys/taskqueue.h> 41174641Skmacy#include <netinet/in.h> 42174641Skmacy#include <netinet/tcp.h> 43237263Snp#include <netinet/toecore.h> 44178302Skmacy 45237263Snp#ifdef TCP_OFFLOAD 46237263Snp#include "cxgb_include.h" 47237263Snp#include "ulp/tom/cxgb_tom.h" 48237263Snp#include "ulp/tom/cxgb_l2t.h" 49237263Snp#include "ulp/tom/cxgb_toepcb.h" 50178302Skmacy 51237263SnpMALLOC_DEFINE(M_CXGB, "cxgb", "Chelsio T3 Offload services"); 52174641Skmacy 53237263Snp/* Module ops */ 54237263Snpstatic int t3_tom_mod_load(void); 55237263Snpstatic int t3_tom_mod_unload(void); 56237263Snpstatic int t3_tom_modevent(module_t, int, void *); 57178302Skmacy 58237263Snp/* ULD ops and helpers */ 59237263Snpstatic int t3_tom_activate(struct adapter *); 60237263Snpstatic int t3_tom_deactivate(struct adapter *); 61174641Skmacy 62237263Snpstatic int alloc_tid_tabs(struct tid_info *, u_int, u_int, u_int, u_int, u_int); 63237263Snpstatic void free_tid_tabs(struct tid_info *); 64237263Snpstatic int write_smt_entry(struct adapter *, int); 65237263Snpstatic void free_tom_data(struct tom_data *); 66178302Skmacy 67237263Snpstatic struct uld_info tom_uld_info = { 68237263Snp .uld_id = ULD_TOM, 69237263Snp .activate = t3_tom_activate, 70237263Snp .deactivate = t3_tom_deactivate, 71174641Skmacy}; 72174641Skmacy 73174641Skmacystruct toepcb * 74237263Snptoepcb_alloc(struct toedev *tod) 75174641Skmacy{ 76174641Skmacy struct toepcb *toep; 77237263Snp 78237263Snp toep = malloc(sizeof(struct toepcb), M_CXGB, M_NOWAIT | M_ZERO); 79174641Skmacy if (toep == NULL) 80174641Skmacy return (NULL); 81174641Skmacy 82237263Snp toep->tp_tod = tod; 83237263Snp toep->tp_wr_max = toep->tp_wr_avail = 15; 84237263Snp toep->tp_wr_unacked = 0; 85237263Snp toep->tp_delack_mode = 0; 86237263Snp 87174641Skmacy return (toep); 88174641Skmacy} 89174641Skmacy 90174641Skmacyvoid 91237263Snptoepcb_free(struct toepcb *toep) 92174641Skmacy{ 93237263Snp free(toep, M_CXGB); 94174641Skmacy} 95174641Skmacy 96178302Skmacystatic int 97237263Snpalloc_tid_tabs(struct tid_info *t, u_int ntids, u_int natids, u_int nstids, 98237263Snp u_int atid_base, u_int stid_base) 99178302Skmacy{ 100178302Skmacy unsigned long size = ntids * sizeof(*t->tid_tab) + 101178302Skmacy natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); 102178302Skmacy 103237263Snp t->tid_tab = malloc(size, M_CXGB, M_NOWAIT | M_ZERO); 104178302Skmacy if (!t->tid_tab) 105178302Skmacy return (ENOMEM); 106178302Skmacy 107178302Skmacy t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; 108178302Skmacy t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; 109178302Skmacy t->ntids = ntids; 110178302Skmacy t->nstids = nstids; 111178302Skmacy t->stid_base = stid_base; 112178302Skmacy t->sfree = NULL; 113178302Skmacy t->natids = natids; 114178302Skmacy t->atid_base = atid_base; 115178302Skmacy t->afree = NULL; 116178302Skmacy t->stids_in_use = t->atids_in_use = 0; 117216373Savg t->tids_in_use = 0; 118237263Snp mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF); 119237263Snp mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF); 120178302Skmacy 121178302Skmacy /* 122178302Skmacy * Setup the free lists for stid_tab and atid_tab. 123178302Skmacy */ 124178302Skmacy if (nstids) { 125178302Skmacy while (--nstids) 126178302Skmacy t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; 127178302Skmacy t->sfree = t->stid_tab; 128178302Skmacy } 129178302Skmacy if (natids) { 130178302Skmacy while (--natids) 131178302Skmacy t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 132178302Skmacy t->afree = t->atid_tab; 133178302Skmacy } 134237263Snp return (0); 135178302Skmacy} 136178302Skmacy 137178302Skmacystatic void 138237263Snpfree_tid_tabs(struct tid_info *t) 139178302Skmacy{ 140237263Snp if (mtx_initialized(&t->stid_lock)) 141237263Snp mtx_destroy(&t->stid_lock); 142237263Snp if (mtx_initialized(&t->atid_lock)) 143237263Snp mtx_destroy(&t->atid_lock); 144237263Snp free(t->tid_tab, M_CXGB); 145178302Skmacy} 146178302Skmacy 147237263Snpstatic int 148237263Snpwrite_smt_entry(struct adapter *sc, int idx) 149178302Skmacy{ 150237263Snp struct port_info *pi = &sc->port[idx]; 151237263Snp struct cpl_smt_write_req *req; 152178302Skmacy struct mbuf *m; 153178302Skmacy 154237263Snp m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, req); 155237263Snp if (m == NULL) { 156237263Snp log(LOG_ERR, "%s: no mbuf, can't write SMT entry for %d\n", 157237263Snp __func__, idx); 158237263Snp return (ENOMEM); 159178302Skmacy } 160178302Skmacy 161237263Snp req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 162237263Snp OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 163237263Snp req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 164237263Snp req->iff = idx; 165237263Snp memset(req->src_mac1, 0, sizeof(req->src_mac1)); 166237263Snp memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 167178302Skmacy 168237263Snp t3_offload_tx(sc, m); 169178302Skmacy 170178302Skmacy return (0); 171178302Skmacy} 172178302Skmacy 173237263Snpstatic void 174237263Snpfree_tom_data(struct tom_data *td) 175178302Skmacy{ 176237263Snp KASSERT(TAILQ_EMPTY(&td->toep_list), 177237263Snp ("%s: toep_list not empty", __func__)); 178178302Skmacy 179237263Snp if (td->listen_mask != 0) 180237263Snp hashdestroy(td->listen_hash, M_CXGB, td->listen_mask); 181178302Skmacy 182237263Snp if (mtx_initialized(&td->toep_list_lock)) 183237263Snp mtx_destroy(&td->toep_list_lock); 184237263Snp if (mtx_initialized(&td->lctx_hash_lock)) 185237263Snp mtx_destroy(&td->lctx_hash_lock); 186237263Snp if (mtx_initialized(&td->tid_release_lock)) 187237263Snp mtx_destroy(&td->tid_release_lock); 188237263Snp if (td->l2t) 189237263Snp t3_free_l2t(td->l2t); 190237263Snp free_tid_tabs(&td->tid_maps); 191237263Snp free(td, M_CXGB); 192178302Skmacy} 193178302Skmacy 194178302Skmacy/* 195237263Snp * Ground control to Major TOM 196237263Snp * Commencing countdown, engines on 197174641Skmacy */ 198178302Skmacystatic int 199237263Snpt3_tom_activate(struct adapter *sc) 200178302Skmacy{ 201237263Snp struct tom_data *td; 202237263Snp struct toedev *tod; 203237263Snp int i, rc = 0; 204237263Snp struct mc5_params *mc5 = &sc->params.mc5; 205237263Snp u_int ntids, natids, mtus; 206178302Skmacy 207237263Snp ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */ 208178302Skmacy 209237263Snp /* per-adapter softc for TOM */ 210237263Snp td = malloc(sizeof(*td), M_CXGB, M_ZERO | M_NOWAIT); 211237263Snp if (td == NULL) 212237263Snp return (ENOMEM); 213178302Skmacy 214237263Snp /* List of TOE PCBs and associated lock */ 215237263Snp mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF); 216237263Snp TAILQ_INIT(&td->toep_list); 217178302Skmacy 218237263Snp /* Listen context */ 219237263Snp mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF); 220237263Snp td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGB, 221237263Snp &td->listen_mask, HASH_NOWAIT); 222178302Skmacy 223237263Snp /* TID release task */ 224237263Snp TASK_INIT(&td->tid_release_task, 0 , t3_process_tid_release_list, td); 225237263Snp mtx_init(&td->tid_release_lock, "tid release", NULL, MTX_DEF); 226178302Skmacy 227237263Snp /* L2 table */ 228237263Snp td->l2t = t3_init_l2t(L2T_SIZE); 229237263Snp if (td->l2t == NULL) { 230237263Snp rc = ENOMEM; 231237263Snp goto done; 232178302Skmacy } 233178302Skmacy 234237263Snp /* TID tables */ 235237263Snp ntids = t3_mc5_size(&sc->mc5) - mc5->nroutes - mc5->nfilters - 236237263Snp mc5->nservers; 237237263Snp natids = min(ntids / 2, 64 * 1024); 238237263Snp rc = alloc_tid_tabs(&td->tid_maps, ntids, natids, mc5->nservers, 239237263Snp 0x100000 /* ATID_BASE */, ntids); 240237263Snp if (rc != 0) 241237263Snp goto done; 242178302Skmacy 243237263Snp /* CPL handlers */ 244237263Snp t3_init_listen_cpl_handlers(sc); 245237263Snp t3_init_l2t_cpl_handlers(sc); 246237263Snp t3_init_cpl_io(sc); 247178302Skmacy 248237263Snp /* toedev ops */ 249237263Snp tod = &td->tod; 250237263Snp init_toedev(tod); 251237263Snp tod->tod_softc = sc; 252237263Snp tod->tod_connect = t3_connect; 253237263Snp tod->tod_listen_start = t3_listen_start; 254237263Snp tod->tod_listen_stop = t3_listen_stop; 255237263Snp tod->tod_rcvd = t3_rcvd; 256237263Snp tod->tod_output = t3_tod_output; 257237263Snp tod->tod_send_rst = t3_send_rst; 258237263Snp tod->tod_send_fin = t3_send_fin; 259237263Snp tod->tod_pcb_detach = t3_pcb_detach; 260237263Snp tod->tod_l2_update = t3_l2_update; 261237263Snp tod->tod_syncache_added = t3_syncache_added; 262237263Snp tod->tod_syncache_removed = t3_syncache_removed; 263237263Snp tod->tod_syncache_respond = t3_syncache_respond; 264237263Snp tod->tod_offload_socket = t3_offload_socket; 265178302Skmacy 266237263Snp /* port MTUs */ 267237263Snp mtus = sc->port[0].ifp->if_mtu; 268237263Snp if (sc->params.nports > 1) 269237263Snp mtus |= sc->port[1].ifp->if_mtu << 16; 270237263Snp t3_write_reg(sc, A_TP_MTU_PORT_TABLE, mtus); 271237263Snp t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd, 272237263Snp sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff); 273178302Skmacy 274237263Snp /* SMT entry for each port */ 275237263Snp for_each_port(sc, i) { 276237263Snp write_smt_entry(sc, i); 277237263Snp TOEDEV(sc->port[i].ifp) = &td->tod; 278178302Skmacy } 279178302Skmacy 280237263Snp /* Switch TP to offload mode */ 281237263Snp t3_tp_set_offload_mode(sc, 1); 282178302Skmacy 283237263Snp sc->tom_softc = td; 284237263Snp sc->flags |= TOM_INIT_DONE; 285237263Snp register_toedev(tod); 286178302Skmacy 287237263Snpdone: 288237263Snp if (rc != 0) 289237263Snp free_tom_data(td); 290178302Skmacy 291237263Snp return (rc); 292174641Skmacy} 293174641Skmacy 294174641Skmacystatic int 295237263Snpt3_tom_deactivate(struct adapter *sc) 296174641Skmacy{ 297237263Snp int rc = 0; 298237263Snp struct tom_data *td = sc->tom_softc; 299174641Skmacy 300237263Snp ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */ 301174641Skmacy 302237263Snp if (td == NULL) 303237263Snp return (0); /* XXX. KASSERT? */ 304174641Skmacy 305237263Snp if (sc->offload_map != 0) 306237263Snp return (EBUSY); /* at least one port has IFCAP_TOE enabled */ 307174641Skmacy 308237263Snp mtx_lock(&td->toep_list_lock); 309237263Snp if (!TAILQ_EMPTY(&td->toep_list)) 310237263Snp rc = EBUSY; 311237263Snp mtx_unlock(&td->toep_list_lock); 312174641Skmacy 313237263Snp mtx_lock(&td->lctx_hash_lock); 314237263Snp if (td->lctx_count > 0) 315237263Snp rc = EBUSY; 316237263Snp mtx_unlock(&td->lctx_hash_lock); 317178302Skmacy 318237263Snp if (rc == 0) { 319237263Snp unregister_toedev(&td->tod); 320237263Snp t3_tp_set_offload_mode(sc, 0); 321237263Snp free_tom_data(td); 322237263Snp sc->tom_softc = NULL; 323237263Snp sc->flags &= ~TOM_INIT_DONE; 324178302Skmacy } 325178302Skmacy 326237263Snp return (rc); 327178302Skmacy} 328178302Skmacy 329178302Skmacystatic int 330237263Snpt3_tom_mod_load(void) 331178302Skmacy{ 332237263Snp int rc; 333178302Skmacy 334237263Snp rc = t3_register_uld(&tom_uld_info); 335237263Snp if (rc != 0) 336237263Snp t3_tom_mod_unload(); 337178302Skmacy 338237263Snp return (rc); 339178302Skmacy} 340178302Skmacy 341178302Skmacystatic void 342237263Snptom_uninit(struct adapter *sc, void *arg __unused) 343178302Skmacy{ 344237263Snp /* Try to free resources (works only if no port has IFCAP_TOE) */ 345237263Snp ADAPTER_LOCK(sc); 346237263Snp if (sc->flags & TOM_INIT_DONE) 347237263Snp t3_deactivate_uld(sc, ULD_TOM); 348237263Snp ADAPTER_UNLOCK(sc); 349178302Skmacy} 350178302Skmacy 351174641Skmacystatic int 352237263Snpt3_tom_mod_unload(void) 353174641Skmacy{ 354237263Snp t3_iterate(tom_uninit, NULL); 355174641Skmacy 356237263Snp if (t3_unregister_uld(&tom_uld_info) == EBUSY) 357237263Snp return (EBUSY); 358174641Skmacy 359174641Skmacy return (0); 360174641Skmacy} 361237263Snp#endif /* ifdef TCP_OFFLOAD */ 362174641Skmacy 363174641Skmacystatic int 364237263Snpt3_tom_modevent(module_t mod, int cmd, void *arg) 365174641Skmacy{ 366237263Snp int rc = 0; 367174641Skmacy 368237263Snp#ifdef TCP_OFFLOAD 369174641Skmacy switch (cmd) { 370174641Skmacy case MOD_LOAD: 371237263Snp rc = t3_tom_mod_load(); 372174641Skmacy break; 373237263Snp 374174641Skmacy case MOD_UNLOAD: 375237263Snp rc = t3_tom_mod_unload(); 376174641Skmacy break; 377237263Snp 378174641Skmacy default: 379237263Snp rc = EINVAL; 380174641Skmacy } 381237263Snp#else 382237263Snp rc = EOPNOTSUPP; 383237263Snp#endif 384237263Snp return (rc); 385174641Skmacy} 386174641Skmacy 387237263Snpstatic moduledata_t t3_tom_moddata= { 388174641Skmacy "t3_tom", 389237263Snp t3_tom_modevent, 390241394Skevlo 0 391174641Skmacy}; 392237263Snp 393174641SkmacyMODULE_VERSION(t3_tom, 1); 394174641SkmacyMODULE_DEPEND(t3_tom, toecore, 1, 1, 1); 395237263SnpMODULE_DEPEND(t3_tom, cxgbc, 1, 1, 1); 396237263SnpDECLARE_MODULE(t3_tom, t3_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY); 397