1/*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: releng/11.0/sys/dev/cxgb/ulp/tom/cxgb_tom.c 241394 2012-10-10 08:36:38Z kevlo $"); 30 31#include "opt_inet.h" 32 33#include <sys/param.h> 34#include <sys/types.h> 35#include <sys/kernel.h> 36#include <sys/queue.h> 37#include <sys/malloc.h> 38#include <sys/module.h> 39#include <sys/socket.h> 40#include <sys/taskqueue.h> 41#include <netinet/in.h> 42#include <netinet/tcp.h> 43#include <netinet/toecore.h> 44 45#ifdef TCP_OFFLOAD 46#include "cxgb_include.h" 47#include "ulp/tom/cxgb_tom.h" 48#include "ulp/tom/cxgb_l2t.h" 49#include "ulp/tom/cxgb_toepcb.h" 50 51MALLOC_DEFINE(M_CXGB, "cxgb", "Chelsio T3 Offload services"); 52 53/* Module ops */ 54static int t3_tom_mod_load(void); 55static int t3_tom_mod_unload(void); 56static int t3_tom_modevent(module_t, int, void *); 57 58/* ULD ops and helpers */ 59static int t3_tom_activate(struct adapter *); 60static int t3_tom_deactivate(struct adapter *); 61 62static int alloc_tid_tabs(struct tid_info *, u_int, u_int, u_int, u_int, u_int); 63static void free_tid_tabs(struct tid_info *); 64static int write_smt_entry(struct adapter *, int); 65static void free_tom_data(struct tom_data *); 66 67static struct uld_info tom_uld_info = { 68 .uld_id = ULD_TOM, 69 .activate = t3_tom_activate, 70 .deactivate = t3_tom_deactivate, 71}; 72 73struct toepcb * 74toepcb_alloc(struct toedev *tod) 75{ 76 struct toepcb *toep; 77 78 toep = malloc(sizeof(struct toepcb), M_CXGB, M_NOWAIT | M_ZERO); 79 if (toep == NULL) 80 return (NULL); 81 82 toep->tp_tod = tod; 83 toep->tp_wr_max = toep->tp_wr_avail = 15; 84 toep->tp_wr_unacked = 0; 85 toep->tp_delack_mode = 0; 86 87 return (toep); 88} 89 90void 91toepcb_free(struct toepcb *toep) 92{ 93 free(toep, M_CXGB); 94} 95 96static int 97alloc_tid_tabs(struct tid_info *t, u_int ntids, u_int natids, u_int nstids, 98 u_int atid_base, u_int stid_base) 99{ 100 unsigned long size = ntids * sizeof(*t->tid_tab) + 101 natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); 102 103 t->tid_tab = malloc(size, M_CXGB, M_NOWAIT | M_ZERO); 104 if (!t->tid_tab) 105 return (ENOMEM); 106 107 t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; 108 t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; 109 t->ntids = ntids; 110 t->nstids = nstids; 111 t->stid_base = stid_base; 112 t->sfree = NULL; 113 t->natids = natids; 114 t->atid_base = atid_base; 115 t->afree = NULL; 116 t->stids_in_use = t->atids_in_use = 0; 117 t->tids_in_use = 0; 118 mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF); 119 mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF); 120 121 /* 122 * Setup the free lists for stid_tab and atid_tab. 123 */ 124 if (nstids) { 125 while (--nstids) 126 t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; 127 t->sfree = t->stid_tab; 128 } 129 if (natids) { 130 while (--natids) 131 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 132 t->afree = t->atid_tab; 133 } 134 return (0); 135} 136 137static void 138free_tid_tabs(struct tid_info *t) 139{ 140 if (mtx_initialized(&t->stid_lock)) 141 mtx_destroy(&t->stid_lock); 142 if (mtx_initialized(&t->atid_lock)) 143 mtx_destroy(&t->atid_lock); 144 free(t->tid_tab, M_CXGB); 145} 146 147static int 148write_smt_entry(struct adapter *sc, int idx) 149{ 150 struct port_info *pi = &sc->port[idx]; 151 struct cpl_smt_write_req *req; 152 struct mbuf *m; 153 154 m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, req); 155 if (m == NULL) { 156 log(LOG_ERR, "%s: no mbuf, can't write SMT entry for %d\n", 157 __func__, idx); 158 return (ENOMEM); 159 } 160 161 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 162 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); 163 req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ 164 req->iff = idx; 165 memset(req->src_mac1, 0, sizeof(req->src_mac1)); 166 memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN); 167 168 t3_offload_tx(sc, m); 169 170 return (0); 171} 172 173static void 174free_tom_data(struct tom_data *td) 175{ 176 KASSERT(TAILQ_EMPTY(&td->toep_list), 177 ("%s: toep_list not empty", __func__)); 178 179 if (td->listen_mask != 0) 180 hashdestroy(td->listen_hash, M_CXGB, td->listen_mask); 181 182 if (mtx_initialized(&td->toep_list_lock)) 183 mtx_destroy(&td->toep_list_lock); 184 if (mtx_initialized(&td->lctx_hash_lock)) 185 mtx_destroy(&td->lctx_hash_lock); 186 if (mtx_initialized(&td->tid_release_lock)) 187 mtx_destroy(&td->tid_release_lock); 188 if (td->l2t) 189 t3_free_l2t(td->l2t); 190 free_tid_tabs(&td->tid_maps); 191 free(td, M_CXGB); 192} 193 194/* 195 * Ground control to Major TOM 196 * Commencing countdown, engines on 197 */ 198static int 199t3_tom_activate(struct adapter *sc) 200{ 201 struct tom_data *td; 202 struct toedev *tod; 203 int i, rc = 0; 204 struct mc5_params *mc5 = &sc->params.mc5; 205 u_int ntids, natids, mtus; 206 207 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */ 208 209 /* per-adapter softc for TOM */ 210 td = malloc(sizeof(*td), M_CXGB, M_ZERO | M_NOWAIT); 211 if (td == NULL) 212 return (ENOMEM); 213 214 /* List of TOE PCBs and associated lock */ 215 mtx_init(&td->toep_list_lock, "PCB list lock", NULL, MTX_DEF); 216 TAILQ_INIT(&td->toep_list); 217 218 /* Listen context */ 219 mtx_init(&td->lctx_hash_lock, "lctx hash lock", NULL, MTX_DEF); 220 td->listen_hash = hashinit_flags(LISTEN_HASH_SIZE, M_CXGB, 221 &td->listen_mask, HASH_NOWAIT); 222 223 /* TID release task */ 224 TASK_INIT(&td->tid_release_task, 0 , t3_process_tid_release_list, td); 225 mtx_init(&td->tid_release_lock, "tid release", NULL, MTX_DEF); 226 227 /* L2 table */ 228 td->l2t = t3_init_l2t(L2T_SIZE); 229 if (td->l2t == NULL) { 230 rc = ENOMEM; 231 goto done; 232 } 233 234 /* TID tables */ 235 ntids = t3_mc5_size(&sc->mc5) - mc5->nroutes - mc5->nfilters - 236 mc5->nservers; 237 natids = min(ntids / 2, 64 * 1024); 238 rc = alloc_tid_tabs(&td->tid_maps, ntids, natids, mc5->nservers, 239 0x100000 /* ATID_BASE */, ntids); 240 if (rc != 0) 241 goto done; 242 243 /* CPL handlers */ 244 t3_init_listen_cpl_handlers(sc); 245 t3_init_l2t_cpl_handlers(sc); 246 t3_init_cpl_io(sc); 247 248 /* toedev ops */ 249 tod = &td->tod; 250 init_toedev(tod); 251 tod->tod_softc = sc; 252 tod->tod_connect = t3_connect; 253 tod->tod_listen_start = t3_listen_start; 254 tod->tod_listen_stop = t3_listen_stop; 255 tod->tod_rcvd = t3_rcvd; 256 tod->tod_output = t3_tod_output; 257 tod->tod_send_rst = t3_send_rst; 258 tod->tod_send_fin = t3_send_fin; 259 tod->tod_pcb_detach = t3_pcb_detach; 260 tod->tod_l2_update = t3_l2_update; 261 tod->tod_syncache_added = t3_syncache_added; 262 tod->tod_syncache_removed = t3_syncache_removed; 263 tod->tod_syncache_respond = t3_syncache_respond; 264 tod->tod_offload_socket = t3_offload_socket; 265 266 /* port MTUs */ 267 mtus = sc->port[0].ifp->if_mtu; 268 if (sc->params.nports > 1) 269 mtus |= sc->port[1].ifp->if_mtu << 16; 270 t3_write_reg(sc, A_TP_MTU_PORT_TABLE, mtus); 271 t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd, 272 sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff); 273 274 /* SMT entry for each port */ 275 for_each_port(sc, i) { 276 write_smt_entry(sc, i); 277 TOEDEV(sc->port[i].ifp) = &td->tod; 278 } 279 280 /* Switch TP to offload mode */ 281 t3_tp_set_offload_mode(sc, 1); 282 283 sc->tom_softc = td; 284 sc->flags |= TOM_INIT_DONE; 285 register_toedev(tod); 286 287done: 288 if (rc != 0) 289 free_tom_data(td); 290 291 return (rc); 292} 293 294static int 295t3_tom_deactivate(struct adapter *sc) 296{ 297 int rc = 0; 298 struct tom_data *td = sc->tom_softc; 299 300 ADAPTER_LOCK_ASSERT_OWNED(sc); /* for sc->flags */ 301 302 if (td == NULL) 303 return (0); /* XXX. KASSERT? */ 304 305 if (sc->offload_map != 0) 306 return (EBUSY); /* at least one port has IFCAP_TOE enabled */ 307 308 mtx_lock(&td->toep_list_lock); 309 if (!TAILQ_EMPTY(&td->toep_list)) 310 rc = EBUSY; 311 mtx_unlock(&td->toep_list_lock); 312 313 mtx_lock(&td->lctx_hash_lock); 314 if (td->lctx_count > 0) 315 rc = EBUSY; 316 mtx_unlock(&td->lctx_hash_lock); 317 318 if (rc == 0) { 319 unregister_toedev(&td->tod); 320 t3_tp_set_offload_mode(sc, 0); 321 free_tom_data(td); 322 sc->tom_softc = NULL; 323 sc->flags &= ~TOM_INIT_DONE; 324 } 325 326 return (rc); 327} 328 329static int 330t3_tom_mod_load(void) 331{ 332 int rc; 333 334 rc = t3_register_uld(&tom_uld_info); 335 if (rc != 0) 336 t3_tom_mod_unload(); 337 338 return (rc); 339} 340 341static void 342tom_uninit(struct adapter *sc, void *arg __unused) 343{ 344 /* Try to free resources (works only if no port has IFCAP_TOE) */ 345 ADAPTER_LOCK(sc); 346 if (sc->flags & TOM_INIT_DONE) 347 t3_deactivate_uld(sc, ULD_TOM); 348 ADAPTER_UNLOCK(sc); 349} 350 351static int 352t3_tom_mod_unload(void) 353{ 354 t3_iterate(tom_uninit, NULL); 355 356 if (t3_unregister_uld(&tom_uld_info) == EBUSY) 357 return (EBUSY); 358 359 return (0); 360} 361#endif /* ifdef TCP_OFFLOAD */ 362 363static int 364t3_tom_modevent(module_t mod, int cmd, void *arg) 365{ 366 int rc = 0; 367 368#ifdef TCP_OFFLOAD 369 switch (cmd) { 370 case MOD_LOAD: 371 rc = t3_tom_mod_load(); 372 break; 373 374 case MOD_UNLOAD: 375 rc = t3_tom_mod_unload(); 376 break; 377 378 default: 379 rc = EINVAL; 380 } 381#else 382 rc = EOPNOTSUPP; 383#endif 384 return (rc); 385} 386 387static moduledata_t t3_tom_moddata= { 388 "t3_tom", 389 t3_tom_modevent, 390 0 391}; 392 393MODULE_VERSION(t3_tom, 1); 394MODULE_DEPEND(t3_tom, toecore, 1, 1, 1); 395MODULE_DEPEND(t3_tom, cxgbc, 1, 1, 1); 396DECLARE_MODULE(t3_tom, t3_tom_moddata, SI_SUB_EXEC, SI_ORDER_ANY); 397