cxgb_tom.c revision 178302
1/************************************************************************** 2 3Copyright (c) 2007, Chelsio Inc. 4All rights reserved. 5 6Redistribution and use in source and binary forms, with or without 7modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Neither the name of the Chelsio Corporation nor the names of its 13 contributors may be used to endorse or promote products derived from 14 this software without specific prior written permission. 15 16THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26POSSIBILITY OF SUCH DAMAGE. 27 28***************************************************************************/ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_tom.c 178302 2008-04-19 03:22:43Z kmacy $"); 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/kernel.h> 36#include <sys/fcntl.h> 37#include <sys/ktr.h> 38#include <sys/limits.h> 39#include <sys/lock.h> 40#include <sys/eventhandler.h> 41#include <sys/mbuf.h> 42#include <sys/module.h> 43#include <sys/condvar.h> 44#include <sys/mutex.h> 45#include <sys/socket.h> 46#include <sys/sysctl.h> 47#include <sys/syslog.h> 48#include <sys/taskqueue.h> 49 50#include <net/if.h> 51#include <net/route.h> 52 53#include <netinet/in.h> 54#include <netinet/in_pcb.h> 55#include <netinet/in_systm.h> 56#include <netinet/in_var.h> 57 58#include <dev/cxgb/cxgb_osdep.h> 59#include <dev/cxgb/sys/mbufq.h> 60 61#include <netinet/in_pcb.h> 62 63#include <dev/cxgb/ulp/tom/cxgb_tcp_offload.h> 64#include <netinet/tcp.h> 65#include <netinet/tcp_var.h> 66#include <netinet/tcp_offload.h> 67#include <netinet/tcp_fsm.h> 68 69#ifdef CONFIG_DEFINED 70#include <cxgb_include.h> 71#else 72#include <dev/cxgb/cxgb_include.h> 73#endif 74 75#include <net/if_vlan_var.h> 76#include <net/route.h> 77 78 79#include <dev/cxgb/t3cdev.h> 80#include <dev/cxgb/common/cxgb_firmware_exports.h> 81#include <dev/cxgb/common/cxgb_tcb.h> 82#include <dev/cxgb/cxgb_include.h> 83#include <dev/cxgb/common/cxgb_ctl_defs.h> 84#include <dev/cxgb/common/cxgb_t3_cpl.h> 85#include <dev/cxgb/cxgb_offload.h> 86#include <dev/cxgb/ulp/toecore/cxgb_toedev.h> 87#include <dev/cxgb/ulp/tom/cxgb_tom.h> 88#include <dev/cxgb/ulp/tom/cxgb_defs.h> 89#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h> 90#include <dev/cxgb/ulp/tom/cxgb_toepcb.h> 91#include <dev/cxgb/ulp/tom/cxgb_tcp.h> 92 93 94 95 96 97static int activated = 1; 98TUNABLE_INT("hw.t3toe.activated", &activated); 99SYSCTL_NODE(_hw, OID_AUTO, t3toe, CTLFLAG_RD, 0, "T3 toe driver parameters"); 100SYSCTL_UINT(_hw_t3toe, OID_AUTO, activated, CTLFLAG_RDTUN, &activated, 0, 101 "enable TOE at init time"); 102 103 104TAILQ_HEAD(, adapter) adapter_list; 105static struct rwlock adapter_list_lock; 106 107static TAILQ_HEAD(, tom_data) cxgb_list; 108static struct mtx cxgb_list_lock; 109static const unsigned int MAX_ATIDS = 64 * 1024; 110static const unsigned int ATID_BASE = 0x100000; 111 112static int t3_toe_attach(struct toedev *dev, const struct offload_id *entry); 113static void cxgb_register_listeners(void); 114static void t3c_tom_add(struct t3cdev *cdev); 115 116/* 117 * Handlers for each CPL opcode 118 */ 119static cxgb_cpl_handler_func tom_cpl_handlers[256]; 120 121 122static eventhandler_tag listen_tag; 123 124static struct offload_id t3_toe_id_tab[] = { 125 { TOE_ID_CHELSIO_T3, 0 }, 126 { TOE_ID_CHELSIO_T3B, 0 }, 127 { TOE_ID_CHELSIO_T3C, 0 }, 128 { 0 } 129}; 130 131static struct tom_info t3_tom_info = { 132 .ti_attach = t3_toe_attach, 133 .ti_id_table = t3_toe_id_tab, 134 .ti_name = "Chelsio-T3" 135}; 136 137struct cxgb_client t3c_tom_client = { 138 .name = "tom_cxgb3", 139 .add = t3c_tom_add, 140 .remove = NULL, 141 .handlers = tom_cpl_handlers, 142 .redirect = NULL 143}; 144 145/* 146 * Add an skb to the deferred skb queue for processing from process context. 147 */ 148void 149t3_defer_reply(struct mbuf *m, struct toedev *dev, defer_handler_t handler) 150{ 151 struct tom_data *td = TOM_DATA(dev); 152 153 m_set_handler(m, handler); 154 mtx_lock(&td->deferq.lock); 155 156 mbufq_tail(&td->deferq, m); 157 if (mbufq_len(&td->deferq) == 1) 158 taskqueue_enqueue(td->tq, &td->deferq_task); 159 mtx_lock(&td->deferq.lock); 160} 161 162struct toepcb * 163toepcb_alloc(void) 164{ 165 struct toepcb *toep; 166 167 toep = malloc(sizeof(struct toepcb), M_CXGB, M_NOWAIT|M_ZERO); 168 169 if (toep == NULL) 170 return (NULL); 171 172 toepcb_init(toep); 173 return (toep); 174} 175 176void 177toepcb_init(struct toepcb *toep) 178{ 179 toep->tp_refcount = 1; 180 cv_init(&toep->tp_cv, "toep cv"); 181} 182 183void 184toepcb_hold(struct toepcb *toep) 185{ 186 atomic_add_acq_int(&toep->tp_refcount, 1); 187} 188 189void 190toepcb_release(struct toepcb *toep) 191{ 192 if (toep->tp_refcount == 1) { 193 free(toep, M_CXGB); 194 return; 195 } 196 atomic_add_acq_int(&toep->tp_refcount, -1); 197} 198 199 200/* 201 * Add a T3 offload device to the list of devices we are managing. 202 */ 203static void 204t3cdev_add(struct tom_data *t) 205{ 206 mtx_lock(&cxgb_list_lock); 207 TAILQ_INSERT_TAIL(&cxgb_list, t, entry); 208 mtx_unlock(&cxgb_list_lock); 209} 210 211static inline int 212cdev2type(struct t3cdev *cdev) 213{ 214 int type = 0; 215 216 switch (cdev->type) { 217 case T3A: 218 type = TOE_ID_CHELSIO_T3; 219 break; 220 case T3B: 221 type = TOE_ID_CHELSIO_T3B; 222 break; 223 case T3C: 224 type = TOE_ID_CHELSIO_T3C; 225 break; 226 } 227 return (type); 228} 229 230/* 231 * Allocate and initialize the TID tables. Returns 0 on success. 232 */ 233static int 234init_tid_tabs(struct tid_info *t, unsigned int ntids, 235 unsigned int natids, unsigned int nstids, 236 unsigned int atid_base, unsigned int stid_base) 237{ 238 unsigned long size = ntids * sizeof(*t->tid_tab) + 239 natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); 240 241 t->tid_tab = cxgb_alloc_mem(size); 242 if (!t->tid_tab) 243 return (ENOMEM); 244 245 t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; 246 t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; 247 t->ntids = ntids; 248 t->nstids = nstids; 249 t->stid_base = stid_base; 250 t->sfree = NULL; 251 t->natids = natids; 252 t->atid_base = atid_base; 253 t->afree = NULL; 254 t->stids_in_use = t->atids_in_use = 0; 255 atomic_set_int(&t->tids_in_use, 0); 256 mtx_init(&t->stid_lock, "stid", NULL, MTX_DUPOK|MTX_DEF); 257 mtx_init(&t->atid_lock, "atid", NULL, MTX_DUPOK|MTX_DEF); 258 259 /* 260 * Setup the free lists for stid_tab and atid_tab. 261 */ 262 if (nstids) { 263 while (--nstids) 264 t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; 265 t->sfree = t->stid_tab; 266 } 267 if (natids) { 268 while (--natids) 269 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 270 t->afree = t->atid_tab; 271 } 272 return 0; 273} 274 275static void 276free_tid_maps(struct tid_info *t) 277{ 278 mtx_destroy(&t->stid_lock); 279 mtx_destroy(&t->atid_lock); 280 cxgb_free_mem(t->tid_tab); 281} 282 283static inline void 284add_adapter(adapter_t *adap) 285{ 286 rw_wlock(&adapter_list_lock); 287 TAILQ_INSERT_TAIL(&adapter_list, adap, adapter_entry); 288 rw_wunlock(&adapter_list_lock); 289} 290 291static inline void 292remove_adapter(adapter_t *adap) 293{ 294 rw_wlock(&adapter_list_lock); 295 TAILQ_REMOVE(&adapter_list, adap, adapter_entry); 296 rw_wunlock(&adapter_list_lock); 297} 298 299/* 300 * Populate a TID_RELEASE WR. The mbuf must be already propely sized. 301 */ 302static inline void 303mk_tid_release(struct mbuf *m, unsigned int tid) 304{ 305 struct cpl_tid_release *req; 306 307 m_set_priority(m, CPL_PRIORITY_SETUP); 308 req = mtod(m, struct cpl_tid_release *); 309 m->m_pkthdr.len = m->m_len = sizeof(*req); 310 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 311 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 312} 313 314static void 315t3_process_tid_release_list(void *data, int pending) 316{ 317 struct mbuf *m; 318 struct t3cdev *tdev = data; 319 struct t3c_data *td = T3C_DATA (tdev); 320 321 mtx_lock(&td->tid_release_lock); 322 while (td->tid_release_list) { 323 struct toe_tid_entry *p = td->tid_release_list; 324 325 td->tid_release_list = (struct toe_tid_entry *)p->ctx; 326 mtx_unlock(&td->tid_release_lock); 327 m = m_get(M_WAIT, MT_DATA); 328 mk_tid_release(m, p - td->tid_maps.tid_tab); 329 cxgb_ofld_send(tdev, m); 330 p->ctx = NULL; 331 mtx_lock(&td->tid_release_lock); 332 } 333 mtx_unlock(&td->tid_release_lock); 334} 335 336int 337cxgb_offload_activate(struct adapter *adapter) 338{ 339 struct t3cdev *dev = &adapter->tdev; 340 int natids, err; 341 struct t3c_data *t; 342 struct tid_range stid_range, tid_range; 343 struct mtutab mtutab; 344 unsigned int l2t_capacity; 345 346 t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO); 347 if (!t) 348 return (ENOMEM); 349 dev->adapter = adapter; 350 351 err = (EOPNOTSUPP); 352 if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 || 353 dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 || 354 dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 || 355 dev->ctl(dev, GET_MTUS, &mtutab) < 0 || 356 dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 || 357 dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0) { 358 device_printf(adapter->dev, "%s: dev->ctl check failed\n", __FUNCTION__); 359 goto out_free; 360 } 361 362 err = (ENOMEM); 363 L2DATA(dev) = t3_init_l2t(l2t_capacity); 364 if (!L2DATA(dev)) { 365 device_printf(adapter->dev, "%s: t3_init_l2t failed\n", __FUNCTION__); 366 goto out_free; 367 } 368 natids = min(tid_range.num / 2, MAX_ATIDS); 369 err = init_tid_tabs(&t->tid_maps, tid_range.num, natids, 370 stid_range.num, ATID_BASE, stid_range.base); 371 if (err) { 372 device_printf(adapter->dev, "%s: init_tid_tabs failed\n", __FUNCTION__); 373 goto out_free_l2t; 374 } 375 376 t->mtus = mtutab.mtus; 377 t->nmtus = mtutab.size; 378 379 TASK_INIT(&t->tid_release_task, 0 /* XXX? */, t3_process_tid_release_list, dev); 380 mtx_init(&t->tid_release_lock, "tid release", NULL, MTX_DUPOK|MTX_DEF); 381 t->dev = dev; 382 383 T3C_DATA (dev) = t; 384 dev->recv = process_rx; 385 dev->arp_update = t3_l2t_update; 386 /* Register netevent handler once */ 387 if (TAILQ_EMPTY(&adapter_list)) { 388#if defined(CONFIG_CHELSIO_T3_MODULE) 389 if (prepare_arp_with_t3core()) 390 log(LOG_ERR, "Unable to set offload capabilities\n"); 391#endif 392 } 393 CTR1(KTR_CXGB, "adding adapter %p", adapter); 394 add_adapter(adapter); 395 device_printf(adapter->dev, "offload started\n"); 396 adapter->flags |= CXGB_OFLD_INIT; 397 return (0); 398 399out_free_l2t: 400 t3_free_l2t(L2DATA(dev)); 401 L2DATA(dev) = NULL; 402out_free: 403 free(t, M_CXGB); 404 return (err); 405} 406 407void 408cxgb_offload_deactivate(struct adapter *adapter) 409{ 410 struct t3cdev *tdev = &adapter->tdev; 411 struct t3c_data *t = T3C_DATA(tdev); 412 413 printf("removing adapter %p\n", adapter); 414 remove_adapter(adapter); 415 if (TAILQ_EMPTY(&adapter_list)) { 416#if defined(CONFIG_CHELSIO_T3_MODULE) 417 restore_arp_sans_t3core(); 418#endif 419 } 420 free_tid_maps(&t->tid_maps); 421 T3C_DATA(tdev) = NULL; 422 t3_free_l2t(L2DATA(tdev)); 423 L2DATA(tdev) = NULL; 424 mtx_destroy(&t->tid_release_lock); 425 free(t, M_CXGB); 426} 427 428/* 429 * Sends an sk_buff to a T3C driver after dealing with any active network taps. 430 */ 431int 432cxgb_ofld_send(struct t3cdev *dev, struct mbuf *m) 433{ 434 int r; 435 436 r = dev->send(dev, m); 437 return r; 438} 439 440static struct ifnet * 441get_iff_from_mac(adapter_t *adapter, const uint8_t *mac, unsigned int vlan) 442{ 443 int i; 444 445 for_each_port(adapter, i) { 446#ifdef notyet 447 const struct vlan_group *grp; 448#endif 449 const struct port_info *p = &adapter->port[i]; 450 struct ifnet *ifp = p->ifp; 451 452 if (!memcmp(p->hw_addr, mac, ETHER_ADDR_LEN)) { 453#ifdef notyet 454 455 if (vlan && vlan != EVL_VLID_MASK) { 456 grp = p->vlan_grp; 457 dev = grp ? grp->vlan_devices[vlan] : NULL; 458 } else 459 while (dev->master) 460 dev = dev->master; 461#endif 462 return (ifp); 463 } 464 } 465 return (NULL); 466} 467 468static inline void 469failover_fixup(adapter_t *adapter, int port) 470{ 471 if (adapter->params.rev == 0) { 472 struct ifnet *ifp = adapter->port[port].ifp; 473 struct cmac *mac = &adapter->port[port].mac; 474 if (!(ifp->if_flags & IFF_UP)) { 475 /* Failover triggered by the interface ifdown */ 476 t3_write_reg(adapter, A_XGM_TX_CTRL + mac->offset, 477 F_TXEN); 478 t3_read_reg(adapter, A_XGM_TX_CTRL + mac->offset); 479 } else { 480 /* Failover triggered by the interface link down */ 481 t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0); 482 t3_read_reg(adapter, A_XGM_RX_CTRL + mac->offset); 483 t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 484 F_RXEN); 485 } 486 } 487} 488 489static int 490cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data) 491{ 492 int ret = 0; 493 struct ulp_iscsi_info *uiip = data; 494 495 switch (req) { 496 case ULP_ISCSI_GET_PARAMS: 497 uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT); 498 uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT); 499 uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK); 500 /* 501 * On tx, the iscsi pdu has to be <= tx page size and has to 502 * fit into the Tx PM FIFO. 503 */ 504 uiip->max_txsz = min(adapter->params.tp.tx_pg_size, 505 t3_read_reg(adapter, A_PM1_TX_CFG) >> 17); 506 /* on rx, the iscsi pdu has to be < rx page size and the 507 whole pdu + cpl headers has to fit into one sge buffer */ 508 /* also check the max rx data length programmed in TP */ 509 uiip->max_rxsz = min(uiip->max_rxsz, 510 ((t3_read_reg(adapter, A_TP_PARA_REG2)) 511 >> S_MAXRXDATA) & M_MAXRXDATA); 512 break; 513 case ULP_ISCSI_SET_PARAMS: 514 t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask); 515 break; 516 default: 517 ret = (EOPNOTSUPP); 518 } 519 return ret; 520} 521 522/* Response queue used for RDMA events. */ 523#define ASYNC_NOTIF_RSPQ 0 524 525static int 526cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data) 527{ 528 int ret = 0; 529 530 switch (req) { 531 case RDMA_GET_PARAMS: { 532 struct rdma_info *req = data; 533 534 req->udbell_physbase = rman_get_start(adapter->udbs_res); 535 req->udbell_len = rman_get_size(adapter->udbs_res); 536 req->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT); 537 req->tpt_top = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT); 538 req->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT); 539 req->pbl_top = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT); 540 req->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT); 541 req->rqt_top = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT); 542 req->kdb_addr = (void *)((unsigned long)rman_get_virtual(adapter->regs_res) + A_SG_KDOORBELL); break; 543 } 544 case RDMA_CQ_OP: { 545 struct rdma_cq_op *req = data; 546 547 /* may be called in any context */ 548 mtx_lock_spin(&adapter->sge.reg_lock); 549 ret = t3_sge_cqcntxt_op(adapter, req->id, req->op, 550 req->credits); 551 mtx_unlock_spin(&adapter->sge.reg_lock); 552 break; 553 } 554 case RDMA_GET_MEM: { 555 struct ch_mem_range *t = data; 556 struct mc7 *mem; 557 558 if ((t->addr & 7) || (t->len & 7)) 559 return (EINVAL); 560 if (t->mem_id == MEM_CM) 561 mem = &adapter->cm; 562 else if (t->mem_id == MEM_PMRX) 563 mem = &adapter->pmrx; 564 else if (t->mem_id == MEM_PMTX) 565 mem = &adapter->pmtx; 566 else 567 return (EINVAL); 568 569 ret = t3_mc7_bd_read(mem, t->addr/8, t->len/8, (u64 *)t->buf); 570 if (ret) 571 return (ret); 572 break; 573 } 574 case RDMA_CQ_SETUP: { 575 struct rdma_cq_setup *req = data; 576 577 mtx_lock_spin(&adapter->sge.reg_lock); 578 ret = t3_sge_init_cqcntxt(adapter, req->id, req->base_addr, 579 req->size, ASYNC_NOTIF_RSPQ, 580 req->ovfl_mode, req->credits, 581 req->credit_thres); 582 mtx_unlock_spin(&adapter->sge.reg_lock); 583 break; 584 } 585 case RDMA_CQ_DISABLE: 586 mtx_lock_spin(&adapter->sge.reg_lock); 587 ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data); 588 mtx_unlock_spin(&adapter->sge.reg_lock); 589 break; 590 case RDMA_CTRL_QP_SETUP: { 591 struct rdma_ctrlqp_setup *req = data; 592 593 mtx_lock_spin(&adapter->sge.reg_lock); 594 ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0, 595 SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ, 596 req->base_addr, req->size, 597 FW_RI_TID_START, 1, 0); 598 mtx_unlock_spin(&adapter->sge.reg_lock); 599 break; 600 } 601 default: 602 ret = EOPNOTSUPP; 603 } 604 return (ret); 605} 606 607static int 608cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data) 609{ 610 struct adapter *adapter = tdev2adap(tdev); 611 struct tid_range *tid; 612 struct mtutab *mtup; 613 struct iff_mac *iffmacp; 614 struct ddp_params *ddpp; 615 struct adap_ports *ports; 616 struct ofld_page_info *rx_page_info; 617 struct tp_params *tp = &adapter->params.tp; 618 int port; 619 620 switch (req) { 621 case GET_MAX_OUTSTANDING_WR: 622 *(unsigned int *)data = FW_WR_NUM; 623 break; 624 case GET_WR_LEN: 625 *(unsigned int *)data = WR_FLITS; 626 break; 627 case GET_TX_MAX_CHUNK: 628 *(unsigned int *)data = 1 << 20; /* 1MB */ 629 break; 630 case GET_TID_RANGE: 631 tid = data; 632 tid->num = t3_mc5_size(&adapter->mc5) - 633 adapter->params.mc5.nroutes - 634 adapter->params.mc5.nfilters - 635 adapter->params.mc5.nservers; 636 tid->base = 0; 637 break; 638 case GET_STID_RANGE: 639 tid = data; 640 tid->num = adapter->params.mc5.nservers; 641 tid->base = t3_mc5_size(&adapter->mc5) - tid->num - 642 adapter->params.mc5.nfilters - 643 adapter->params.mc5.nroutes; 644 break; 645 case GET_L2T_CAPACITY: 646 *(unsigned int *)data = 2048; 647 break; 648 case GET_MTUS: 649 mtup = data; 650 mtup->size = NMTUS; 651 mtup->mtus = adapter->params.mtus; 652 break; 653 case GET_IFF_FROM_MAC: 654 iffmacp = data; 655 iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr, 656 iffmacp->vlan_tag & EVL_VLID_MASK); 657 break; 658 case GET_DDP_PARAMS: 659 ddpp = data; 660 ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT); 661 ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT); 662 ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK); 663 break; 664 case GET_PORTS: 665 ports = data; 666 ports->nports = adapter->params.nports; 667 for_each_port(adapter, port) 668 ports->lldevs[port] = adapter->port[port].ifp; 669 break; 670 case FAILOVER: 671 port = *(int *)data; 672 t3_port_failover(adapter, port); 673 failover_fixup(adapter, port); 674 break; 675 case FAILOVER_DONE: 676 port = *(int *)data; 677 t3_failover_done(adapter, port); 678 break; 679 case FAILOVER_CLEAR: 680 t3_failover_clear(adapter); 681 break; 682 case GET_RX_PAGE_INFO: 683 rx_page_info = data; 684 rx_page_info->page_size = tp->rx_pg_size; 685 rx_page_info->num = tp->rx_num_pgs; 686 break; 687 case ULP_ISCSI_GET_PARAMS: 688 case ULP_ISCSI_SET_PARAMS: 689 if (!offload_running(adapter)) 690 return (EAGAIN); 691 return cxgb_ulp_iscsi_ctl(adapter, req, data); 692 case RDMA_GET_PARAMS: 693 case RDMA_CQ_OP: 694 case RDMA_CQ_SETUP: 695 case RDMA_CQ_DISABLE: 696 case RDMA_CTRL_QP_SETUP: 697 case RDMA_GET_MEM: 698 if (!offload_running(adapter)) 699 return (EAGAIN); 700 return cxgb_rdma_ctl(adapter, req, data); 701 default: 702 return (EOPNOTSUPP); 703 } 704 return 0; 705} 706 707/* 708 * Allocate a TOM data structure, 709 * initialize its cpl_handlers 710 * and register it as a T3C client 711 */ 712static void 713t3c_tom_add(struct t3cdev *cdev) 714{ 715 int i; 716 unsigned int wr_len; 717 struct tom_data *t; 718 struct toedev *tdev; 719 struct adap_ports *port_info; 720 721 t = malloc(sizeof(*t), M_CXGB, M_NOWAIT|M_ZERO); 722 if (t == NULL) 723 return; 724 725 cdev->send = t3_offload_tx; 726 cdev->ctl = cxgb_offload_ctl; 727 728 if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0) 729 goto out_free_tom; 730 731 port_info = malloc(sizeof(*port_info), M_CXGB, M_NOWAIT|M_ZERO); 732 if (!port_info) 733 goto out_free_tom; 734 735 if (cdev->ctl(cdev, GET_PORTS, port_info) < 0) 736 goto out_free_all; 737 738 t3_init_wr_tab(wr_len); 739 t->cdev = cdev; 740 t->client = &t3c_tom_client; 741 742 /* Register TCP offload device */ 743 tdev = &t->tdev; 744 tdev->tod_ttid = cdev2type(cdev); 745 tdev->tod_lldev = cdev->lldev; 746 747 if (register_toedev(tdev, "toe%d")) { 748 printf("unable to register offload device"); 749 goto out_free_all; 750 } 751 TOM_DATA(tdev) = t; 752 753 for (i = 0; i < port_info->nports; i++) { 754 struct ifnet *ifp = port_info->lldevs[i]; 755 TOEDEV(ifp) = tdev; 756 757 CTR1(KTR_TOM, "enabling toe on %p", ifp); 758 ifp->if_capabilities |= IFCAP_TOE4; 759 ifp->if_capenable |= IFCAP_TOE4; 760 } 761 t->ports = port_info; 762 763 /* Add device to the list of offload devices */ 764 t3cdev_add(t); 765 766 /* Activate TCP offload device */ 767 cxgb_offload_activate(TOM_DATA(tdev)->cdev->adapter); 768 769 activate_offload(tdev); 770 cxgb_register_listeners(); 771 return; 772 773out_free_all: 774 printf("out_free_all fail\n"); 775 free(port_info, M_CXGB); 776out_free_tom: 777 printf("out_free_tom fail\n"); 778 free(t, M_CXGB); 779 return; 780} 781 782 783 784static int 785do_act_open_rpl(struct t3cdev *dev, struct mbuf *m) 786{ 787 struct cpl_act_open_rpl *rpl = cplhdr(m); 788 unsigned int atid = G_TID(ntohl(rpl->atid)); 789 struct toe_tid_entry *toe_tid; 790 791 toe_tid = lookup_atid(&(T3C_DATA (dev))->tid_maps, atid); 792 if (toe_tid->ctx && toe_tid->client && toe_tid->client->handlers && 793 toe_tid->client->handlers[CPL_ACT_OPEN_RPL]) { 794 return toe_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, m, 795 toe_tid->ctx); 796 } else { 797 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 798 dev->name, CPL_ACT_OPEN_RPL); 799 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 800 } 801} 802 803static int 804do_stid_rpl(struct t3cdev *dev, struct mbuf *m) 805{ 806 union opcode_tid *p = cplhdr(m); 807 unsigned int stid = G_TID(ntohl(p->opcode_tid)); 808 struct toe_tid_entry *toe_tid; 809 810 toe_tid = lookup_stid(&(T3C_DATA (dev))->tid_maps, stid); 811 if (toe_tid->ctx && toe_tid->client->handlers && 812 toe_tid->client->handlers[p->opcode]) { 813 return toe_tid->client->handlers[p->opcode] (dev, m, toe_tid->ctx); 814 } else { 815 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 816 dev->name, p->opcode); 817 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 818 } 819} 820 821static int 822do_hwtid_rpl(struct t3cdev *dev, struct mbuf *m) 823{ 824 union opcode_tid *p = cplhdr(m); 825 unsigned int hwtid; 826 struct toe_tid_entry *toe_tid; 827 828 DPRINTF("do_hwtid_rpl opcode=0x%x\n", p->opcode); 829 hwtid = G_TID(ntohl(p->opcode_tid)); 830 831 toe_tid = lookup_tid(&(T3C_DATA (dev))->tid_maps, hwtid); 832 if (toe_tid->ctx && toe_tid->client->handlers && 833 toe_tid->client->handlers[p->opcode]) { 834 return toe_tid->client->handlers[p->opcode] 835 (dev, m, toe_tid->ctx); 836 } else { 837 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 838 dev->name, p->opcode); 839 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 840 } 841} 842 843static int 844do_cr(struct t3cdev *dev, struct mbuf *m) 845{ 846 struct cpl_pass_accept_req *req = cplhdr(m); 847 unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 848 struct toe_tid_entry *toe_tid; 849 850 toe_tid = lookup_stid(&(T3C_DATA (dev))->tid_maps, stid); 851 if (toe_tid->ctx && toe_tid->client->handlers && 852 toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) { 853 return toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ] 854 (dev, m, toe_tid->ctx); 855 } else { 856 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 857 dev->name, CPL_PASS_ACCEPT_REQ); 858 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 859 } 860} 861 862static int 863do_abort_req_rss(struct t3cdev *dev, struct mbuf *m) 864{ 865 union opcode_tid *p = cplhdr(m); 866 unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); 867 struct toe_tid_entry *toe_tid; 868 869 toe_tid = lookup_tid(&(T3C_DATA (dev))->tid_maps, hwtid); 870 if (toe_tid->ctx && toe_tid->client->handlers && 871 toe_tid->client->handlers[p->opcode]) { 872 return toe_tid->client->handlers[p->opcode] 873 (dev, m, toe_tid->ctx); 874 } else { 875 struct cpl_abort_req_rss *req = cplhdr(m); 876 struct cpl_abort_rpl *rpl; 877 878 struct mbuf *m = m_get(M_NOWAIT, MT_DATA); 879 if (!m) { 880 log(LOG_NOTICE, "do_abort_req_rss: couldn't get mbuf!\n"); 881 goto out; 882 } 883 884 m_set_priority(m, CPL_PRIORITY_DATA); 885 rpl = cplhdr(m); 886 rpl->wr.wr_hi = 887 htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 888 rpl->wr.wr_lo = htonl(V_WR_TID(GET_TID(req))); 889 OPCODE_TID(rpl) = 890 htonl(MK_OPCODE_TID(CPL_ABORT_RPL, GET_TID(req))); 891 rpl->cmd = req->status; 892 cxgb_ofld_send(dev, m); 893 out: 894 return (CPL_RET_BUF_DONE); 895 } 896} 897 898static int 899do_act_establish(struct t3cdev *dev, struct mbuf *m) 900{ 901 struct cpl_act_establish *req; 902 unsigned int atid; 903 struct toe_tid_entry *toe_tid; 904 905 req = cplhdr(m); 906 atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 907 toe_tid = lookup_atid(&(T3C_DATA (dev))->tid_maps, atid); 908 if (toe_tid && toe_tid->ctx && toe_tid->client->handlers && 909 toe_tid->client->handlers[CPL_ACT_ESTABLISH]) { 910 911 return toe_tid->client->handlers[CPL_ACT_ESTABLISH] 912 (dev, m, toe_tid->ctx); 913 } else { 914 915 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 916 dev->name, CPL_PASS_ACCEPT_REQ); 917 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 918 } 919} 920 921 922static int 923do_term(struct t3cdev *dev, struct mbuf *m) 924{ 925 unsigned int hwtid = ntohl(m_get_priority(m)) >> 8 & 0xfffff; 926 unsigned int opcode = G_OPCODE(ntohl(m->m_pkthdr.csum_data)); 927 struct toe_tid_entry *toe_tid; 928 929 toe_tid = lookup_tid(&(T3C_DATA (dev))->tid_maps, hwtid); 930 if (toe_tid && toe_tid->ctx && toe_tid->client->handlers && 931 toe_tid->client->handlers[opcode]) { 932 return toe_tid->client->handlers[opcode](dev, m, toe_tid->ctx); 933 } else { 934 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 935 dev->name, opcode); 936 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 937 } 938 return (0); 939} 940 941/* 942 * Process a received packet with an unknown/unexpected CPL opcode. 943 */ 944static int 945do_bad_cpl(struct t3cdev *cdev, struct mbuf *m, void *ctx) 946{ 947 log(LOG_ERR, "%s: received bad CPL command %u\n", cdev->name, 948 0xFF & *mtod(m, unsigned int *)); 949 return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG); 950} 951 952/* 953 * Add a new handler to the CPL dispatch table. A NULL handler may be supplied 954 * to unregister an existing handler. 955 */ 956void 957t3tom_register_cpl_handler(unsigned int opcode, cxgb_cpl_handler_func h) 958{ 959 if (opcode < UCHAR_MAX) 960 tom_cpl_handlers[opcode] = h ? h : do_bad_cpl; 961 else 962 log(LOG_ERR, "Chelsio T3 TOM: handler registration for " 963 "opcode %u failed\n", opcode); 964} 965 966/* 967 * Make a preliminary determination if a connection can be offloaded. It's OK 968 * to fail the offload later if we say we can offload here. For now this 969 * always accepts the offload request unless there are IP options. 970 */ 971static int 972can_offload(struct toedev *dev, struct socket *so) 973{ 974 struct tom_data *tomd = TOM_DATA(dev); 975 struct t3cdev *cdev = T3CDEV(dev->tod_lldev); 976 struct tid_info *t = &(T3C_DATA(cdev))->tid_maps; 977 978 return so_sotoinpcb(so)->inp_depend4.inp4_options == NULL && 979 tomd->conf.activated && 980 (tomd->conf.max_conn < 0 || 981 atomic_load_acq_int(&t->tids_in_use) + t->atids_in_use < tomd->conf.max_conn); 982} 983 984static int 985tom_ctl(struct toedev *dev, unsigned int req, void *data) 986{ 987 struct tom_data *t = TOM_DATA(dev); 988 struct t3cdev *cdev = t->cdev; 989 990 if (cdev->ctl) 991 return cdev->ctl(cdev, req, data); 992 993 return (EOPNOTSUPP); 994} 995 996/* 997 * Free an active-open TID. 998 */ 999void * 1000cxgb_free_atid(struct t3cdev *tdev, int atid) 1001{ 1002 struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; 1003 union active_open_entry *p = atid2entry(t, atid); 1004 void *ctx = p->toe_tid.ctx; 1005 1006 mtx_lock(&t->atid_lock); 1007 p->next = t->afree; 1008 t->afree = p; 1009 t->atids_in_use--; 1010 mtx_unlock(&t->atid_lock); 1011 1012 return ctx; 1013} 1014 1015/* 1016 * Free a server TID and return it to the free pool. 1017 */ 1018void 1019cxgb_free_stid(struct t3cdev *tdev, int stid) 1020{ 1021 struct tid_info *t = &(T3C_DATA (tdev))->tid_maps; 1022 union listen_entry *p = stid2entry(t, stid); 1023 1024 mtx_lock(&t->stid_lock); 1025 p->next = t->sfree; 1026 t->sfree = p; 1027 t->stids_in_use--; 1028 mtx_unlock(&t->stid_lock); 1029} 1030 1031/* 1032 * Free a server TID and return it to the free pool. 1033 */ 1034void * 1035cxgb_get_lctx(struct t3cdev *tdev, int stid) 1036{ 1037 struct tid_info *t = &(T3C_DATA (tdev))->tid_maps; 1038 union listen_entry *p = stid2entry(t, stid); 1039 1040 return (p->toe_tid.ctx); 1041} 1042 1043void 1044cxgb_insert_tid(struct t3cdev *tdev, struct cxgb_client *client, 1045 void *ctx, unsigned int tid) 1046{ 1047 struct tid_info *t = &(T3C_DATA (tdev))->tid_maps; 1048 1049 t->tid_tab[tid].client = client; 1050 t->tid_tab[tid].ctx = ctx; 1051 atomic_add_int(&t->tids_in_use, 1); 1052} 1053 1054/* use ctx as a next pointer in the tid release list */ 1055void 1056cxgb_queue_tid_release(struct t3cdev *tdev, unsigned int tid) 1057{ 1058 struct t3c_data *td = T3C_DATA (tdev); 1059 struct toe_tid_entry *p = &td->tid_maps.tid_tab[tid]; 1060 1061 CTR0(KTR_TOM, "queuing tid release\n"); 1062 1063 mtx_lock(&td->tid_release_lock); 1064 p->ctx = td->tid_release_list; 1065 td->tid_release_list = p; 1066 1067 if (!p->ctx) 1068 taskqueue_enqueue(tdev->adapter->tq, &td->tid_release_task); 1069 1070 mtx_unlock(&td->tid_release_lock); 1071} 1072 1073/* 1074 * Remove a tid from the TID table. A client may defer processing its last 1075 * CPL message if it is locked at the time it arrives, and while the message 1076 * sits in the client's backlog the TID may be reused for another connection. 1077 * To handle this we atomically switch the TID association if it still points 1078 * to the original client context. 1079 */ 1080void 1081cxgb_remove_tid(struct t3cdev *tdev, void *ctx, unsigned int tid) 1082{ 1083 struct tid_info *t = &(T3C_DATA (tdev))->tid_maps; 1084 1085 if (tid >= t->ntids) 1086 panic("tid=%d >= t->ntids=%d", tid, t->ntids); 1087 1088 if (tdev->type == T3A) 1089 atomic_cmpset_ptr((uintptr_t *)&t->tid_tab[tid].ctx, (long)NULL, (long)ctx); 1090 else { 1091 struct mbuf *m; 1092 1093 m = m_get(M_NOWAIT, MT_DATA); 1094 if (__predict_true(m != NULL)) { 1095 mk_tid_release(m, tid); 1096 CTR1(KTR_CXGB, "releasing tid=%u", tid); 1097 1098 cxgb_ofld_send(tdev, m); 1099 t->tid_tab[tid].ctx = NULL; 1100 } else 1101 cxgb_queue_tid_release(tdev, tid); 1102 } 1103 atomic_add_int(&t->tids_in_use, -1); 1104} 1105 1106int 1107cxgb_alloc_atid(struct t3cdev *tdev, struct cxgb_client *client, 1108 void *ctx) 1109{ 1110 int atid = -1; 1111 struct tid_info *t = &(T3C_DATA (tdev))->tid_maps; 1112 1113 mtx_lock(&t->atid_lock); 1114 if (t->afree) { 1115 union active_open_entry *p = t->afree; 1116 1117 atid = (p - t->atid_tab) + t->atid_base; 1118 t->afree = p->next; 1119 p->toe_tid.ctx = ctx; 1120 p->toe_tid.client = client; 1121 t->atids_in_use++; 1122 } 1123 mtx_unlock(&t->atid_lock); 1124 return atid; 1125} 1126 1127int 1128cxgb_alloc_stid(struct t3cdev *tdev, struct cxgb_client *client, 1129 void *ctx) 1130{ 1131 int stid = -1; 1132 struct tid_info *t = &(T3C_DATA (tdev))->tid_maps; 1133 1134 mtx_lock(&t->stid_lock); 1135 if (t->sfree) { 1136 union listen_entry *p = t->sfree; 1137 1138 stid = (p - t->stid_tab) + t->stid_base; 1139 t->sfree = p->next; 1140 p->toe_tid.ctx = ctx; 1141 p->toe_tid.client = client; 1142 t->stids_in_use++; 1143 } 1144 mtx_unlock(&t->stid_lock); 1145 return stid; 1146} 1147 1148 1149static int 1150is_offloading(struct ifnet *ifp) 1151{ 1152 struct adapter *adapter; 1153 int port; 1154 1155 rw_rlock(&adapter_list_lock); 1156 TAILQ_FOREACH(adapter, &adapter_list, adapter_entry) { 1157 for_each_port(adapter, port) { 1158 if (ifp == adapter->port[port].ifp) { 1159 rw_runlock(&adapter_list_lock); 1160 return 1; 1161 } 1162 } 1163 } 1164 rw_runlock(&adapter_list_lock); 1165 return 0; 1166} 1167 1168 1169static void 1170cxgb_arp_update_event(void *unused, struct rtentry *rt0, 1171 uint8_t *enaddr, struct sockaddr *sa) 1172{ 1173 1174 if (!is_offloading(rt0->rt_ifp)) 1175 return; 1176 1177 RT_ADDREF(rt0); 1178 RT_UNLOCK(rt0); 1179 cxgb_neigh_update(rt0, enaddr, sa); 1180 RT_LOCK(rt0); 1181 RT_REMREF(rt0); 1182} 1183 1184static void 1185cxgb_redirect_event(void *unused, int event, struct rtentry *rt0, 1186 struct rtentry *rt1, struct sockaddr *sa) 1187{ 1188 /* 1189 * ignore events on non-offloaded interfaces 1190 */ 1191 if (!is_offloading(rt0->rt_ifp)) 1192 return; 1193 1194 /* 1195 * Cannot redirect to non-offload device. 1196 */ 1197 if (!is_offloading(rt1->rt_ifp)) { 1198 log(LOG_WARNING, "%s: Redirect to non-offload" 1199 "device ignored.\n", __FUNCTION__); 1200 return; 1201 } 1202 1203 /* 1204 * avoid LORs by dropping the route lock but keeping a reference 1205 * 1206 */ 1207 RT_ADDREF(rt0); 1208 RT_UNLOCK(rt0); 1209 RT_ADDREF(rt1); 1210 RT_UNLOCK(rt1); 1211 1212 cxgb_redirect(rt0, rt1, sa); 1213 cxgb_neigh_update(rt1, NULL, sa); 1214 1215 RT_LOCK(rt0); 1216 RT_REMREF(rt0); 1217 RT_LOCK(rt1); 1218 RT_REMREF(rt1); 1219} 1220 1221void 1222cxgb_neigh_update(struct rtentry *rt, uint8_t *enaddr, struct sockaddr *sa) 1223{ 1224 1225 if (rt->rt_ifp && is_offloading(rt->rt_ifp) && (rt->rt_ifp->if_flags & IFCAP_TOE)) { 1226 struct t3cdev *tdev = T3CDEV(rt->rt_ifp); 1227 1228 PANIC_IF(!tdev); 1229 t3_l2t_update(tdev, rt, enaddr, sa); 1230 } 1231} 1232 1233static void 1234set_l2t_ix(struct t3cdev *tdev, u32 tid, struct l2t_entry *e) 1235{ 1236 struct mbuf *m; 1237 struct cpl_set_tcb_field *req; 1238 1239 m = m_gethdr(M_NOWAIT, MT_DATA); 1240 if (!m) { 1241 log(LOG_ERR, "%s: cannot allocate mbuf!\n", __FUNCTION__); 1242 return; 1243 } 1244 1245 m_set_priority(m, CPL_PRIORITY_CONTROL); 1246 req = mtod(m, struct cpl_set_tcb_field *); 1247 m->m_pkthdr.len = m->m_len = sizeof(*req); 1248 1249 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1250 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1251 req->reply = 0; 1252 req->cpu_idx = 0; 1253 req->word = htons(W_TCB_L2T_IX); 1254 req->mask = htobe64(V_TCB_L2T_IX(M_TCB_L2T_IX)); 1255 req->val = htobe64(V_TCB_L2T_IX(e->idx)); 1256 tdev->send(tdev, m); 1257} 1258 1259void 1260cxgb_redirect(struct rtentry *old, struct rtentry *new, struct sockaddr *sa) 1261{ 1262 struct ifnet *olddev, *newdev; 1263 struct tid_info *ti; 1264 struct t3cdev *tdev; 1265 u32 tid; 1266 int update_tcb; 1267 struct l2t_entry *e; 1268 struct toe_tid_entry *te; 1269 1270 olddev = old->rt_ifp; 1271 newdev = new->rt_ifp; 1272 if (!is_offloading(olddev)) 1273 return; 1274 if (!is_offloading(newdev)) { 1275 log(LOG_WARNING, "%s: Redirect to non-offload" 1276 "device ignored.\n", __FUNCTION__); 1277 return; 1278 } 1279 tdev = T3CDEV(olddev); 1280 PANIC_IF(!tdev); 1281 if (tdev != T3CDEV(newdev)) { 1282 log(LOG_WARNING, "%s: Redirect to different " 1283 "offload device ignored.\n", __FUNCTION__); 1284 return; 1285 } 1286 1287 /* Add new L2T entry */ 1288 e = t3_l2t_get(tdev, new, new->rt_ifp, sa); 1289 if (!e) { 1290 log(LOG_ERR, "%s: couldn't allocate new l2t entry!\n", 1291 __FUNCTION__); 1292 return; 1293 } 1294 1295 /* Walk tid table and notify clients of dst change. */ 1296 ti = &(T3C_DATA (tdev))->tid_maps; 1297 for (tid=0; tid < ti->ntids; tid++) { 1298 te = lookup_tid(ti, tid); 1299 PANIC_IF(!te); 1300 if (te->ctx && te->client && te->client->redirect) { 1301 update_tcb = te->client->redirect(te->ctx, old, new, 1302 e); 1303 if (update_tcb) { 1304 l2t_hold(L2DATA(tdev), e); 1305 set_l2t_ix(tdev, tid, e); 1306 } 1307 } 1308 } 1309 l2t_release(L2DATA(tdev), e); 1310} 1311 1312/* 1313 * Initialize the CPL dispatch table. 1314 */ 1315static void 1316init_cpl_handlers(void) 1317{ 1318 int i; 1319 1320 for (i = 0; i < 256; ++i) 1321 tom_cpl_handlers[i] = do_bad_cpl; 1322 1323 t3_init_listen_cpl_handlers(); 1324} 1325 1326static int 1327t3_toe_attach(struct toedev *dev, const struct offload_id *entry) 1328{ 1329 struct tom_data *t = TOM_DATA(dev); 1330 struct t3cdev *cdev = t->cdev; 1331 struct ddp_params ddp; 1332 struct ofld_page_info rx_page_info; 1333 int err; 1334 1335 t3_init_tunables(t); 1336 mtx_init(&t->listen_lock, "tom data listeners", NULL, MTX_DEF); 1337 CTR2(KTR_TOM, "t3_toe_attach dev=%p entry=%p", dev, entry); 1338 /* Adjust TOE activation for this module */ 1339 t->conf.activated = activated; 1340 1341 dev->tod_can_offload = can_offload; 1342 dev->tod_connect = t3_connect; 1343 dev->tod_ctl = tom_ctl; 1344#if 0 1345 dev->tod_failover = t3_failover; 1346#endif 1347 err = cdev->ctl(cdev, GET_DDP_PARAMS, &ddp); 1348 if (err) 1349 return err; 1350 1351 err = cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info); 1352 if (err) 1353 return err; 1354 1355 t->ddp_llimit = ddp.llimit; 1356 t->ddp_ulimit = ddp.ulimit; 1357 t->pdev = ddp.pdev; 1358 t->rx_page_size = rx_page_info.page_size; 1359 /* OK if this fails, we just can't do DDP */ 1360 t->nppods = (ddp.ulimit + 1 - ddp.llimit) / PPOD_SIZE; 1361 t->ppod_map = malloc(t->nppods, M_DEVBUF, M_NOWAIT|M_ZERO); 1362 1363 mtx_init(&t->ppod_map_lock, "ppod map", NULL, MTX_DEF); 1364 1365 1366 t3_sysctl_register(cdev->adapter, &t->conf); 1367 return (0); 1368} 1369 1370static void 1371cxgb_toe_listen_start(void *unused, struct tcpcb *tp) 1372{ 1373 struct socket *so = inp_inpcbtosocket(tp->t_inpcb); 1374 struct tom_data *p; 1375 1376 mtx_lock(&cxgb_list_lock); 1377 TAILQ_FOREACH(p, &cxgb_list, entry) { 1378 t3_listen_start(&p->tdev, so, p->cdev); 1379 } 1380 mtx_unlock(&cxgb_list_lock); 1381} 1382 1383static void 1384cxgb_toe_listen_stop(void *unused, struct tcpcb *tp) 1385{ 1386 struct socket *so = inp_inpcbtosocket(tp->t_inpcb); 1387 struct tom_data *p; 1388 1389 mtx_lock(&cxgb_list_lock); 1390 TAILQ_FOREACH(p, &cxgb_list, entry) { 1391 if (tp->t_state == TCPS_LISTEN) 1392 t3_listen_stop(&p->tdev, so, p->cdev); 1393 } 1394 mtx_unlock(&cxgb_list_lock); 1395} 1396 1397static void 1398cxgb_toe_listen_start_handler(struct inpcb *inp, void *arg) 1399{ 1400 struct tcpcb *tp = intotcpcb(inp); 1401 1402 if (tp->t_state == TCPS_LISTEN) 1403 cxgb_toe_listen_start(NULL, tp); 1404} 1405 1406static void 1407cxgb_register_listeners(void) 1408{ 1409 1410 inp_apply_all(cxgb_toe_listen_start_handler, NULL); 1411} 1412 1413static int 1414t3_tom_init(void) 1415{ 1416 init_cpl_handlers(); 1417 if (t3_init_cpl_io() < 0) { 1418 log(LOG_ERR, 1419 "Unable to initialize cpl io ops\n"); 1420 return -1; 1421 } 1422 t3_init_socket_ops(); 1423 1424 /* Register with the TOE device layer. */ 1425 1426 if (register_tom(&t3_tom_info) != 0) { 1427 log(LOG_ERR, 1428 "Unable to register Chelsio T3 TCP offload module.\n"); 1429 return -1; 1430 } 1431 1432 rw_init(&adapter_list_lock, "ofld adap list"); 1433 TAILQ_INIT(&adapter_list); 1434 EVENTHANDLER_REGISTER(route_arp_update_event, cxgb_arp_update_event, 1435 NULL, EVENTHANDLER_PRI_ANY); 1436 EVENTHANDLER_REGISTER(route_redirect_event, cxgb_redirect_event, 1437 NULL, EVENTHANDLER_PRI_ANY); 1438 1439 mtx_init(&cxgb_list_lock, "cxgb tom list", NULL, MTX_DEF); 1440 listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 1441 cxgb_toe_listen_start, NULL, EVENTHANDLER_PRI_ANY); 1442 listen_tag = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 1443 cxgb_toe_listen_stop, NULL, EVENTHANDLER_PRI_ANY); 1444 TAILQ_INIT(&cxgb_list); 1445 1446 1447 1448 t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl); 1449 t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl); 1450 t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr); 1451 t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl); 1452 t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl); 1453 t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl); 1454 t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl); 1455 t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl); 1456 t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl); 1457 t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl); 1458 t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); 1459 t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl); 1460 t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl); 1461 t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss); 1462 t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); 1463 t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term); 1464 t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl); 1465 t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl); 1466 t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl); 1467 t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl); 1468 t3_register_cpl_handler(CPL_GET_TCB_RPL, do_hwtid_rpl); 1469 t3_register_cpl_handler(CPL_SET_TCB_RPL, do_hwtid_rpl); 1470 1471 /* Register to offloading devices */ 1472 cxgb_register_client(&t3c_tom_client); 1473 1474 return (0); 1475} 1476 1477static int 1478t3_tom_load(module_t mod, int cmd, void *arg) 1479{ 1480 int err = 0; 1481 1482 switch (cmd) { 1483 case MOD_LOAD: 1484 t3_tom_init(); 1485 break; 1486 case MOD_QUIESCE: 1487 break; 1488 case MOD_UNLOAD: 1489 printf("uhm, ... unloading isn't really supported for toe\n"); 1490 break; 1491 case MOD_SHUTDOWN: 1492 break; 1493 default: 1494 err = EOPNOTSUPP; 1495 break; 1496 } 1497 1498 return (err); 1499} 1500 1501static moduledata_t mod_data= { 1502 "t3_tom", 1503 t3_tom_load, 1504 0 1505}; 1506MODULE_VERSION(t3_tom, 1); 1507MODULE_DEPEND(t3_tom, toecore, 1, 1, 1); 1508MODULE_DEPEND(t3_tom, if_cxgb, 1, 1, 1); 1509DECLARE_MODULE(t3_tom, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 1510 1511