t4_main.c revision 219944
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_main.c 219944 2011-03-24 01:03:01Z np $"); 30 31#include "opt_inet.h" 32 33#include <sys/param.h> 34#include <sys/conf.h> 35#include <sys/priv.h> 36#include <sys/kernel.h> 37#include <sys/bus.h> 38#include <sys/module.h> 39#include <sys/malloc.h> 40#include <sys/queue.h> 41#include <sys/taskqueue.h> 42#include <sys/pciio.h> 43#include <dev/pci/pcireg.h> 44#include <dev/pci/pcivar.h> 45#include <dev/pci/pci_private.h> 46#include <sys/firmware.h> 47#include <sys/sbuf.h> 48#include <sys/smp.h> 49#include <sys/socket.h> 50#include <sys/sockio.h> 51#include <sys/sysctl.h> 52#include <net/ethernet.h> 53#include <net/if.h> 54#include <net/if_types.h> 55#include <net/if_dl.h> 56 57#include "common/t4_hw.h" 58#include "common/common.h" 59#include "common/t4_regs.h" 60#include "common/t4_regs_values.h" 61#include "common/t4fw_interface.h" 62#include "t4_ioctl.h" 63 64/* T4 bus driver interface */ 65static int t4_probe(device_t); 66static int t4_attach(device_t); 67static int t4_detach(device_t); 68static device_method_t t4_methods[] = { 69 DEVMETHOD(device_probe, t4_probe), 70 DEVMETHOD(device_attach, t4_attach), 71 DEVMETHOD(device_detach, t4_detach), 72 73 /* bus interface */ 74 DEVMETHOD(bus_print_child, bus_generic_print_child), 75 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 76 77 { 0, 0 } 78}; 79static driver_t t4_driver = { 80 "t4nex", 81 t4_methods, 82 sizeof(struct adapter) 83}; 84 85 86/* T4 port (cxgbe) interface */ 87static int cxgbe_probe(device_t); 88static int cxgbe_attach(device_t); 89static int cxgbe_detach(device_t); 90static device_method_t cxgbe_methods[] = { 91 DEVMETHOD(device_probe, cxgbe_probe), 92 DEVMETHOD(device_attach, cxgbe_attach), 93 DEVMETHOD(device_detach, cxgbe_detach), 94 { 0, 0 } 95}; 96static driver_t cxgbe_driver = { 97 "cxgbe", 98 cxgbe_methods, 99 sizeof(struct port_info) 100}; 101 102static d_ioctl_t t4_ioctl; 103static d_open_t t4_open; 104static d_close_t t4_close; 105 106static struct cdevsw t4_cdevsw = { 107 .d_version = D_VERSION, 108 .d_flags = 0, 109 .d_open = t4_open, 110 .d_close = t4_close, 111 .d_ioctl = t4_ioctl, 112 .d_name = "t4nex", 113}; 114 115/* ifnet + media interface */ 116static void cxgbe_init(void *); 117static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 118static void cxgbe_start(struct ifnet *); 119static int cxgbe_transmit(struct ifnet *, struct mbuf *); 120static void cxgbe_qflush(struct ifnet *); 121static int cxgbe_media_change(struct ifnet *); 122static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 123 124MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services"); 125 126/* 127 * Tunables. 128 */ 129SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD, 0, "cxgbe driver parameters"); 130 131static int force_firmware_install = 0; 132TUNABLE_INT("hw.cxgbe.force_firmware_install", &force_firmware_install); 133SYSCTL_UINT(_hw_cxgbe, OID_AUTO, force_firmware_install, CTLFLAG_RDTUN, 134 &force_firmware_install, 0, "install firmware on every attach."); 135 136/* 137 * Holdoff timer and packet counter values. 138 */ 139static unsigned int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200}; 140static unsigned int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 141 142/* 143 * Max # of tx and rx queues to use for each 10G and 1G port. 144 */ 145static unsigned int max_ntxq_10g = 8; 146TUNABLE_INT("hw.cxgbe.max_ntxq_10G_port", &max_ntxq_10g); 147SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_ntxq_10G_port, CTLFLAG_RDTUN, 148 &max_ntxq_10g, 0, "maximum number of tx queues per 10G port."); 149 150static unsigned int max_nrxq_10g = 8; 151TUNABLE_INT("hw.cxgbe.max_nrxq_10G_port", &max_nrxq_10g); 152SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_nrxq_10G_port, CTLFLAG_RDTUN, 153 &max_nrxq_10g, 0, "maximum number of rxq's (per 10G port)."); 154 155static unsigned int max_ntxq_1g = 2; 156TUNABLE_INT("hw.cxgbe.max_ntxq_1G_port", &max_ntxq_1g); 157SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_ntxq_1G_port, CTLFLAG_RDTUN, 158 &max_ntxq_1g, 0, "maximum number of tx queues per 1G port."); 159 160static unsigned int max_nrxq_1g = 2; 161TUNABLE_INT("hw.cxgbe.max_nrxq_1G_port", &max_nrxq_1g); 162SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_nrxq_1G_port, CTLFLAG_RDTUN, 163 &max_nrxq_1g, 0, "maximum number of rxq's (per 1G port)."); 164 165/* 166 * Holdoff parameters for 10G and 1G ports. 167 */ 168static unsigned int tmr_idx_10g = 1; 169TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &tmr_idx_10g); 170SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_10G, CTLFLAG_RDTUN, 171 &tmr_idx_10g, 0, 172 "default timer index for interrupt holdoff (10G ports)."); 173 174static int pktc_idx_10g = 2; 175TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &pktc_idx_10g); 176SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_10G, CTLFLAG_RDTUN, 177 &pktc_idx_10g, 0, 178 "default pkt counter index for interrupt holdoff (10G ports)."); 179 180static unsigned int tmr_idx_1g = 1; 181TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &tmr_idx_1g); 182SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_1G, CTLFLAG_RDTUN, 183 &tmr_idx_1g, 0, 184 "default timer index for interrupt holdoff (1G ports)."); 185 186static int pktc_idx_1g = 2; 187TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &pktc_idx_1g); 188SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_1G, CTLFLAG_RDTUN, 189 &pktc_idx_1g, 0, 190 "default pkt counter index for interrupt holdoff (1G ports)."); 191 192/* 193 * Size (# of entries) of each tx and rx queue. 194 */ 195static unsigned int qsize_txq = TX_EQ_QSIZE; 196TUNABLE_INT("hw.cxgbe.qsize_txq", &qsize_txq); 197SYSCTL_UINT(_hw_cxgbe, OID_AUTO, qsize_txq, CTLFLAG_RDTUN, 198 &qsize_txq, 0, "default queue size of NIC tx queues."); 199 200static unsigned int qsize_rxq = RX_IQ_QSIZE; 201TUNABLE_INT("hw.cxgbe.qsize_rxq", &qsize_rxq); 202SYSCTL_UINT(_hw_cxgbe, OID_AUTO, qsize_rxq, CTLFLAG_RDTUN, 203 &qsize_rxq, 0, "default queue size of NIC rx queues."); 204 205/* 206 * Interrupt types allowed. 207 */ 208static int intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 209TUNABLE_INT("hw.cxgbe.interrupt_types", &intr_types); 210SYSCTL_UINT(_hw_cxgbe, OID_AUTO, interrupt_types, CTLFLAG_RDTUN, &intr_types, 0, 211 "interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively)"); 212 213/* 214 * Force the driver to use interrupt forwarding. 215 */ 216static int intr_fwd = 0; 217TUNABLE_INT("hw.cxgbe.interrupt_forwarding", &intr_fwd); 218SYSCTL_UINT(_hw_cxgbe, OID_AUTO, interrupt_forwarding, CTLFLAG_RDTUN, 219 &intr_fwd, 0, "always use forwarded interrupts"); 220 221struct intrs_and_queues { 222 int intr_type; /* INTx, MSI, or MSI-X */ 223 int nirq; /* Number of vectors */ 224 int intr_fwd; /* Interrupts forwarded */ 225 int ntxq10g; /* # of NIC txq's for each 10G port */ 226 int nrxq10g; /* # of NIC rxq's for each 10G port */ 227 int ntxq1g; /* # of NIC txq's for each 1G port */ 228 int nrxq1g; /* # of NIC rxq's for each 1G port */ 229}; 230 231enum { 232 MEMWIN0_APERTURE = 2048, 233 MEMWIN0_BASE = 0x1b800, 234 MEMWIN1_APERTURE = 32768, 235 MEMWIN1_BASE = 0x28000, 236 MEMWIN2_APERTURE = 65536, 237 MEMWIN2_BASE = 0x30000, 238}; 239 240enum { 241 XGMAC_MTU = (1 << 0), 242 XGMAC_PROMISC = (1 << 1), 243 XGMAC_ALLMULTI = (1 << 2), 244 XGMAC_VLANEX = (1 << 3), 245 XGMAC_UCADDR = (1 << 4), 246 XGMAC_MCADDRS = (1 << 5), 247 248 XGMAC_ALL = 0xffff 249}; 250 251static int map_bars(struct adapter *); 252static void setup_memwin(struct adapter *); 253static int cfg_itype_and_nqueues(struct adapter *, int, int, 254 struct intrs_and_queues *); 255static int prep_firmware(struct adapter *); 256static int get_capabilities(struct adapter *, struct fw_caps_config_cmd *); 257static int get_params(struct adapter *, struct fw_caps_config_cmd *); 258static void t4_set_desc(struct adapter *); 259static void build_medialist(struct port_info *); 260static int update_mac_settings(struct port_info *, int); 261static int cxgbe_init_locked(struct port_info *); 262static int cxgbe_init_synchronized(struct port_info *); 263static int cxgbe_uninit_locked(struct port_info *); 264static int cxgbe_uninit_synchronized(struct port_info *); 265static int first_port_up(struct adapter *); 266static int last_port_down(struct adapter *); 267static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 268 iq_intr_handler_t *, void *, char *); 269static int t4_free_irq(struct adapter *, struct irq *); 270static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 271 unsigned int); 272static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 273static void cxgbe_tick(void *); 274static int t4_sysctls(struct adapter *); 275static int cxgbe_sysctls(struct port_info *); 276static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 277static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 278static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 279static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 280static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 281static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 282static inline void txq_start(struct ifnet *, struct sge_txq *); 283static int t4_mod_event(module_t, int, void *); 284 285struct t4_pciids { 286 uint16_t device; 287 uint8_t mpf; 288 char *desc; 289} t4_pciids[] = { 290 {0xa000, 0, "Chelsio Terminator 4 FPGA"}, 291 {0x4400, 4, "Chelsio T440-dbg"}, 292 {0x4401, 4, "Chelsio T420-CR"}, 293 {0x4402, 4, "Chelsio T422-CR"}, 294 {0x4403, 4, "Chelsio T440-CR"}, 295 {0x4404, 4, "Chelsio T420-BCH"}, 296 {0x4405, 4, "Chelsio T440-BCH"}, 297 {0x4406, 4, "Chelsio T440-CH"}, 298 {0x4407, 4, "Chelsio T420-SO"}, 299 {0x4408, 4, "Chelsio T420-CX"}, 300 {0x4409, 4, "Chelsio T420-BT"}, 301 {0x440a, 4, "Chelsio T404-BT"}, 302}; 303 304static int 305t4_probe(device_t dev) 306{ 307 int i; 308 uint16_t v = pci_get_vendor(dev); 309 uint16_t d = pci_get_device(dev); 310 311 if (v != PCI_VENDOR_ID_CHELSIO) 312 return (ENXIO); 313 314 for (i = 0; i < ARRAY_SIZE(t4_pciids); i++) { 315 if (d == t4_pciids[i].device && 316 pci_get_function(dev) == t4_pciids[i].mpf) { 317 device_set_desc(dev, t4_pciids[i].desc); 318 return (BUS_PROBE_DEFAULT); 319 } 320 } 321 322 return (ENXIO); 323} 324 325static int 326t4_attach(device_t dev) 327{ 328 struct adapter *sc; 329 int rc = 0, i, n10g, n1g, rqidx, tqidx; 330 struct fw_caps_config_cmd caps; 331 uint32_t p, v; 332 struct intrs_and_queues iaq; 333 struct sge *s; 334 335 sc = device_get_softc(dev); 336 sc->dev = dev; 337 sc->pf = pci_get_function(dev); 338 sc->mbox = sc->pf; 339 340 pci_enable_busmaster(dev); 341 pci_set_max_read_req(dev, 4096); 342 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 343 device_get_nameunit(dev)); 344 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 345 346 rc = map_bars(sc); 347 if (rc != 0) 348 goto done; /* error message displayed already */ 349 350 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 351 352 /* Prepare the adapter for operation */ 353 rc = -t4_prep_adapter(sc); 354 if (rc != 0) { 355 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 356 goto done; 357 } 358 359 /* Do this really early */ 360 sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT, 361 GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); 362 sc->cdev->si_drv1 = sc; 363 364 /* Prepare the firmware for operation */ 365 rc = prep_firmware(sc); 366 if (rc != 0) 367 goto done; /* error message displayed already */ 368 369 /* Get device capabilities and select which ones we'll use */ 370 rc = get_capabilities(sc, &caps); 371 if (rc != 0) { 372 device_printf(dev, 373 "failed to initialize adapter capabilities: %d.\n", rc); 374 goto done; 375 } 376 377 /* Choose the global RSS mode. */ 378 rc = -t4_config_glbl_rss(sc, sc->mbox, 379 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 380 F_FW_RSS_GLB_CONFIG_CMD_TNLMAPEN | 381 F_FW_RSS_GLB_CONFIG_CMD_TNLALLLKP); 382 if (rc != 0) { 383 device_printf(dev, 384 "failed to select global RSS mode: %d.\n", rc); 385 goto done; 386 } 387 388 /* These are total (sum of all ports) limits for a bus driver */ 389 rc = -t4_cfg_pfvf(sc, sc->mbox, sc->pf, 0, 390 64, /* max # of egress queues */ 391 64, /* max # of egress Ethernet or control queues */ 392 64, /* max # of ingress queues with fl/interrupt */ 393 0, /* max # of ingress queues without interrupt */ 394 0, /* PCIe traffic class */ 395 4, /* max # of virtual interfaces */ 396 M_FW_PFVF_CMD_CMASK, M_FW_PFVF_CMD_PMASK, 16, 397 FW_CMD_CAP_PF, FW_CMD_CAP_PF); 398 if (rc != 0) { 399 device_printf(dev, 400 "failed to configure pf/vf resources: %d.\n", rc); 401 goto done; 402 } 403 404 /* Need this before sge_init */ 405 for (i = 0; i < SGE_NTIMERS; i++) 406 sc->sge.timer_val[i] = min(intr_timer[i], 200U); 407 for (i = 0; i < SGE_NCOUNTERS; i++) 408 sc->sge.counter_val[i] = min(intr_pktcount[i], M_THRESHOLD_0); 409 410 /* Also need the cooked value of cclk before sge_init */ 411 p = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 412 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CCLK)); 413 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &p, &v); 414 if (rc != 0) { 415 device_printf(sc->dev, 416 "failed to obtain core clock value: %d.\n", rc); 417 goto done; 418 } 419 sc->params.vpd.cclk = v; 420 421 t4_sge_init(sc); 422 423 /* 424 * XXX: This is the place to call t4_set_filter_mode() 425 */ 426 427 /* get basic stuff going */ 428 rc = -t4_early_init(sc, sc->mbox); 429 if (rc != 0) { 430 device_printf(dev, "early init failed: %d.\n", rc); 431 goto done; 432 } 433 434 rc = get_params(sc, &caps); 435 if (rc != 0) 436 goto done; /* error message displayed already */ 437 438 /* These are finalized by FW initialization, load their values now */ 439 v = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 440 sc->params.tp.tre = G_TIMERRESOLUTION(v); 441 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(v); 442 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 443 444 /* tweak some settings */ 445 t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) | 446 V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) | 447 V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9)); 448 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 449 450 setup_memwin(sc); 451 452 rc = t4_create_dma_tag(sc); 453 if (rc != 0) 454 goto done; /* error message displayed already */ 455 456 /* 457 * First pass over all the ports - allocate VIs and initialize some 458 * basic parameters like mac address, port type, etc. We also figure 459 * out whether a port is 10G or 1G and use that information when 460 * calculating how many interrupts to attempt to allocate. 461 */ 462 n10g = n1g = 0; 463 for_each_port(sc, i) { 464 struct port_info *pi; 465 466 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 467 sc->port[i] = pi; 468 469 /* These must be set before t4_port_init */ 470 pi->adapter = sc; 471 pi->port_id = i; 472 473 /* Allocate the vi and initialize parameters like mac addr */ 474 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 475 if (rc != 0) { 476 device_printf(dev, "unable to initialize port %d: %d\n", 477 i, rc); 478 free(pi, M_CXGBE); 479 sc->port[i] = NULL; /* indicates init failed */ 480 continue; 481 } 482 483 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 484 device_get_nameunit(dev), i); 485 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 486 487 if (is_10G_port(pi)) { 488 n10g++; 489 pi->tmr_idx = tmr_idx_10g; 490 pi->pktc_idx = pktc_idx_10g; 491 } else { 492 n1g++; 493 pi->tmr_idx = tmr_idx_1g; 494 pi->pktc_idx = pktc_idx_1g; 495 } 496 497 pi->xact_addr_filt = -1; 498 499 pi->qsize_rxq = max(qsize_rxq, 128); 500 while (pi->qsize_rxq & 7) 501 pi->qsize_rxq++; 502 pi->qsize_txq = max(qsize_txq, 128); 503 504 if (pi->qsize_rxq != qsize_rxq) { 505 device_printf(dev, 506 "using %d instead of %d as the rx queue size.\n", 507 pi->qsize_rxq, qsize_rxq); 508 } 509 if (pi->qsize_txq != qsize_txq) { 510 device_printf(dev, 511 "using %d instead of %d as the tx queue size.\n", 512 pi->qsize_txq, qsize_txq); 513 } 514 515 pi->dev = device_add_child(dev, "cxgbe", -1); 516 if (pi->dev == NULL) { 517 device_printf(dev, 518 "failed to add device for port %d.\n", i); 519 rc = ENXIO; 520 goto done; 521 } 522 device_set_softc(pi->dev, pi); 523 524 setbit(&sc->registered_device_map, i); 525 } 526 527 if (sc->registered_device_map == 0) { 528 device_printf(dev, "no usable ports\n"); 529 rc = ENXIO; 530 goto done; 531 } 532 533 /* 534 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 535 */ 536 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 537 if (rc != 0) 538 goto done; /* error message displayed already */ 539 540 sc->intr_type = iaq.intr_type; 541 sc->intr_count = iaq.nirq; 542 543 s = &sc->sge; 544 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 545 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 546 s->neq = s->ntxq + s->nrxq; /* the fl in an rxq is an eq */ 547 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 548 if (iaq.intr_fwd) { 549 sc->flags |= INTR_FWD; 550 s->niq += NFIQ(sc); /* forwarded interrupt queues */ 551 s->fiq = malloc(NFIQ(sc) * sizeof(struct sge_iq), M_CXGBE, 552 M_ZERO | M_WAITOK); 553 } 554 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 555 M_ZERO | M_WAITOK); 556 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 557 M_ZERO | M_WAITOK); 558 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 559 M_ZERO | M_WAITOK); 560 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 561 M_ZERO | M_WAITOK); 562 563 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 564 M_ZERO | M_WAITOK); 565 566 t4_sysctls(sc); 567 568 /* 569 * Second pass over the ports. This time we know the number of rx and 570 * tx queues that each port should get. 571 */ 572 rqidx = tqidx = 0; 573 for_each_port(sc, i) { 574 struct port_info *pi = sc->port[i]; 575 576 if (pi == NULL) 577 continue; 578 579 pi->first_rxq = rqidx; 580 pi->nrxq = is_10G_port(pi) ? iaq.nrxq10g : iaq.nrxq1g; 581 582 pi->first_txq = tqidx; 583 pi->ntxq = is_10G_port(pi) ? iaq.ntxq10g : iaq.ntxq1g; 584 585 rqidx += pi->nrxq; 586 tqidx += pi->ntxq; 587 } 588 589 rc = bus_generic_attach(dev); 590 if (rc != 0) { 591 device_printf(dev, 592 "failed to attach all child ports: %d\n", rc); 593 goto done; 594 } 595 596#ifdef INVARIANTS 597 device_printf(dev, 598 "%p, %d ports (0x%x), %d intr_type, %d intr_count\n", 599 sc, sc->params.nports, sc->params.portvec, 600 sc->intr_type, sc->intr_count); 601#endif 602 t4_set_desc(sc); 603 604done: 605 if (rc != 0) 606 t4_detach(dev); 607 608 return (rc); 609} 610 611/* 612 * Idempotent 613 */ 614static int 615t4_detach(device_t dev) 616{ 617 struct adapter *sc; 618 struct port_info *pi; 619 int i; 620 621 sc = device_get_softc(dev); 622 623 if (sc->cdev) 624 destroy_dev(sc->cdev); 625 626 bus_generic_detach(dev); 627 for (i = 0; i < MAX_NPORTS; i++) { 628 pi = sc->port[i]; 629 if (pi) { 630 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 631 if (pi->dev) 632 device_delete_child(dev, pi->dev); 633 634 mtx_destroy(&pi->pi_lock); 635 free(pi, M_CXGBE); 636 } 637 } 638 639 if (sc->flags & FW_OK) 640 t4_fw_bye(sc, sc->mbox); 641 642 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 643 pci_release_msi(dev); 644 645 if (sc->regs_res) 646 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 647 sc->regs_res); 648 649 if (sc->msix_res) 650 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 651 sc->msix_res); 652 653 free(sc->irq, M_CXGBE); 654 free(sc->sge.rxq, M_CXGBE); 655 free(sc->sge.txq, M_CXGBE); 656 free(sc->sge.fiq, M_CXGBE); 657 free(sc->sge.iqmap, M_CXGBE); 658 free(sc->sge.eqmap, M_CXGBE); 659 t4_destroy_dma_tag(sc); 660 mtx_destroy(&sc->sc_lock); 661 662 bzero(sc, sizeof(*sc)); 663 664 return (0); 665} 666 667 668static int 669cxgbe_probe(device_t dev) 670{ 671 char buf[128]; 672 struct port_info *pi = device_get_softc(dev); 673 674 snprintf(buf, sizeof(buf), "Port %d", pi->port_id); 675 device_set_desc_copy(dev, buf); 676 677 return (BUS_PROBE_DEFAULT); 678} 679 680#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 681 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 682 IFCAP_VLAN_HWTSO) 683#define T4_CAP_ENABLE (T4_CAP & ~IFCAP_TSO6) 684 685static int 686cxgbe_attach(device_t dev) 687{ 688 struct port_info *pi = device_get_softc(dev); 689 struct ifnet *ifp; 690 691 /* Allocate an ifnet and set it up */ 692 ifp = if_alloc(IFT_ETHER); 693 if (ifp == NULL) { 694 device_printf(dev, "Cannot allocate ifnet\n"); 695 return (ENOMEM); 696 } 697 pi->ifp = ifp; 698 ifp->if_softc = pi; 699 700 callout_init(&pi->tick, CALLOUT_MPSAFE); 701 pi->tq = taskqueue_create("cxgbe_taskq", M_NOWAIT, 702 taskqueue_thread_enqueue, &pi->tq); 703 if (pi->tq == NULL) { 704 device_printf(dev, "failed to allocate port task queue\n"); 705 if_free(pi->ifp); 706 return (ENOMEM); 707 } 708 taskqueue_start_threads(&pi->tq, 1, PI_NET, "%s taskq", 709 device_get_nameunit(dev)); 710 711 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 712 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 713 714 ifp->if_init = cxgbe_init; 715 ifp->if_ioctl = cxgbe_ioctl; 716 ifp->if_start = cxgbe_start; 717 ifp->if_transmit = cxgbe_transmit; 718 ifp->if_qflush = cxgbe_qflush; 719 720 ifp->if_snd.ifq_drv_maxlen = 1024; 721 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); 722 IFQ_SET_READY(&ifp->if_snd); 723 724 ifp->if_capabilities = T4_CAP; 725 ifp->if_capenable = T4_CAP_ENABLE; 726 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; 727 728 /* Initialize ifmedia for this port */ 729 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 730 cxgbe_media_status); 731 build_medialist(pi); 732 733 ether_ifattach(ifp, pi->hw_addr); 734 735#ifdef INVARIANTS 736 device_printf(dev, "%p, %d txq, %d rxq\n", pi, pi->ntxq, pi->nrxq); 737#endif 738 739 cxgbe_sysctls(pi); 740 741 return (0); 742} 743 744static int 745cxgbe_detach(device_t dev) 746{ 747 struct port_info *pi = device_get_softc(dev); 748 struct adapter *sc = pi->adapter; 749 int rc; 750 751 /* Tell if_ioctl and if_init that the port is going away */ 752 ADAPTER_LOCK(sc); 753 SET_DOOMED(pi); 754 wakeup(&sc->flags); 755 while (IS_BUSY(sc)) 756 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 757 SET_BUSY(sc); 758 ADAPTER_UNLOCK(sc); 759 760 rc = cxgbe_uninit_synchronized(pi); 761 if (rc != 0) 762 device_printf(dev, "port uninit failed: %d.\n", rc); 763 764 taskqueue_free(pi->tq); 765 766 ifmedia_removeall(&pi->media); 767 ether_ifdetach(pi->ifp); 768 if_free(pi->ifp); 769 770 ADAPTER_LOCK(sc); 771 CLR_BUSY(sc); 772 wakeup_one(&sc->flags); 773 ADAPTER_UNLOCK(sc); 774 775 return (0); 776} 777 778static void 779cxgbe_init(void *arg) 780{ 781 struct port_info *pi = arg; 782 struct adapter *sc = pi->adapter; 783 784 ADAPTER_LOCK(sc); 785 cxgbe_init_locked(pi); /* releases adapter lock */ 786 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 787} 788 789static int 790cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 791{ 792 int rc = 0, mtu, flags; 793 struct port_info *pi = ifp->if_softc; 794 struct adapter *sc = pi->adapter; 795 struct ifreq *ifr = (struct ifreq *)data; 796 uint32_t mask; 797 798 switch (cmd) { 799 case SIOCSIFMTU: 800 ADAPTER_LOCK(sc); 801 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 802 if (rc) { 803fail: 804 ADAPTER_UNLOCK(sc); 805 return (rc); 806 } 807 808 mtu = ifr->ifr_mtu; 809 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 810 rc = EINVAL; 811 } else { 812 ifp->if_mtu = mtu; 813 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 814 t4_update_fl_bufsize(ifp); 815 PORT_LOCK(pi); 816 rc = update_mac_settings(pi, XGMAC_MTU); 817 PORT_UNLOCK(pi); 818 } 819 } 820 ADAPTER_UNLOCK(sc); 821 break; 822 823 case SIOCSIFFLAGS: 824 ADAPTER_LOCK(sc); 825 if (IS_DOOMED(pi)) { 826 rc = ENXIO; 827 goto fail; 828 } 829 if (ifp->if_flags & IFF_UP) { 830 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 831 flags = pi->if_flags; 832 if ((ifp->if_flags ^ flags) & 833 (IFF_PROMISC | IFF_ALLMULTI)) { 834 if (IS_BUSY(sc)) { 835 rc = EBUSY; 836 goto fail; 837 } 838 PORT_LOCK(pi); 839 rc = update_mac_settings(pi, 840 XGMAC_PROMISC | XGMAC_ALLMULTI); 841 PORT_UNLOCK(pi); 842 } 843 ADAPTER_UNLOCK(sc); 844 } else 845 rc = cxgbe_init_locked(pi); 846 pi->if_flags = ifp->if_flags; 847 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 848 rc = cxgbe_uninit_locked(pi); 849 else 850 ADAPTER_UNLOCK(sc); 851 852 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 853 break; 854 855 case SIOCADDMULTI: 856 case SIOCDELMULTI: /* these two can be called with a mutex held :-( */ 857 ADAPTER_LOCK(sc); 858 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 859 if (rc) 860 goto fail; 861 862 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 863 PORT_LOCK(pi); 864 rc = update_mac_settings(pi, XGMAC_MCADDRS); 865 PORT_UNLOCK(pi); 866 } 867 ADAPTER_UNLOCK(sc); 868 break; 869 870 case SIOCSIFCAP: 871 ADAPTER_LOCK(sc); 872 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 873 if (rc) 874 goto fail; 875 876 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 877 if (mask & IFCAP_TXCSUM) { 878 ifp->if_capenable ^= IFCAP_TXCSUM; 879 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 880 881 if (IFCAP_TSO & ifp->if_capenable && 882 !(IFCAP_TXCSUM & ifp->if_capenable)) { 883 ifp->if_capenable &= ~IFCAP_TSO; 884 ifp->if_hwassist &= ~CSUM_TSO; 885 if_printf(ifp, 886 "tso disabled due to -txcsum.\n"); 887 } 888 } 889 if (mask & IFCAP_RXCSUM) 890 ifp->if_capenable ^= IFCAP_RXCSUM; 891 if (mask & IFCAP_TSO4) { 892 ifp->if_capenable ^= IFCAP_TSO4; 893 894 if (IFCAP_TSO & ifp->if_capenable) { 895 if (IFCAP_TXCSUM & ifp->if_capenable) 896 ifp->if_hwassist |= CSUM_TSO; 897 else { 898 ifp->if_capenable &= ~IFCAP_TSO; 899 ifp->if_hwassist &= ~CSUM_TSO; 900 if_printf(ifp, 901 "enable txcsum first.\n"); 902 rc = EAGAIN; 903 } 904 } else 905 ifp->if_hwassist &= ~CSUM_TSO; 906 } 907 if (mask & IFCAP_LRO) { 908#ifdef INET 909 int i; 910 struct sge_rxq *rxq; 911 912 ifp->if_capenable ^= IFCAP_LRO; 913 for_each_rxq(pi, i, rxq) { 914 if (ifp->if_capenable & IFCAP_LRO) 915 rxq->flags |= RXQ_LRO_ENABLED; 916 else 917 rxq->flags &= ~RXQ_LRO_ENABLED; 918 } 919#endif 920 } 921#ifndef TCP_OFFLOAD_DISABLE 922 if (mask & IFCAP_TOE4) { 923 rc = EOPNOTSUPP; 924 } 925#endif 926 if (mask & IFCAP_VLAN_HWTAGGING) { 927 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 928 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 929 PORT_LOCK(pi); 930 rc = update_mac_settings(pi, XGMAC_VLANEX); 931 PORT_UNLOCK(pi); 932 } 933 } 934 if (mask & IFCAP_VLAN_MTU) { 935 ifp->if_capenable ^= IFCAP_VLAN_MTU; 936 937 /* Need to find out how to disable auto-mtu-inflation */ 938 } 939 if (mask & IFCAP_VLAN_HWTSO) 940 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 941 if (mask & IFCAP_VLAN_HWCSUM) 942 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 943 944#ifdef VLAN_CAPABILITIES 945 VLAN_CAPABILITIES(ifp); 946#endif 947 ADAPTER_UNLOCK(sc); 948 break; 949 950 case SIOCSIFMEDIA: 951 case SIOCGIFMEDIA: 952 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 953 break; 954 955 default: 956 rc = ether_ioctl(ifp, cmd, data); 957 } 958 959 return (rc); 960} 961 962static void 963cxgbe_start(struct ifnet *ifp) 964{ 965 struct port_info *pi = ifp->if_softc; 966 struct sge_txq *txq; 967 int i; 968 969 for_each_txq(pi, i, txq) { 970 if (TXQ_TRYLOCK(txq)) { 971 txq_start(ifp, txq); 972 TXQ_UNLOCK(txq); 973 } 974 } 975} 976 977static int 978cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 979{ 980 struct port_info *pi = ifp->if_softc; 981 struct adapter *sc = pi->adapter; 982 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 983 struct buf_ring *br; 984 int rc; 985 986 M_ASSERTPKTHDR(m); 987 988 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 989 m_freem(m); 990 return (0); 991 } 992 993 if (m->m_flags & M_FLOWID) 994 txq += (m->m_pkthdr.flowid % pi->ntxq); 995 br = txq->eq.br; 996 997 if (TXQ_TRYLOCK(txq) == 0) { 998 /* 999 * XXX: make sure that this packet really is sent out. There is 1000 * a small race where t4_eth_tx may stop draining the drbr and 1001 * goes away, just before we enqueued this mbuf. 1002 */ 1003 1004 return (drbr_enqueue(ifp, br, m)); 1005 } 1006 1007 /* 1008 * txq->m is the mbuf that is held up due to a temporary shortage of 1009 * resources and it should be put on the wire first. Then what's in 1010 * drbr and finally the mbuf that was just passed in to us. 1011 * 1012 * Return code should indicate the fate of the mbuf that was passed in 1013 * this time. 1014 */ 1015 1016 TXQ_LOCK_ASSERT_OWNED(txq); 1017 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1018 1019 /* Queued for transmission. */ 1020 1021 rc = drbr_enqueue(ifp, br, m); 1022 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1023 (void) t4_eth_tx(ifp, txq, m); 1024 TXQ_UNLOCK(txq); 1025 return (rc); 1026 } 1027 1028 /* Direct transmission. */ 1029 rc = t4_eth_tx(ifp, txq, m); 1030 if (rc != 0 && txq->m) 1031 rc = 0; /* held, will be transmitted soon (hopefully) */ 1032 1033 TXQ_UNLOCK(txq); 1034 return (rc); 1035} 1036 1037static void 1038cxgbe_qflush(struct ifnet *ifp) 1039{ 1040 struct port_info *pi = ifp->if_softc; 1041 1042 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1043} 1044 1045static int 1046cxgbe_media_change(struct ifnet *ifp) 1047{ 1048 struct port_info *pi = ifp->if_softc; 1049 1050 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1051 1052 return (EOPNOTSUPP); 1053} 1054 1055static void 1056cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1057{ 1058 struct port_info *pi = ifp->if_softc; 1059 struct ifmedia_entry *cur = pi->media.ifm_cur; 1060 int speed = pi->link_cfg.speed; 1061 int data = (pi->port_type << 8) | pi->mod_type; 1062 1063 if (cur->ifm_data != data) { 1064 build_medialist(pi); 1065 cur = pi->media.ifm_cur; 1066 } 1067 1068 ifmr->ifm_status = IFM_AVALID; 1069 if (!pi->link_cfg.link_ok) 1070 return; 1071 1072 ifmr->ifm_status |= IFM_ACTIVE; 1073 1074 /* active and current will differ iff current media is autoselect. */ 1075 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1076 return; 1077 1078 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1079 if (speed == SPEED_10000) 1080 ifmr->ifm_active |= IFM_10G_T; 1081 else if (speed == SPEED_1000) 1082 ifmr->ifm_active |= IFM_1000_T; 1083 else if (speed == SPEED_100) 1084 ifmr->ifm_active |= IFM_100_TX; 1085 else if (speed == SPEED_10) 1086 ifmr->ifm_active |= IFM_10_T; 1087 else 1088 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1089 speed)); 1090} 1091 1092void 1093t4_fatal_err(struct adapter *sc) 1094{ 1095 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1096 t4_intr_disable(sc); 1097 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1098 device_get_nameunit(sc->dev)); 1099} 1100 1101static int 1102map_bars(struct adapter *sc) 1103{ 1104 sc->regs_rid = PCIR_BAR(0); 1105 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1106 &sc->regs_rid, RF_ACTIVE); 1107 if (sc->regs_res == NULL) { 1108 device_printf(sc->dev, "cannot map registers.\n"); 1109 return (ENXIO); 1110 } 1111 sc->bt = rman_get_bustag(sc->regs_res); 1112 sc->bh = rman_get_bushandle(sc->regs_res); 1113 sc->mmio_len = rman_get_size(sc->regs_res); 1114 1115 sc->msix_rid = PCIR_BAR(4); 1116 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1117 &sc->msix_rid, RF_ACTIVE); 1118 if (sc->msix_res == NULL) { 1119 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1120 return (ENXIO); 1121 } 1122 1123 return (0); 1124} 1125 1126static void 1127setup_memwin(struct adapter *sc) 1128{ 1129 u_long bar0; 1130 1131 bar0 = rman_get_start(sc->regs_res); 1132 1133 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1134 (bar0 + MEMWIN0_BASE) | V_BIR(0) | 1135 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1136 1137 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1138 (bar0 + MEMWIN1_BASE) | V_BIR(0) | 1139 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1140 1141 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1142 (bar0 + MEMWIN2_BASE) | V_BIR(0) | 1143 V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10)); 1144} 1145 1146static int 1147cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1148 struct intrs_and_queues *iaq) 1149{ 1150 int rc, itype, navail, nc, nrxq10g, nrxq1g; 1151 1152 bzero(iaq, sizeof(*iaq)); 1153 nc = mp_ncpus; /* our snapshot of the number of CPUs */ 1154 1155 for (itype = INTR_MSIX; itype; itype >>= 1) { 1156 1157 if ((itype & intr_types) == 0) 1158 continue; /* not allowed */ 1159 1160 if (itype == INTR_MSIX) 1161 navail = pci_msix_count(sc->dev); 1162 else if (itype == INTR_MSI) 1163 navail = pci_msi_count(sc->dev); 1164 else 1165 navail = 1; 1166 1167 if (navail == 0) 1168 continue; 1169 1170 iaq->intr_type = itype; 1171 1172 iaq->ntxq10g = min(nc, max_ntxq_10g); 1173 iaq->ntxq1g = min(nc, max_ntxq_1g); 1174 1175 nrxq10g = min(nc, max_nrxq_10g); 1176 nrxq1g = min(nc, max_nrxq_1g); 1177 1178 /* Extra 2 is for a) error interrupt b) firmware event */ 1179 iaq->nirq = n10g * nrxq10g + n1g * nrxq1g + 2; 1180 if (iaq->nirq <= navail && intr_fwd == 0) { 1181 1182 if (itype == INTR_MSI && !powerof2(iaq->nirq)) 1183 goto fwd; 1184 1185 /* One for err, one for fwq, and one for each rxq */ 1186 1187 iaq->intr_fwd = 0; 1188 iaq->nrxq10g = nrxq10g; 1189 iaq->nrxq1g = nrxq1g; 1190 1191 } else { 1192fwd: 1193 iaq->intr_fwd = 1; 1194 1195 if (navail > nc) { 1196 if (itype == INTR_MSIX) 1197 navail = nc + 1; 1198 1199 /* navail is and must remain a pow2 for MSI */ 1200 if (itype == INTR_MSI) { 1201 KASSERT(powerof2(navail), 1202 ("%d not power of 2", navail)); 1203 1204 while (navail / 2 > nc) 1205 navail /= 2; 1206 } 1207 } 1208 iaq->nirq = navail; /* total # of interrupts */ 1209 1210 /* 1211 * If we have multiple vectors available reserve one 1212 * exclusively for errors. The rest will be shared by 1213 * the fwq and data. 1214 */ 1215 if (navail > 1) 1216 navail--; 1217 iaq->nrxq10g = min(nrxq10g, navail); 1218 iaq->nrxq1g = min(nrxq1g, navail); 1219 } 1220 1221 navail = iaq->nirq; 1222 rc = 0; 1223 if (itype == INTR_MSIX) 1224 rc = pci_alloc_msix(sc->dev, &navail); 1225 else if (itype == INTR_MSI) 1226 rc = pci_alloc_msi(sc->dev, &navail); 1227 1228 if (rc == 0) { 1229 if (navail == iaq->nirq) 1230 return (0); 1231 1232 /* 1233 * Didn't get the number requested. Use whatever number 1234 * the kernel is willing to allocate (it's in navail). 1235 */ 1236 pci_release_msi(sc->dev); 1237 goto fwd; 1238 } 1239 1240 device_printf(sc->dev, 1241 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1242 itype, rc, iaq->nirq, navail); 1243 } 1244 1245 device_printf(sc->dev, 1246 "failed to find a usable interrupt type. " 1247 "allowed=%d, msi-x=%d, msi=%d, intx=1", intr_types, 1248 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1249 1250 return (ENXIO); 1251} 1252 1253/* 1254 * Install a compatible firmware (if required), establish contact with it, 1255 * become the master, and reset the device. 1256 */ 1257static int 1258prep_firmware(struct adapter *sc) 1259{ 1260 const struct firmware *fw; 1261 int rc; 1262 enum dev_state state; 1263 1264 /* Check firmware version and install a different one if necessary */ 1265 rc = t4_check_fw_version(sc); 1266 if (rc != 0 || force_firmware_install) { 1267 uint32_t v = 0; 1268 1269 fw = firmware_get(T4_FWNAME); 1270 if (fw != NULL) { 1271 const struct fw_hdr *hdr = (const void *)fw->data; 1272 1273 v = ntohl(hdr->fw_ver); 1274 1275 /* 1276 * The firmware module will not be used if it isn't the 1277 * same major version as what the driver was compiled 1278 * with. This check trumps force_firmware_install. 1279 */ 1280 if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) { 1281 device_printf(sc->dev, 1282 "Found firmware image but version %d " 1283 "can not be used with this driver (%d)\n", 1284 G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR); 1285 1286 firmware_put(fw, FIRMWARE_UNLOAD); 1287 fw = NULL; 1288 } 1289 } 1290 1291 if (fw == NULL && (rc < 0 || force_firmware_install)) { 1292 device_printf(sc->dev, "No usable firmware. " 1293 "card has %d.%d.%d, driver compiled with %d.%d.%d, " 1294 "force_firmware_install%s set", 1295 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1296 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1297 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1298 FW_VERSION_MAJOR, FW_VERSION_MINOR, 1299 FW_VERSION_MICRO, 1300 force_firmware_install ? "" : " not"); 1301 return (EAGAIN); 1302 } 1303 1304 /* 1305 * Always upgrade, even for minor/micro/build mismatches. 1306 * Downgrade only for a major version mismatch or if 1307 * force_firmware_install was specified. 1308 */ 1309 if (fw != NULL && (rc < 0 || force_firmware_install || 1310 v > sc->params.fw_vers)) { 1311 device_printf(sc->dev, 1312 "installing firmware %d.%d.%d.%d on card.\n", 1313 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1314 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1315 1316 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1317 if (rc != 0) { 1318 device_printf(sc->dev, 1319 "failed to install firmware: %d\n", rc); 1320 firmware_put(fw, FIRMWARE_UNLOAD); 1321 return (rc); 1322 } else { 1323 /* refresh */ 1324 (void) t4_check_fw_version(sc); 1325 } 1326 } 1327 1328 if (fw != NULL) 1329 firmware_put(fw, FIRMWARE_UNLOAD); 1330 } 1331 1332 /* Contact firmware, request master */ 1333 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state); 1334 if (rc < 0) { 1335 rc = -rc; 1336 device_printf(sc->dev, 1337 "failed to connect to the firmware: %d.\n", rc); 1338 return (rc); 1339 } 1340 1341 /* Reset device */ 1342 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1343 if (rc != 0) { 1344 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 1345 if (rc != ETIMEDOUT && rc != EIO) 1346 t4_fw_bye(sc, sc->mbox); 1347 return (rc); 1348 } 1349 1350 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 1351 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1352 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1353 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1354 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1355 sc->flags |= FW_OK; 1356 1357 return (0); 1358} 1359 1360static int 1361get_capabilities(struct adapter *sc, struct fw_caps_config_cmd *caps) 1362{ 1363 int rc; 1364 1365 bzero(caps, sizeof(*caps)); 1366 caps->op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1367 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1368 caps->retval_len16 = htobe32(FW_LEN16(*caps)); 1369 1370 rc = -t4_wr_mbox(sc, sc->mbox, caps, sizeof(*caps), caps); 1371 if (rc != 0) 1372 return (rc); 1373 1374 if (caps->niccaps & htobe16(FW_CAPS_CONFIG_NIC_VM)) 1375 caps->niccaps ^= htobe16(FW_CAPS_CONFIG_NIC_VM); 1376 1377 caps->op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1378 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1379 rc = -t4_wr_mbox(sc, sc->mbox, caps, sizeof(*caps), NULL); 1380 1381 return (rc); 1382} 1383 1384static int 1385get_params(struct adapter *sc, struct fw_caps_config_cmd *caps) 1386{ 1387 int rc; 1388 uint32_t params[7], val[7]; 1389 1390#define FW_PARAM_DEV(param) \ 1391 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1392 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1393#define FW_PARAM_PFVF(param) \ 1394 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1395 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1396 1397 params[0] = FW_PARAM_DEV(PORTVEC); 1398 params[1] = FW_PARAM_PFVF(IQFLINT_START); 1399 params[2] = FW_PARAM_PFVF(EQ_START); 1400 params[3] = FW_PARAM_PFVF(FILTER_START); 1401 params[4] = FW_PARAM_PFVF(FILTER_END); 1402 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 5, params, val); 1403 if (rc != 0) { 1404 device_printf(sc->dev, 1405 "failed to query parameters: %d.\n", rc); 1406 goto done; 1407 } 1408 1409 sc->params.portvec = val[0]; 1410 sc->params.nports = 0; 1411 while (val[0]) { 1412 sc->params.nports++; 1413 val[0] &= val[0] - 1; 1414 } 1415 1416 sc->sge.iq_start = val[1]; 1417 sc->sge.eq_start = val[2]; 1418 sc->tids.ftid_base = val[3]; 1419 sc->tids.nftids = val[4] - val[3] + 1; 1420 1421 if (caps->toecaps) { 1422 /* query offload-related parameters */ 1423 params[0] = FW_PARAM_DEV(NTID); 1424 params[1] = FW_PARAM_PFVF(SERVER_START); 1425 params[2] = FW_PARAM_PFVF(SERVER_END); 1426 params[3] = FW_PARAM_PFVF(TDDP_START); 1427 params[4] = FW_PARAM_PFVF(TDDP_END); 1428 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1429 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, params, val); 1430 if (rc != 0) { 1431 device_printf(sc->dev, 1432 "failed to query TOE parameters: %d.\n", rc); 1433 goto done; 1434 } 1435 sc->tids.ntids = val[0]; 1436 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1437 sc->tids.stid_base = val[1]; 1438 sc->tids.nstids = val[2] - val[1] + 1; 1439 sc->vres.ddp.start = val[3]; 1440 sc->vres.ddp.size = val[4] - val[3] + 1; 1441 sc->params.ofldq_wr_cred = val[5]; 1442 sc->params.offload = 1; 1443 } 1444 if (caps->rdmacaps) { 1445 params[0] = FW_PARAM_PFVF(STAG_START); 1446 params[1] = FW_PARAM_PFVF(STAG_END); 1447 params[2] = FW_PARAM_PFVF(RQ_START); 1448 params[3] = FW_PARAM_PFVF(RQ_END); 1449 params[4] = FW_PARAM_PFVF(PBL_START); 1450 params[5] = FW_PARAM_PFVF(PBL_END); 1451 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, params, val); 1452 if (rc != 0) { 1453 device_printf(sc->dev, 1454 "failed to query RDMA parameters: %d.\n", rc); 1455 goto done; 1456 } 1457 sc->vres.stag.start = val[0]; 1458 sc->vres.stag.size = val[1] - val[0] + 1; 1459 sc->vres.rq.start = val[2]; 1460 sc->vres.rq.size = val[3] - val[2] + 1; 1461 sc->vres.pbl.start = val[4]; 1462 sc->vres.pbl.size = val[5] - val[4] + 1; 1463 } 1464 if (caps->iscsicaps) { 1465 params[0] = FW_PARAM_PFVF(ISCSI_START); 1466 params[1] = FW_PARAM_PFVF(ISCSI_END); 1467 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, params, val); 1468 if (rc != 0) { 1469 device_printf(sc->dev, 1470 "failed to query iSCSI parameters: %d.\n", rc); 1471 goto done; 1472 } 1473 sc->vres.iscsi.start = val[0]; 1474 sc->vres.iscsi.size = val[1] - val[0] + 1; 1475 } 1476#undef FW_PARAM_PFVF 1477#undef FW_PARAM_DEV 1478 1479done: 1480 return (rc); 1481} 1482 1483static void 1484t4_set_desc(struct adapter *sc) 1485{ 1486 char buf[128]; 1487 struct adapter_params *p = &sc->params; 1488 1489 snprintf(buf, sizeof(buf), 1490 "Chelsio %s (rev %d) %d port %sNIC PCIe-x%d %d %s, S/N:%s, E/C:%s", 1491 p->vpd.id, p->rev, p->nports, is_offload(sc) ? "R" : "", 1492 p->pci.width, sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1493 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), p->vpd.sn, p->vpd.ec); 1494 1495 device_set_desc_copy(sc->dev, buf); 1496} 1497 1498static void 1499build_medialist(struct port_info *pi) 1500{ 1501 struct ifmedia *media = &pi->media; 1502 int data, m; 1503 1504 PORT_LOCK(pi); 1505 1506 ifmedia_removeall(media); 1507 1508 m = IFM_ETHER | IFM_FDX; 1509 data = (pi->port_type << 8) | pi->mod_type; 1510 1511 switch(pi->port_type) { 1512 case FW_PORT_TYPE_BT_XFI: 1513 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1514 break; 1515 1516 case FW_PORT_TYPE_BT_XAUI: 1517 ifmedia_add(media, m | IFM_10G_T, data, NULL); 1518 /* fall through */ 1519 1520 case FW_PORT_TYPE_BT_SGMII: 1521 ifmedia_add(media, m | IFM_1000_T, data, NULL); 1522 ifmedia_add(media, m | IFM_100_TX, data, NULL); 1523 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 1524 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 1525 break; 1526 1527 case FW_PORT_TYPE_CX4: 1528 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 1529 ifmedia_set(media, m | IFM_10G_CX4); 1530 break; 1531 1532 case FW_PORT_TYPE_SFP: 1533 case FW_PORT_TYPE_FIBER_XFI: 1534 case FW_PORT_TYPE_FIBER_XAUI: 1535 switch (pi->mod_type) { 1536 1537 case FW_PORT_MOD_TYPE_LR: 1538 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 1539 ifmedia_set(media, m | IFM_10G_LR); 1540 break; 1541 1542 case FW_PORT_MOD_TYPE_SR: 1543 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 1544 ifmedia_set(media, m | IFM_10G_SR); 1545 break; 1546 1547 case FW_PORT_MOD_TYPE_LRM: 1548 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 1549 ifmedia_set(media, m | IFM_10G_LRM); 1550 break; 1551 1552 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 1553 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 1554 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 1555 ifmedia_set(media, m | IFM_10G_TWINAX); 1556 break; 1557 1558 case FW_PORT_MOD_TYPE_NONE: 1559 m &= ~IFM_FDX; 1560 ifmedia_add(media, m | IFM_NONE, data, NULL); 1561 ifmedia_set(media, m | IFM_NONE); 1562 break; 1563 1564 case FW_PORT_MOD_TYPE_NA: 1565 case FW_PORT_MOD_TYPE_ER: 1566 default: 1567 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 1568 ifmedia_set(media, m | IFM_UNKNOWN); 1569 break; 1570 } 1571 break; 1572 1573 case FW_PORT_TYPE_KX4: 1574 case FW_PORT_TYPE_KX: 1575 case FW_PORT_TYPE_KR: 1576 default: 1577 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 1578 ifmedia_set(media, m | IFM_UNKNOWN); 1579 break; 1580 } 1581 1582 PORT_UNLOCK(pi); 1583} 1584 1585/* 1586 * Program the port's XGMAC based on parameters in ifnet. The caller also 1587 * indicates which parameters should be programmed (the rest are left alone). 1588 */ 1589static int 1590update_mac_settings(struct port_info *pi, int flags) 1591{ 1592 int rc; 1593 struct ifnet *ifp = pi->ifp; 1594 struct adapter *sc = pi->adapter; 1595 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 1596 1597 PORT_LOCK_ASSERT_OWNED(pi); 1598 KASSERT(flags, ("%s: not told what to update.", __func__)); 1599 1600 if (flags & XGMAC_MTU) 1601 mtu = ifp->if_mtu; 1602 1603 if (flags & XGMAC_PROMISC) 1604 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 1605 1606 if (flags & XGMAC_ALLMULTI) 1607 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 1608 1609 if (flags & XGMAC_VLANEX) 1610 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 1611 1612 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 1613 vlanex, false); 1614 if (rc) { 1615 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 1616 return (rc); 1617 } 1618 1619 if (flags & XGMAC_UCADDR) { 1620 uint8_t ucaddr[ETHER_ADDR_LEN]; 1621 1622 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 1623 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 1624 ucaddr, true, true); 1625 if (rc < 0) { 1626 rc = -rc; 1627 if_printf(ifp, "change_mac failed: %d\n", rc); 1628 return (rc); 1629 } else { 1630 pi->xact_addr_filt = rc; 1631 rc = 0; 1632 } 1633 } 1634 1635 if (flags & XGMAC_MCADDRS) { 1636 const uint8_t *mcaddr; 1637 int del = 1; 1638 uint64_t hash = 0; 1639 struct ifmultiaddr *ifma; 1640 1641 if_maddr_rlock(ifp); 1642 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1643 if (ifma->ifma_addr->sa_family != AF_LINK) 1644 continue; 1645 mcaddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 1646 1647 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, del, 1, 1648 &mcaddr, NULL, &hash, 0); 1649 if (rc < 0) { 1650 rc = -rc; 1651 if_printf(ifp, "failed to add mc address" 1652 " %02x:%02x:%02x:%02x:%02x:%02x rc=%d\n", 1653 mcaddr[0], mcaddr[1], mcaddr[2], mcaddr[3], 1654 mcaddr[4], mcaddr[5], rc); 1655 goto mcfail; 1656 } 1657 del = 0; 1658 } 1659 1660 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 1661 if (rc != 0) 1662 if_printf(ifp, "failed to set mc address hash: %d", rc); 1663mcfail: 1664 if_maddr_runlock(ifp); 1665 } 1666 1667 return (rc); 1668} 1669 1670static int 1671cxgbe_init_locked(struct port_info *pi) 1672{ 1673 struct adapter *sc = pi->adapter; 1674 int rc = 0; 1675 1676 ADAPTER_LOCK_ASSERT_OWNED(sc); 1677 1678 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 1679 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) { 1680 rc = EINTR; 1681 goto done; 1682 } 1683 } 1684 if (IS_DOOMED(pi)) { 1685 rc = ENXIO; 1686 goto done; 1687 } 1688 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1689 1690 /* Give up the adapter lock, port init code can sleep. */ 1691 SET_BUSY(sc); 1692 ADAPTER_UNLOCK(sc); 1693 1694 rc = cxgbe_init_synchronized(pi); 1695 1696done: 1697 ADAPTER_LOCK(sc); 1698 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1699 CLR_BUSY(sc); 1700 wakeup_one(&sc->flags); 1701 ADAPTER_UNLOCK(sc); 1702 return (rc); 1703} 1704 1705static int 1706cxgbe_init_synchronized(struct port_info *pi) 1707{ 1708 struct adapter *sc = pi->adapter; 1709 struct ifnet *ifp = pi->ifp; 1710 int rc = 0, i; 1711 uint16_t *rss; 1712 struct sge_rxq *rxq; 1713 1714 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1715 1716 if (isset(&sc->open_device_map, pi->port_id)) { 1717 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 1718 ("mismatch between open_device_map and if_drv_flags")); 1719 return (0); /* already running */ 1720 } 1721 1722 if (sc->open_device_map == 0 && ((rc = first_port_up(sc)) != 0)) 1723 return (rc); /* error message displayed already */ 1724 1725 /* 1726 * Allocate tx/rx/fl queues for this port. 1727 */ 1728 rc = t4_setup_eth_queues(pi); 1729 if (rc != 0) 1730 goto done; /* error message displayed already */ 1731 1732 /* 1733 * Setup RSS for this port. 1734 */ 1735 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 1736 for_each_rxq(pi, i, rxq) { 1737 rss[i] = rxq->iq.abs_id; 1738 } 1739 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 1740 pi->nrxq); 1741 free(rss, M_CXGBE); 1742 if (rc != 0) { 1743 if_printf(ifp, "rss_config failed: %d\n", rc); 1744 goto done; 1745 } 1746 1747 PORT_LOCK(pi); 1748 rc = update_mac_settings(pi, XGMAC_ALL); 1749 PORT_UNLOCK(pi); 1750 if (rc) 1751 goto done; /* error message displayed already */ 1752 1753 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 1754 if (rc != 0) { 1755 if_printf(ifp, "start_link failed: %d\n", rc); 1756 goto done; 1757 } 1758 1759 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 1760 if (rc != 0) { 1761 if_printf(ifp, "enable_vi failed: %d\n", rc); 1762 goto done; 1763 } 1764 pi->flags |= VI_ENABLED; 1765 1766 /* all ok */ 1767 setbit(&sc->open_device_map, pi->port_id); 1768 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1769 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1770 1771 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 1772done: 1773 if (rc != 0) 1774 cxgbe_uninit_synchronized(pi); 1775 1776 return (rc); 1777} 1778 1779static int 1780cxgbe_uninit_locked(struct port_info *pi) 1781{ 1782 struct adapter *sc = pi->adapter; 1783 int rc; 1784 1785 ADAPTER_LOCK_ASSERT_OWNED(sc); 1786 1787 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 1788 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) { 1789 rc = EINTR; 1790 goto done; 1791 } 1792 } 1793 if (IS_DOOMED(pi)) { 1794 rc = ENXIO; 1795 goto done; 1796 } 1797 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 1798 SET_BUSY(sc); 1799 ADAPTER_UNLOCK(sc); 1800 1801 rc = cxgbe_uninit_synchronized(pi); 1802 1803 ADAPTER_LOCK(sc); 1804 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 1805 CLR_BUSY(sc); 1806 wakeup_one(&sc->flags); 1807done: 1808 ADAPTER_UNLOCK(sc); 1809 return (rc); 1810} 1811 1812/* 1813 * Idempotent. 1814 */ 1815static int 1816cxgbe_uninit_synchronized(struct port_info *pi) 1817{ 1818 struct adapter *sc = pi->adapter; 1819 struct ifnet *ifp = pi->ifp; 1820 int rc; 1821 1822 /* 1823 * taskqueue_drain may cause a deadlock if the adapter lock is held. 1824 */ 1825 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1826 1827 /* 1828 * Clear this port's bit from the open device map, and then drain 1829 * tasks and callouts. 1830 */ 1831 clrbit(&sc->open_device_map, pi->port_id); 1832 1833 PORT_LOCK(pi); 1834 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1835 callout_stop(&pi->tick); 1836 PORT_UNLOCK(pi); 1837 callout_drain(&pi->tick); 1838 1839 /* 1840 * Stop and then free the queues' resources, including the queues 1841 * themselves. 1842 * 1843 * XXX: we could just stop the queues here (on ifconfig down) and free 1844 * them later (on port detach), but having up/down go through the entire 1845 * allocate/activate/deactivate/free sequence is a good way to find 1846 * leaks and bugs. 1847 */ 1848 rc = t4_teardown_eth_queues(pi); 1849 if (rc != 0) 1850 if_printf(ifp, "teardown failed: %d\n", rc); 1851 1852 if (pi->flags & VI_ENABLED) { 1853 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 1854 if (rc) 1855 if_printf(ifp, "disable_vi failed: %d\n", rc); 1856 else 1857 pi->flags &= ~VI_ENABLED; 1858 } 1859 1860 pi->link_cfg.link_ok = 0; 1861 pi->link_cfg.speed = 0; 1862 t4_os_link_changed(sc, pi->port_id, 0); 1863 1864 if (sc->open_device_map == 0) 1865 last_port_down(sc); 1866 1867 return (0); 1868} 1869 1870#define T4_ALLOC_IRQ(sc, irqid, rid, handler, arg, name) do { \ 1871 rc = t4_alloc_irq(sc, &sc->irq[irqid], rid, handler, arg, name); \ 1872 if (rc != 0) \ 1873 goto done; \ 1874} while (0) 1875static int 1876first_port_up(struct adapter *sc) 1877{ 1878 int rc, i; 1879 char name[8]; 1880 1881 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1882 1883 /* 1884 * The firmware event queue and the optional forwarded interrupt queues. 1885 */ 1886 rc = t4_setup_adapter_iqs(sc); 1887 if (rc != 0) 1888 goto done; 1889 1890 /* 1891 * Setup interrupts. 1892 */ 1893 if (sc->intr_count == 1) { 1894 KASSERT(sc->flags & INTR_FWD, 1895 ("%s: single interrupt but not forwarded?", __func__)); 1896 T4_ALLOC_IRQ(sc, 0, 0, t4_intr_all, sc, "all"); 1897 } else { 1898 /* Multiple interrupts. The first one is always error intr */ 1899 T4_ALLOC_IRQ(sc, 0, 1, t4_intr_err, sc, "err"); 1900 1901 if (sc->flags & INTR_FWD) { 1902 /* The rest are shared by the fwq and all data intr */ 1903 for (i = 1; i < sc->intr_count; i++) { 1904 snprintf(name, sizeof(name), "mux%d", i - 1); 1905 T4_ALLOC_IRQ(sc, i, i + 1, t4_intr_fwd, 1906 &sc->sge.fiq[i - 1], name); 1907 } 1908 } else { 1909 struct port_info *pi; 1910 int p, q; 1911 1912 T4_ALLOC_IRQ(sc, 1, 2, t4_intr_evt, &sc->sge.fwq, 1913 "evt"); 1914 1915 p = q = 0; 1916 pi = sc->port[p]; 1917 for (i = 2; i < sc->intr_count; i++) { 1918 snprintf(name, sizeof(name), "p%dq%d", p, q); 1919 if (++q >= pi->nrxq) { 1920 p++; 1921 q = 0; 1922 pi = sc->port[p]; 1923 } 1924 T4_ALLOC_IRQ(sc, i, i + 1, t4_intr_data, 1925 &sc->sge.rxq[i - 2], name); 1926 } 1927 } 1928 } 1929 1930 t4_intr_enable(sc); 1931 sc->flags |= FULL_INIT_DONE; 1932 1933done: 1934 if (rc != 0) 1935 last_port_down(sc); 1936 1937 return (rc); 1938} 1939#undef T4_ALLOC_IRQ 1940 1941/* 1942 * Idempotent. 1943 */ 1944static int 1945last_port_down(struct adapter *sc) 1946{ 1947 int i; 1948 1949 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1950 1951 t4_intr_disable(sc); 1952 1953 t4_teardown_adapter_iqs(sc); 1954 1955 for (i = 0; i < sc->intr_count; i++) 1956 t4_free_irq(sc, &sc->irq[i]); 1957 1958 sc->flags &= ~FULL_INIT_DONE; 1959 1960 return (0); 1961} 1962 1963static int 1964t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 1965 iq_intr_handler_t *handler, void *arg, char *name) 1966{ 1967 int rc; 1968 1969 irq->rid = rid; 1970 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 1971 RF_SHAREABLE | RF_ACTIVE); 1972 if (irq->res == NULL) { 1973 device_printf(sc->dev, 1974 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 1975 return (ENOMEM); 1976 } 1977 1978 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 1979 NULL, handler, arg, &irq->tag); 1980 if (rc != 0) { 1981 device_printf(sc->dev, 1982 "failed to setup interrupt for rid %d, name %s: %d\n", 1983 rid, name, rc); 1984 } else if (name) 1985 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 1986 1987 return (rc); 1988} 1989 1990static int 1991t4_free_irq(struct adapter *sc, struct irq *irq) 1992{ 1993 if (irq->tag) 1994 bus_teardown_intr(sc->dev, irq->res, irq->tag); 1995 if (irq->res) 1996 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 1997 1998 bzero(irq, sizeof(*irq)); 1999 2000 return (0); 2001} 2002 2003static void 2004reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 2005 unsigned int end) 2006{ 2007 uint32_t *p = (uint32_t *)(buf + start); 2008 2009 for ( ; start <= end; start += sizeof(uint32_t)) 2010 *p++ = t4_read_reg(sc, start); 2011} 2012 2013static void 2014t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 2015{ 2016 int i; 2017 static const unsigned int reg_ranges[] = { 2018 0x1008, 0x1108, 2019 0x1180, 0x11b4, 2020 0x11fc, 0x123c, 2021 0x1300, 0x173c, 2022 0x1800, 0x18fc, 2023 0x3000, 0x30d8, 2024 0x30e0, 0x5924, 2025 0x5960, 0x59d4, 2026 0x5a00, 0x5af8, 2027 0x6000, 0x6098, 2028 0x6100, 0x6150, 2029 0x6200, 0x6208, 2030 0x6240, 0x6248, 2031 0x6280, 0x6338, 2032 0x6370, 0x638c, 2033 0x6400, 0x643c, 2034 0x6500, 0x6524, 2035 0x6a00, 0x6a38, 2036 0x6a60, 0x6a78, 2037 0x6b00, 0x6b84, 2038 0x6bf0, 0x6c84, 2039 0x6cf0, 0x6d84, 2040 0x6df0, 0x6e84, 2041 0x6ef0, 0x6f84, 2042 0x6ff0, 0x7084, 2043 0x70f0, 0x7184, 2044 0x71f0, 0x7284, 2045 0x72f0, 0x7384, 2046 0x73f0, 0x7450, 2047 0x7500, 0x7530, 2048 0x7600, 0x761c, 2049 0x7680, 0x76cc, 2050 0x7700, 0x7798, 2051 0x77c0, 0x77fc, 2052 0x7900, 0x79fc, 2053 0x7b00, 0x7c38, 2054 0x7d00, 0x7efc, 2055 0x8dc0, 0x8e1c, 2056 0x8e30, 0x8e78, 2057 0x8ea0, 0x8f6c, 2058 0x8fc0, 0x9074, 2059 0x90fc, 0x90fc, 2060 0x9400, 0x9458, 2061 0x9600, 0x96bc, 2062 0x9800, 0x9808, 2063 0x9820, 0x983c, 2064 0x9850, 0x9864, 2065 0x9c00, 0x9c6c, 2066 0x9c80, 0x9cec, 2067 0x9d00, 0x9d6c, 2068 0x9d80, 0x9dec, 2069 0x9e00, 0x9e6c, 2070 0x9e80, 0x9eec, 2071 0x9f00, 0x9f6c, 2072 0x9f80, 0x9fec, 2073 0xd004, 0xd03c, 2074 0xdfc0, 0xdfe0, 2075 0xe000, 0xea7c, 2076 0xf000, 0x11190, 2077 0x19040, 0x19124, 2078 0x19150, 0x191b0, 2079 0x191d0, 0x191e8, 2080 0x19238, 0x1924c, 2081 0x193f8, 0x19474, 2082 0x19490, 0x194f8, 2083 0x19800, 0x19f30, 2084 0x1a000, 0x1a06c, 2085 0x1a0b0, 0x1a120, 2086 0x1a128, 0x1a138, 2087 0x1a190, 0x1a1c4, 2088 0x1a1fc, 0x1a1fc, 2089 0x1e040, 0x1e04c, 2090 0x1e240, 0x1e28c, 2091 0x1e2c0, 0x1e2c0, 2092 0x1e2e0, 0x1e2e0, 2093 0x1e300, 0x1e384, 2094 0x1e3c0, 0x1e3c8, 2095 0x1e440, 0x1e44c, 2096 0x1e640, 0x1e68c, 2097 0x1e6c0, 0x1e6c0, 2098 0x1e6e0, 0x1e6e0, 2099 0x1e700, 0x1e784, 2100 0x1e7c0, 0x1e7c8, 2101 0x1e840, 0x1e84c, 2102 0x1ea40, 0x1ea8c, 2103 0x1eac0, 0x1eac0, 2104 0x1eae0, 0x1eae0, 2105 0x1eb00, 0x1eb84, 2106 0x1ebc0, 0x1ebc8, 2107 0x1ec40, 0x1ec4c, 2108 0x1ee40, 0x1ee8c, 2109 0x1eec0, 0x1eec0, 2110 0x1eee0, 0x1eee0, 2111 0x1ef00, 0x1ef84, 2112 0x1efc0, 0x1efc8, 2113 0x1f040, 0x1f04c, 2114 0x1f240, 0x1f28c, 2115 0x1f2c0, 0x1f2c0, 2116 0x1f2e0, 0x1f2e0, 2117 0x1f300, 0x1f384, 2118 0x1f3c0, 0x1f3c8, 2119 0x1f440, 0x1f44c, 2120 0x1f640, 0x1f68c, 2121 0x1f6c0, 0x1f6c0, 2122 0x1f6e0, 0x1f6e0, 2123 0x1f700, 0x1f784, 2124 0x1f7c0, 0x1f7c8, 2125 0x1f840, 0x1f84c, 2126 0x1fa40, 0x1fa8c, 2127 0x1fac0, 0x1fac0, 2128 0x1fae0, 0x1fae0, 2129 0x1fb00, 0x1fb84, 2130 0x1fbc0, 0x1fbc8, 2131 0x1fc40, 0x1fc4c, 2132 0x1fe40, 0x1fe8c, 2133 0x1fec0, 0x1fec0, 2134 0x1fee0, 0x1fee0, 2135 0x1ff00, 0x1ff84, 2136 0x1ffc0, 0x1ffc8, 2137 0x20000, 0x2002c, 2138 0x20100, 0x2013c, 2139 0x20190, 0x201c8, 2140 0x20200, 0x20318, 2141 0x20400, 0x20528, 2142 0x20540, 0x20614, 2143 0x21000, 0x21040, 2144 0x2104c, 0x21060, 2145 0x210c0, 0x210ec, 2146 0x21200, 0x21268, 2147 0x21270, 0x21284, 2148 0x212fc, 0x21388, 2149 0x21400, 0x21404, 2150 0x21500, 0x21518, 2151 0x2152c, 0x2153c, 2152 0x21550, 0x21554, 2153 0x21600, 0x21600, 2154 0x21608, 0x21628, 2155 0x21630, 0x2163c, 2156 0x21700, 0x2171c, 2157 0x21780, 0x2178c, 2158 0x21800, 0x21c38, 2159 0x21c80, 0x21d7c, 2160 0x21e00, 0x21e04, 2161 0x22000, 0x2202c, 2162 0x22100, 0x2213c, 2163 0x22190, 0x221c8, 2164 0x22200, 0x22318, 2165 0x22400, 0x22528, 2166 0x22540, 0x22614, 2167 0x23000, 0x23040, 2168 0x2304c, 0x23060, 2169 0x230c0, 0x230ec, 2170 0x23200, 0x23268, 2171 0x23270, 0x23284, 2172 0x232fc, 0x23388, 2173 0x23400, 0x23404, 2174 0x23500, 0x23518, 2175 0x2352c, 0x2353c, 2176 0x23550, 0x23554, 2177 0x23600, 0x23600, 2178 0x23608, 0x23628, 2179 0x23630, 0x2363c, 2180 0x23700, 0x2371c, 2181 0x23780, 0x2378c, 2182 0x23800, 0x23c38, 2183 0x23c80, 0x23d7c, 2184 0x23e00, 0x23e04, 2185 0x24000, 0x2402c, 2186 0x24100, 0x2413c, 2187 0x24190, 0x241c8, 2188 0x24200, 0x24318, 2189 0x24400, 0x24528, 2190 0x24540, 0x24614, 2191 0x25000, 0x25040, 2192 0x2504c, 0x25060, 2193 0x250c0, 0x250ec, 2194 0x25200, 0x25268, 2195 0x25270, 0x25284, 2196 0x252fc, 0x25388, 2197 0x25400, 0x25404, 2198 0x25500, 0x25518, 2199 0x2552c, 0x2553c, 2200 0x25550, 0x25554, 2201 0x25600, 0x25600, 2202 0x25608, 0x25628, 2203 0x25630, 0x2563c, 2204 0x25700, 0x2571c, 2205 0x25780, 0x2578c, 2206 0x25800, 0x25c38, 2207 0x25c80, 0x25d7c, 2208 0x25e00, 0x25e04, 2209 0x26000, 0x2602c, 2210 0x26100, 0x2613c, 2211 0x26190, 0x261c8, 2212 0x26200, 0x26318, 2213 0x26400, 0x26528, 2214 0x26540, 0x26614, 2215 0x27000, 0x27040, 2216 0x2704c, 0x27060, 2217 0x270c0, 0x270ec, 2218 0x27200, 0x27268, 2219 0x27270, 0x27284, 2220 0x272fc, 0x27388, 2221 0x27400, 0x27404, 2222 0x27500, 0x27518, 2223 0x2752c, 0x2753c, 2224 0x27550, 0x27554, 2225 0x27600, 0x27600, 2226 0x27608, 0x27628, 2227 0x27630, 0x2763c, 2228 0x27700, 0x2771c, 2229 0x27780, 0x2778c, 2230 0x27800, 0x27c38, 2231 0x27c80, 0x27d7c, 2232 0x27e00, 0x27e04 2233 }; 2234 2235 regs->version = 4 | (sc->params.rev << 10); 2236 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2) 2237 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 2238} 2239 2240static void 2241cxgbe_tick(void *arg) 2242{ 2243 struct port_info *pi = arg; 2244 struct ifnet *ifp = pi->ifp; 2245 struct sge_txq *txq; 2246 int i, drops; 2247 struct port_stats *s = &pi->stats; 2248 2249 PORT_LOCK(pi); 2250 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2251 PORT_UNLOCK(pi); 2252 return; /* without scheduling another callout */ 2253 } 2254 2255 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 2256 2257 ifp->if_opackets = s->tx_frames; 2258 ifp->if_ipackets = s->rx_frames; 2259 ifp->if_obytes = s->tx_octets; 2260 ifp->if_ibytes = s->rx_octets; 2261 ifp->if_omcasts = s->tx_mcast_frames; 2262 ifp->if_imcasts = s->rx_mcast_frames; 2263 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2264 s->rx_ovflow3; 2265 2266 drops = s->tx_drop; 2267 for_each_txq(pi, i, txq) 2268 drops += txq->eq.br->br_drops; 2269 ifp->if_snd.ifq_drops = drops; 2270 2271 ifp->if_oerrors = s->tx_error_frames; 2272 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 2273 s->rx_fcs_err + s->rx_len_err; 2274 2275 callout_schedule(&pi->tick, hz); 2276 PORT_UNLOCK(pi); 2277} 2278 2279static int 2280t4_sysctls(struct adapter *sc) 2281{ 2282 struct sysctl_ctx_list *ctx; 2283 struct sysctl_oid *oid; 2284 struct sysctl_oid_list *children; 2285 2286 ctx = device_get_sysctl_ctx(sc->dev); 2287 oid = device_get_sysctl_tree(sc->dev); 2288 children = SYSCTL_CHILDREN(oid); 2289 2290 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, 2291 &sc->params.nports, 0, "# of ports"); 2292 2293 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 2294 &sc->params.rev, 0, "chip hardware revision"); 2295 2296 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 2297 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 2298 2299 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "TOE", CTLFLAG_RD, 2300 &sc->params.offload, 0, "hardware is capable of TCP offload"); 2301 2302 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, 2303 &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)"); 2304 2305 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 2306 CTLTYPE_STRING | CTLFLAG_RD, &intr_timer, sizeof(intr_timer), 2307 sysctl_int_array, "A", "interrupt holdoff timer values (us)"); 2308 2309 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 2310 CTLTYPE_STRING | CTLFLAG_RD, &intr_pktcount, sizeof(intr_pktcount), 2311 sysctl_int_array, "A", "interrupt holdoff packet counter values"); 2312 2313 return (0); 2314} 2315 2316static int 2317cxgbe_sysctls(struct port_info *pi) 2318{ 2319 struct sysctl_ctx_list *ctx; 2320 struct sysctl_oid *oid; 2321 struct sysctl_oid_list *children; 2322 2323 ctx = device_get_sysctl_ctx(pi->dev); 2324 2325 /* 2326 * dev.cxgbe.X. 2327 */ 2328 oid = device_get_sysctl_tree(pi->dev); 2329 children = SYSCTL_CHILDREN(oid); 2330 2331 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 2332 &pi->nrxq, 0, "# of rx queues"); 2333 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 2334 &pi->ntxq, 0, "# of tx queues"); 2335 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 2336 &pi->first_rxq, 0, "index of first rx queue"); 2337 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 2338 &pi->first_txq, 0, "index of first tx queue"); 2339 2340 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 2341 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 2342 "holdoff timer index"); 2343 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 2344 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 2345 "holdoff packet counter index"); 2346 2347 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 2348 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 2349 "rx queue size"); 2350 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 2351 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 2352 "tx queue size"); 2353 2354 /* 2355 * dev.cxgbe.X.stats. 2356 */ 2357 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 2358 NULL, "port statistics"); 2359 children = SYSCTL_CHILDREN(oid); 2360 2361#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 2362 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 2363 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 2364 sysctl_handle_t4_reg64, "QU", desc) 2365 2366 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 2367 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 2368 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 2369 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 2370 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 2371 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 2372 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 2373 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 2374 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 2375 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 2376 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 2377 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 2378 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 2379 "# of tx frames in this range", 2380 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 2381 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 2382 "# of tx frames in this range", 2383 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 2384 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 2385 "# of tx frames in this range", 2386 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 2387 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 2388 "# of tx frames in this range", 2389 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 2390 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 2391 "# of tx frames in this range", 2392 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 2393 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 2394 "# of tx frames in this range", 2395 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 2396 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 2397 "# of tx frames in this range", 2398 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 2399 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 2400 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 2401 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 2402 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 2403 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 2404 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 2405 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 2406 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 2407 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 2408 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 2409 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 2410 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 2411 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 2412 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 2413 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 2414 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 2415 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 2416 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 2417 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 2418 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 2419 2420 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 2421 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 2422 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 2423 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 2424 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 2425 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 2426 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 2427 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 2428 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 2429 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 2430 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 2431 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 2432 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 2433 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 2434 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 2435 "# of frames received with bad FCS", 2436 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 2437 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 2438 "# of frames received with length error", 2439 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 2440 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 2441 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 2442 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 2443 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 2444 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 2445 "# of rx frames in this range", 2446 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 2447 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 2448 "# of rx frames in this range", 2449 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 2450 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 2451 "# of rx frames in this range", 2452 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 2453 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 2454 "# of rx frames in this range", 2455 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 2456 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 2457 "# of rx frames in this range", 2458 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 2459 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 2460 "# of rx frames in this range", 2461 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 2462 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 2463 "# of rx frames in this range", 2464 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 2465 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 2466 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 2467 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 2468 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 2469 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 2470 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 2471 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 2472 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 2473 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 2474 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 2475 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 2476 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 2477 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 2478 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 2479 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 2480 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 2481 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 2482 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 2483 2484#undef SYSCTL_ADD_T4_REG64 2485 2486#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 2487 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 2488 &pi->stats.name, desc) 2489 2490 /* We get these from port_stats and they may be stale by upto 1s */ 2491 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 2492 "# drops due to buffer-group 0 overflows"); 2493 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 2494 "# drops due to buffer-group 1 overflows"); 2495 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 2496 "# drops due to buffer-group 2 overflows"); 2497 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 2498 "# drops due to buffer-group 3 overflows"); 2499 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 2500 "# of buffer-group 0 truncated packets"); 2501 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 2502 "# of buffer-group 1 truncated packets"); 2503 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 2504 "# of buffer-group 2 truncated packets"); 2505 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 2506 "# of buffer-group 3 truncated packets"); 2507 2508#undef SYSCTL_ADD_T4_PORTSTAT 2509 2510 return (0); 2511} 2512 2513static int 2514sysctl_int_array(SYSCTL_HANDLER_ARGS) 2515{ 2516 int rc, *i; 2517 struct sbuf sb; 2518 2519 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 2520 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 2521 sbuf_printf(&sb, "%d ", *i); 2522 sbuf_trim(&sb); 2523 sbuf_finish(&sb); 2524 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 2525 sbuf_delete(&sb); 2526 return (rc); 2527} 2528 2529static int 2530sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 2531{ 2532 struct port_info *pi = arg1; 2533 struct adapter *sc = pi->adapter; 2534 struct sge_rxq *rxq; 2535 int idx, rc, i; 2536 2537 idx = pi->tmr_idx; 2538 2539 rc = sysctl_handle_int(oidp, &idx, 0, req); 2540 if (rc != 0 || req->newptr == NULL) 2541 return (rc); 2542 2543 if (idx < 0 || idx >= SGE_NTIMERS) 2544 return (EINVAL); 2545 2546 ADAPTER_LOCK(sc); 2547 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 2548 if (rc == 0) { 2549 for_each_rxq(pi, i, rxq) { 2550 rxq->iq.intr_params = V_QINTR_TIMER_IDX(idx) | 2551 V_QINTR_CNT_EN(pi->pktc_idx != -1); 2552 } 2553 pi->tmr_idx = idx; 2554 } 2555 2556 ADAPTER_UNLOCK(sc); 2557 return (rc); 2558} 2559 2560static int 2561sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 2562{ 2563 struct port_info *pi = arg1; 2564 struct adapter *sc = pi->adapter; 2565 int idx, rc; 2566 2567 idx = pi->pktc_idx; 2568 2569 rc = sysctl_handle_int(oidp, &idx, 0, req); 2570 if (rc != 0 || req->newptr == NULL) 2571 return (rc); 2572 2573 if (idx < -1 || idx >= SGE_NCOUNTERS) 2574 return (EINVAL); 2575 2576 ADAPTER_LOCK(sc); 2577 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 2578 if (rc == 0 && pi->ifp->if_drv_flags & IFF_DRV_RUNNING) 2579 rc = EBUSY; /* can be changed only when port is down */ 2580 2581 if (rc == 0) 2582 pi->pktc_idx = idx; 2583 2584 ADAPTER_UNLOCK(sc); 2585 return (rc); 2586} 2587 2588static int 2589sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 2590{ 2591 struct port_info *pi = arg1; 2592 struct adapter *sc = pi->adapter; 2593 int qsize, rc; 2594 2595 qsize = pi->qsize_rxq; 2596 2597 rc = sysctl_handle_int(oidp, &qsize, 0, req); 2598 if (rc != 0 || req->newptr == NULL) 2599 return (rc); 2600 2601 if (qsize < 128 || (qsize & 7)) 2602 return (EINVAL); 2603 2604 ADAPTER_LOCK(sc); 2605 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 2606 if (rc == 0 && pi->ifp->if_drv_flags & IFF_DRV_RUNNING) 2607 rc = EBUSY; /* can be changed only when port is down */ 2608 2609 if (rc == 0) 2610 pi->qsize_rxq = qsize; 2611 2612 ADAPTER_UNLOCK(sc); 2613 return (rc); 2614} 2615 2616static int 2617sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 2618{ 2619 struct port_info *pi = arg1; 2620 struct adapter *sc = pi->adapter; 2621 int qsize, rc; 2622 2623 qsize = pi->qsize_txq; 2624 2625 rc = sysctl_handle_int(oidp, &qsize, 0, req); 2626 if (rc != 0 || req->newptr == NULL) 2627 return (rc); 2628 2629 if (qsize < 128) 2630 return (EINVAL); 2631 2632 ADAPTER_LOCK(sc); 2633 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 2634 if (rc == 0 && pi->ifp->if_drv_flags & IFF_DRV_RUNNING) 2635 rc = EBUSY; /* can be changed only when port is down */ 2636 2637 if (rc == 0) 2638 pi->qsize_txq = qsize; 2639 2640 ADAPTER_UNLOCK(sc); 2641 return (rc); 2642} 2643 2644static int 2645sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 2646{ 2647 struct adapter *sc = arg1; 2648 int reg = arg2; 2649 uint64_t val; 2650 2651 val = t4_read_reg64(sc, reg); 2652 2653 return (sysctl_handle_64(oidp, &val, 0, req)); 2654} 2655 2656static inline void 2657txq_start(struct ifnet *ifp, struct sge_txq *txq) 2658{ 2659 struct buf_ring *br; 2660 struct mbuf *m; 2661 2662 TXQ_LOCK_ASSERT_OWNED(txq); 2663 2664 br = txq->eq.br; 2665 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 2666 if (m) 2667 t4_eth_tx(ifp, txq, m); 2668} 2669 2670void 2671cxgbe_txq_start(void *arg, int count) 2672{ 2673 struct sge_txq *txq = arg; 2674 2675 TXQ_LOCK(txq); 2676 txq_start(txq->ifp, txq); 2677 TXQ_UNLOCK(txq); 2678} 2679 2680int 2681t4_os_find_pci_capability(struct adapter *sc, int cap) 2682{ 2683 device_t dev; 2684 struct pci_devinfo *dinfo; 2685 pcicfgregs *cfg; 2686 uint32_t status; 2687 uint8_t ptr; 2688 2689 dev = sc->dev; 2690 dinfo = device_get_ivars(dev); 2691 cfg = &dinfo->cfg; 2692 2693 status = pci_read_config(dev, PCIR_STATUS, 2); 2694 if (!(status & PCIM_STATUS_CAPPRESENT)) 2695 return (0); 2696 2697 switch (cfg->hdrtype & PCIM_HDRTYPE) { 2698 case 0: 2699 case 1: 2700 ptr = PCIR_CAP_PTR; 2701 break; 2702 case 2: 2703 ptr = PCIR_CAP_PTR_2; 2704 break; 2705 default: 2706 return (0); 2707 break; 2708 } 2709 ptr = pci_read_config(dev, ptr, 1); 2710 2711 while (ptr != 0) { 2712 if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap) 2713 return (ptr); 2714 ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1); 2715 } 2716 2717 return (0); 2718} 2719 2720int 2721t4_os_pci_save_state(struct adapter *sc) 2722{ 2723 device_t dev; 2724 struct pci_devinfo *dinfo; 2725 2726 dev = sc->dev; 2727 dinfo = device_get_ivars(dev); 2728 2729 pci_cfg_save(dev, dinfo, 0); 2730 return (0); 2731} 2732 2733int 2734t4_os_pci_restore_state(struct adapter *sc) 2735{ 2736 device_t dev; 2737 struct pci_devinfo *dinfo; 2738 2739 dev = sc->dev; 2740 dinfo = device_get_ivars(dev); 2741 2742 pci_cfg_restore(dev, dinfo); 2743 return (0); 2744} 2745 2746void 2747t4_os_portmod_changed(const struct adapter *sc, int idx) 2748{ 2749 struct port_info *pi = sc->port[idx]; 2750 static const char *mod_str[] = { 2751 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX" 2752 }; 2753 2754 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 2755 if_printf(pi->ifp, "transceiver unplugged.\n"); 2756 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) { 2757 if_printf(pi->ifp, "%s transceiver inserted.\n", 2758 mod_str[pi->mod_type]); 2759 } else { 2760 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 2761 pi->mod_type); 2762 } 2763} 2764 2765void 2766t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 2767{ 2768 struct port_info *pi = sc->port[idx]; 2769 struct ifnet *ifp = pi->ifp; 2770 2771 if (link_stat) { 2772 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 2773 if_link_state_change(ifp, LINK_STATE_UP); 2774 } else 2775 if_link_state_change(ifp, LINK_STATE_DOWN); 2776} 2777 2778static int 2779t4_open(struct cdev *dev, int flags, int type, struct thread *td) 2780{ 2781 return (0); 2782} 2783 2784static int 2785t4_close(struct cdev *dev, int flags, int type, struct thread *td) 2786{ 2787 return (0); 2788} 2789 2790static int 2791t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 2792 struct thread *td) 2793{ 2794 int rc; 2795 struct adapter *sc = dev->si_drv1; 2796 2797 rc = priv_check(td, PRIV_DRIVER); 2798 if (rc != 0) 2799 return (rc); 2800 2801 switch (cmd) { 2802 case CHELSIO_T4_GETREG32: { 2803 struct t4_reg32 *edata = (struct t4_reg32 *)data; 2804 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2805 return (EFAULT); 2806 edata->val = t4_read_reg(sc, edata->addr); 2807 break; 2808 } 2809 case CHELSIO_T4_SETREG32: { 2810 struct t4_reg32 *edata = (struct t4_reg32 *)data; 2811 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 2812 return (EFAULT); 2813 t4_write_reg(sc, edata->addr, edata->val); 2814 break; 2815 } 2816 case CHELSIO_T4_REGDUMP: { 2817 struct t4_regdump *regs = (struct t4_regdump *)data; 2818 int reglen = T4_REGDUMP_SIZE; 2819 uint8_t *buf; 2820 2821 if (regs->len < reglen) { 2822 regs->len = reglen; /* hint to the caller */ 2823 return (ENOBUFS); 2824 } 2825 2826 regs->len = reglen; 2827 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 2828 t4_get_regs(sc, regs, buf); 2829 rc = copyout(buf, regs->data, reglen); 2830 free(buf, M_CXGBE); 2831 break; 2832 } 2833 default: 2834 rc = EINVAL; 2835 } 2836 2837 return (rc); 2838} 2839 2840static int 2841t4_mod_event(module_t mod, int cmd, void *arg) 2842{ 2843 2844 if (cmd == MOD_LOAD) 2845 t4_sge_modload(); 2846 2847 return (0); 2848} 2849 2850static devclass_t t4_devclass; 2851static devclass_t cxgbe_devclass; 2852 2853DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 2854MODULE_VERSION(t4nex, 1); 2855 2856DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 2857MODULE_VERSION(cxgbe, 1); 2858