t4_main.c revision 240680
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_main.c 240680 2012-09-18 22:04:59Z gavin $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58 59#include "common/common.h" 60#include "common/t4_msg.h" 61#include "common/t4_regs.h" 62#include "common/t4_regs_values.h" 63#include "t4_ioctl.h" 64#include "t4_l2t.h" 65 66/* T4 bus driver interface */ 67static int t4_probe(device_t); 68static int t4_attach(device_t); 69static int t4_detach(device_t); 70static device_method_t t4_methods[] = { 71 DEVMETHOD(device_probe, t4_probe), 72 DEVMETHOD(device_attach, t4_attach), 73 DEVMETHOD(device_detach, t4_detach), 74 75 DEVMETHOD_END 76}; 77static driver_t t4_driver = { 78 "t4nex", 79 t4_methods, 80 sizeof(struct adapter) 81}; 82 83 84/* T4 port (cxgbe) interface */ 85static int cxgbe_probe(device_t); 86static int cxgbe_attach(device_t); 87static int cxgbe_detach(device_t); 88static device_method_t cxgbe_methods[] = { 89 DEVMETHOD(device_probe, cxgbe_probe), 90 DEVMETHOD(device_attach, cxgbe_attach), 91 DEVMETHOD(device_detach, cxgbe_detach), 92 { 0, 0 } 93}; 94static driver_t cxgbe_driver = { 95 "cxgbe", 96 cxgbe_methods, 97 sizeof(struct port_info) 98}; 99 100static d_ioctl_t t4_ioctl; 101static d_open_t t4_open; 102static d_close_t t4_close; 103 104static struct cdevsw t4_cdevsw = { 105 .d_version = D_VERSION, 106 .d_flags = 0, 107 .d_open = t4_open, 108 .d_close = t4_close, 109 .d_ioctl = t4_ioctl, 110 .d_name = "t4nex", 111}; 112 113/* ifnet + media interface */ 114static void cxgbe_init(void *); 115static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 116static int cxgbe_transmit(struct ifnet *, struct mbuf *); 117static void cxgbe_qflush(struct ifnet *); 118static int cxgbe_media_change(struct ifnet *); 119static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 120 121MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services"); 122 123/* 124 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 125 * then ADAPTER_LOCK, then t4_uld_list_lock. 126 */ 127static struct mtx t4_list_lock; 128static SLIST_HEAD(, adapter) t4_list; 129#ifdef TCP_OFFLOAD 130static struct mtx t4_uld_list_lock; 131static SLIST_HEAD(, uld_info) t4_uld_list; 132#endif 133 134/* 135 * Tunables. See tweak_tunables() too. 136 */ 137 138/* 139 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 140 */ 141#define NTXQ_10G 16 142static int t4_ntxq10g = -1; 143TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 144 145#define NRXQ_10G 8 146static int t4_nrxq10g = -1; 147TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 148 149#define NTXQ_1G 4 150static int t4_ntxq1g = -1; 151TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 152 153#define NRXQ_1G 2 154static int t4_nrxq1g = -1; 155TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 156 157#ifdef TCP_OFFLOAD 158#define NOFLDTXQ_10G 8 159static int t4_nofldtxq10g = -1; 160TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 161 162#define NOFLDRXQ_10G 2 163static int t4_nofldrxq10g = -1; 164TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 165 166#define NOFLDTXQ_1G 2 167static int t4_nofldtxq1g = -1; 168TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 169 170#define NOFLDRXQ_1G 1 171static int t4_nofldrxq1g = -1; 172TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 173#endif 174 175/* 176 * Holdoff parameters for 10G and 1G ports. 177 */ 178#define TMR_IDX_10G 1 179static int t4_tmr_idx_10g = TMR_IDX_10G; 180TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 181 182#define PKTC_IDX_10G (-1) 183static int t4_pktc_idx_10g = PKTC_IDX_10G; 184TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 185 186#define TMR_IDX_1G 1 187static int t4_tmr_idx_1g = TMR_IDX_1G; 188TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 189 190#define PKTC_IDX_1G (-1) 191static int t4_pktc_idx_1g = PKTC_IDX_1G; 192TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 193 194/* 195 * Size (# of entries) of each tx and rx queue. 196 */ 197static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 198TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 199 200static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 201TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 202 203/* 204 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 205 */ 206static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 207TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 208 209/* 210 * Configuration file. 211 */ 212static char t4_cfg_file[32] = "default"; 213TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 214 215/* 216 * ASIC features that will be used. Disable the ones you don't want so that the 217 * chip resources aren't wasted on features that will not be used. 218 */ 219static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 220TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 221 222static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 223TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 224 225static int t4_toecaps_allowed = -1; 226TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 227 228static int t4_rdmacaps_allowed = 0; 229TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 230 231static int t4_iscsicaps_allowed = 0; 232TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 233 234static int t4_fcoecaps_allowed = 0; 235TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 236 237struct intrs_and_queues { 238 int intr_type; /* INTx, MSI, or MSI-X */ 239 int nirq; /* Number of vectors */ 240 int intr_flags; 241 int ntxq10g; /* # of NIC txq's for each 10G port */ 242 int nrxq10g; /* # of NIC rxq's for each 10G port */ 243 int ntxq1g; /* # of NIC txq's for each 1G port */ 244 int nrxq1g; /* # of NIC rxq's for each 1G port */ 245#ifdef TCP_OFFLOAD 246 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 247 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 248 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 249 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 250#endif 251}; 252 253struct filter_entry { 254 uint32_t valid:1; /* filter allocated and valid */ 255 uint32_t locked:1; /* filter is administratively locked */ 256 uint32_t pending:1; /* filter action is pending firmware reply */ 257 uint32_t smtidx:8; /* Source MAC Table index for smac */ 258 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 259 260 struct t4_filter_specification fs; 261}; 262 263enum { 264 XGMAC_MTU = (1 << 0), 265 XGMAC_PROMISC = (1 << 1), 266 XGMAC_ALLMULTI = (1 << 2), 267 XGMAC_VLANEX = (1 << 3), 268 XGMAC_UCADDR = (1 << 4), 269 XGMAC_MCADDRS = (1 << 5), 270 271 XGMAC_ALL = 0xffff 272}; 273 274static int map_bars(struct adapter *); 275static void setup_memwin(struct adapter *); 276static int cfg_itype_and_nqueues(struct adapter *, int, int, 277 struct intrs_and_queues *); 278static int prep_firmware(struct adapter *); 279static int upload_config_file(struct adapter *, const struct firmware *, 280 uint32_t *, uint32_t *); 281static int partition_resources(struct adapter *, const struct firmware *); 282static int get_params__pre_init(struct adapter *); 283static int get_params__post_init(struct adapter *); 284static void t4_set_desc(struct adapter *); 285static void build_medialist(struct port_info *); 286static int update_mac_settings(struct port_info *, int); 287static int cxgbe_init_locked(struct port_info *); 288static int cxgbe_init_synchronized(struct port_info *); 289static int cxgbe_uninit_locked(struct port_info *); 290static int cxgbe_uninit_synchronized(struct port_info *); 291static int setup_intr_handlers(struct adapter *); 292static int adapter_full_init(struct adapter *); 293static int adapter_full_uninit(struct adapter *); 294static int port_full_init(struct port_info *); 295static int port_full_uninit(struct port_info *); 296static void quiesce_eq(struct adapter *, struct sge_eq *); 297static void quiesce_iq(struct adapter *, struct sge_iq *); 298static void quiesce_fl(struct adapter *, struct sge_fl *); 299static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 300 driver_intr_t *, void *, char *); 301static int t4_free_irq(struct adapter *, struct irq *); 302static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 303 unsigned int); 304static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 305static void cxgbe_tick(void *); 306static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 307static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 308 struct mbuf *); 309static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 310static int fw_msg_not_handled(struct adapter *, const __be64 *); 311static int t4_sysctls(struct adapter *); 312static int cxgbe_sysctls(struct port_info *); 313static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 314static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 315static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 316static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 317static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 318static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 319static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 320#ifdef SBUF_DRAIN 321static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 322static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 323static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 324static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 325static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 326static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 327static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 328static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 329static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 330static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 331static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 332static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 333static int sysctl_tids(SYSCTL_HANDLER_ARGS); 334static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 335static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 336#endif 337static inline void txq_start(struct ifnet *, struct sge_txq *); 338static uint32_t fconf_to_mode(uint32_t); 339static uint32_t mode_to_fconf(uint32_t); 340static uint32_t fspec_to_fconf(struct t4_filter_specification *); 341static int get_filter_mode(struct adapter *, uint32_t *); 342static int set_filter_mode(struct adapter *, uint32_t); 343static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 344static int get_filter(struct adapter *, struct t4_filter *); 345static int set_filter(struct adapter *, struct t4_filter *); 346static int del_filter(struct adapter *, struct t4_filter *); 347static void clear_filter(struct filter_entry *); 348static int set_filter_wr(struct adapter *, int); 349static int del_filter_wr(struct adapter *, int); 350static int get_sge_context(struct adapter *, struct t4_sge_context *); 351static int read_card_mem(struct adapter *, struct t4_mem_range *); 352#ifdef TCP_OFFLOAD 353static int toe_capability(struct port_info *, int); 354#endif 355static int t4_mod_event(module_t, int, void *); 356 357struct t4_pciids { 358 uint16_t device; 359 char *desc; 360} t4_pciids[] = { 361 {0xa000, "Chelsio Terminator 4 FPGA"}, 362 {0x4400, "Chelsio T440-dbg"}, 363 {0x4401, "Chelsio T420-CR"}, 364 {0x4402, "Chelsio T422-CR"}, 365 {0x4403, "Chelsio T440-CR"}, 366 {0x4404, "Chelsio T420-BCH"}, 367 {0x4405, "Chelsio T440-BCH"}, 368 {0x4406, "Chelsio T440-CH"}, 369 {0x4407, "Chelsio T420-SO"}, 370 {0x4408, "Chelsio T420-CX"}, 371 {0x4409, "Chelsio T420-BT"}, 372 {0x440a, "Chelsio T404-BT"}, 373}; 374 375#ifdef TCP_OFFLOAD 376/* 377 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 378 * exactly the same for both rxq and ofld_rxq. 379 */ 380CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 381CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 382#endif 383 384/* No easy way to include t4_msg.h before adapter.h so we check this way */ 385CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 386CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 387 388static int 389t4_probe(device_t dev) 390{ 391 int i; 392 uint16_t v = pci_get_vendor(dev); 393 uint16_t d = pci_get_device(dev); 394 uint8_t f = pci_get_function(dev); 395 396 if (v != PCI_VENDOR_ID_CHELSIO) 397 return (ENXIO); 398 399 /* Attach only to PF0 of the FPGA */ 400 if (d == 0xa000 && f != 0) 401 return (ENXIO); 402 403 for (i = 0; i < nitems(t4_pciids); i++) { 404 if (d == t4_pciids[i].device) { 405 device_set_desc(dev, t4_pciids[i].desc); 406 return (BUS_PROBE_DEFAULT); 407 } 408 } 409 410 return (ENXIO); 411} 412 413static int 414t4_attach(device_t dev) 415{ 416 struct adapter *sc; 417 int rc = 0, i, n10g, n1g, rqidx, tqidx; 418 struct intrs_and_queues iaq; 419 struct sge *s; 420#ifdef TCP_OFFLOAD 421 int ofld_rqidx, ofld_tqidx; 422#endif 423 424 sc = device_get_softc(dev); 425 sc->dev = dev; 426 427 pci_enable_busmaster(dev); 428 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 429 uint32_t v; 430 431 pci_set_max_read_req(dev, 4096); 432 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 433 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 434 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 435 } 436 437 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 438 device_get_nameunit(dev)); 439 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 440 mtx_lock(&t4_list_lock); 441 SLIST_INSERT_HEAD(&t4_list, sc, link); 442 mtx_unlock(&t4_list_lock); 443 444 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 445 TAILQ_INIT(&sc->sfl); 446 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 447 448 rc = map_bars(sc); 449 if (rc != 0) 450 goto done; /* error message displayed already */ 451 452 /* 453 * This is the real PF# to which we're attaching. Works from within PCI 454 * passthrough environments too, where pci_get_function() could return a 455 * different PF# depending on the passthrough configuration. We need to 456 * use the real PF# in all our communication with the firmware. 457 */ 458 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 459 sc->mbox = sc->pf; 460 461 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 462 sc->an_handler = an_not_handled; 463 for (i = 0; i < nitems(sc->cpl_handler); i++) 464 sc->cpl_handler[i] = cpl_not_handled; 465 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 466 sc->fw_msg_handler[i] = fw_msg_not_handled; 467 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 468 469 /* Prepare the adapter for operation */ 470 rc = -t4_prep_adapter(sc); 471 if (rc != 0) { 472 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 473 goto done; 474 } 475 476 /* 477 * Do this really early, with the memory windows set up even before the 478 * character device. The userland tool's register i/o and mem read 479 * will work even in "recovery mode". 480 */ 481 setup_memwin(sc); 482 sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT, 483 GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); 484 sc->cdev->si_drv1 = sc; 485 486 /* Go no further if recovery mode has been requested. */ 487 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 488 device_printf(dev, "recovery mode.\n"); 489 goto done; 490 } 491 492 /* Prepare the firmware for operation */ 493 rc = prep_firmware(sc); 494 if (rc != 0) 495 goto done; /* error message displayed already */ 496 497 rc = get_params__pre_init(sc); 498 if (rc != 0) 499 goto done; /* error message displayed already */ 500 501 rc = t4_sge_init(sc); 502 if (rc != 0) 503 goto done; /* error message displayed already */ 504 505 if (sc->flags & MASTER_PF) { 506 /* get basic stuff going */ 507 rc = -t4_fw_initialize(sc, sc->mbox); 508 if (rc != 0) { 509 device_printf(dev, "early init failed: %d.\n", rc); 510 goto done; 511 } 512 } 513 514 rc = get_params__post_init(sc); 515 if (rc != 0) 516 goto done; /* error message displayed already */ 517 518 if (sc->flags & MASTER_PF) { 519 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 520 521 /* final tweaks to some settings */ 522 523 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, 524 sc->params.b_wnd); 525 /* 4K, 16K, 64K, 256K DDP "page sizes" */ 526 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(0) | V_HPZ1(2) | 527 V_HPZ2(4) | V_HPZ3(6)); 528 t4_set_reg_field(sc, A_ULP_RX_CTL, F_TDDPTAGTCB, F_TDDPTAGTCB); 529 t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 | 530 F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 531 F_TUNNELCNGDROP0 | F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | 532 F_TUNNELCNGDROP3); 533 t4_set_reg_field(sc, A_TP_PARA_REG5, 534 V_INDICATESIZE(M_INDICATESIZE) | 535 F_REARMDDPOFFSET | F_RESETDDPOFFSET, 536 V_INDICATESIZE(indsz) | 537 F_REARMDDPOFFSET | F_RESETDDPOFFSET); 538 } else { 539 /* 540 * XXX: Verify that we can live with whatever the master driver 541 * has done so far, and hope that it doesn't change any global 542 * setting from underneath us in the future. 543 */ 544 } 545 546 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 547 A_TP_VLAN_PRI_MAP); 548 549 for (i = 0; i < NCHAN; i++) 550 sc->params.tp.tx_modq[i] = i; 551 552 rc = t4_create_dma_tag(sc); 553 if (rc != 0) 554 goto done; /* error message displayed already */ 555 556 /* 557 * First pass over all the ports - allocate VIs and initialize some 558 * basic parameters like mac address, port type, etc. We also figure 559 * out whether a port is 10G or 1G and use that information when 560 * calculating how many interrupts to attempt to allocate. 561 */ 562 n10g = n1g = 0; 563 for_each_port(sc, i) { 564 struct port_info *pi; 565 566 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 567 sc->port[i] = pi; 568 569 /* These must be set before t4_port_init */ 570 pi->adapter = sc; 571 pi->port_id = i; 572 573 /* Allocate the vi and initialize parameters like mac addr */ 574 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 575 if (rc != 0) { 576 device_printf(dev, "unable to initialize port %d: %d\n", 577 i, rc); 578 free(pi, M_CXGBE); 579 sc->port[i] = NULL; 580 goto done; 581 } 582 583 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 584 device_get_nameunit(dev), i); 585 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 586 587 if (is_10G_port(pi)) { 588 n10g++; 589 pi->tmr_idx = t4_tmr_idx_10g; 590 pi->pktc_idx = t4_pktc_idx_10g; 591 } else { 592 n1g++; 593 pi->tmr_idx = t4_tmr_idx_1g; 594 pi->pktc_idx = t4_pktc_idx_1g; 595 } 596 597 pi->xact_addr_filt = -1; 598 599 pi->qsize_rxq = t4_qsize_rxq; 600 pi->qsize_txq = t4_qsize_txq; 601 602 pi->dev = device_add_child(dev, "cxgbe", -1); 603 if (pi->dev == NULL) { 604 device_printf(dev, 605 "failed to add device for port %d.\n", i); 606 rc = ENXIO; 607 goto done; 608 } 609 device_set_softc(pi->dev, pi); 610 } 611 612 /* 613 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 614 */ 615 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 616 if (rc != 0) 617 goto done; /* error message displayed already */ 618 619 sc->intr_type = iaq.intr_type; 620 sc->intr_count = iaq.nirq; 621 sc->flags |= iaq.intr_flags; 622 623 s = &sc->sge; 624 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 625 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 626 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 627 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 628 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 629 630#ifdef TCP_OFFLOAD 631 if (is_offload(sc)) { 632 633 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 634 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 635 s->neq += s->nofldtxq + s->nofldrxq; 636 s->niq += s->nofldrxq; 637 638 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 639 M_CXGBE, M_ZERO | M_WAITOK); 640 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 641 M_CXGBE, M_ZERO | M_WAITOK); 642 } 643#endif 644 645 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 646 M_ZERO | M_WAITOK); 647 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 648 M_ZERO | M_WAITOK); 649 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 650 M_ZERO | M_WAITOK); 651 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 652 M_ZERO | M_WAITOK); 653 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 654 M_ZERO | M_WAITOK); 655 656 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 657 M_ZERO | M_WAITOK); 658 659 t4_init_l2t(sc, M_WAITOK); 660 661 /* 662 * Second pass over the ports. This time we know the number of rx and 663 * tx queues that each port should get. 664 */ 665 rqidx = tqidx = 0; 666#ifdef TCP_OFFLOAD 667 ofld_rqidx = ofld_tqidx = 0; 668#endif 669 for_each_port(sc, i) { 670 struct port_info *pi = sc->port[i]; 671 672 if (pi == NULL) 673 continue; 674 675 pi->first_rxq = rqidx; 676 pi->first_txq = tqidx; 677 if (is_10G_port(pi)) { 678 pi->nrxq = iaq.nrxq10g; 679 pi->ntxq = iaq.ntxq10g; 680 } else { 681 pi->nrxq = iaq.nrxq1g; 682 pi->ntxq = iaq.ntxq1g; 683 } 684 685 rqidx += pi->nrxq; 686 tqidx += pi->ntxq; 687 688#ifdef TCP_OFFLOAD 689 if (is_offload(sc)) { 690 pi->first_ofld_rxq = ofld_rqidx; 691 pi->first_ofld_txq = ofld_tqidx; 692 if (is_10G_port(pi)) { 693 pi->nofldrxq = iaq.nofldrxq10g; 694 pi->nofldtxq = iaq.nofldtxq10g; 695 } else { 696 pi->nofldrxq = iaq.nofldrxq1g; 697 pi->nofldtxq = iaq.nofldtxq1g; 698 } 699 ofld_rqidx += pi->nofldrxq; 700 ofld_tqidx += pi->nofldtxq; 701 } 702#endif 703 } 704 705 rc = setup_intr_handlers(sc); 706 if (rc != 0) { 707 device_printf(dev, 708 "failed to setup interrupt handlers: %d\n", rc); 709 goto done; 710 } 711 712 rc = bus_generic_attach(dev); 713 if (rc != 0) { 714 device_printf(dev, 715 "failed to attach all child ports: %d\n", rc); 716 goto done; 717 } 718 719 device_printf(dev, 720 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 721 sc->params.pci.width, sc->params.nports, sc->intr_count, 722 sc->intr_type == INTR_MSIX ? "MSI-X" : 723 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 724 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 725 726 t4_set_desc(sc); 727 728done: 729 if (rc != 0 && sc->cdev) { 730 /* cdev was created and so cxgbetool works; recover that way. */ 731 device_printf(dev, 732 "error during attach, adapter is now in recovery mode.\n"); 733 rc = 0; 734 } 735 736 if (rc != 0) 737 t4_detach(dev); 738 else 739 t4_sysctls(sc); 740 741 return (rc); 742} 743 744/* 745 * Idempotent 746 */ 747static int 748t4_detach(device_t dev) 749{ 750 struct adapter *sc; 751 struct port_info *pi; 752 int i, rc; 753 754 sc = device_get_softc(dev); 755 756 if (sc->flags & FULL_INIT_DONE) 757 t4_intr_disable(sc); 758 759 if (sc->cdev) { 760 destroy_dev(sc->cdev); 761 sc->cdev = NULL; 762 } 763 764 rc = bus_generic_detach(dev); 765 if (rc) { 766 device_printf(dev, 767 "failed to detach child devices: %d\n", rc); 768 return (rc); 769 } 770 771 for (i = 0; i < sc->intr_count; i++) 772 t4_free_irq(sc, &sc->irq[i]); 773 774 for (i = 0; i < MAX_NPORTS; i++) { 775 pi = sc->port[i]; 776 if (pi) { 777 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 778 if (pi->dev) 779 device_delete_child(dev, pi->dev); 780 781 mtx_destroy(&pi->pi_lock); 782 free(pi, M_CXGBE); 783 } 784 } 785 786 if (sc->flags & FULL_INIT_DONE) 787 adapter_full_uninit(sc); 788 789 if (sc->flags & FW_OK) 790 t4_fw_bye(sc, sc->mbox); 791 792 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 793 pci_release_msi(dev); 794 795 if (sc->regs_res) 796 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 797 sc->regs_res); 798 799 if (sc->msix_res) 800 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 801 sc->msix_res); 802 803 if (sc->l2t) 804 t4_free_l2t(sc->l2t); 805 806#ifdef TCP_OFFLOAD 807 free(sc->sge.ofld_rxq, M_CXGBE); 808 free(sc->sge.ofld_txq, M_CXGBE); 809#endif 810 free(sc->irq, M_CXGBE); 811 free(sc->sge.rxq, M_CXGBE); 812 free(sc->sge.txq, M_CXGBE); 813 free(sc->sge.ctrlq, M_CXGBE); 814 free(sc->sge.iqmap, M_CXGBE); 815 free(sc->sge.eqmap, M_CXGBE); 816 free(sc->tids.ftid_tab, M_CXGBE); 817 t4_destroy_dma_tag(sc); 818 if (mtx_initialized(&sc->sc_lock)) { 819 mtx_lock(&t4_list_lock); 820 SLIST_REMOVE(&t4_list, sc, adapter, link); 821 mtx_unlock(&t4_list_lock); 822 mtx_destroy(&sc->sc_lock); 823 } 824 825 if (mtx_initialized(&sc->sfl_lock)) 826 mtx_destroy(&sc->sfl_lock); 827 828 bzero(sc, sizeof(*sc)); 829 830 return (0); 831} 832 833 834static int 835cxgbe_probe(device_t dev) 836{ 837 char buf[128]; 838 struct port_info *pi = device_get_softc(dev); 839 840 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 841 device_set_desc_copy(dev, buf); 842 843 return (BUS_PROBE_DEFAULT); 844} 845 846#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 847 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 848 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 849#define T4_CAP_ENABLE (T4_CAP) 850 851static int 852cxgbe_attach(device_t dev) 853{ 854 struct port_info *pi = device_get_softc(dev); 855 struct ifnet *ifp; 856 857 /* Allocate an ifnet and set it up */ 858 ifp = if_alloc(IFT_ETHER); 859 if (ifp == NULL) { 860 device_printf(dev, "Cannot allocate ifnet\n"); 861 return (ENOMEM); 862 } 863 pi->ifp = ifp; 864 ifp->if_softc = pi; 865 866 callout_init(&pi->tick, CALLOUT_MPSAFE); 867 868 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 869 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 870 871 ifp->if_init = cxgbe_init; 872 ifp->if_ioctl = cxgbe_ioctl; 873 ifp->if_transmit = cxgbe_transmit; 874 ifp->if_qflush = cxgbe_qflush; 875 876 ifp->if_capabilities = T4_CAP; 877#ifdef TCP_OFFLOAD 878 if (is_offload(pi->adapter)) 879 ifp->if_capabilities |= IFCAP_TOE4; 880#endif 881 ifp->if_capenable = T4_CAP_ENABLE; 882 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 883 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 884 885 /* Initialize ifmedia for this port */ 886 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 887 cxgbe_media_status); 888 build_medialist(pi); 889 890 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 891 EVENTHANDLER_PRI_ANY); 892 893 ether_ifattach(ifp, pi->hw_addr); 894 895#ifdef TCP_OFFLOAD 896 if (is_offload(pi->adapter)) { 897 device_printf(dev, 898 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 899 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 900 } else 901#endif 902 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 903 904 cxgbe_sysctls(pi); 905 906 return (0); 907} 908 909static int 910cxgbe_detach(device_t dev) 911{ 912 struct port_info *pi = device_get_softc(dev); 913 struct adapter *sc = pi->adapter; 914 struct ifnet *ifp = pi->ifp; 915 916 /* Tell if_ioctl and if_init that the port is going away */ 917 ADAPTER_LOCK(sc); 918 SET_DOOMED(pi); 919 wakeup(&sc->flags); 920 while (IS_BUSY(sc)) 921 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 922 SET_BUSY(sc); 923 ADAPTER_UNLOCK(sc); 924 925 if (pi->vlan_c) 926 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 927 928 PORT_LOCK(pi); 929 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 930 callout_stop(&pi->tick); 931 PORT_UNLOCK(pi); 932 callout_drain(&pi->tick); 933 934 /* Let detach proceed even if these fail. */ 935 cxgbe_uninit_synchronized(pi); 936 port_full_uninit(pi); 937 938 ifmedia_removeall(&pi->media); 939 ether_ifdetach(pi->ifp); 940 if_free(pi->ifp); 941 942 ADAPTER_LOCK(sc); 943 CLR_BUSY(sc); 944 wakeup_one(&sc->flags); 945 ADAPTER_UNLOCK(sc); 946 947 return (0); 948} 949 950static void 951cxgbe_init(void *arg) 952{ 953 struct port_info *pi = arg; 954 struct adapter *sc = pi->adapter; 955 956 ADAPTER_LOCK(sc); 957 cxgbe_init_locked(pi); /* releases adapter lock */ 958 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 959} 960 961static int 962cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 963{ 964 int rc = 0, mtu, flags; 965 struct port_info *pi = ifp->if_softc; 966 struct adapter *sc = pi->adapter; 967 struct ifreq *ifr = (struct ifreq *)data; 968 uint32_t mask; 969 970 switch (cmd) { 971 case SIOCSIFMTU: 972 ADAPTER_LOCK(sc); 973 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 974 if (rc) { 975fail: 976 ADAPTER_UNLOCK(sc); 977 return (rc); 978 } 979 980 mtu = ifr->ifr_mtu; 981 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) { 982 rc = EINVAL; 983 } else { 984 ifp->if_mtu = mtu; 985 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 986 t4_update_fl_bufsize(ifp); 987 PORT_LOCK(pi); 988 rc = update_mac_settings(pi, XGMAC_MTU); 989 PORT_UNLOCK(pi); 990 } 991 } 992 ADAPTER_UNLOCK(sc); 993 break; 994 995 case SIOCSIFFLAGS: 996 ADAPTER_LOCK(sc); 997 if (IS_DOOMED(pi)) { 998 rc = ENXIO; 999 goto fail; 1000 } 1001 if (ifp->if_flags & IFF_UP) { 1002 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1003 flags = pi->if_flags; 1004 if ((ifp->if_flags ^ flags) & 1005 (IFF_PROMISC | IFF_ALLMULTI)) { 1006 if (IS_BUSY(sc)) { 1007 rc = EBUSY; 1008 goto fail; 1009 } 1010 PORT_LOCK(pi); 1011 rc = update_mac_settings(pi, 1012 XGMAC_PROMISC | XGMAC_ALLMULTI); 1013 PORT_UNLOCK(pi); 1014 } 1015 ADAPTER_UNLOCK(sc); 1016 } else 1017 rc = cxgbe_init_locked(pi); 1018 pi->if_flags = ifp->if_flags; 1019 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1020 rc = cxgbe_uninit_locked(pi); 1021 else 1022 ADAPTER_UNLOCK(sc); 1023 1024 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1025 break; 1026 1027 case SIOCADDMULTI: 1028 case SIOCDELMULTI: /* these two can be called with a mutex held :-( */ 1029 ADAPTER_LOCK(sc); 1030 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1031 if (rc) 1032 goto fail; 1033 1034 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1035 PORT_LOCK(pi); 1036 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1037 PORT_UNLOCK(pi); 1038 } 1039 ADAPTER_UNLOCK(sc); 1040 break; 1041 1042 case SIOCSIFCAP: 1043 ADAPTER_LOCK(sc); 1044 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 1045 if (rc) 1046 goto fail; 1047 1048 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1049 if (mask & IFCAP_TXCSUM) { 1050 ifp->if_capenable ^= IFCAP_TXCSUM; 1051 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1052 1053 if (IFCAP_TSO4 & ifp->if_capenable && 1054 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1055 ifp->if_capenable &= ~IFCAP_TSO4; 1056 if_printf(ifp, 1057 "tso4 disabled due to -txcsum.\n"); 1058 } 1059 } 1060 if (mask & IFCAP_TXCSUM_IPV6) { 1061 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1062 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1063 1064 if (IFCAP_TSO6 & ifp->if_capenable && 1065 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1066 ifp->if_capenable &= ~IFCAP_TSO6; 1067 if_printf(ifp, 1068 "tso6 disabled due to -txcsum6.\n"); 1069 } 1070 } 1071 if (mask & IFCAP_RXCSUM) 1072 ifp->if_capenable ^= IFCAP_RXCSUM; 1073 if (mask & IFCAP_RXCSUM_IPV6) 1074 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1075 1076 /* 1077 * Note that we leave CSUM_TSO alone (it is always set). The 1078 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1079 * sending a TSO request our way, so it's sufficient to toggle 1080 * IFCAP_TSOx only. 1081 */ 1082 if (mask & IFCAP_TSO4) { 1083 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1084 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1085 if_printf(ifp, "enable txcsum first.\n"); 1086 rc = EAGAIN; 1087 goto fail; 1088 } 1089 ifp->if_capenable ^= IFCAP_TSO4; 1090 } 1091 if (mask & IFCAP_TSO6) { 1092 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1093 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1094 if_printf(ifp, "enable txcsum6 first.\n"); 1095 rc = EAGAIN; 1096 goto fail; 1097 } 1098 ifp->if_capenable ^= IFCAP_TSO6; 1099 } 1100 if (mask & IFCAP_LRO) { 1101#if defined(INET) || defined(INET6) 1102 int i; 1103 struct sge_rxq *rxq; 1104 1105 ifp->if_capenable ^= IFCAP_LRO; 1106 for_each_rxq(pi, i, rxq) { 1107 if (ifp->if_capenable & IFCAP_LRO) 1108 rxq->iq.flags |= IQ_LRO_ENABLED; 1109 else 1110 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1111 } 1112#endif 1113 } 1114#ifdef TCP_OFFLOAD 1115 if (mask & IFCAP_TOE) { 1116 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1117 1118 rc = toe_capability(pi, enable); 1119 if (rc != 0) 1120 goto fail; 1121 1122 ifp->if_capenable ^= mask; 1123 } 1124#endif 1125 if (mask & IFCAP_VLAN_HWTAGGING) { 1126 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1127 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1128 PORT_LOCK(pi); 1129 rc = update_mac_settings(pi, XGMAC_VLANEX); 1130 PORT_UNLOCK(pi); 1131 } 1132 } 1133 if (mask & IFCAP_VLAN_MTU) { 1134 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1135 1136 /* Need to find out how to disable auto-mtu-inflation */ 1137 } 1138 if (mask & IFCAP_VLAN_HWTSO) 1139 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1140 if (mask & IFCAP_VLAN_HWCSUM) 1141 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1142 1143#ifdef VLAN_CAPABILITIES 1144 VLAN_CAPABILITIES(ifp); 1145#endif 1146 ADAPTER_UNLOCK(sc); 1147 break; 1148 1149 case SIOCSIFMEDIA: 1150 case SIOCGIFMEDIA: 1151 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1152 break; 1153 1154 default: 1155 rc = ether_ioctl(ifp, cmd, data); 1156 } 1157 1158 return (rc); 1159} 1160 1161static int 1162cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1163{ 1164 struct port_info *pi = ifp->if_softc; 1165 struct adapter *sc = pi->adapter; 1166 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1167 struct buf_ring *br; 1168 int rc; 1169 1170 M_ASSERTPKTHDR(m); 1171 1172 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1173 m_freem(m); 1174 return (ENETDOWN); 1175 } 1176 1177 if (m->m_flags & M_FLOWID) 1178 txq += (m->m_pkthdr.flowid % pi->ntxq); 1179 br = txq->br; 1180 1181 if (TXQ_TRYLOCK(txq) == 0) { 1182 struct sge_eq *eq = &txq->eq; 1183 1184 /* 1185 * It is possible that t4_eth_tx finishes up and releases the 1186 * lock between the TRYLOCK above and the drbr_enqueue here. We 1187 * need to make sure that this mbuf doesn't just sit there in 1188 * the drbr. 1189 */ 1190 1191 rc = drbr_enqueue(ifp, br, m); 1192 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1193 !(eq->flags & EQ_DOOMED)) 1194 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1195 return (rc); 1196 } 1197 1198 /* 1199 * txq->m is the mbuf that is held up due to a temporary shortage of 1200 * resources and it should be put on the wire first. Then what's in 1201 * drbr and finally the mbuf that was just passed in to us. 1202 * 1203 * Return code should indicate the fate of the mbuf that was passed in 1204 * this time. 1205 */ 1206 1207 TXQ_LOCK_ASSERT_OWNED(txq); 1208 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1209 1210 /* Queued for transmission. */ 1211 1212 rc = drbr_enqueue(ifp, br, m); 1213 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1214 (void) t4_eth_tx(ifp, txq, m); 1215 TXQ_UNLOCK(txq); 1216 return (rc); 1217 } 1218 1219 /* Direct transmission. */ 1220 rc = t4_eth_tx(ifp, txq, m); 1221 if (rc != 0 && txq->m) 1222 rc = 0; /* held, will be transmitted soon (hopefully) */ 1223 1224 TXQ_UNLOCK(txq); 1225 return (rc); 1226} 1227 1228static void 1229cxgbe_qflush(struct ifnet *ifp) 1230{ 1231 struct port_info *pi = ifp->if_softc; 1232 struct sge_txq *txq; 1233 int i; 1234 struct mbuf *m; 1235 1236 /* queues do not exist if !PORT_INIT_DONE. */ 1237 if (pi->flags & PORT_INIT_DONE) { 1238 for_each_txq(pi, i, txq) { 1239 TXQ_LOCK(txq); 1240 m_freem(txq->m); 1241 txq->m = NULL; 1242 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1243 m_freem(m); 1244 TXQ_UNLOCK(txq); 1245 } 1246 } 1247 if_qflush(ifp); 1248} 1249 1250static int 1251cxgbe_media_change(struct ifnet *ifp) 1252{ 1253 struct port_info *pi = ifp->if_softc; 1254 1255 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1256 1257 return (EOPNOTSUPP); 1258} 1259 1260static void 1261cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1262{ 1263 struct port_info *pi = ifp->if_softc; 1264 struct ifmedia_entry *cur = pi->media.ifm_cur; 1265 int speed = pi->link_cfg.speed; 1266 int data = (pi->port_type << 8) | pi->mod_type; 1267 1268 if (cur->ifm_data != data) { 1269 build_medialist(pi); 1270 cur = pi->media.ifm_cur; 1271 } 1272 1273 ifmr->ifm_status = IFM_AVALID; 1274 if (!pi->link_cfg.link_ok) 1275 return; 1276 1277 ifmr->ifm_status |= IFM_ACTIVE; 1278 1279 /* active and current will differ iff current media is autoselect. */ 1280 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1281 return; 1282 1283 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1284 if (speed == SPEED_10000) 1285 ifmr->ifm_active |= IFM_10G_T; 1286 else if (speed == SPEED_1000) 1287 ifmr->ifm_active |= IFM_1000_T; 1288 else if (speed == SPEED_100) 1289 ifmr->ifm_active |= IFM_100_TX; 1290 else if (speed == SPEED_10) 1291 ifmr->ifm_active |= IFM_10_T; 1292 else 1293 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1294 speed)); 1295} 1296 1297void 1298t4_fatal_err(struct adapter *sc) 1299{ 1300 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1301 t4_intr_disable(sc); 1302 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1303 device_get_nameunit(sc->dev)); 1304} 1305 1306static int 1307map_bars(struct adapter *sc) 1308{ 1309 sc->regs_rid = PCIR_BAR(0); 1310 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1311 &sc->regs_rid, RF_ACTIVE); 1312 if (sc->regs_res == NULL) { 1313 device_printf(sc->dev, "cannot map registers.\n"); 1314 return (ENXIO); 1315 } 1316 sc->bt = rman_get_bustag(sc->regs_res); 1317 sc->bh = rman_get_bushandle(sc->regs_res); 1318 sc->mmio_len = rman_get_size(sc->regs_res); 1319 1320 sc->msix_rid = PCIR_BAR(4); 1321 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1322 &sc->msix_rid, RF_ACTIVE); 1323 if (sc->msix_res == NULL) { 1324 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1325 return (ENXIO); 1326 } 1327 1328 return (0); 1329} 1330 1331static void 1332setup_memwin(struct adapter *sc) 1333{ 1334 uint32_t bar0; 1335 1336 /* 1337 * Read low 32b of bar0 indirectly via the hardware backdoor mechanism. 1338 * Works from within PCI passthrough environments too, where 1339 * rman_get_start() can return a different value. We need to program 1340 * the memory window decoders with the actual addresses that will be 1341 * coming across the PCIe link. 1342 */ 1343 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1344 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1345 1346 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1347 (bar0 + MEMWIN0_BASE) | V_BIR(0) | 1348 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1349 1350 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1351 (bar0 + MEMWIN1_BASE) | V_BIR(0) | 1352 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1353 1354 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1355 (bar0 + MEMWIN2_BASE) | V_BIR(0) | 1356 V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10)); 1357 1358 /* flush */ 1359 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1360} 1361 1362static int 1363cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1364 struct intrs_and_queues *iaq) 1365{ 1366 int rc, itype, navail, nrxq10g, nrxq1g, n; 1367 int nofldrxq10g = 0, nofldrxq1g = 0; 1368 1369 bzero(iaq, sizeof(*iaq)); 1370 1371 iaq->ntxq10g = t4_ntxq10g; 1372 iaq->ntxq1g = t4_ntxq1g; 1373 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1374 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1375#ifdef TCP_OFFLOAD 1376 if (is_offload(sc)) { 1377 iaq->nofldtxq10g = t4_nofldtxq10g; 1378 iaq->nofldtxq1g = t4_nofldtxq1g; 1379 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1380 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1381 } 1382#endif 1383 1384 for (itype = INTR_MSIX; itype; itype >>= 1) { 1385 1386 if ((itype & t4_intr_types) == 0) 1387 continue; /* not allowed */ 1388 1389 if (itype == INTR_MSIX) 1390 navail = pci_msix_count(sc->dev); 1391 else if (itype == INTR_MSI) 1392 navail = pci_msi_count(sc->dev); 1393 else 1394 navail = 1; 1395restart: 1396 if (navail == 0) 1397 continue; 1398 1399 iaq->intr_type = itype; 1400 iaq->intr_flags = 0; 1401 1402 /* 1403 * Best option: an interrupt vector for errors, one for the 1404 * firmware event queue, and one each for each rxq (NIC as well 1405 * as offload). 1406 */ 1407 iaq->nirq = T4_EXTRA_INTR; 1408 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1409 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1410 if (iaq->nirq <= navail && 1411 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1412 iaq->intr_flags |= INTR_DIRECT; 1413 goto allocate; 1414 } 1415 1416 /* 1417 * Second best option: an interrupt vector for errors, one for 1418 * the firmware event queue, and one each for either NIC or 1419 * offload rxq's. 1420 */ 1421 iaq->nirq = T4_EXTRA_INTR; 1422 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1423 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1424 if (iaq->nirq <= navail && 1425 (itype != INTR_MSI || powerof2(iaq->nirq))) 1426 goto allocate; 1427 1428 /* 1429 * Next best option: an interrupt vector for errors, one for the 1430 * firmware event queue, and at least one per port. At this 1431 * point we know we'll have to downsize nrxq or nofldrxq to fit 1432 * what's available to us. 1433 */ 1434 iaq->nirq = T4_EXTRA_INTR; 1435 iaq->nirq += n10g + n1g; 1436 if (iaq->nirq <= navail) { 1437 int leftover = navail - iaq->nirq; 1438 1439 if (n10g > 0) { 1440 int target = max(nrxq10g, nofldrxq10g); 1441 1442 n = 1; 1443 while (n < target && leftover >= n10g) { 1444 leftover -= n10g; 1445 iaq->nirq += n10g; 1446 n++; 1447 } 1448 iaq->nrxq10g = min(n, nrxq10g); 1449#ifdef TCP_OFFLOAD 1450 if (is_offload(sc)) 1451 iaq->nofldrxq10g = min(n, nofldrxq10g); 1452#endif 1453 } 1454 1455 if (n1g > 0) { 1456 int target = max(nrxq1g, nofldrxq1g); 1457 1458 n = 1; 1459 while (n < target && leftover >= n1g) { 1460 leftover -= n1g; 1461 iaq->nirq += n1g; 1462 n++; 1463 } 1464 iaq->nrxq1g = min(n, nrxq1g); 1465#ifdef TCP_OFFLOAD 1466 if (is_offload(sc)) 1467 iaq->nofldrxq1g = min(n, nofldrxq1g); 1468#endif 1469 } 1470 1471 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1472 goto allocate; 1473 } 1474 1475 /* 1476 * Least desirable option: one interrupt vector for everything. 1477 */ 1478 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1479#ifdef TCP_OFFLOAD 1480 if (is_offload(sc)) 1481 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1482#endif 1483 1484allocate: 1485 navail = iaq->nirq; 1486 rc = 0; 1487 if (itype == INTR_MSIX) 1488 rc = pci_alloc_msix(sc->dev, &navail); 1489 else if (itype == INTR_MSI) 1490 rc = pci_alloc_msi(sc->dev, &navail); 1491 1492 if (rc == 0) { 1493 if (navail == iaq->nirq) 1494 return (0); 1495 1496 /* 1497 * Didn't get the number requested. Use whatever number 1498 * the kernel is willing to allocate (it's in navail). 1499 */ 1500 device_printf(sc->dev, "fewer vectors than requested, " 1501 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1502 itype, iaq->nirq, navail); 1503 pci_release_msi(sc->dev); 1504 goto restart; 1505 } 1506 1507 device_printf(sc->dev, 1508 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1509 itype, rc, iaq->nirq, navail); 1510 } 1511 1512 device_printf(sc->dev, 1513 "failed to find a usable interrupt type. " 1514 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1515 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1516 1517 return (ENXIO); 1518} 1519 1520/* 1521 * Install a compatible firmware (if required), establish contact with it (by 1522 * saying hello), and reset the device. If we end up as the master driver, 1523 * partition adapter resources by providing a configuration file to the 1524 * firmware. 1525 */ 1526static int 1527prep_firmware(struct adapter *sc) 1528{ 1529 const struct firmware *fw = NULL, *cfg = NULL, *default_cfg; 1530 int rc; 1531 enum dev_state state; 1532 1533 default_cfg = firmware_get(T4_CFGNAME); 1534 1535 /* Check firmware version and install a different one if necessary */ 1536 rc = t4_check_fw_version(sc); 1537 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 1538 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1539 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1540 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1541 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1542 if (rc != 0) { 1543 uint32_t v = 0; 1544 1545 fw = firmware_get(T4_FWNAME); 1546 if (fw != NULL) { 1547 const struct fw_hdr *hdr = (const void *)fw->data; 1548 1549 v = ntohl(hdr->fw_ver); 1550 1551 /* 1552 * The firmware module will not be used if it isn't the 1553 * same major version as what the driver was compiled 1554 * with. 1555 */ 1556 if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) { 1557 device_printf(sc->dev, 1558 "Found firmware image but version %d " 1559 "can not be used with this driver (%d)\n", 1560 G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR); 1561 1562 firmware_put(fw, FIRMWARE_UNLOAD); 1563 fw = NULL; 1564 } 1565 } 1566 1567 if (fw == NULL && rc < 0) { 1568 device_printf(sc->dev, "No usable firmware. " 1569 "card has %d.%d.%d, driver compiled with %d.%d.%d", 1570 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1571 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1572 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1573 FW_VERSION_MAJOR, FW_VERSION_MINOR, 1574 FW_VERSION_MICRO); 1575 rc = EAGAIN; 1576 goto done; 1577 } 1578 1579 /* 1580 * Always upgrade, even for minor/micro/build mismatches. 1581 * Downgrade only for a major version mismatch or if 1582 * force_firmware_install was specified. 1583 */ 1584 if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) { 1585 device_printf(sc->dev, 1586 "installing firmware %d.%d.%d.%d on card.\n", 1587 G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v), 1588 G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v)); 1589 1590 rc = -t4_load_fw(sc, fw->data, fw->datasize); 1591 if (rc != 0) { 1592 device_printf(sc->dev, 1593 "failed to install firmware: %d\n", rc); 1594 goto done; 1595 } else { 1596 /* refresh */ 1597 (void) t4_check_fw_version(sc); 1598 snprintf(sc->fw_version, 1599 sizeof(sc->fw_version), "%u.%u.%u.%u", 1600 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 1601 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 1602 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 1603 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 1604 } 1605 } 1606 } 1607 1608 /* Contact firmware. */ 1609 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1610 if (rc < 0) { 1611 rc = -rc; 1612 device_printf(sc->dev, 1613 "failed to connect to the firmware: %d.\n", rc); 1614 goto done; 1615 } 1616 if (rc == sc->mbox) 1617 sc->flags |= MASTER_PF; 1618 1619 /* Reset device */ 1620 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1621 if (rc != 0) { 1622 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 1623 if (rc != ETIMEDOUT && rc != EIO) 1624 t4_fw_bye(sc, sc->mbox); 1625 goto done; 1626 } 1627 1628 /* Partition adapter resources as specified in the config file. */ 1629 if (sc->flags & MASTER_PF) { 1630 if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) { 1631 char s[32]; 1632 1633 snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file); 1634 cfg = firmware_get(s); 1635 if (cfg == NULL) { 1636 device_printf(sc->dev, 1637 "unable to locate %s module, " 1638 "will use default config file.\n", s); 1639 } 1640 } 1641 1642 rc = partition_resources(sc, cfg ? cfg : default_cfg); 1643 if (rc != 0) 1644 goto done; /* error message displayed already */ 1645 } 1646 1647 sc->flags |= FW_OK; 1648 1649done: 1650 if (fw != NULL) 1651 firmware_put(fw, FIRMWARE_UNLOAD); 1652 if (cfg != NULL) 1653 firmware_put(cfg, FIRMWARE_UNLOAD); 1654 if (default_cfg != NULL) 1655 firmware_put(default_cfg, FIRMWARE_UNLOAD); 1656 1657 return (rc); 1658} 1659 1660#define FW_PARAM_DEV(param) \ 1661 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1662 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1663#define FW_PARAM_PFVF(param) \ 1664 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1665 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1666 1667/* 1668 * Upload configuration file to card's memory. 1669 */ 1670static int 1671upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt, 1672 uint32_t *ma) 1673{ 1674 int rc, i; 1675 uint32_t param, val, mtype, maddr, bar, off, win, remaining; 1676 const uint32_t *b; 1677 1678 /* Figure out where the firmware wants us to upload it. */ 1679 param = FW_PARAM_DEV(CF); 1680 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1681 if (rc != 0) { 1682 /* Firmwares without config file support will fail this way */ 1683 device_printf(sc->dev, 1684 "failed to query config file location: %d.\n", rc); 1685 return (rc); 1686 } 1687 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1688 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1689 1690 if (maddr & 3) { 1691 device_printf(sc->dev, 1692 "cannot upload config file (type %u, addr %x).\n", 1693 mtype, maddr); 1694 return (EFAULT); 1695 } 1696 1697 /* Translate mtype/maddr to an address suitable for the PCIe window */ 1698 val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1699 val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE; 1700 switch (mtype) { 1701 case FW_MEMTYPE_CF_EDC0: 1702 if (!(val & F_EDRAM0_ENABLE)) 1703 goto err; 1704 bar = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1705 maddr += G_EDRAM0_BASE(bar) << 20; 1706 break; 1707 1708 case FW_MEMTYPE_CF_EDC1: 1709 if (!(val & F_EDRAM1_ENABLE)) 1710 goto err; 1711 bar = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1712 maddr += G_EDRAM1_BASE(bar) << 20; 1713 break; 1714 1715 case FW_MEMTYPE_CF_EXTMEM: 1716 if (!(val & F_EXT_MEM_ENABLE)) 1717 goto err; 1718 bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1719 maddr += G_EXT_MEM_BASE(bar) << 20; 1720 break; 1721 1722 default: 1723err: 1724 device_printf(sc->dev, 1725 "cannot upload config file (type %u, enabled %u).\n", 1726 mtype, val); 1727 return (EFAULT); 1728 } 1729 1730 /* 1731 * Position the PCIe window (we use memwin2) to the 16B aligned area 1732 * just at/before the upload location. 1733 */ 1734 win = maddr & ~0xf; 1735 off = maddr - win; /* offset from the start of the window. */ 1736 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 1737 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 1738 1739 remaining = fw->datasize; 1740 if (remaining > FLASH_CFG_MAX_SIZE || 1741 remaining > MEMWIN2_APERTURE - off) { 1742 device_printf(sc->dev, "cannot upload config file all at once " 1743 "(size %u, max %u, room %u).\n", 1744 remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off); 1745 return (EFBIG); 1746 } 1747 1748 /* 1749 * XXX: sheer laziness. We deliberately added 4 bytes of useless 1750 * stuffing/comments at the end of the config file so it's ok to simply 1751 * throw away the last remaining bytes when the config file is not an 1752 * exact multiple of 4. 1753 */ 1754 b = fw->data; 1755 for (i = 0; remaining >= 4; i += 4, remaining -= 4) 1756 t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++); 1757 1758 return (rc); 1759} 1760 1761/* 1762 * Partition chip resources for use between various PFs, VFs, etc. This is done 1763 * by uploading the firmware configuration file to the adapter and instructing 1764 * the firmware to process it. 1765 */ 1766static int 1767partition_resources(struct adapter *sc, const struct firmware *cfg) 1768{ 1769 int rc; 1770 struct fw_caps_config_cmd caps; 1771 uint32_t mtype, maddr, finicsum, cfcsum; 1772 1773 rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT; 1774 if (rc != 0) { 1775 mtype = FW_MEMTYPE_CF_FLASH; 1776 maddr = t4_flash_cfg_addr(sc); 1777 } 1778 1779 bzero(&caps, sizeof(caps)); 1780 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1781 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1782 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1783 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1784 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1785 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1786 if (rc != 0) { 1787 device_printf(sc->dev, 1788 "failed to pre-process config file: %d.\n", rc); 1789 return (rc); 1790 } 1791 1792 finicsum = be32toh(caps.finicsum); 1793 cfcsum = be32toh(caps.cfcsum); 1794 if (finicsum != cfcsum) { 1795 device_printf(sc->dev, 1796 "WARNING: config file checksum mismatch: %08x %08x\n", 1797 finicsum, cfcsum); 1798 } 1799 sc->cfcsum = cfcsum; 1800 1801#define LIMIT_CAPS(x) do { \ 1802 caps.x &= htobe16(t4_##x##_allowed); \ 1803 sc->x = htobe16(caps.x); \ 1804} while (0) 1805 1806 /* 1807 * Let the firmware know what features will (not) be used so it can tune 1808 * things accordingly. 1809 */ 1810 LIMIT_CAPS(linkcaps); 1811 LIMIT_CAPS(niccaps); 1812 LIMIT_CAPS(toecaps); 1813 LIMIT_CAPS(rdmacaps); 1814 LIMIT_CAPS(iscsicaps); 1815 LIMIT_CAPS(fcoecaps); 1816#undef LIMIT_CAPS 1817 1818 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1819 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1820 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1821 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 1822 if (rc != 0) { 1823 device_printf(sc->dev, 1824 "failed to process config file: %d.\n", rc); 1825 return (rc); 1826 } 1827 1828 return (0); 1829} 1830 1831/* 1832 * Retrieve parameters that are needed (or nice to have) prior to calling 1833 * t4_sge_init and t4_fw_initialize. 1834 */ 1835static int 1836get_params__pre_init(struct adapter *sc) 1837{ 1838 int rc; 1839 uint32_t param[2], val[2]; 1840 struct fw_devlog_cmd cmd; 1841 struct devlog_params *dlog = &sc->params.devlog; 1842 1843 param[0] = FW_PARAM_DEV(PORTVEC); 1844 param[1] = FW_PARAM_DEV(CCLK); 1845 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1846 if (rc != 0) { 1847 device_printf(sc->dev, 1848 "failed to query parameters (pre_init): %d.\n", rc); 1849 return (rc); 1850 } 1851 1852 sc->params.portvec = val[0]; 1853 sc->params.nports = bitcount32(val[0]); 1854 sc->params.vpd.cclk = val[1]; 1855 1856 /* Read device log parameters. */ 1857 bzero(&cmd, sizeof(cmd)); 1858 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1859 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1860 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 1861 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 1862 if (rc != 0) { 1863 device_printf(sc->dev, 1864 "failed to get devlog parameters: %d.\n", rc); 1865 bzero(dlog, sizeof (*dlog)); 1866 rc = 0; /* devlog isn't critical for device operation */ 1867 } else { 1868 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 1869 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1870 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1871 dlog->size = be32toh(cmd.memsize_devlog); 1872 } 1873 1874 return (rc); 1875} 1876 1877/* 1878 * Retrieve various parameters that are of interest to the driver. The device 1879 * has been initialized by the firmware at this point. 1880 */ 1881static int 1882get_params__post_init(struct adapter *sc) 1883{ 1884 int rc; 1885 uint32_t param[7], val[7]; 1886 struct fw_caps_config_cmd caps; 1887 1888 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1889 param[1] = FW_PARAM_PFVF(EQ_START); 1890 param[2] = FW_PARAM_PFVF(FILTER_START); 1891 param[3] = FW_PARAM_PFVF(FILTER_END); 1892 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val); 1893 if (rc != 0) { 1894 device_printf(sc->dev, 1895 "failed to query parameters (post_init): %d.\n", rc); 1896 return (rc); 1897 } 1898 1899 sc->sge.iq_start = val[0]; 1900 sc->sge.eq_start = val[1]; 1901 sc->tids.ftid_base = val[2]; 1902 sc->tids.nftids = val[3] - val[2] + 1; 1903 1904 /* get capabilites */ 1905 bzero(&caps, sizeof(caps)); 1906 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1907 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1908 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 1909 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 1910 if (rc != 0) { 1911 device_printf(sc->dev, 1912 "failed to get card capabilities: %d.\n", rc); 1913 return (rc); 1914 } 1915 1916 if (caps.toecaps) { 1917 /* query offload-related parameters */ 1918 param[0] = FW_PARAM_DEV(NTID); 1919 param[1] = FW_PARAM_PFVF(SERVER_START); 1920 param[2] = FW_PARAM_PFVF(SERVER_END); 1921 param[3] = FW_PARAM_PFVF(TDDP_START); 1922 param[4] = FW_PARAM_PFVF(TDDP_END); 1923 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1924 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1925 if (rc != 0) { 1926 device_printf(sc->dev, 1927 "failed to query TOE parameters: %d.\n", rc); 1928 return (rc); 1929 } 1930 sc->tids.ntids = val[0]; 1931 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1932 sc->tids.stid_base = val[1]; 1933 sc->tids.nstids = val[2] - val[1] + 1; 1934 sc->vres.ddp.start = val[3]; 1935 sc->vres.ddp.size = val[4] - val[3] + 1; 1936 sc->params.ofldq_wr_cred = val[5]; 1937 sc->params.offload = 1; 1938 } 1939 if (caps.rdmacaps) { 1940 param[0] = FW_PARAM_PFVF(STAG_START); 1941 param[1] = FW_PARAM_PFVF(STAG_END); 1942 param[2] = FW_PARAM_PFVF(RQ_START); 1943 param[3] = FW_PARAM_PFVF(RQ_END); 1944 param[4] = FW_PARAM_PFVF(PBL_START); 1945 param[5] = FW_PARAM_PFVF(PBL_END); 1946 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1947 if (rc != 0) { 1948 device_printf(sc->dev, 1949 "failed to query RDMA parameters(1): %d.\n", rc); 1950 return (rc); 1951 } 1952 sc->vres.stag.start = val[0]; 1953 sc->vres.stag.size = val[1] - val[0] + 1; 1954 sc->vres.rq.start = val[2]; 1955 sc->vres.rq.size = val[3] - val[2] + 1; 1956 sc->vres.pbl.start = val[4]; 1957 sc->vres.pbl.size = val[5] - val[4] + 1; 1958 1959 param[0] = FW_PARAM_PFVF(SQRQ_START); 1960 param[1] = FW_PARAM_PFVF(SQRQ_END); 1961 param[2] = FW_PARAM_PFVF(CQ_START); 1962 param[3] = FW_PARAM_PFVF(CQ_END); 1963 param[4] = FW_PARAM_PFVF(OCQ_START); 1964 param[5] = FW_PARAM_PFVF(OCQ_END); 1965 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 1966 if (rc != 0) { 1967 device_printf(sc->dev, 1968 "failed to query RDMA parameters(2): %d.\n", rc); 1969 return (rc); 1970 } 1971 sc->vres.qp.start = val[0]; 1972 sc->vres.qp.size = val[1] - val[0] + 1; 1973 sc->vres.cq.start = val[2]; 1974 sc->vres.cq.size = val[3] - val[2] + 1; 1975 sc->vres.ocq.start = val[4]; 1976 sc->vres.ocq.size = val[5] - val[4] + 1; 1977 } 1978 if (caps.iscsicaps) { 1979 param[0] = FW_PARAM_PFVF(ISCSI_START); 1980 param[1] = FW_PARAM_PFVF(ISCSI_END); 1981 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1982 if (rc != 0) { 1983 device_printf(sc->dev, 1984 "failed to query iSCSI parameters: %d.\n", rc); 1985 return (rc); 1986 } 1987 sc->vres.iscsi.start = val[0]; 1988 sc->vres.iscsi.size = val[1] - val[0] + 1; 1989 } 1990 1991 /* These are finalized by FW initialization, load their values now */ 1992 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1993 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1994 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1995 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1996 1997 return (rc); 1998} 1999 2000#undef FW_PARAM_PFVF 2001#undef FW_PARAM_DEV 2002 2003static void 2004t4_set_desc(struct adapter *sc) 2005{ 2006 char buf[128]; 2007 struct adapter_params *p = &sc->params; 2008 2009 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 2010 p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec); 2011 2012 device_set_desc_copy(sc->dev, buf); 2013} 2014 2015static void 2016build_medialist(struct port_info *pi) 2017{ 2018 struct ifmedia *media = &pi->media; 2019 int data, m; 2020 2021 PORT_LOCK(pi); 2022 2023 ifmedia_removeall(media); 2024 2025 m = IFM_ETHER | IFM_FDX; 2026 data = (pi->port_type << 8) | pi->mod_type; 2027 2028 switch(pi->port_type) { 2029 case FW_PORT_TYPE_BT_XFI: 2030 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2031 break; 2032 2033 case FW_PORT_TYPE_BT_XAUI: 2034 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2035 /* fall through */ 2036 2037 case FW_PORT_TYPE_BT_SGMII: 2038 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2039 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2040 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2041 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2042 break; 2043 2044 case FW_PORT_TYPE_CX4: 2045 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2046 ifmedia_set(media, m | IFM_10G_CX4); 2047 break; 2048 2049 case FW_PORT_TYPE_SFP: 2050 case FW_PORT_TYPE_FIBER_XFI: 2051 case FW_PORT_TYPE_FIBER_XAUI: 2052 switch (pi->mod_type) { 2053 2054 case FW_PORT_MOD_TYPE_LR: 2055 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2056 ifmedia_set(media, m | IFM_10G_LR); 2057 break; 2058 2059 case FW_PORT_MOD_TYPE_SR: 2060 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2061 ifmedia_set(media, m | IFM_10G_SR); 2062 break; 2063 2064 case FW_PORT_MOD_TYPE_LRM: 2065 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2066 ifmedia_set(media, m | IFM_10G_LRM); 2067 break; 2068 2069 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2070 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2071 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2072 ifmedia_set(media, m | IFM_10G_TWINAX); 2073 break; 2074 2075 case FW_PORT_MOD_TYPE_NONE: 2076 m &= ~IFM_FDX; 2077 ifmedia_add(media, m | IFM_NONE, data, NULL); 2078 ifmedia_set(media, m | IFM_NONE); 2079 break; 2080 2081 case FW_PORT_MOD_TYPE_NA: 2082 case FW_PORT_MOD_TYPE_ER: 2083 default: 2084 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2085 ifmedia_set(media, m | IFM_UNKNOWN); 2086 break; 2087 } 2088 break; 2089 2090 case FW_PORT_TYPE_KX4: 2091 case FW_PORT_TYPE_KX: 2092 case FW_PORT_TYPE_KR: 2093 default: 2094 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2095 ifmedia_set(media, m | IFM_UNKNOWN); 2096 break; 2097 } 2098 2099 PORT_UNLOCK(pi); 2100} 2101 2102#define FW_MAC_EXACT_CHUNK 7 2103 2104/* 2105 * Program the port's XGMAC based on parameters in ifnet. The caller also 2106 * indicates which parameters should be programmed (the rest are left alone). 2107 */ 2108static int 2109update_mac_settings(struct port_info *pi, int flags) 2110{ 2111 int rc; 2112 struct ifnet *ifp = pi->ifp; 2113 struct adapter *sc = pi->adapter; 2114 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2115 2116 PORT_LOCK_ASSERT_OWNED(pi); 2117 KASSERT(flags, ("%s: not told what to update.", __func__)); 2118 2119 if (flags & XGMAC_MTU) 2120 mtu = ifp->if_mtu; 2121 2122 if (flags & XGMAC_PROMISC) 2123 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2124 2125 if (flags & XGMAC_ALLMULTI) 2126 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2127 2128 if (flags & XGMAC_VLANEX) 2129 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2130 2131 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2132 vlanex, false); 2133 if (rc) { 2134 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2135 return (rc); 2136 } 2137 2138 if (flags & XGMAC_UCADDR) { 2139 uint8_t ucaddr[ETHER_ADDR_LEN]; 2140 2141 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2142 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2143 ucaddr, true, true); 2144 if (rc < 0) { 2145 rc = -rc; 2146 if_printf(ifp, "change_mac failed: %d\n", rc); 2147 return (rc); 2148 } else { 2149 pi->xact_addr_filt = rc; 2150 rc = 0; 2151 } 2152 } 2153 2154 if (flags & XGMAC_MCADDRS) { 2155 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2156 int del = 1; 2157 uint64_t hash = 0; 2158 struct ifmultiaddr *ifma; 2159 int i = 0, j; 2160 2161 if_maddr_rlock(ifp); 2162 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2163 if (ifma->ifma_addr->sa_family != AF_LINK) 2164 continue; 2165 mcaddr[i++] = 2166 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2167 2168 if (i == FW_MAC_EXACT_CHUNK) { 2169 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2170 del, i, mcaddr, NULL, &hash, 0); 2171 if (rc < 0) { 2172 rc = -rc; 2173 for (j = 0; j < i; j++) { 2174 if_printf(ifp, 2175 "failed to add mc address" 2176 " %02x:%02x:%02x:" 2177 "%02x:%02x:%02x rc=%d\n", 2178 mcaddr[j][0], mcaddr[j][1], 2179 mcaddr[j][2], mcaddr[j][3], 2180 mcaddr[j][4], mcaddr[j][5], 2181 rc); 2182 } 2183 goto mcfail; 2184 } 2185 del = 0; 2186 i = 0; 2187 } 2188 } 2189 if (i > 0) { 2190 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2191 del, i, mcaddr, NULL, &hash, 0); 2192 if (rc < 0) { 2193 rc = -rc; 2194 for (j = 0; j < i; j++) { 2195 if_printf(ifp, 2196 "failed to add mc address" 2197 " %02x:%02x:%02x:" 2198 "%02x:%02x:%02x rc=%d\n", 2199 mcaddr[j][0], mcaddr[j][1], 2200 mcaddr[j][2], mcaddr[j][3], 2201 mcaddr[j][4], mcaddr[j][5], 2202 rc); 2203 } 2204 goto mcfail; 2205 } 2206 } 2207 2208 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2209 if (rc != 0) 2210 if_printf(ifp, "failed to set mc address hash: %d", rc); 2211mcfail: 2212 if_maddr_runlock(ifp); 2213 } 2214 2215 return (rc); 2216} 2217 2218static int 2219cxgbe_init_locked(struct port_info *pi) 2220{ 2221 struct adapter *sc = pi->adapter; 2222 int rc = 0; 2223 2224 ADAPTER_LOCK_ASSERT_OWNED(sc); 2225 2226 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2227 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) { 2228 rc = EINTR; 2229 goto done; 2230 } 2231 } 2232 if (IS_DOOMED(pi)) { 2233 rc = ENXIO; 2234 goto done; 2235 } 2236 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2237 2238 /* Give up the adapter lock, port init code can sleep. */ 2239 SET_BUSY(sc); 2240 ADAPTER_UNLOCK(sc); 2241 2242 rc = cxgbe_init_synchronized(pi); 2243 2244done: 2245 ADAPTER_LOCK(sc); 2246 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2247 CLR_BUSY(sc); 2248 wakeup_one(&sc->flags); 2249 ADAPTER_UNLOCK(sc); 2250 return (rc); 2251} 2252 2253static int 2254cxgbe_init_synchronized(struct port_info *pi) 2255{ 2256 struct adapter *sc = pi->adapter; 2257 struct ifnet *ifp = pi->ifp; 2258 int rc = 0; 2259 2260 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2261 2262 if (isset(&sc->open_device_map, pi->port_id)) { 2263 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2264 ("mismatch between open_device_map and if_drv_flags")); 2265 return (0); /* already running */ 2266 } 2267 2268 if (!(sc->flags & FULL_INIT_DONE) && 2269 ((rc = adapter_full_init(sc)) != 0)) 2270 return (rc); /* error message displayed already */ 2271 2272 if (!(pi->flags & PORT_INIT_DONE) && 2273 ((rc = port_full_init(pi)) != 0)) 2274 return (rc); /* error message displayed already */ 2275 2276 PORT_LOCK(pi); 2277 rc = update_mac_settings(pi, XGMAC_ALL); 2278 PORT_UNLOCK(pi); 2279 if (rc) 2280 goto done; /* error message displayed already */ 2281 2282 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2283 if (rc != 0) { 2284 if_printf(ifp, "start_link failed: %d\n", rc); 2285 goto done; 2286 } 2287 2288 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2289 if (rc != 0) { 2290 if_printf(ifp, "enable_vi failed: %d\n", rc); 2291 goto done; 2292 } 2293 2294 /* all ok */ 2295 setbit(&sc->open_device_map, pi->port_id); 2296 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2297 2298 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2299done: 2300 if (rc != 0) 2301 cxgbe_uninit_synchronized(pi); 2302 2303 return (rc); 2304} 2305 2306static int 2307cxgbe_uninit_locked(struct port_info *pi) 2308{ 2309 struct adapter *sc = pi->adapter; 2310 int rc; 2311 2312 ADAPTER_LOCK_ASSERT_OWNED(sc); 2313 2314 while (!IS_DOOMED(pi) && IS_BUSY(sc)) { 2315 if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) { 2316 rc = EINTR; 2317 goto done; 2318 } 2319 } 2320 if (IS_DOOMED(pi)) { 2321 rc = ENXIO; 2322 goto done; 2323 } 2324 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2325 SET_BUSY(sc); 2326 ADAPTER_UNLOCK(sc); 2327 2328 rc = cxgbe_uninit_synchronized(pi); 2329 2330 ADAPTER_LOCK(sc); 2331 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2332 CLR_BUSY(sc); 2333 wakeup_one(&sc->flags); 2334done: 2335 ADAPTER_UNLOCK(sc); 2336 return (rc); 2337} 2338 2339/* 2340 * Idempotent. 2341 */ 2342static int 2343cxgbe_uninit_synchronized(struct port_info *pi) 2344{ 2345 struct adapter *sc = pi->adapter; 2346 struct ifnet *ifp = pi->ifp; 2347 int rc; 2348 2349 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2350 2351 /* 2352 * Disable the VI so that all its data in either direction is discarded 2353 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2354 * tick) intact as the TP can deliver negative advice or data that it's 2355 * holding in its RAM (for an offloaded connection) even after the VI is 2356 * disabled. 2357 */ 2358 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2359 if (rc) { 2360 if_printf(ifp, "disable_vi failed: %d\n", rc); 2361 return (rc); 2362 } 2363 2364 clrbit(&sc->open_device_map, pi->port_id); 2365 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2366 2367 pi->link_cfg.link_ok = 0; 2368 pi->link_cfg.speed = 0; 2369 t4_os_link_changed(sc, pi->port_id, 0); 2370 2371 return (0); 2372} 2373 2374/* 2375 * It is ok for this function to fail midway and return right away. t4_detach 2376 * will walk the entire sc->irq list and clean up whatever is valid. 2377 */ 2378static int 2379setup_intr_handlers(struct adapter *sc) 2380{ 2381 int rc, rid, p, q; 2382 char s[8]; 2383 struct irq *irq; 2384 struct port_info *pi; 2385 struct sge_rxq *rxq; 2386#ifdef TCP_OFFLOAD 2387 struct sge_ofld_rxq *ofld_rxq; 2388#endif 2389 2390 /* 2391 * Setup interrupts. 2392 */ 2393 irq = &sc->irq[0]; 2394 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2395 if (sc->intr_count == 1) { 2396 KASSERT(!(sc->flags & INTR_DIRECT), 2397 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2398 2399 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 2400 if (rc != 0) 2401 return (rc); 2402 } else { 2403 /* Multiple interrupts. */ 2404 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2405 ("%s: too few intr.", __func__)); 2406 2407 /* The first one is always error intr */ 2408 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 2409 if (rc != 0) 2410 return (rc); 2411 irq++; 2412 rid++; 2413 2414 /* The second one is always the firmware event queue */ 2415 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 2416 "evt"); 2417 if (rc != 0) 2418 return (rc); 2419 irq++; 2420 rid++; 2421 2422 /* 2423 * Note that if INTR_DIRECT is not set then either the NIC rx 2424 * queues or (exclusive or) the TOE rx queueus will be taking 2425 * direct interrupts. 2426 * 2427 * There is no need to check for is_offload(sc) as nofldrxq 2428 * will be 0 if offload is disabled. 2429 */ 2430 for_each_port(sc, p) { 2431 pi = sc->port[p]; 2432 2433#ifdef TCP_OFFLOAD 2434 /* 2435 * Skip over the NIC queues if they aren't taking direct 2436 * interrupts. 2437 */ 2438 if (!(sc->flags & INTR_DIRECT) && 2439 pi->nofldrxq > pi->nrxq) 2440 goto ofld_queues; 2441#endif 2442 rxq = &sc->sge.rxq[pi->first_rxq]; 2443 for (q = 0; q < pi->nrxq; q++, rxq++) { 2444 snprintf(s, sizeof(s), "%d.%d", p, q); 2445 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 2446 s); 2447 if (rc != 0) 2448 return (rc); 2449 irq++; 2450 rid++; 2451 } 2452 2453#ifdef TCP_OFFLOAD 2454 /* 2455 * Skip over the offload queues if they aren't taking 2456 * direct interrupts. 2457 */ 2458 if (!(sc->flags & INTR_DIRECT)) 2459 continue; 2460ofld_queues: 2461 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2462 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2463 snprintf(s, sizeof(s), "%d,%d", p, q); 2464 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 2465 ofld_rxq, s); 2466 if (rc != 0) 2467 return (rc); 2468 irq++; 2469 rid++; 2470 } 2471#endif 2472 } 2473 } 2474 2475 return (0); 2476} 2477 2478static int 2479adapter_full_init(struct adapter *sc) 2480{ 2481 int rc, i; 2482 2483 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2484 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2485 ("%s: FULL_INIT_DONE already", __func__)); 2486 2487 /* 2488 * queues that belong to the adapter (not any particular port). 2489 */ 2490 rc = t4_setup_adapter_queues(sc); 2491 if (rc != 0) 2492 goto done; 2493 2494 for (i = 0; i < nitems(sc->tq); i++) { 2495 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2496 taskqueue_thread_enqueue, &sc->tq[i]); 2497 if (sc->tq[i] == NULL) { 2498 device_printf(sc->dev, 2499 "failed to allocate task queue %d\n", i); 2500 rc = ENOMEM; 2501 goto done; 2502 } 2503 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 2504 device_get_nameunit(sc->dev), i); 2505 } 2506 2507 t4_intr_enable(sc); 2508 sc->flags |= FULL_INIT_DONE; 2509done: 2510 if (rc != 0) 2511 adapter_full_uninit(sc); 2512 2513 return (rc); 2514} 2515 2516static int 2517adapter_full_uninit(struct adapter *sc) 2518{ 2519 int i; 2520 2521 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2522 2523 t4_teardown_adapter_queues(sc); 2524 2525 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 2526 taskqueue_free(sc->tq[i]); 2527 sc->tq[i] = NULL; 2528 } 2529 2530 sc->flags &= ~FULL_INIT_DONE; 2531 2532 return (0); 2533} 2534 2535static int 2536port_full_init(struct port_info *pi) 2537{ 2538 struct adapter *sc = pi->adapter; 2539 struct ifnet *ifp = pi->ifp; 2540 uint16_t *rss; 2541 struct sge_rxq *rxq; 2542 int rc, i; 2543 2544 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2545 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 2546 ("%s: PORT_INIT_DONE already", __func__)); 2547 2548 sysctl_ctx_init(&pi->ctx); 2549 pi->flags |= PORT_SYSCTL_CTX; 2550 2551 /* 2552 * Allocate tx/rx/fl queues for this port. 2553 */ 2554 rc = t4_setup_port_queues(pi); 2555 if (rc != 0) 2556 goto done; /* error message displayed already */ 2557 2558 /* 2559 * Setup RSS for this port. 2560 */ 2561 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 2562 M_ZERO | M_WAITOK); 2563 for_each_rxq(pi, i, rxq) { 2564 rss[i] = rxq->iq.abs_id; 2565 } 2566 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2567 pi->rss_size, rss, pi->nrxq); 2568 free(rss, M_CXGBE); 2569 if (rc != 0) { 2570 if_printf(ifp, "rss_config failed: %d\n", rc); 2571 goto done; 2572 } 2573 2574 pi->flags |= PORT_INIT_DONE; 2575done: 2576 if (rc != 0) 2577 port_full_uninit(pi); 2578 2579 return (rc); 2580} 2581 2582/* 2583 * Idempotent. 2584 */ 2585static int 2586port_full_uninit(struct port_info *pi) 2587{ 2588 struct adapter *sc = pi->adapter; 2589 int i; 2590 struct sge_rxq *rxq; 2591 struct sge_txq *txq; 2592#ifdef TCP_OFFLOAD 2593 struct sge_ofld_rxq *ofld_rxq; 2594 struct sge_wrq *ofld_txq; 2595#endif 2596 2597 if (pi->flags & PORT_INIT_DONE) { 2598 2599 /* Need to quiesce queues. XXX: ctrl queues? */ 2600 2601 for_each_txq(pi, i, txq) { 2602 quiesce_eq(sc, &txq->eq); 2603 } 2604 2605#ifdef TCP_OFFLOAD 2606 for_each_ofld_txq(pi, i, ofld_txq) { 2607 quiesce_eq(sc, &ofld_txq->eq); 2608 } 2609#endif 2610 2611 for_each_rxq(pi, i, rxq) { 2612 quiesce_iq(sc, &rxq->iq); 2613 quiesce_fl(sc, &rxq->fl); 2614 } 2615 2616#ifdef TCP_OFFLOAD 2617 for_each_ofld_rxq(pi, i, ofld_rxq) { 2618 quiesce_iq(sc, &ofld_rxq->iq); 2619 quiesce_fl(sc, &ofld_rxq->fl); 2620 } 2621#endif 2622 } 2623 2624 t4_teardown_port_queues(pi); 2625 pi->flags &= ~PORT_INIT_DONE; 2626 2627 return (0); 2628} 2629 2630static void 2631quiesce_eq(struct adapter *sc, struct sge_eq *eq) 2632{ 2633 EQ_LOCK(eq); 2634 eq->flags |= EQ_DOOMED; 2635 2636 /* 2637 * Wait for the response to a credit flush if one's 2638 * pending. 2639 */ 2640 while (eq->flags & EQ_CRFLUSHED) 2641 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 2642 EQ_UNLOCK(eq); 2643 2644 callout_drain(&eq->tx_callout); /* XXX: iffy */ 2645 pause("callout", 10); /* Still iffy */ 2646 2647 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 2648} 2649 2650static void 2651quiesce_iq(struct adapter *sc, struct sge_iq *iq) 2652{ 2653 (void) sc; /* unused */ 2654 2655 /* Synchronize with the interrupt handler */ 2656 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 2657 pause("iqfree", 1); 2658} 2659 2660static void 2661quiesce_fl(struct adapter *sc, struct sge_fl *fl) 2662{ 2663 mtx_lock(&sc->sfl_lock); 2664 FL_LOCK(fl); 2665 fl->flags |= FL_DOOMED; 2666 FL_UNLOCK(fl); 2667 mtx_unlock(&sc->sfl_lock); 2668 2669 callout_drain(&sc->sfl_callout); 2670 KASSERT((fl->flags & FL_STARVING) == 0, 2671 ("%s: still starving", __func__)); 2672} 2673 2674static int 2675t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 2676 driver_intr_t *handler, void *arg, char *name) 2677{ 2678 int rc; 2679 2680 irq->rid = rid; 2681 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 2682 RF_SHAREABLE | RF_ACTIVE); 2683 if (irq->res == NULL) { 2684 device_printf(sc->dev, 2685 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 2686 return (ENOMEM); 2687 } 2688 2689 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 2690 NULL, handler, arg, &irq->tag); 2691 if (rc != 0) { 2692 device_printf(sc->dev, 2693 "failed to setup interrupt for rid %d, name %s: %d\n", 2694 rid, name, rc); 2695 } else if (name) 2696 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 2697 2698 return (rc); 2699} 2700 2701static int 2702t4_free_irq(struct adapter *sc, struct irq *irq) 2703{ 2704 if (irq->tag) 2705 bus_teardown_intr(sc->dev, irq->res, irq->tag); 2706 if (irq->res) 2707 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 2708 2709 bzero(irq, sizeof(*irq)); 2710 2711 return (0); 2712} 2713 2714static void 2715reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 2716 unsigned int end) 2717{ 2718 uint32_t *p = (uint32_t *)(buf + start); 2719 2720 for ( ; start <= end; start += sizeof(uint32_t)) 2721 *p++ = t4_read_reg(sc, start); 2722} 2723 2724static void 2725t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 2726{ 2727 int i; 2728 static const unsigned int reg_ranges[] = { 2729 0x1008, 0x1108, 2730 0x1180, 0x11b4, 2731 0x11fc, 0x123c, 2732 0x1300, 0x173c, 2733 0x1800, 0x18fc, 2734 0x3000, 0x30d8, 2735 0x30e0, 0x5924, 2736 0x5960, 0x59d4, 2737 0x5a00, 0x5af8, 2738 0x6000, 0x6098, 2739 0x6100, 0x6150, 2740 0x6200, 0x6208, 2741 0x6240, 0x6248, 2742 0x6280, 0x6338, 2743 0x6370, 0x638c, 2744 0x6400, 0x643c, 2745 0x6500, 0x6524, 2746 0x6a00, 0x6a38, 2747 0x6a60, 0x6a78, 2748 0x6b00, 0x6b84, 2749 0x6bf0, 0x6c84, 2750 0x6cf0, 0x6d84, 2751 0x6df0, 0x6e84, 2752 0x6ef0, 0x6f84, 2753 0x6ff0, 0x7084, 2754 0x70f0, 0x7184, 2755 0x71f0, 0x7284, 2756 0x72f0, 0x7384, 2757 0x73f0, 0x7450, 2758 0x7500, 0x7530, 2759 0x7600, 0x761c, 2760 0x7680, 0x76cc, 2761 0x7700, 0x7798, 2762 0x77c0, 0x77fc, 2763 0x7900, 0x79fc, 2764 0x7b00, 0x7c38, 2765 0x7d00, 0x7efc, 2766 0x8dc0, 0x8e1c, 2767 0x8e30, 0x8e78, 2768 0x8ea0, 0x8f6c, 2769 0x8fc0, 0x9074, 2770 0x90fc, 0x90fc, 2771 0x9400, 0x9458, 2772 0x9600, 0x96bc, 2773 0x9800, 0x9808, 2774 0x9820, 0x983c, 2775 0x9850, 0x9864, 2776 0x9c00, 0x9c6c, 2777 0x9c80, 0x9cec, 2778 0x9d00, 0x9d6c, 2779 0x9d80, 0x9dec, 2780 0x9e00, 0x9e6c, 2781 0x9e80, 0x9eec, 2782 0x9f00, 0x9f6c, 2783 0x9f80, 0x9fec, 2784 0xd004, 0xd03c, 2785 0xdfc0, 0xdfe0, 2786 0xe000, 0xea7c, 2787 0xf000, 0x11190, 2788 0x19040, 0x1906c, 2789 0x19078, 0x19080, 2790 0x1908c, 0x19124, 2791 0x19150, 0x191b0, 2792 0x191d0, 0x191e8, 2793 0x19238, 0x1924c, 2794 0x193f8, 0x19474, 2795 0x19490, 0x194f8, 2796 0x19800, 0x19f30, 2797 0x1a000, 0x1a06c, 2798 0x1a0b0, 0x1a120, 2799 0x1a128, 0x1a138, 2800 0x1a190, 0x1a1c4, 2801 0x1a1fc, 0x1a1fc, 2802 0x1e040, 0x1e04c, 2803 0x1e284, 0x1e28c, 2804 0x1e2c0, 0x1e2c0, 2805 0x1e2e0, 0x1e2e0, 2806 0x1e300, 0x1e384, 2807 0x1e3c0, 0x1e3c8, 2808 0x1e440, 0x1e44c, 2809 0x1e684, 0x1e68c, 2810 0x1e6c0, 0x1e6c0, 2811 0x1e6e0, 0x1e6e0, 2812 0x1e700, 0x1e784, 2813 0x1e7c0, 0x1e7c8, 2814 0x1e840, 0x1e84c, 2815 0x1ea84, 0x1ea8c, 2816 0x1eac0, 0x1eac0, 2817 0x1eae0, 0x1eae0, 2818 0x1eb00, 0x1eb84, 2819 0x1ebc0, 0x1ebc8, 2820 0x1ec40, 0x1ec4c, 2821 0x1ee84, 0x1ee8c, 2822 0x1eec0, 0x1eec0, 2823 0x1eee0, 0x1eee0, 2824 0x1ef00, 0x1ef84, 2825 0x1efc0, 0x1efc8, 2826 0x1f040, 0x1f04c, 2827 0x1f284, 0x1f28c, 2828 0x1f2c0, 0x1f2c0, 2829 0x1f2e0, 0x1f2e0, 2830 0x1f300, 0x1f384, 2831 0x1f3c0, 0x1f3c8, 2832 0x1f440, 0x1f44c, 2833 0x1f684, 0x1f68c, 2834 0x1f6c0, 0x1f6c0, 2835 0x1f6e0, 0x1f6e0, 2836 0x1f700, 0x1f784, 2837 0x1f7c0, 0x1f7c8, 2838 0x1f840, 0x1f84c, 2839 0x1fa84, 0x1fa8c, 2840 0x1fac0, 0x1fac0, 2841 0x1fae0, 0x1fae0, 2842 0x1fb00, 0x1fb84, 2843 0x1fbc0, 0x1fbc8, 2844 0x1fc40, 0x1fc4c, 2845 0x1fe84, 0x1fe8c, 2846 0x1fec0, 0x1fec0, 2847 0x1fee0, 0x1fee0, 2848 0x1ff00, 0x1ff84, 2849 0x1ffc0, 0x1ffc8, 2850 0x20000, 0x2002c, 2851 0x20100, 0x2013c, 2852 0x20190, 0x201c8, 2853 0x20200, 0x20318, 2854 0x20400, 0x20528, 2855 0x20540, 0x20614, 2856 0x21000, 0x21040, 2857 0x2104c, 0x21060, 2858 0x210c0, 0x210ec, 2859 0x21200, 0x21268, 2860 0x21270, 0x21284, 2861 0x212fc, 0x21388, 2862 0x21400, 0x21404, 2863 0x21500, 0x21518, 2864 0x2152c, 0x2153c, 2865 0x21550, 0x21554, 2866 0x21600, 0x21600, 2867 0x21608, 0x21628, 2868 0x21630, 0x2163c, 2869 0x21700, 0x2171c, 2870 0x21780, 0x2178c, 2871 0x21800, 0x21c38, 2872 0x21c80, 0x21d7c, 2873 0x21e00, 0x21e04, 2874 0x22000, 0x2202c, 2875 0x22100, 0x2213c, 2876 0x22190, 0x221c8, 2877 0x22200, 0x22318, 2878 0x22400, 0x22528, 2879 0x22540, 0x22614, 2880 0x23000, 0x23040, 2881 0x2304c, 0x23060, 2882 0x230c0, 0x230ec, 2883 0x23200, 0x23268, 2884 0x23270, 0x23284, 2885 0x232fc, 0x23388, 2886 0x23400, 0x23404, 2887 0x23500, 0x23518, 2888 0x2352c, 0x2353c, 2889 0x23550, 0x23554, 2890 0x23600, 0x23600, 2891 0x23608, 0x23628, 2892 0x23630, 0x2363c, 2893 0x23700, 0x2371c, 2894 0x23780, 0x2378c, 2895 0x23800, 0x23c38, 2896 0x23c80, 0x23d7c, 2897 0x23e00, 0x23e04, 2898 0x24000, 0x2402c, 2899 0x24100, 0x2413c, 2900 0x24190, 0x241c8, 2901 0x24200, 0x24318, 2902 0x24400, 0x24528, 2903 0x24540, 0x24614, 2904 0x25000, 0x25040, 2905 0x2504c, 0x25060, 2906 0x250c0, 0x250ec, 2907 0x25200, 0x25268, 2908 0x25270, 0x25284, 2909 0x252fc, 0x25388, 2910 0x25400, 0x25404, 2911 0x25500, 0x25518, 2912 0x2552c, 0x2553c, 2913 0x25550, 0x25554, 2914 0x25600, 0x25600, 2915 0x25608, 0x25628, 2916 0x25630, 0x2563c, 2917 0x25700, 0x2571c, 2918 0x25780, 0x2578c, 2919 0x25800, 0x25c38, 2920 0x25c80, 0x25d7c, 2921 0x25e00, 0x25e04, 2922 0x26000, 0x2602c, 2923 0x26100, 0x2613c, 2924 0x26190, 0x261c8, 2925 0x26200, 0x26318, 2926 0x26400, 0x26528, 2927 0x26540, 0x26614, 2928 0x27000, 0x27040, 2929 0x2704c, 0x27060, 2930 0x270c0, 0x270ec, 2931 0x27200, 0x27268, 2932 0x27270, 0x27284, 2933 0x272fc, 0x27388, 2934 0x27400, 0x27404, 2935 0x27500, 0x27518, 2936 0x2752c, 0x2753c, 2937 0x27550, 0x27554, 2938 0x27600, 0x27600, 2939 0x27608, 0x27628, 2940 0x27630, 0x2763c, 2941 0x27700, 0x2771c, 2942 0x27780, 0x2778c, 2943 0x27800, 0x27c38, 2944 0x27c80, 0x27d7c, 2945 0x27e00, 0x27e04 2946 }; 2947 2948 regs->version = 4 | (sc->params.rev << 10); 2949 for (i = 0; i < nitems(reg_ranges); i += 2) 2950 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 2951} 2952 2953static void 2954cxgbe_tick(void *arg) 2955{ 2956 struct port_info *pi = arg; 2957 struct ifnet *ifp = pi->ifp; 2958 struct sge_txq *txq; 2959 int i, drops; 2960 struct port_stats *s = &pi->stats; 2961 2962 PORT_LOCK(pi); 2963 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 2964 PORT_UNLOCK(pi); 2965 return; /* without scheduling another callout */ 2966 } 2967 2968 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 2969 2970 ifp->if_opackets = s->tx_frames - s->tx_pause; 2971 ifp->if_ipackets = s->rx_frames - s->rx_pause; 2972 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 2973 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 2974 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 2975 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 2976 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 2977 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 2978 s->rx_trunc3; 2979 2980 drops = s->tx_drop; 2981 for_each_txq(pi, i, txq) 2982 drops += txq->br->br_drops; 2983 ifp->if_snd.ifq_drops = drops; 2984 2985 ifp->if_oerrors = s->tx_error_frames; 2986 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 2987 s->rx_fcs_err + s->rx_len_err; 2988 2989 callout_schedule(&pi->tick, hz); 2990 PORT_UNLOCK(pi); 2991} 2992 2993static void 2994cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 2995{ 2996 struct ifnet *vlan; 2997 2998 if (arg != ifp) 2999 return; 3000 3001 vlan = VLAN_DEVAT(ifp, vid); 3002 VLAN_SETCOOKIE(vlan, ifp); 3003} 3004 3005static int 3006cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 3007{ 3008 3009#ifdef INVARIANTS 3010 panic("%s: opcode 0x%02x on iq %p with payload %p", 3011 __func__, rss->opcode, iq, m); 3012#else 3013 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 3014 __func__, rss->opcode, iq, m); 3015 m_freem(m); 3016#endif 3017 return (EDOOFUS); 3018} 3019 3020int 3021t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3022{ 3023 uintptr_t *loc, new; 3024 3025 if (opcode >= nitems(sc->cpl_handler)) 3026 return (EINVAL); 3027 3028 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3029 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3030 atomic_store_rel_ptr(loc, new); 3031 3032 return (0); 3033} 3034 3035static int 3036an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 3037{ 3038 3039#ifdef INVARIANTS 3040 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 3041#else 3042 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 3043 __func__, iq, ctrl); 3044#endif 3045 return (EDOOFUS); 3046} 3047 3048int 3049t4_register_an_handler(struct adapter *sc, an_handler_t h) 3050{ 3051 uintptr_t *loc, new; 3052 3053 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 3054 loc = (uintptr_t *) &sc->an_handler; 3055 atomic_store_rel_ptr(loc, new); 3056 3057 return (0); 3058} 3059 3060static int 3061fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 3062{ 3063 __be64 *r = __DECONST(__be64 *, rpl); 3064 struct cpl_fw6_msg *cpl = member2struct(cpl_fw6_msg, data, r); 3065 3066#ifdef INVARIANTS 3067 panic("%s: fw_msg type %d", __func__, cpl->type); 3068#else 3069 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 3070#endif 3071 return (EDOOFUS); 3072} 3073 3074int 3075t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 3076{ 3077 uintptr_t *loc, new; 3078 3079 if (type >= nitems(sc->fw_msg_handler)) 3080 return (EINVAL); 3081 3082 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 3083 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 3084 atomic_store_rel_ptr(loc, new); 3085 3086 return (0); 3087} 3088 3089static int 3090t4_sysctls(struct adapter *sc) 3091{ 3092 struct sysctl_ctx_list *ctx; 3093 struct sysctl_oid *oid; 3094 struct sysctl_oid_list *children, *c0; 3095 static char *caps[] = { 3096 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 3097 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 3098 "\20\1TOE", /* caps[2] toecaps */ 3099 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 3100 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 3101 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 3102 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 3103 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 3104 }; 3105 3106 ctx = device_get_sysctl_ctx(sc->dev); 3107 3108 /* 3109 * dev.t4nex.X. 3110 */ 3111 oid = device_get_sysctl_tree(sc->dev); 3112 c0 = children = SYSCTL_CHILDREN(oid); 3113 3114 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, 3115 &sc->params.nports, 0, "# of ports"); 3116 3117 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 3118 &sc->params.rev, 0, "chip hardware revision"); 3119 3120 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 3121 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 3122 3123 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 3124 CTLFLAG_RD, &t4_cfg_file, 0, "configuration file"); 3125 3126 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, 3127 &sc->cfcsum, 0, "config file checksum"); 3128 3129 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 3130 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 3131 sysctl_bitfield, "A", "available link capabilities"); 3132 3133 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 3134 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 3135 sysctl_bitfield, "A", "available NIC capabilities"); 3136 3137 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 3138 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 3139 sysctl_bitfield, "A", "available TCP offload capabilities"); 3140 3141 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 3142 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 3143 sysctl_bitfield, "A", "available RDMA capabilities"); 3144 3145 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 3146 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 3147 sysctl_bitfield, "A", "available iSCSI capabilities"); 3148 3149 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 3150 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 3151 sysctl_bitfield, "A", "available FCoE capabilities"); 3152 3153 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, 3154 &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)"); 3155 3156 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 3157 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 3158 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 3159 "interrupt holdoff timer values (us)"); 3160 3161 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 3162 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 3163 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 3164 "interrupt holdoff packet counter values"); 3165 3166#ifdef SBUF_DRAIN 3167 /* 3168 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 3169 */ 3170 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 3171 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 3172 "logs and miscellaneous information"); 3173 children = SYSCTL_CHILDREN(oid); 3174 3175 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 3176 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3177 sysctl_cctrl, "A", "congestion control"); 3178 3179 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 3180 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3181 sysctl_cpl_stats, "A", "CPL statistics"); 3182 3183 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 3184 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3185 sysctl_ddp_stats, "A", "DDP statistics"); 3186 3187 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 3188 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3189 sysctl_devlog, "A", "firmware's device log"); 3190 3191 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 3192 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3193 sysctl_fcoe_stats, "A", "FCoE statistics"); 3194 3195 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 3196 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3197 sysctl_hw_sched, "A", "hardware scheduler "); 3198 3199 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 3200 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3201 sysctl_l2t, "A", "hardware L2 table"); 3202 3203 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 3204 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3205 sysctl_lb_stats, "A", "loopback statistics"); 3206 3207 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 3208 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3209 sysctl_meminfo, "A", "memory regions"); 3210 3211 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 3212 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3213 sysctl_path_mtus, "A", "path MTUs"); 3214 3215 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 3216 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3217 sysctl_pm_stats, "A", "PM statistics"); 3218 3219 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 3220 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3221 sysctl_rdma_stats, "A", "RDMA statistics"); 3222 3223 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 3224 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3225 sysctl_tcp_stats, "A", "TCP statistics"); 3226 3227 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 3228 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3229 sysctl_tids, "A", "TID information"); 3230 3231 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 3232 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3233 sysctl_tp_err_stats, "A", "TP error statistics"); 3234 3235 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 3236 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 3237 sysctl_tx_rate, "A", "Tx rate"); 3238#endif 3239 3240#ifdef TCP_OFFLOAD 3241 if (is_offload(sc)) { 3242 /* 3243 * dev.t4nex.X.toe. 3244 */ 3245 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 3246 NULL, "TOE parameters"); 3247 children = SYSCTL_CHILDREN(oid); 3248 3249 sc->tt.sndbuf = 256 * 1024; 3250 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 3251 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 3252 3253 sc->tt.ddp = 0; 3254 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 3255 &sc->tt.ddp, 0, "DDP allowed"); 3256 3257 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 3258 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 3259 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 3260 3261 sc->tt.ddp_thres = 3262 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 3263 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 3264 &sc->tt.ddp_thres, 0, "DDP threshold"); 3265 } 3266#endif 3267 3268 3269 return (0); 3270} 3271 3272static int 3273cxgbe_sysctls(struct port_info *pi) 3274{ 3275 struct sysctl_ctx_list *ctx; 3276 struct sysctl_oid *oid; 3277 struct sysctl_oid_list *children; 3278 3279 ctx = device_get_sysctl_ctx(pi->dev); 3280 3281 /* 3282 * dev.cxgbe.X. 3283 */ 3284 oid = device_get_sysctl_tree(pi->dev); 3285 children = SYSCTL_CHILDREN(oid); 3286 3287 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 3288 &pi->nrxq, 0, "# of rx queues"); 3289 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 3290 &pi->ntxq, 0, "# of tx queues"); 3291 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 3292 &pi->first_rxq, 0, "index of first rx queue"); 3293 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 3294 &pi->first_txq, 0, "index of first tx queue"); 3295 3296#ifdef TCP_OFFLOAD 3297 if (is_offload(pi->adapter)) { 3298 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 3299 &pi->nofldrxq, 0, 3300 "# of rx queues for offloaded TCP connections"); 3301 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 3302 &pi->nofldtxq, 0, 3303 "# of tx queues for offloaded TCP connections"); 3304 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 3305 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 3306 "index of first TOE rx queue"); 3307 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 3308 CTLFLAG_RD, &pi->first_ofld_txq, 0, 3309 "index of first TOE tx queue"); 3310 } 3311#endif 3312 3313 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 3314 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 3315 "holdoff timer index"); 3316 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 3317 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 3318 "holdoff packet counter index"); 3319 3320 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 3321 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 3322 "rx queue size"); 3323 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 3324 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 3325 "tx queue size"); 3326 3327 /* 3328 * dev.cxgbe.X.stats. 3329 */ 3330 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 3331 NULL, "port statistics"); 3332 children = SYSCTL_CHILDREN(oid); 3333 3334#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 3335 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 3336 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 3337 sysctl_handle_t4_reg64, "QU", desc) 3338 3339 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 3340 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 3341 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 3342 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 3343 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 3344 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 3345 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 3346 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 3347 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 3348 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 3349 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 3350 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 3351 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 3352 "# of tx frames in this range", 3353 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 3354 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 3355 "# of tx frames in this range", 3356 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 3357 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 3358 "# of tx frames in this range", 3359 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 3360 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 3361 "# of tx frames in this range", 3362 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 3363 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 3364 "# of tx frames in this range", 3365 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 3366 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 3367 "# of tx frames in this range", 3368 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 3369 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 3370 "# of tx frames in this range", 3371 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 3372 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 3373 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 3374 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 3375 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 3376 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 3377 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 3378 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 3379 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 3380 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 3381 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 3382 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 3383 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 3384 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 3385 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 3386 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 3387 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 3388 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 3389 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 3390 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 3391 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 3392 3393 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 3394 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 3395 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 3396 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 3397 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 3398 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 3399 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 3400 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 3401 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 3402 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 3403 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 3404 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 3405 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 3406 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 3407 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 3408 "# of frames received with bad FCS", 3409 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 3410 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 3411 "# of frames received with length error", 3412 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 3413 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 3414 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 3415 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 3416 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 3417 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 3418 "# of rx frames in this range", 3419 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 3420 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 3421 "# of rx frames in this range", 3422 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 3423 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 3424 "# of rx frames in this range", 3425 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 3426 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 3427 "# of rx frames in this range", 3428 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 3429 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 3430 "# of rx frames in this range", 3431 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 3432 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 3433 "# of rx frames in this range", 3434 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 3435 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 3436 "# of rx frames in this range", 3437 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 3438 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 3439 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 3440 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 3441 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 3442 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 3443 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 3444 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 3445 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 3446 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 3447 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 3448 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 3449 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 3450 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 3451 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 3452 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 3453 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 3454 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 3455 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 3456 3457#undef SYSCTL_ADD_T4_REG64 3458 3459#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 3460 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 3461 &pi->stats.name, desc) 3462 3463 /* We get these from port_stats and they may be stale by upto 1s */ 3464 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 3465 "# drops due to buffer-group 0 overflows"); 3466 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 3467 "# drops due to buffer-group 1 overflows"); 3468 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 3469 "# drops due to buffer-group 2 overflows"); 3470 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 3471 "# drops due to buffer-group 3 overflows"); 3472 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 3473 "# of buffer-group 0 truncated packets"); 3474 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 3475 "# of buffer-group 1 truncated packets"); 3476 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 3477 "# of buffer-group 2 truncated packets"); 3478 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 3479 "# of buffer-group 3 truncated packets"); 3480 3481#undef SYSCTL_ADD_T4_PORTSTAT 3482 3483 return (0); 3484} 3485 3486static int 3487sysctl_int_array(SYSCTL_HANDLER_ARGS) 3488{ 3489 int rc, *i; 3490 struct sbuf sb; 3491 3492 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 3493 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 3494 sbuf_printf(&sb, "%d ", *i); 3495 sbuf_trim(&sb); 3496 sbuf_finish(&sb); 3497 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 3498 sbuf_delete(&sb); 3499 return (rc); 3500} 3501 3502static int 3503sysctl_bitfield(SYSCTL_HANDLER_ARGS) 3504{ 3505 int rc; 3506 struct sbuf *sb; 3507 3508 rc = sysctl_wire_old_buffer(req, 0); 3509 if (rc != 0) 3510 return(rc); 3511 3512 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3513 if (sb == NULL) 3514 return (ENOMEM); 3515 3516 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 3517 rc = sbuf_finish(sb); 3518 sbuf_delete(sb); 3519 3520 return (rc); 3521} 3522 3523static int 3524sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 3525{ 3526 struct port_info *pi = arg1; 3527 struct adapter *sc = pi->adapter; 3528 int idx, rc, i; 3529 3530 idx = pi->tmr_idx; 3531 3532 rc = sysctl_handle_int(oidp, &idx, 0, req); 3533 if (rc != 0 || req->newptr == NULL) 3534 return (rc); 3535 3536 if (idx < 0 || idx >= SGE_NTIMERS) 3537 return (EINVAL); 3538 3539 ADAPTER_LOCK(sc); 3540 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3541 if (rc == 0) { 3542 struct sge_rxq *rxq; 3543 uint8_t v; 3544 3545 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 3546 for_each_rxq(pi, i, rxq) { 3547#ifdef atomic_store_rel_8 3548 atomic_store_rel_8(&rxq->iq.intr_params, v); 3549#else 3550 rxq->iq.intr_params = v; 3551#endif 3552 } 3553 pi->tmr_idx = idx; 3554 } 3555 3556 ADAPTER_UNLOCK(sc); 3557 return (rc); 3558} 3559 3560static int 3561sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 3562{ 3563 struct port_info *pi = arg1; 3564 struct adapter *sc = pi->adapter; 3565 int idx, rc; 3566 3567 idx = pi->pktc_idx; 3568 3569 rc = sysctl_handle_int(oidp, &idx, 0, req); 3570 if (rc != 0 || req->newptr == NULL) 3571 return (rc); 3572 3573 if (idx < -1 || idx >= SGE_NCOUNTERS) 3574 return (EINVAL); 3575 3576 ADAPTER_LOCK(sc); 3577 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3578 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3579 rc = EBUSY; /* cannot be changed once the queues are created */ 3580 3581 if (rc == 0) 3582 pi->pktc_idx = idx; 3583 3584 ADAPTER_UNLOCK(sc); 3585 return (rc); 3586} 3587 3588static int 3589sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 3590{ 3591 struct port_info *pi = arg1; 3592 struct adapter *sc = pi->adapter; 3593 int qsize, rc; 3594 3595 qsize = pi->qsize_rxq; 3596 3597 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3598 if (rc != 0 || req->newptr == NULL) 3599 return (rc); 3600 3601 if (qsize < 128 || (qsize & 7)) 3602 return (EINVAL); 3603 3604 ADAPTER_LOCK(sc); 3605 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3606 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3607 rc = EBUSY; /* cannot be changed once the queues are created */ 3608 3609 if (rc == 0) 3610 pi->qsize_rxq = qsize; 3611 3612 ADAPTER_UNLOCK(sc); 3613 return (rc); 3614} 3615 3616static int 3617sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 3618{ 3619 struct port_info *pi = arg1; 3620 struct adapter *sc = pi->adapter; 3621 int qsize, rc; 3622 3623 qsize = pi->qsize_txq; 3624 3625 rc = sysctl_handle_int(oidp, &qsize, 0, req); 3626 if (rc != 0 || req->newptr == NULL) 3627 return (rc); 3628 3629 if (qsize < 128) 3630 return (EINVAL); 3631 3632 ADAPTER_LOCK(sc); 3633 rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0); 3634 if (rc == 0 && pi->flags & PORT_INIT_DONE) 3635 rc = EBUSY; /* cannot be changed once the queues are created */ 3636 3637 if (rc == 0) 3638 pi->qsize_txq = qsize; 3639 3640 ADAPTER_UNLOCK(sc); 3641 return (rc); 3642} 3643 3644static int 3645sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 3646{ 3647 struct adapter *sc = arg1; 3648 int reg = arg2; 3649 uint64_t val; 3650 3651 val = t4_read_reg64(sc, reg); 3652 3653 return (sysctl_handle_64(oidp, &val, 0, req)); 3654} 3655 3656#ifdef SBUF_DRAIN 3657static int 3658sysctl_cctrl(SYSCTL_HANDLER_ARGS) 3659{ 3660 struct adapter *sc = arg1; 3661 struct sbuf *sb; 3662 int rc, i; 3663 uint16_t incr[NMTUS][NCCTRL_WIN]; 3664 static const char *dec_fac[] = { 3665 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 3666 "0.9375" 3667 }; 3668 3669 rc = sysctl_wire_old_buffer(req, 0); 3670 if (rc != 0) 3671 return (rc); 3672 3673 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3674 if (sb == NULL) 3675 return (ENOMEM); 3676 3677 t4_read_cong_tbl(sc, incr); 3678 3679 for (i = 0; i < NCCTRL_WIN; ++i) { 3680 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 3681 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 3682 incr[5][i], incr[6][i], incr[7][i]); 3683 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 3684 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 3685 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 3686 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 3687 } 3688 3689 rc = sbuf_finish(sb); 3690 sbuf_delete(sb); 3691 3692 return (rc); 3693} 3694 3695static int 3696sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 3697{ 3698 struct adapter *sc = arg1; 3699 struct sbuf *sb; 3700 int rc; 3701 struct tp_cpl_stats stats; 3702 3703 rc = sysctl_wire_old_buffer(req, 0); 3704 if (rc != 0) 3705 return (rc); 3706 3707 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3708 if (sb == NULL) 3709 return (ENOMEM); 3710 3711 t4_tp_get_cpl_stats(sc, &stats); 3712 3713 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 3714 "channel 3\n"); 3715 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 3716 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 3717 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 3718 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 3719 3720 rc = sbuf_finish(sb); 3721 sbuf_delete(sb); 3722 3723 return (rc); 3724} 3725 3726static int 3727sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 3728{ 3729 struct adapter *sc = arg1; 3730 struct sbuf *sb; 3731 int rc; 3732 struct tp_usm_stats stats; 3733 3734 rc = sysctl_wire_old_buffer(req, 0); 3735 if (rc != 0) 3736 return(rc); 3737 3738 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3739 if (sb == NULL) 3740 return (ENOMEM); 3741 3742 t4_get_usm_stats(sc, &stats); 3743 3744 sbuf_printf(sb, "Frames: %u\n", stats.frames); 3745 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 3746 sbuf_printf(sb, "Drops: %u", stats.drops); 3747 3748 rc = sbuf_finish(sb); 3749 sbuf_delete(sb); 3750 3751 return (rc); 3752} 3753 3754const char *devlog_level_strings[] = { 3755 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 3756 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 3757 [FW_DEVLOG_LEVEL_ERR] = "ERR", 3758 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 3759 [FW_DEVLOG_LEVEL_INFO] = "INFO", 3760 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 3761}; 3762 3763const char *devlog_facility_strings[] = { 3764 [FW_DEVLOG_FACILITY_CORE] = "CORE", 3765 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 3766 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 3767 [FW_DEVLOG_FACILITY_RES] = "RES", 3768 [FW_DEVLOG_FACILITY_HW] = "HW", 3769 [FW_DEVLOG_FACILITY_FLR] = "FLR", 3770 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 3771 [FW_DEVLOG_FACILITY_PHY] = "PHY", 3772 [FW_DEVLOG_FACILITY_MAC] = "MAC", 3773 [FW_DEVLOG_FACILITY_PORT] = "PORT", 3774 [FW_DEVLOG_FACILITY_VI] = "VI", 3775 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 3776 [FW_DEVLOG_FACILITY_ACL] = "ACL", 3777 [FW_DEVLOG_FACILITY_TM] = "TM", 3778 [FW_DEVLOG_FACILITY_QFC] = "QFC", 3779 [FW_DEVLOG_FACILITY_DCB] = "DCB", 3780 [FW_DEVLOG_FACILITY_ETH] = "ETH", 3781 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 3782 [FW_DEVLOG_FACILITY_RI] = "RI", 3783 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 3784 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 3785 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 3786 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 3787}; 3788 3789static int 3790sysctl_devlog(SYSCTL_HANDLER_ARGS) 3791{ 3792 struct adapter *sc = arg1; 3793 struct devlog_params *dparams = &sc->params.devlog; 3794 struct fw_devlog_e *buf, *e; 3795 int i, j, rc, nentries, first = 0; 3796 struct sbuf *sb; 3797 uint64_t ftstamp = UINT64_MAX; 3798 3799 if (dparams->start == 0) 3800 return (ENXIO); 3801 3802 nentries = dparams->size / sizeof(struct fw_devlog_e); 3803 3804 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 3805 if (buf == NULL) 3806 return (ENOMEM); 3807 3808 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 3809 (void *)buf); 3810 if (rc != 0) 3811 goto done; 3812 3813 for (i = 0; i < nentries; i++) { 3814 e = &buf[i]; 3815 3816 if (e->timestamp == 0) 3817 break; /* end */ 3818 3819 e->timestamp = be64toh(e->timestamp); 3820 e->seqno = be32toh(e->seqno); 3821 for (j = 0; j < 8; j++) 3822 e->params[j] = be32toh(e->params[j]); 3823 3824 if (e->timestamp < ftstamp) { 3825 ftstamp = e->timestamp; 3826 first = i; 3827 } 3828 } 3829 3830 if (buf[first].timestamp == 0) 3831 goto done; /* nothing in the log */ 3832 3833 rc = sysctl_wire_old_buffer(req, 0); 3834 if (rc != 0) 3835 goto done; 3836 3837 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3838 if (sb == NULL) { 3839 rc = ENOMEM; 3840 goto done; 3841 } 3842 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 3843 "Seq#", "Tstamp", "Level", "Facility", "Message"); 3844 3845 i = first; 3846 do { 3847 e = &buf[i]; 3848 if (e->timestamp == 0) 3849 break; /* end */ 3850 3851 sbuf_printf(sb, "%10d %15ju %8s %8s ", 3852 e->seqno, e->timestamp, 3853 (e->level < nitems(devlog_level_strings) ? 3854 devlog_level_strings[e->level] : "UNKNOWN"), 3855 (e->facility < nitems(devlog_facility_strings) ? 3856 devlog_facility_strings[e->facility] : "UNKNOWN")); 3857 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 3858 e->params[2], e->params[3], e->params[4], 3859 e->params[5], e->params[6], e->params[7]); 3860 3861 if (++i == nentries) 3862 i = 0; 3863 } while (i != first); 3864 3865 rc = sbuf_finish(sb); 3866 sbuf_delete(sb); 3867done: 3868 free(buf, M_CXGBE); 3869 return (rc); 3870} 3871 3872static int 3873sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 3874{ 3875 struct adapter *sc = arg1; 3876 struct sbuf *sb; 3877 int rc; 3878 struct tp_fcoe_stats stats[4]; 3879 3880 rc = sysctl_wire_old_buffer(req, 0); 3881 if (rc != 0) 3882 return (rc); 3883 3884 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3885 if (sb == NULL) 3886 return (ENOMEM); 3887 3888 t4_get_fcoe_stats(sc, 0, &stats[0]); 3889 t4_get_fcoe_stats(sc, 1, &stats[1]); 3890 t4_get_fcoe_stats(sc, 2, &stats[2]); 3891 t4_get_fcoe_stats(sc, 3, &stats[3]); 3892 3893 sbuf_printf(sb, " channel 0 channel 1 " 3894 "channel 2 channel 3\n"); 3895 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 3896 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 3897 stats[3].octetsDDP); 3898 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 3899 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 3900 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 3901 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 3902 stats[3].framesDrop); 3903 3904 rc = sbuf_finish(sb); 3905 sbuf_delete(sb); 3906 3907 return (rc); 3908} 3909 3910static int 3911sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 3912{ 3913 struct adapter *sc = arg1; 3914 struct sbuf *sb; 3915 int rc, i; 3916 unsigned int map, kbps, ipg, mode; 3917 unsigned int pace_tab[NTX_SCHED]; 3918 3919 rc = sysctl_wire_old_buffer(req, 0); 3920 if (rc != 0) 3921 return (rc); 3922 3923 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 3924 if (sb == NULL) 3925 return (ENOMEM); 3926 3927 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 3928 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 3929 t4_read_pace_tbl(sc, pace_tab); 3930 3931 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 3932 "Class IPG (0.1 ns) Flow IPG (us)"); 3933 3934 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 3935 t4_get_tx_sched(sc, i, &kbps, &ipg); 3936 sbuf_printf(sb, "\n %u %-5s %u ", i, 3937 (mode & (1 << i)) ? "flow" : "class", map & 3); 3938 if (kbps) 3939 sbuf_printf(sb, "%9u ", kbps); 3940 else 3941 sbuf_printf(sb, " disabled "); 3942 3943 if (ipg) 3944 sbuf_printf(sb, "%13u ", ipg); 3945 else 3946 sbuf_printf(sb, " disabled "); 3947 3948 if (pace_tab[i]) 3949 sbuf_printf(sb, "%10u", pace_tab[i]); 3950 else 3951 sbuf_printf(sb, " disabled"); 3952 } 3953 3954 rc = sbuf_finish(sb); 3955 sbuf_delete(sb); 3956 3957 return (rc); 3958} 3959 3960static int 3961sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 3962{ 3963 struct adapter *sc = arg1; 3964 struct sbuf *sb; 3965 int rc, i, j; 3966 uint64_t *p0, *p1; 3967 struct lb_port_stats s[2]; 3968 static const char *stat_name[] = { 3969 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 3970 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 3971 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 3972 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 3973 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 3974 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 3975 "BG2FramesTrunc:", "BG3FramesTrunc:" 3976 }; 3977 3978 rc = sysctl_wire_old_buffer(req, 0); 3979 if (rc != 0) 3980 return (rc); 3981 3982 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 3983 if (sb == NULL) 3984 return (ENOMEM); 3985 3986 memset(s, 0, sizeof(s)); 3987 3988 for (i = 0; i < 4; i += 2) { 3989 t4_get_lb_stats(sc, i, &s[0]); 3990 t4_get_lb_stats(sc, i + 1, &s[1]); 3991 3992 p0 = &s[0].octets; 3993 p1 = &s[1].octets; 3994 sbuf_printf(sb, "%s Loopback %u" 3995 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 3996 3997 for (j = 0; j < nitems(stat_name); j++) 3998 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 3999 *p0++, *p1++); 4000 } 4001 4002 rc = sbuf_finish(sb); 4003 sbuf_delete(sb); 4004 4005 return (rc); 4006} 4007 4008struct mem_desc { 4009 unsigned int base; 4010 unsigned int limit; 4011 unsigned int idx; 4012}; 4013 4014static int 4015mem_desc_cmp(const void *a, const void *b) 4016{ 4017 return ((const struct mem_desc *)a)->base - 4018 ((const struct mem_desc *)b)->base; 4019} 4020 4021static void 4022mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 4023 unsigned int to) 4024{ 4025 unsigned int size; 4026 4027 size = to - from + 1; 4028 if (size == 0) 4029 return; 4030 4031 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 4032 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 4033} 4034 4035static int 4036sysctl_meminfo(SYSCTL_HANDLER_ARGS) 4037{ 4038 struct adapter *sc = arg1; 4039 struct sbuf *sb; 4040 int rc, i, n; 4041 uint32_t lo, hi; 4042 static const char *memory[] = { "EDC0:", "EDC1:", "MC:" }; 4043 static const char *region[] = { 4044 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 4045 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 4046 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 4047 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 4048 "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:", 4049 "ULPTX state:", "On-chip queues:" 4050 }; 4051 struct mem_desc avail[3]; 4052 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 4053 struct mem_desc *md = mem; 4054 4055 rc = sysctl_wire_old_buffer(req, 0); 4056 if (rc != 0) 4057 return (rc); 4058 4059 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4060 if (sb == NULL) 4061 return (ENOMEM); 4062 4063 for (i = 0; i < nitems(mem); i++) { 4064 mem[i].limit = 0; 4065 mem[i].idx = i; 4066 } 4067 4068 /* Find and sort the populated memory ranges */ 4069 i = 0; 4070 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 4071 if (lo & F_EDRAM0_ENABLE) { 4072 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 4073 avail[i].base = G_EDRAM0_BASE(hi) << 20; 4074 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 4075 avail[i].idx = 0; 4076 i++; 4077 } 4078 if (lo & F_EDRAM1_ENABLE) { 4079 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 4080 avail[i].base = G_EDRAM1_BASE(hi) << 20; 4081 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 4082 avail[i].idx = 1; 4083 i++; 4084 } 4085 if (lo & F_EXT_MEM_ENABLE) { 4086 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 4087 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 4088 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20); 4089 avail[i].idx = 2; 4090 i++; 4091 } 4092 if (!i) /* no memory available */ 4093 return 0; 4094 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 4095 4096 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 4097 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 4098 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 4099 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4100 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 4101 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 4102 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 4103 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 4104 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 4105 4106 /* the next few have explicit upper bounds */ 4107 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 4108 md->limit = md->base - 1 + 4109 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 4110 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 4111 md++; 4112 4113 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 4114 md->limit = md->base - 1 + 4115 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 4116 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 4117 md++; 4118 4119 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4120 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 4121 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 4122 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 4123 } else { 4124 md->base = 0; 4125 md->idx = nitems(region); /* hide it */ 4126 } 4127 md++; 4128 4129#define ulp_region(reg) \ 4130 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 4131 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 4132 4133 ulp_region(RX_ISCSI); 4134 ulp_region(RX_TDDP); 4135 ulp_region(TX_TPT); 4136 ulp_region(RX_STAG); 4137 ulp_region(RX_RQ); 4138 ulp_region(RX_RQUDP); 4139 ulp_region(RX_PBL); 4140 ulp_region(TX_PBL); 4141#undef ulp_region 4142 4143 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 4144 md->limit = md->base + sc->tids.ntids - 1; 4145 md++; 4146 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 4147 md->limit = md->base + sc->tids.ntids - 1; 4148 md++; 4149 4150 md->base = sc->vres.ocq.start; 4151 if (sc->vres.ocq.size) 4152 md->limit = md->base + sc->vres.ocq.size - 1; 4153 else 4154 md->idx = nitems(region); /* hide it */ 4155 md++; 4156 4157 /* add any address-space holes, there can be up to 3 */ 4158 for (n = 0; n < i - 1; n++) 4159 if (avail[n].limit < avail[n + 1].base) 4160 (md++)->base = avail[n].limit; 4161 if (avail[n].limit) 4162 (md++)->base = avail[n].limit; 4163 4164 n = md - mem; 4165 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 4166 4167 for (lo = 0; lo < i; lo++) 4168 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 4169 avail[lo].limit - 1); 4170 4171 sbuf_printf(sb, "\n"); 4172 for (i = 0; i < n; i++) { 4173 if (mem[i].idx >= nitems(region)) 4174 continue; /* skip holes */ 4175 if (!mem[i].limit) 4176 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 4177 mem_region_show(sb, region[mem[i].idx], mem[i].base, 4178 mem[i].limit); 4179 } 4180 4181 sbuf_printf(sb, "\n"); 4182 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 4183 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 4184 mem_region_show(sb, "uP RAM:", lo, hi); 4185 4186 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 4187 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 4188 mem_region_show(sb, "uP Extmem2:", lo, hi); 4189 4190 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 4191 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 4192 G_PMRXMAXPAGE(lo), 4193 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 4194 (lo & F_PMRXNUMCHN) ? 2 : 1); 4195 4196 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 4197 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 4198 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 4199 G_PMTXMAXPAGE(lo), 4200 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 4201 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 4202 sbuf_printf(sb, "%u p-structs\n", 4203 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 4204 4205 for (i = 0; i < 4; i++) { 4206 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 4207 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 4208 i, G_USED(lo), G_ALLOC(lo)); 4209 } 4210 for (i = 0; i < 4; i++) { 4211 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 4212 sbuf_printf(sb, 4213 "\nLoopback %d using %u pages out of %u allocated", 4214 i, G_USED(lo), G_ALLOC(lo)); 4215 } 4216 4217 rc = sbuf_finish(sb); 4218 sbuf_delete(sb); 4219 4220 return (rc); 4221} 4222 4223static int 4224sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 4225{ 4226 struct adapter *sc = arg1; 4227 struct sbuf *sb; 4228 int rc; 4229 uint16_t mtus[NMTUS]; 4230 4231 rc = sysctl_wire_old_buffer(req, 0); 4232 if (rc != 0) 4233 return (rc); 4234 4235 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4236 if (sb == NULL) 4237 return (ENOMEM); 4238 4239 t4_read_mtu_tbl(sc, mtus, NULL); 4240 4241 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 4242 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 4243 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 4244 mtus[14], mtus[15]); 4245 4246 rc = sbuf_finish(sb); 4247 sbuf_delete(sb); 4248 4249 return (rc); 4250} 4251 4252static int 4253sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 4254{ 4255 struct adapter *sc = arg1; 4256 struct sbuf *sb; 4257 int rc, i; 4258 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 4259 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 4260 static const char *pm_stats[] = { 4261 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 4262 }; 4263 4264 rc = sysctl_wire_old_buffer(req, 0); 4265 if (rc != 0) 4266 return (rc); 4267 4268 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4269 if (sb == NULL) 4270 return (ENOMEM); 4271 4272 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 4273 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 4274 4275 sbuf_printf(sb, " Tx count Tx cycles " 4276 "Rx count Rx cycles"); 4277 for (i = 0; i < PM_NSTATS; i++) 4278 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 4279 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 4280 4281 rc = sbuf_finish(sb); 4282 sbuf_delete(sb); 4283 4284 return (rc); 4285} 4286 4287static int 4288sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 4289{ 4290 struct adapter *sc = arg1; 4291 struct sbuf *sb; 4292 int rc; 4293 struct tp_rdma_stats stats; 4294 4295 rc = sysctl_wire_old_buffer(req, 0); 4296 if (rc != 0) 4297 return (rc); 4298 4299 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4300 if (sb == NULL) 4301 return (ENOMEM); 4302 4303 t4_tp_get_rdma_stats(sc, &stats); 4304 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 4305 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 4306 4307 rc = sbuf_finish(sb); 4308 sbuf_delete(sb); 4309 4310 return (rc); 4311} 4312 4313static int 4314sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 4315{ 4316 struct adapter *sc = arg1; 4317 struct sbuf *sb; 4318 int rc; 4319 struct tp_tcp_stats v4, v6; 4320 4321 rc = sysctl_wire_old_buffer(req, 0); 4322 if (rc != 0) 4323 return (rc); 4324 4325 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4326 if (sb == NULL) 4327 return (ENOMEM); 4328 4329 t4_tp_get_tcp_stats(sc, &v4, &v6); 4330 sbuf_printf(sb, 4331 " IP IPv6\n"); 4332 sbuf_printf(sb, "OutRsts: %20u %20u\n", 4333 v4.tcpOutRsts, v6.tcpOutRsts); 4334 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 4335 v4.tcpInSegs, v6.tcpInSegs); 4336 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 4337 v4.tcpOutSegs, v6.tcpOutSegs); 4338 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 4339 v4.tcpRetransSegs, v6.tcpRetransSegs); 4340 4341 rc = sbuf_finish(sb); 4342 sbuf_delete(sb); 4343 4344 return (rc); 4345} 4346 4347static int 4348sysctl_tids(SYSCTL_HANDLER_ARGS) 4349{ 4350 struct adapter *sc = arg1; 4351 struct sbuf *sb; 4352 int rc; 4353 struct tid_info *t = &sc->tids; 4354 4355 rc = sysctl_wire_old_buffer(req, 0); 4356 if (rc != 0) 4357 return (rc); 4358 4359 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4360 if (sb == NULL) 4361 return (ENOMEM); 4362 4363 if (t->natids) { 4364 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 4365 t->atids_in_use); 4366 } 4367 4368 if (t->ntids) { 4369 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 4370 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 4371 4372 if (b) { 4373 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 4374 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4375 t->ntids - 1); 4376 } else { 4377 sbuf_printf(sb, "TID range: %u-%u", 4378 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 4379 t->ntids - 1); 4380 } 4381 } else 4382 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 4383 sbuf_printf(sb, ", in use: %u\n", 4384 atomic_load_acq_int(&t->tids_in_use)); 4385 } 4386 4387 if (t->nstids) { 4388 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 4389 t->stid_base + t->nstids - 1, t->stids_in_use); 4390 } 4391 4392 if (t->nftids) { 4393 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 4394 t->ftid_base + t->nftids - 1); 4395 } 4396 4397 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 4398 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 4399 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 4400 4401 rc = sbuf_finish(sb); 4402 sbuf_delete(sb); 4403 4404 return (rc); 4405} 4406 4407static int 4408sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 4409{ 4410 struct adapter *sc = arg1; 4411 struct sbuf *sb; 4412 int rc; 4413 struct tp_err_stats stats; 4414 4415 rc = sysctl_wire_old_buffer(req, 0); 4416 if (rc != 0) 4417 return (rc); 4418 4419 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4420 if (sb == NULL) 4421 return (ENOMEM); 4422 4423 t4_tp_get_err_stats(sc, &stats); 4424 4425 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4426 "channel 3\n"); 4427 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 4428 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 4429 stats.macInErrs[3]); 4430 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 4431 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 4432 stats.hdrInErrs[3]); 4433 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 4434 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 4435 stats.tcpInErrs[3]); 4436 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 4437 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 4438 stats.tcp6InErrs[3]); 4439 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 4440 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 4441 stats.tnlCongDrops[3]); 4442 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 4443 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 4444 stats.tnlTxDrops[3]); 4445 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 4446 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 4447 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 4448 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 4449 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 4450 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 4451 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 4452 stats.ofldNoNeigh, stats.ofldCongDefer); 4453 4454 rc = sbuf_finish(sb); 4455 sbuf_delete(sb); 4456 4457 return (rc); 4458} 4459 4460static int 4461sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 4462{ 4463 struct adapter *sc = arg1; 4464 struct sbuf *sb; 4465 int rc; 4466 u64 nrate[NCHAN], orate[NCHAN]; 4467 4468 rc = sysctl_wire_old_buffer(req, 0); 4469 if (rc != 0) 4470 return (rc); 4471 4472 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4473 if (sb == NULL) 4474 return (ENOMEM); 4475 4476 t4_get_chan_txrate(sc, nrate, orate); 4477 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4478 "channel 3\n"); 4479 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 4480 nrate[0], nrate[1], nrate[2], nrate[3]); 4481 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 4482 orate[0], orate[1], orate[2], orate[3]); 4483 4484 rc = sbuf_finish(sb); 4485 sbuf_delete(sb); 4486 4487 return (rc); 4488} 4489#endif 4490 4491static inline void 4492txq_start(struct ifnet *ifp, struct sge_txq *txq) 4493{ 4494 struct buf_ring *br; 4495 struct mbuf *m; 4496 4497 TXQ_LOCK_ASSERT_OWNED(txq); 4498 4499 br = txq->br; 4500 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 4501 if (m) 4502 t4_eth_tx(ifp, txq, m); 4503} 4504 4505void 4506t4_tx_callout(void *arg) 4507{ 4508 struct sge_eq *eq = arg; 4509 struct adapter *sc; 4510 4511 if (EQ_TRYLOCK(eq) == 0) 4512 goto reschedule; 4513 4514 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 4515 EQ_UNLOCK(eq); 4516reschedule: 4517 if (__predict_true(!(eq->flags && EQ_DOOMED))) 4518 callout_schedule(&eq->tx_callout, 1); 4519 return; 4520 } 4521 4522 EQ_LOCK_ASSERT_OWNED(eq); 4523 4524 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 4525 4526 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4527 struct sge_txq *txq = arg; 4528 struct port_info *pi = txq->ifp->if_softc; 4529 4530 sc = pi->adapter; 4531 } else { 4532 struct sge_wrq *wrq = arg; 4533 4534 sc = wrq->adapter; 4535 } 4536 4537 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 4538 } 4539 4540 EQ_UNLOCK(eq); 4541} 4542 4543void 4544t4_tx_task(void *arg, int count) 4545{ 4546 struct sge_eq *eq = arg; 4547 4548 EQ_LOCK(eq); 4549 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 4550 struct sge_txq *txq = arg; 4551 txq_start(txq->ifp, txq); 4552 } else { 4553 struct sge_wrq *wrq = arg; 4554 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 4555 } 4556 EQ_UNLOCK(eq); 4557} 4558 4559static uint32_t 4560fconf_to_mode(uint32_t fconf) 4561{ 4562 uint32_t mode; 4563 4564 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 4565 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 4566 4567 if (fconf & F_FRAGMENTATION) 4568 mode |= T4_FILTER_IP_FRAGMENT; 4569 4570 if (fconf & F_MPSHITTYPE) 4571 mode |= T4_FILTER_MPS_HIT_TYPE; 4572 4573 if (fconf & F_MACMATCH) 4574 mode |= T4_FILTER_MAC_IDX; 4575 4576 if (fconf & F_ETHERTYPE) 4577 mode |= T4_FILTER_ETH_TYPE; 4578 4579 if (fconf & F_PROTOCOL) 4580 mode |= T4_FILTER_IP_PROTO; 4581 4582 if (fconf & F_TOS) 4583 mode |= T4_FILTER_IP_TOS; 4584 4585 if (fconf & F_VLAN) 4586 mode |= T4_FILTER_VLAN; 4587 4588 if (fconf & F_VNIC_ID) 4589 mode |= T4_FILTER_VNIC; 4590 4591 if (fconf & F_PORT) 4592 mode |= T4_FILTER_PORT; 4593 4594 if (fconf & F_FCOE) 4595 mode |= T4_FILTER_FCoE; 4596 4597 return (mode); 4598} 4599 4600static uint32_t 4601mode_to_fconf(uint32_t mode) 4602{ 4603 uint32_t fconf = 0; 4604 4605 if (mode & T4_FILTER_IP_FRAGMENT) 4606 fconf |= F_FRAGMENTATION; 4607 4608 if (mode & T4_FILTER_MPS_HIT_TYPE) 4609 fconf |= F_MPSHITTYPE; 4610 4611 if (mode & T4_FILTER_MAC_IDX) 4612 fconf |= F_MACMATCH; 4613 4614 if (mode & T4_FILTER_ETH_TYPE) 4615 fconf |= F_ETHERTYPE; 4616 4617 if (mode & T4_FILTER_IP_PROTO) 4618 fconf |= F_PROTOCOL; 4619 4620 if (mode & T4_FILTER_IP_TOS) 4621 fconf |= F_TOS; 4622 4623 if (mode & T4_FILTER_VLAN) 4624 fconf |= F_VLAN; 4625 4626 if (mode & T4_FILTER_VNIC) 4627 fconf |= F_VNIC_ID; 4628 4629 if (mode & T4_FILTER_PORT) 4630 fconf |= F_PORT; 4631 4632 if (mode & T4_FILTER_FCoE) 4633 fconf |= F_FCOE; 4634 4635 return (fconf); 4636} 4637 4638static uint32_t 4639fspec_to_fconf(struct t4_filter_specification *fs) 4640{ 4641 uint32_t fconf = 0; 4642 4643 if (fs->val.frag || fs->mask.frag) 4644 fconf |= F_FRAGMENTATION; 4645 4646 if (fs->val.matchtype || fs->mask.matchtype) 4647 fconf |= F_MPSHITTYPE; 4648 4649 if (fs->val.macidx || fs->mask.macidx) 4650 fconf |= F_MACMATCH; 4651 4652 if (fs->val.ethtype || fs->mask.ethtype) 4653 fconf |= F_ETHERTYPE; 4654 4655 if (fs->val.proto || fs->mask.proto) 4656 fconf |= F_PROTOCOL; 4657 4658 if (fs->val.tos || fs->mask.tos) 4659 fconf |= F_TOS; 4660 4661 if (fs->val.vlan_vld || fs->mask.vlan_vld) 4662 fconf |= F_VLAN; 4663 4664 if (fs->val.vnic_vld || fs->mask.vnic_vld) 4665 fconf |= F_VNIC_ID; 4666 4667 if (fs->val.iport || fs->mask.iport) 4668 fconf |= F_PORT; 4669 4670 if (fs->val.fcoe || fs->mask.fcoe) 4671 fconf |= F_FCOE; 4672 4673 return (fconf); 4674} 4675 4676static int 4677get_filter_mode(struct adapter *sc, uint32_t *mode) 4678{ 4679 uint32_t fconf; 4680 4681 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 4682 A_TP_VLAN_PRI_MAP); 4683 4684 if (sc->filter_mode != fconf) { 4685 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 4686 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 4687 sc->filter_mode = fconf; 4688 } 4689 4690 *mode = fconf_to_mode(sc->filter_mode); 4691 4692 return (0); 4693} 4694 4695static int 4696set_filter_mode(struct adapter *sc, uint32_t mode) 4697{ 4698 uint32_t fconf; 4699 int rc; 4700 4701 fconf = mode_to_fconf(mode); 4702 4703 ADAPTER_LOCK(sc); 4704 if (IS_BUSY(sc)) { 4705 rc = EAGAIN; 4706 goto done; 4707 } 4708 4709 if (sc->tids.ftids_in_use > 0) { 4710 rc = EBUSY; 4711 goto done; 4712 } 4713 4714#ifdef TCP_OFFLOAD 4715 if (sc->offload_map) { 4716 rc = EBUSY; 4717 goto done; 4718 } 4719#endif 4720 4721#ifdef notyet 4722 rc = -t4_set_filter_mode(sc, fconf); 4723 if (rc == 0) 4724 sc->filter_mode = fconf; 4725#else 4726 rc = ENOTSUP; 4727#endif 4728 4729done: 4730 ADAPTER_UNLOCK(sc); 4731 return (rc); 4732} 4733 4734static inline uint64_t 4735get_filter_hits(struct adapter *sc, uint32_t fid) 4736{ 4737 uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 4738 uint64_t hits; 4739 4740 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0), 4741 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 4742 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0)); 4743 hits = t4_read_reg64(sc, MEMWIN0_BASE + 16); 4744 4745 return (be64toh(hits)); 4746} 4747 4748static int 4749get_filter(struct adapter *sc, struct t4_filter *t) 4750{ 4751 int i, nfilters = sc->tids.nftids; 4752 struct filter_entry *f; 4753 4754 ADAPTER_LOCK_ASSERT_OWNED(sc); 4755 4756 if (IS_BUSY(sc)) 4757 return (EAGAIN); 4758 4759 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 4760 t->idx >= nfilters) { 4761 t->idx = 0xffffffff; 4762 return (0); 4763 } 4764 4765 f = &sc->tids.ftid_tab[t->idx]; 4766 for (i = t->idx; i < nfilters; i++, f++) { 4767 if (f->valid) { 4768 t->idx = i; 4769 t->l2tidx = f->l2t ? f->l2t->idx : 0; 4770 t->smtidx = f->smtidx; 4771 if (f->fs.hitcnts) 4772 t->hits = get_filter_hits(sc, t->idx); 4773 else 4774 t->hits = UINT64_MAX; 4775 t->fs = f->fs; 4776 4777 return (0); 4778 } 4779 } 4780 4781 t->idx = 0xffffffff; 4782 return (0); 4783} 4784 4785static int 4786set_filter(struct adapter *sc, struct t4_filter *t) 4787{ 4788 unsigned int nfilters, nports; 4789 struct filter_entry *f; 4790 int i; 4791 4792 ADAPTER_LOCK_ASSERT_OWNED(sc); 4793 4794 nfilters = sc->tids.nftids; 4795 nports = sc->params.nports; 4796 4797 if (nfilters == 0) 4798 return (ENOTSUP); 4799 4800 if (!(sc->flags & FULL_INIT_DONE)) 4801 return (EAGAIN); 4802 4803 if (t->idx >= nfilters) 4804 return (EINVAL); 4805 4806 /* Validate against the global filter mode */ 4807 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) 4808 return (E2BIG); 4809 4810 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) 4811 return (EINVAL); 4812 4813 if (t->fs.val.iport >= nports) 4814 return (EINVAL); 4815 4816 /* Can't specify an iq if not steering to it */ 4817 if (!t->fs.dirsteer && t->fs.iq) 4818 return (EINVAL); 4819 4820 /* IPv6 filter idx must be 4 aligned */ 4821 if (t->fs.type == 1 && 4822 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) 4823 return (EINVAL); 4824 4825 if (sc->tids.ftid_tab == NULL) { 4826 KASSERT(sc->tids.ftids_in_use == 0, 4827 ("%s: no memory allocated but filters_in_use > 0", 4828 __func__)); 4829 4830 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 4831 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 4832 if (sc->tids.ftid_tab == NULL) 4833 return (ENOMEM); 4834 } 4835 4836 for (i = 0; i < 4; i++) { 4837 f = &sc->tids.ftid_tab[t->idx + i]; 4838 4839 if (f->pending || f->valid) 4840 return (EBUSY); 4841 if (f->locked) 4842 return (EPERM); 4843 4844 if (t->fs.type == 0) 4845 break; 4846 } 4847 4848 f = &sc->tids.ftid_tab[t->idx]; 4849 f->fs = t->fs; 4850 4851 return set_filter_wr(sc, t->idx); 4852} 4853 4854static int 4855del_filter(struct adapter *sc, struct t4_filter *t) 4856{ 4857 unsigned int nfilters; 4858 struct filter_entry *f; 4859 4860 ADAPTER_LOCK_ASSERT_OWNED(sc); 4861 4862 if (IS_BUSY(sc)) 4863 return (EAGAIN); 4864 4865 nfilters = sc->tids.nftids; 4866 4867 if (nfilters == 0) 4868 return (ENOTSUP); 4869 4870 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 4871 t->idx >= nfilters) 4872 return (EINVAL); 4873 4874 if (!(sc->flags & FULL_INIT_DONE)) 4875 return (EAGAIN); 4876 4877 f = &sc->tids.ftid_tab[t->idx]; 4878 4879 if (f->pending) 4880 return (EBUSY); 4881 if (f->locked) 4882 return (EPERM); 4883 4884 if (f->valid) { 4885 t->fs = f->fs; /* extra info for the caller */ 4886 return del_filter_wr(sc, t->idx); 4887 } 4888 4889 return (0); 4890} 4891 4892static void 4893clear_filter(struct filter_entry *f) 4894{ 4895 if (f->l2t) 4896 t4_l2t_release(f->l2t); 4897 4898 bzero(f, sizeof (*f)); 4899} 4900 4901static int 4902set_filter_wr(struct adapter *sc, int fidx) 4903{ 4904 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 4905 struct wrqe *wr; 4906 struct fw_filter_wr *fwr; 4907 unsigned int ftid; 4908 4909 ADAPTER_LOCK_ASSERT_OWNED(sc); 4910 4911 if (f->fs.newdmac || f->fs.newvlan) { 4912 /* This filter needs an L2T entry; allocate one. */ 4913 f->l2t = t4_l2t_alloc_switching(sc->l2t); 4914 if (f->l2t == NULL) 4915 return (EAGAIN); 4916 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 4917 f->fs.dmac)) { 4918 t4_l2t_release(f->l2t); 4919 f->l2t = NULL; 4920 return (ENOMEM); 4921 } 4922 } 4923 4924 ftid = sc->tids.ftid_base + fidx; 4925 4926 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 4927 if (wr == NULL) 4928 return (ENOMEM); 4929 4930 fwr = wrtod(wr); 4931 bzero(fwr, sizeof (*fwr)); 4932 4933 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 4934 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 4935 fwr->tid_to_iq = 4936 htobe32(V_FW_FILTER_WR_TID(ftid) | 4937 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 4938 V_FW_FILTER_WR_NOREPLY(0) | 4939 V_FW_FILTER_WR_IQ(f->fs.iq)); 4940 fwr->del_filter_to_l2tix = 4941 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 4942 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 4943 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 4944 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 4945 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 4946 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 4947 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 4948 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 4949 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 4950 f->fs.newvlan == VLAN_REWRITE) | 4951 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 4952 f->fs.newvlan == VLAN_REWRITE) | 4953 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 4954 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 4955 V_FW_FILTER_WR_PRIO(f->fs.prio) | 4956 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 4957 fwr->ethtype = htobe16(f->fs.val.ethtype); 4958 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 4959 fwr->frag_to_ovlan_vldm = 4960 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 4961 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 4962 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 4963 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 4964 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 4965 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 4966 fwr->smac_sel = 0; 4967 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 4968 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 4969 fwr->maci_to_matchtypem = 4970 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 4971 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 4972 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 4973 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 4974 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 4975 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 4976 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 4977 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 4978 fwr->ptcl = f->fs.val.proto; 4979 fwr->ptclm = f->fs.mask.proto; 4980 fwr->ttyp = f->fs.val.tos; 4981 fwr->ttypm = f->fs.mask.tos; 4982 fwr->ivlan = htobe16(f->fs.val.vlan); 4983 fwr->ivlanm = htobe16(f->fs.mask.vlan); 4984 fwr->ovlan = htobe16(f->fs.val.vnic); 4985 fwr->ovlanm = htobe16(f->fs.mask.vnic); 4986 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 4987 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 4988 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 4989 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 4990 fwr->lp = htobe16(f->fs.val.dport); 4991 fwr->lpm = htobe16(f->fs.mask.dport); 4992 fwr->fp = htobe16(f->fs.val.sport); 4993 fwr->fpm = htobe16(f->fs.mask.sport); 4994 if (f->fs.newsmac) 4995 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 4996 4997 f->pending = 1; 4998 sc->tids.ftids_in_use++; 4999 5000 t4_wrq_tx(sc, wr); 5001 return (0); 5002} 5003 5004static int 5005del_filter_wr(struct adapter *sc, int fidx) 5006{ 5007 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 5008 struct wrqe *wr; 5009 struct fw_filter_wr *fwr; 5010 unsigned int ftid; 5011 5012 ADAPTER_LOCK_ASSERT_OWNED(sc); 5013 5014 ftid = sc->tids.ftid_base + fidx; 5015 5016 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 5017 if (wr == NULL) 5018 return (ENOMEM); 5019 fwr = wrtod(wr); 5020 bzero(fwr, sizeof (*fwr)); 5021 5022 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 5023 5024 f->pending = 1; 5025 t4_wrq_tx(sc, wr); 5026 return (0); 5027} 5028 5029int 5030t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 5031{ 5032 struct adapter *sc = iq->adapter; 5033 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 5034 unsigned int idx = GET_TID(rpl); 5035 5036 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 5037 rss->opcode)); 5038 5039 if (idx >= sc->tids.ftid_base && 5040 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 5041 unsigned int rc = G_COOKIE(rpl->cookie); 5042 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 5043 5044 ADAPTER_LOCK(sc); 5045 if (rc == FW_FILTER_WR_FLT_ADDED) { 5046 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 5047 f->pending = 0; /* asynchronous setup completed */ 5048 f->valid = 1; 5049 } else { 5050 if (rc != FW_FILTER_WR_FLT_DELETED) { 5051 /* Add or delete failed, display an error */ 5052 log(LOG_ERR, 5053 "filter %u setup failed with error %u\n", 5054 idx, rc); 5055 } 5056 5057 clear_filter(f); 5058 sc->tids.ftids_in_use--; 5059 } 5060 ADAPTER_UNLOCK(sc); 5061 } 5062 5063 return (0); 5064} 5065 5066static int 5067get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 5068{ 5069 int rc = EINVAL; 5070 5071 if (cntxt->cid > M_CTXTQID) 5072 return (rc); 5073 5074 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 5075 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 5076 return (rc); 5077 5078 if (sc->flags & FW_OK) { 5079 ADAPTER_LOCK(sc); /* Avoid parallel t4_wr_mbox */ 5080 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 5081 &cntxt->data[0]); 5082 ADAPTER_UNLOCK(sc); 5083 } 5084 5085 if (rc != 0) { 5086 /* Read via firmware failed or wasn't even attempted */ 5087 5088 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, 5089 &cntxt->data[0]); 5090 } 5091 5092 return (rc); 5093} 5094 5095static int 5096read_card_mem(struct adapter *sc, struct t4_mem_range *mr) 5097{ 5098 uint32_t base, size, lo, hi, win, off, remaining, i, n; 5099 uint32_t *buf, *b; 5100 int rc; 5101 5102 /* reads are in multiples of 32 bits */ 5103 if (mr->addr & 3 || mr->len & 3 || mr->len == 0) 5104 return (EINVAL); 5105 5106 /* 5107 * We don't want to deal with potential holes so we mandate that the 5108 * requested region must lie entirely within one of the 3 memories. 5109 */ 5110 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5111 if (lo & F_EDRAM0_ENABLE) { 5112 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5113 base = G_EDRAM0_BASE(hi) << 20; 5114 size = G_EDRAM0_SIZE(hi) << 20; 5115 if (size > 0 && 5116 mr->addr >= base && mr->addr < base + size && 5117 mr->addr + mr->len <= base + size) 5118 goto proceed; 5119 } 5120 if (lo & F_EDRAM1_ENABLE) { 5121 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5122 base = G_EDRAM1_BASE(hi) << 20; 5123 size = G_EDRAM1_SIZE(hi) << 20; 5124 if (size > 0 && 5125 mr->addr >= base && mr->addr < base + size && 5126 mr->addr + mr->len <= base + size) 5127 goto proceed; 5128 } 5129 if (lo & F_EXT_MEM_ENABLE) { 5130 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5131 base = G_EXT_MEM_BASE(hi) << 20; 5132 size = G_EXT_MEM_SIZE(hi) << 20; 5133 if (size > 0 && 5134 mr->addr >= base && mr->addr < base + size && 5135 mr->addr + mr->len <= base + size) 5136 goto proceed; 5137 } 5138 return (ENXIO); 5139 5140proceed: 5141 buf = b = malloc(mr->len, M_CXGBE, M_WAITOK); 5142 5143 /* 5144 * Position the PCIe window (we use memwin2) to the 16B aligned area 5145 * just at/before the requested region. 5146 */ 5147 win = mr->addr & ~0xf; 5148 off = mr->addr - win; /* offset of the requested region in the win */ 5149 remaining = mr->len; 5150 5151 while (remaining) { 5152 t4_write_reg(sc, 5153 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win); 5154 t4_read_reg(sc, 5155 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2)); 5156 5157 /* number of bytes that we'll copy in the inner loop */ 5158 n = min(remaining, MEMWIN2_APERTURE - off); 5159 5160 for (i = 0; i < n; i += 4, remaining -= 4) 5161 *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i); 5162 5163 win += MEMWIN2_APERTURE; 5164 off = 0; 5165 } 5166 5167 rc = copyout(buf, mr->data, mr->len); 5168 free(buf, M_CXGBE); 5169 5170 return (rc); 5171} 5172 5173int 5174t4_os_find_pci_capability(struct adapter *sc, int cap) 5175{ 5176 int i; 5177 5178 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 5179} 5180 5181int 5182t4_os_pci_save_state(struct adapter *sc) 5183{ 5184 device_t dev; 5185 struct pci_devinfo *dinfo; 5186 5187 dev = sc->dev; 5188 dinfo = device_get_ivars(dev); 5189 5190 pci_cfg_save(dev, dinfo, 0); 5191 return (0); 5192} 5193 5194int 5195t4_os_pci_restore_state(struct adapter *sc) 5196{ 5197 device_t dev; 5198 struct pci_devinfo *dinfo; 5199 5200 dev = sc->dev; 5201 dinfo = device_get_ivars(dev); 5202 5203 pci_cfg_restore(dev, dinfo); 5204 return (0); 5205} 5206 5207void 5208t4_os_portmod_changed(const struct adapter *sc, int idx) 5209{ 5210 struct port_info *pi = sc->port[idx]; 5211 static const char *mod_str[] = { 5212 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 5213 }; 5214 5215 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 5216 if_printf(pi->ifp, "transceiver unplugged.\n"); 5217 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 5218 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 5219 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 5220 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 5221 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 5222 if_printf(pi->ifp, "%s transceiver inserted.\n", 5223 mod_str[pi->mod_type]); 5224 } else { 5225 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 5226 pi->mod_type); 5227 } 5228} 5229 5230void 5231t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 5232{ 5233 struct port_info *pi = sc->port[idx]; 5234 struct ifnet *ifp = pi->ifp; 5235 5236 if (link_stat) { 5237 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 5238 if_link_state_change(ifp, LINK_STATE_UP); 5239 } else 5240 if_link_state_change(ifp, LINK_STATE_DOWN); 5241} 5242 5243void 5244t4_iterate(void (*func)(struct adapter *, void *), void *arg) 5245{ 5246 struct adapter *sc; 5247 5248 mtx_lock(&t4_list_lock); 5249 SLIST_FOREACH(sc, &t4_list, link) { 5250 /* 5251 * func should not make any assumptions about what state sc is 5252 * in - the only guarantee is that sc->sc_lock is a valid lock. 5253 */ 5254 func(sc, arg); 5255 } 5256 mtx_unlock(&t4_list_lock); 5257} 5258 5259static int 5260t4_open(struct cdev *dev, int flags, int type, struct thread *td) 5261{ 5262 return (0); 5263} 5264 5265static int 5266t4_close(struct cdev *dev, int flags, int type, struct thread *td) 5267{ 5268 return (0); 5269} 5270 5271static int 5272t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 5273 struct thread *td) 5274{ 5275 int rc; 5276 struct adapter *sc = dev->si_drv1; 5277 5278 rc = priv_check(td, PRIV_DRIVER); 5279 if (rc != 0) 5280 return (rc); 5281 5282 switch (cmd) { 5283 case CHELSIO_T4_GETREG: { 5284 struct t4_reg *edata = (struct t4_reg *)data; 5285 5286 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5287 return (EFAULT); 5288 5289 if (edata->size == 4) 5290 edata->val = t4_read_reg(sc, edata->addr); 5291 else if (edata->size == 8) 5292 edata->val = t4_read_reg64(sc, edata->addr); 5293 else 5294 return (EINVAL); 5295 5296 break; 5297 } 5298 case CHELSIO_T4_SETREG: { 5299 struct t4_reg *edata = (struct t4_reg *)data; 5300 5301 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 5302 return (EFAULT); 5303 5304 if (edata->size == 4) { 5305 if (edata->val & 0xffffffff00000000) 5306 return (EINVAL); 5307 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 5308 } else if (edata->size == 8) 5309 t4_write_reg64(sc, edata->addr, edata->val); 5310 else 5311 return (EINVAL); 5312 break; 5313 } 5314 case CHELSIO_T4_REGDUMP: { 5315 struct t4_regdump *regs = (struct t4_regdump *)data; 5316 int reglen = T4_REGDUMP_SIZE; 5317 uint8_t *buf; 5318 5319 if (regs->len < reglen) { 5320 regs->len = reglen; /* hint to the caller */ 5321 return (ENOBUFS); 5322 } 5323 5324 regs->len = reglen; 5325 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 5326 t4_get_regs(sc, regs, buf); 5327 rc = copyout(buf, regs->data, reglen); 5328 free(buf, M_CXGBE); 5329 break; 5330 } 5331 case CHELSIO_T4_GET_FILTER_MODE: 5332 rc = get_filter_mode(sc, (uint32_t *)data); 5333 break; 5334 case CHELSIO_T4_SET_FILTER_MODE: 5335 rc = set_filter_mode(sc, *(uint32_t *)data); 5336 break; 5337 case CHELSIO_T4_GET_FILTER: 5338 ADAPTER_LOCK(sc); 5339 rc = get_filter(sc, (struct t4_filter *)data); 5340 ADAPTER_UNLOCK(sc); 5341 break; 5342 case CHELSIO_T4_SET_FILTER: 5343 ADAPTER_LOCK(sc); 5344 rc = set_filter(sc, (struct t4_filter *)data); 5345 ADAPTER_UNLOCK(sc); 5346 break; 5347 case CHELSIO_T4_DEL_FILTER: 5348 ADAPTER_LOCK(sc); 5349 rc = del_filter(sc, (struct t4_filter *)data); 5350 ADAPTER_UNLOCK(sc); 5351 break; 5352 case CHELSIO_T4_GET_SGE_CONTEXT: 5353 rc = get_sge_context(sc, (struct t4_sge_context *)data); 5354 break; 5355 case CHELSIO_T4_LOAD_FW: { 5356 struct t4_data *fw = (struct t4_data *)data; 5357 uint8_t *fw_data; 5358 5359 if (sc->flags & FULL_INIT_DONE) 5360 return (EBUSY); 5361 5362 fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT); 5363 if (fw_data == NULL) 5364 return (ENOMEM); 5365 5366 rc = copyin(fw->data, fw_data, fw->len); 5367 if (rc == 0) 5368 rc = -t4_load_fw(sc, fw_data, fw->len); 5369 5370 free(fw_data, M_CXGBE); 5371 break; 5372 } 5373 case CHELSIO_T4_GET_MEM: 5374 rc = read_card_mem(sc, (struct t4_mem_range *)data); 5375 break; 5376 default: 5377 rc = EINVAL; 5378 } 5379 5380 return (rc); 5381} 5382 5383#ifdef TCP_OFFLOAD 5384static int 5385toe_capability(struct port_info *pi, int enable) 5386{ 5387 int rc; 5388 struct adapter *sc = pi->adapter; 5389 5390 ADAPTER_LOCK_ASSERT_OWNED(sc); 5391 5392 if (!is_offload(sc)) 5393 return (ENODEV); 5394 5395 if (enable) { 5396 if (!(sc->flags & FULL_INIT_DONE)) { 5397 log(LOG_WARNING, 5398 "You must enable a cxgbe interface first\n"); 5399 return (EAGAIN); 5400 } 5401 5402 if (isset(&sc->offload_map, pi->port_id)) 5403 return (0); 5404 5405 if (!(sc->flags & TOM_INIT_DONE)) { 5406 rc = t4_activate_uld(sc, ULD_TOM); 5407 if (rc == EAGAIN) { 5408 log(LOG_WARNING, 5409 "You must kldload t4_tom.ko before trying " 5410 "to enable TOE on a cxgbe interface.\n"); 5411 } 5412 if (rc != 0) 5413 return (rc); 5414 KASSERT(sc->tom_softc != NULL, 5415 ("%s: TOM activated but softc NULL", __func__)); 5416 KASSERT(sc->flags & TOM_INIT_DONE, 5417 ("%s: TOM activated but flag not set", __func__)); 5418 } 5419 5420 setbit(&sc->offload_map, pi->port_id); 5421 } else { 5422 if (!isset(&sc->offload_map, pi->port_id)) 5423 return (0); 5424 5425 KASSERT(sc->flags & TOM_INIT_DONE, 5426 ("%s: TOM never initialized?", __func__)); 5427 clrbit(&sc->offload_map, pi->port_id); 5428 } 5429 5430 return (0); 5431} 5432 5433/* 5434 * Add an upper layer driver to the global list. 5435 */ 5436int 5437t4_register_uld(struct uld_info *ui) 5438{ 5439 int rc = 0; 5440 struct uld_info *u; 5441 5442 mtx_lock(&t4_uld_list_lock); 5443 SLIST_FOREACH(u, &t4_uld_list, link) { 5444 if (u->uld_id == ui->uld_id) { 5445 rc = EEXIST; 5446 goto done; 5447 } 5448 } 5449 5450 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 5451 ui->refcount = 0; 5452done: 5453 mtx_unlock(&t4_uld_list_lock); 5454 return (rc); 5455} 5456 5457int 5458t4_unregister_uld(struct uld_info *ui) 5459{ 5460 int rc = EINVAL; 5461 struct uld_info *u; 5462 5463 mtx_lock(&t4_uld_list_lock); 5464 5465 SLIST_FOREACH(u, &t4_uld_list, link) { 5466 if (u == ui) { 5467 if (ui->refcount > 0) { 5468 rc = EBUSY; 5469 goto done; 5470 } 5471 5472 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 5473 rc = 0; 5474 goto done; 5475 } 5476 } 5477done: 5478 mtx_unlock(&t4_uld_list_lock); 5479 return (rc); 5480} 5481 5482int 5483t4_activate_uld(struct adapter *sc, int id) 5484{ 5485 int rc = EAGAIN; 5486 struct uld_info *ui; 5487 5488 mtx_lock(&t4_uld_list_lock); 5489 5490 SLIST_FOREACH(ui, &t4_uld_list, link) { 5491 if (ui->uld_id == id) { 5492 rc = ui->activate(sc); 5493 if (rc == 0) 5494 ui->refcount++; 5495 goto done; 5496 } 5497 } 5498done: 5499 mtx_unlock(&t4_uld_list_lock); 5500 5501 return (rc); 5502} 5503 5504int 5505t4_deactivate_uld(struct adapter *sc, int id) 5506{ 5507 int rc = EINVAL; 5508 struct uld_info *ui; 5509 5510 mtx_lock(&t4_uld_list_lock); 5511 5512 SLIST_FOREACH(ui, &t4_uld_list, link) { 5513 if (ui->uld_id == id) { 5514 rc = ui->deactivate(sc); 5515 if (rc == 0) 5516 ui->refcount--; 5517 goto done; 5518 } 5519 } 5520done: 5521 mtx_unlock(&t4_uld_list_lock); 5522 5523 return (rc); 5524} 5525#endif 5526 5527/* 5528 * Come up with reasonable defaults for some of the tunables, provided they're 5529 * not set by the user (in which case we'll use the values as is). 5530 */ 5531static void 5532tweak_tunables(void) 5533{ 5534 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 5535 5536 if (t4_ntxq10g < 1) 5537 t4_ntxq10g = min(nc, NTXQ_10G); 5538 5539 if (t4_ntxq1g < 1) 5540 t4_ntxq1g = min(nc, NTXQ_1G); 5541 5542 if (t4_nrxq10g < 1) 5543 t4_nrxq10g = min(nc, NRXQ_10G); 5544 5545 if (t4_nrxq1g < 1) 5546 t4_nrxq1g = min(nc, NRXQ_1G); 5547 5548#ifdef TCP_OFFLOAD 5549 if (t4_nofldtxq10g < 1) 5550 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 5551 5552 if (t4_nofldtxq1g < 1) 5553 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 5554 5555 if (t4_nofldrxq10g < 1) 5556 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 5557 5558 if (t4_nofldrxq1g < 1) 5559 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 5560 5561 if (t4_toecaps_allowed == -1) 5562 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 5563#else 5564 if (t4_toecaps_allowed == -1) 5565 t4_toecaps_allowed = 0; 5566#endif 5567 5568 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 5569 t4_tmr_idx_10g = TMR_IDX_10G; 5570 5571 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 5572 t4_pktc_idx_10g = PKTC_IDX_10G; 5573 5574 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 5575 t4_tmr_idx_1g = TMR_IDX_1G; 5576 5577 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 5578 t4_pktc_idx_1g = PKTC_IDX_1G; 5579 5580 if (t4_qsize_txq < 128) 5581 t4_qsize_txq = 128; 5582 5583 if (t4_qsize_rxq < 128) 5584 t4_qsize_rxq = 128; 5585 while (t4_qsize_rxq & 7) 5586 t4_qsize_rxq++; 5587 5588 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 5589} 5590 5591static int 5592t4_mod_event(module_t mod, int cmd, void *arg) 5593{ 5594 int rc = 0; 5595 5596 switch (cmd) { 5597 case MOD_LOAD: 5598 t4_sge_modload(); 5599 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 5600 SLIST_INIT(&t4_list); 5601#ifdef TCP_OFFLOAD 5602 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 5603 SLIST_INIT(&t4_uld_list); 5604#endif 5605 tweak_tunables(); 5606 break; 5607 5608 case MOD_UNLOAD: 5609#ifdef TCP_OFFLOAD 5610 mtx_lock(&t4_uld_list_lock); 5611 if (!SLIST_EMPTY(&t4_uld_list)) { 5612 rc = EBUSY; 5613 mtx_unlock(&t4_uld_list_lock); 5614 break; 5615 } 5616 mtx_unlock(&t4_uld_list_lock); 5617 mtx_destroy(&t4_uld_list_lock); 5618#endif 5619 mtx_lock(&t4_list_lock); 5620 if (!SLIST_EMPTY(&t4_list)) { 5621 rc = EBUSY; 5622 mtx_unlock(&t4_list_lock); 5623 break; 5624 } 5625 mtx_unlock(&t4_list_lock); 5626 mtx_destroy(&t4_list_lock); 5627 break; 5628 } 5629 5630 return (rc); 5631} 5632 5633static devclass_t t4_devclass; 5634static devclass_t cxgbe_devclass; 5635 5636DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0); 5637MODULE_VERSION(t4nex, 1); 5638 5639DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 5640MODULE_VERSION(cxgbe, 1); 5641