t4_main.c revision 249383
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_main.c 249383 2013-04-11 21:17:49Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58#if defined(__i386__) || defined(__amd64__) 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" 69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80}; 81static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85}; 86 87 88/* T4 port (cxgbe) interface */ 89static int cxgbe_probe(device_t); 90static int cxgbe_attach(device_t); 91static int cxgbe_detach(device_t); 92static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97}; 98static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102}; 103 104static d_ioctl_t t4_ioctl; 105static d_open_t t4_open; 106static d_close_t t4_close; 107 108static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115}; 116 117/* T5 bus driver interface */ 118static int t5_probe(device_t); 119static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125}; 126static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130}; 131 132 133/* T5 port (cxl) interface */ 134static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138}; 139 140static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147}; 148 149/* ifnet + media interface */ 150static void cxgbe_init(void *); 151static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153static void cxgbe_qflush(struct ifnet *); 154static int cxgbe_media_change(struct ifnet *); 155static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159/* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163static struct mtx t4_list_lock; 164static SLIST_HEAD(, adapter) t4_list; 165#ifdef TCP_OFFLOAD 166static struct mtx t4_uld_list_lock; 167static SLIST_HEAD(, uld_info) t4_uld_list; 168#endif 169 170/* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181/* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184#define NTXQ_10G 16 185static int t4_ntxq10g = -1; 186TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188#define NRXQ_10G 8 189static int t4_nrxq10g = -1; 190TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192#define NTXQ_1G 4 193static int t4_ntxq1g = -1; 194TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196#define NRXQ_1G 2 197static int t4_nrxq1g = -1; 198TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200#ifdef TCP_OFFLOAD 201#define NOFLDTXQ_10G 8 202static int t4_nofldtxq10g = -1; 203TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 204 205#define NOFLDRXQ_10G 2 206static int t4_nofldrxq10g = -1; 207TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 208 209#define NOFLDTXQ_1G 2 210static int t4_nofldtxq1g = -1; 211TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 212 213#define NOFLDRXQ_1G 1 214static int t4_nofldrxq1g = -1; 215TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 216#endif 217 218/* 219 * Holdoff parameters for 10G and 1G ports. 220 */ 221#define TMR_IDX_10G 1 222static int t4_tmr_idx_10g = TMR_IDX_10G; 223TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 224 225#define PKTC_IDX_10G (-1) 226static int t4_pktc_idx_10g = PKTC_IDX_10G; 227TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 228 229#define TMR_IDX_1G 1 230static int t4_tmr_idx_1g = TMR_IDX_1G; 231TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 232 233#define PKTC_IDX_1G (-1) 234static int t4_pktc_idx_1g = PKTC_IDX_1G; 235TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 236 237/* 238 * Size (# of entries) of each tx and rx queue. 239 */ 240static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 241TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 242 243static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 244TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 245 246/* 247 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 248 */ 249static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 250TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 251 252/* 253 * Configuration file. 254 */ 255#define DEFAULT_CF "default" 256#define FLASH_CF "flash" 257#define UWIRE_CF "uwire" 258#define FPGA_CF "fpga" 259static char t4_cfg_file[32] = DEFAULT_CF; 260TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 261 262/* 263 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 264 * encouraged respectively). 265 */ 266static unsigned int t4_fw_install = 1; 267TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 268 269/* 270 * ASIC features that will be used. Disable the ones you don't want so that the 271 * chip resources aren't wasted on features that will not be used. 272 */ 273static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 274TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 275 276static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 277TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 278 279static int t4_toecaps_allowed = -1; 280TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 281 282static int t4_rdmacaps_allowed = 0; 283TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 284 285static int t4_iscsicaps_allowed = 0; 286TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 287 288static int t4_fcoecaps_allowed = 0; 289TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 290 291static int t5_write_combine = 0; 292TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 293 294struct intrs_and_queues { 295 int intr_type; /* INTx, MSI, or MSI-X */ 296 int nirq; /* Number of vectors */ 297 int intr_flags; 298 int ntxq10g; /* # of NIC txq's for each 10G port */ 299 int nrxq10g; /* # of NIC rxq's for each 10G port */ 300 int ntxq1g; /* # of NIC txq's for each 1G port */ 301 int nrxq1g; /* # of NIC rxq's for each 1G port */ 302#ifdef TCP_OFFLOAD 303 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 304 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 305 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 306 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 307#endif 308}; 309 310struct filter_entry { 311 uint32_t valid:1; /* filter allocated and valid */ 312 uint32_t locked:1; /* filter is administratively locked */ 313 uint32_t pending:1; /* filter action is pending firmware reply */ 314 uint32_t smtidx:8; /* Source MAC Table index for smac */ 315 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 316 317 struct t4_filter_specification fs; 318}; 319 320enum { 321 XGMAC_MTU = (1 << 0), 322 XGMAC_PROMISC = (1 << 1), 323 XGMAC_ALLMULTI = (1 << 2), 324 XGMAC_VLANEX = (1 << 3), 325 XGMAC_UCADDR = (1 << 4), 326 XGMAC_MCADDRS = (1 << 5), 327 328 XGMAC_ALL = 0xffff 329}; 330 331static int map_bars_0_and_4(struct adapter *); 332static int map_bar_2(struct adapter *); 333static void setup_memwin(struct adapter *); 334static int validate_mem_range(struct adapter *, uint32_t, int); 335static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 336 uint32_t *); 337static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 338static uint32_t position_memwin(struct adapter *, int, uint32_t); 339static int cfg_itype_and_nqueues(struct adapter *, int, int, 340 struct intrs_and_queues *); 341static int prep_firmware(struct adapter *); 342static int partition_resources(struct adapter *, const struct firmware *, 343 const char *); 344static int get_params__pre_init(struct adapter *); 345static int get_params__post_init(struct adapter *); 346static int set_params__post_init(struct adapter *); 347static void t4_set_desc(struct adapter *); 348static void build_medialist(struct port_info *); 349static int update_mac_settings(struct port_info *, int); 350static int cxgbe_init_synchronized(struct port_info *); 351static int cxgbe_uninit_synchronized(struct port_info *); 352static int setup_intr_handlers(struct adapter *); 353static int adapter_full_init(struct adapter *); 354static int adapter_full_uninit(struct adapter *); 355static int port_full_init(struct port_info *); 356static int port_full_uninit(struct port_info *); 357static void quiesce_eq(struct adapter *, struct sge_eq *); 358static void quiesce_iq(struct adapter *, struct sge_iq *); 359static void quiesce_fl(struct adapter *, struct sge_fl *); 360static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 361 driver_intr_t *, void *, char *); 362static int t4_free_irq(struct adapter *, struct irq *); 363static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 364 unsigned int); 365static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 366static void cxgbe_tick(void *); 367static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 368static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 369 struct mbuf *); 370static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 371static int fw_msg_not_handled(struct adapter *, const __be64 *); 372static int t4_sysctls(struct adapter *); 373static int cxgbe_sysctls(struct port_info *); 374static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 375static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 376static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 377static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 378static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 379static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 380static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 381#ifdef SBUF_DRAIN 382static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 383static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 384static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 385static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 386static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 387static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 388static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 389static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 390static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 391static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 392static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 393static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 394static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 395static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 396static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 397static int sysctl_tids(SYSCTL_HANDLER_ARGS); 398static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 399static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 400static int sysctl_wrwc_stats(SYSCTL_HANDLER_ARGS); 401#endif 402static inline void txq_start(struct ifnet *, struct sge_txq *); 403static uint32_t fconf_to_mode(uint32_t); 404static uint32_t mode_to_fconf(uint32_t); 405static uint32_t fspec_to_fconf(struct t4_filter_specification *); 406static int get_filter_mode(struct adapter *, uint32_t *); 407static int set_filter_mode(struct adapter *, uint32_t); 408static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 409static int get_filter(struct adapter *, struct t4_filter *); 410static int set_filter(struct adapter *, struct t4_filter *); 411static int del_filter(struct adapter *, struct t4_filter *); 412static void clear_filter(struct filter_entry *); 413static int set_filter_wr(struct adapter *, int); 414static int del_filter_wr(struct adapter *, int); 415static int get_sge_context(struct adapter *, struct t4_sge_context *); 416static int load_fw(struct adapter *, struct t4_data *); 417static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 418static int read_i2c(struct adapter *, struct t4_i2c_data *); 419#ifdef TCP_OFFLOAD 420static int toe_capability(struct port_info *, int); 421#endif 422static int mod_event(module_t, int, void *); 423 424struct { 425 uint16_t device; 426 char *desc; 427} t4_pciids[] = { 428 {0xa000, "Chelsio Terminator 4 FPGA"}, 429 {0x4400, "Chelsio T440-dbg"}, 430 {0x4401, "Chelsio T420-CR"}, 431 {0x4402, "Chelsio T422-CR"}, 432 {0x4403, "Chelsio T440-CR"}, 433 {0x4404, "Chelsio T420-BCH"}, 434 {0x4405, "Chelsio T440-BCH"}, 435 {0x4406, "Chelsio T440-CH"}, 436 {0x4407, "Chelsio T420-SO"}, 437 {0x4408, "Chelsio T420-CX"}, 438 {0x4409, "Chelsio T420-BT"}, 439 {0x440a, "Chelsio T404-BT"}, 440 {0x440e, "Chelsio T440-LP-CR"}, 441}, t5_pciids[] = { 442 {0xb000, "Chelsio Terminator 5 FPGA"}, 443 {0x5400, "Chelsio T580-dbg"}, 444}; 445 446#ifdef TCP_OFFLOAD 447/* 448 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 449 * exactly the same for both rxq and ofld_rxq. 450 */ 451CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 452CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 453#endif 454 455/* No easy way to include t4_msg.h before adapter.h so we check this way */ 456CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 457CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 458 459static int 460t4_probe(device_t dev) 461{ 462 int i; 463 uint16_t v = pci_get_vendor(dev); 464 uint16_t d = pci_get_device(dev); 465 uint8_t f = pci_get_function(dev); 466 467 if (v != PCI_VENDOR_ID_CHELSIO) 468 return (ENXIO); 469 470 /* Attach only to PF0 of the FPGA */ 471 if (d == 0xa000 && f != 0) 472 return (ENXIO); 473 474 for (i = 0; i < nitems(t4_pciids); i++) { 475 if (d == t4_pciids[i].device) { 476 device_set_desc(dev, t4_pciids[i].desc); 477 return (BUS_PROBE_DEFAULT); 478 } 479 } 480 481 return (ENXIO); 482} 483 484static int 485t5_probe(device_t dev) 486{ 487 int i; 488 uint16_t v = pci_get_vendor(dev); 489 uint16_t d = pci_get_device(dev); 490 uint8_t f = pci_get_function(dev); 491 492 if (v != PCI_VENDOR_ID_CHELSIO) 493 return (ENXIO); 494 495 /* Attach only to PF0 of the FPGA */ 496 if (d == 0xb000 && f != 0) 497 return (ENXIO); 498 499 for (i = 0; i < nitems(t5_pciids); i++) { 500 if (d == t5_pciids[i].device) { 501 device_set_desc(dev, t5_pciids[i].desc); 502 return (BUS_PROBE_DEFAULT); 503 } 504 } 505 506 return (ENXIO); 507} 508 509static int 510t4_attach(device_t dev) 511{ 512 struct adapter *sc; 513 int rc = 0, i, n10g, n1g, rqidx, tqidx; 514 struct intrs_and_queues iaq; 515 struct sge *s; 516#ifdef TCP_OFFLOAD 517 int ofld_rqidx, ofld_tqidx; 518#endif 519 520 sc = device_get_softc(dev); 521 sc->dev = dev; 522 523 pci_enable_busmaster(dev); 524 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 525 uint32_t v; 526 527 pci_set_max_read_req(dev, 4096); 528 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 529 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 530 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 531 } 532 533 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 534 device_get_nameunit(dev)); 535 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 536 mtx_lock(&t4_list_lock); 537 SLIST_INSERT_HEAD(&t4_list, sc, link); 538 mtx_unlock(&t4_list_lock); 539 540 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 541 TAILQ_INIT(&sc->sfl); 542 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 543 544 rc = map_bars_0_and_4(sc); 545 if (rc != 0) 546 goto done; /* error message displayed already */ 547 548 /* 549 * This is the real PF# to which we're attaching. Works from within PCI 550 * passthrough environments too, where pci_get_function() could return a 551 * different PF# depending on the passthrough configuration. We need to 552 * use the real PF# in all our communication with the firmware. 553 */ 554 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 555 sc->mbox = sc->pf; 556 557 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 558 sc->an_handler = an_not_handled; 559 for (i = 0; i < nitems(sc->cpl_handler); i++) 560 sc->cpl_handler[i] = cpl_not_handled; 561 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 562 sc->fw_msg_handler[i] = fw_msg_not_handled; 563 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 564 t4_init_sge_cpl_handlers(sc); 565 566 /* Prepare the adapter for operation */ 567 rc = -t4_prep_adapter(sc); 568 if (rc != 0) { 569 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 570 goto done; 571 } 572 573 /* 574 * Do this really early, with the memory windows set up even before the 575 * character device. The userland tool's register i/o and mem read 576 * will work even in "recovery mode". 577 */ 578 setup_memwin(sc); 579 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 580 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 581 device_get_nameunit(dev)); 582 if (sc->cdev == NULL) 583 device_printf(dev, "failed to create nexus char device.\n"); 584 else 585 sc->cdev->si_drv1 = sc; 586 587 /* Go no further if recovery mode has been requested. */ 588 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 589 device_printf(dev, "recovery mode.\n"); 590 goto done; 591 } 592 593 /* Prepare the firmware for operation */ 594 rc = prep_firmware(sc); 595 if (rc != 0) 596 goto done; /* error message displayed already */ 597 598 rc = get_params__post_init(sc); 599 if (rc != 0) 600 goto done; /* error message displayed already */ 601 602 rc = set_params__post_init(sc); 603 if (rc != 0) 604 goto done; /* error message displayed already */ 605 606 rc = map_bar_2(sc); 607 if (rc != 0) 608 goto done; /* error message displayed already */ 609 610 for (i = 0; i < NCHAN; i++) 611 sc->params.tp.tx_modq[i] = i; 612 613 rc = t4_create_dma_tag(sc); 614 if (rc != 0) 615 goto done; /* error message displayed already */ 616 617 /* 618 * First pass over all the ports - allocate VIs and initialize some 619 * basic parameters like mac address, port type, etc. We also figure 620 * out whether a port is 10G or 1G and use that information when 621 * calculating how many interrupts to attempt to allocate. 622 */ 623 n10g = n1g = 0; 624 for_each_port(sc, i) { 625 struct port_info *pi; 626 627 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 628 sc->port[i] = pi; 629 630 /* These must be set before t4_port_init */ 631 pi->adapter = sc; 632 pi->port_id = i; 633 634 /* Allocate the vi and initialize parameters like mac addr */ 635 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 636 if (rc != 0) { 637 device_printf(dev, "unable to initialize port %d: %d\n", 638 i, rc); 639 free(pi, M_CXGBE); 640 sc->port[i] = NULL; 641 goto done; 642 } 643 644 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 645 device_get_nameunit(dev), i); 646 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 647 648 if (is_10G_port(pi)) { 649 n10g++; 650 pi->tmr_idx = t4_tmr_idx_10g; 651 pi->pktc_idx = t4_pktc_idx_10g; 652 } else { 653 n1g++; 654 pi->tmr_idx = t4_tmr_idx_1g; 655 pi->pktc_idx = t4_pktc_idx_1g; 656 } 657 658 pi->xact_addr_filt = -1; 659 660 pi->qsize_rxq = t4_qsize_rxq; 661 pi->qsize_txq = t4_qsize_txq; 662 663 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 664 if (pi->dev == NULL) { 665 device_printf(dev, 666 "failed to add device for port %d.\n", i); 667 rc = ENXIO; 668 goto done; 669 } 670 device_set_softc(pi->dev, pi); 671 } 672 673 /* 674 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 675 */ 676 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 677 if (rc != 0) 678 goto done; /* error message displayed already */ 679 680 sc->intr_type = iaq.intr_type; 681 sc->intr_count = iaq.nirq; 682 sc->flags |= iaq.intr_flags; 683 684 s = &sc->sge; 685 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 686 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 687 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 688 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 689 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 690 691#ifdef TCP_OFFLOAD 692 if (is_offload(sc)) { 693 694 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 695 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 696 s->neq += s->nofldtxq + s->nofldrxq; 697 s->niq += s->nofldrxq; 698 699 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 700 M_CXGBE, M_ZERO | M_WAITOK); 701 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 702 M_CXGBE, M_ZERO | M_WAITOK); 703 } 704#endif 705 706 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 707 M_ZERO | M_WAITOK); 708 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 709 M_ZERO | M_WAITOK); 710 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 711 M_ZERO | M_WAITOK); 712 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 713 M_ZERO | M_WAITOK); 714 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 715 M_ZERO | M_WAITOK); 716 717 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 718 M_ZERO | M_WAITOK); 719 720 t4_init_l2t(sc, M_WAITOK); 721 722 /* 723 * Second pass over the ports. This time we know the number of rx and 724 * tx queues that each port should get. 725 */ 726 rqidx = tqidx = 0; 727#ifdef TCP_OFFLOAD 728 ofld_rqidx = ofld_tqidx = 0; 729#endif 730 for_each_port(sc, i) { 731 struct port_info *pi = sc->port[i]; 732 733 if (pi == NULL) 734 continue; 735 736 pi->first_rxq = rqidx; 737 pi->first_txq = tqidx; 738 if (is_10G_port(pi)) { 739 pi->nrxq = iaq.nrxq10g; 740 pi->ntxq = iaq.ntxq10g; 741 } else { 742 pi->nrxq = iaq.nrxq1g; 743 pi->ntxq = iaq.ntxq1g; 744 } 745 746 rqidx += pi->nrxq; 747 tqidx += pi->ntxq; 748 749#ifdef TCP_OFFLOAD 750 if (is_offload(sc)) { 751 pi->first_ofld_rxq = ofld_rqidx; 752 pi->first_ofld_txq = ofld_tqidx; 753 if (is_10G_port(pi)) { 754 pi->nofldrxq = iaq.nofldrxq10g; 755 pi->nofldtxq = iaq.nofldtxq10g; 756 } else { 757 pi->nofldrxq = iaq.nofldrxq1g; 758 pi->nofldtxq = iaq.nofldtxq1g; 759 } 760 ofld_rqidx += pi->nofldrxq; 761 ofld_tqidx += pi->nofldtxq; 762 } 763#endif 764 } 765 766 rc = setup_intr_handlers(sc); 767 if (rc != 0) { 768 device_printf(dev, 769 "failed to setup interrupt handlers: %d\n", rc); 770 goto done; 771 } 772 773 rc = bus_generic_attach(dev); 774 if (rc != 0) { 775 device_printf(dev, 776 "failed to attach all child ports: %d\n", rc); 777 goto done; 778 } 779 780 device_printf(dev, 781 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 782 sc->params.pci.width, sc->params.nports, sc->intr_count, 783 sc->intr_type == INTR_MSIX ? "MSI-X" : 784 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 785 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 786 787 t4_set_desc(sc); 788 789done: 790 if (rc != 0 && sc->cdev) { 791 /* cdev was created and so cxgbetool works; recover that way. */ 792 device_printf(dev, 793 "error during attach, adapter is now in recovery mode.\n"); 794 rc = 0; 795 } 796 797 if (rc != 0) 798 t4_detach(dev); 799 else 800 t4_sysctls(sc); 801 802 return (rc); 803} 804 805/* 806 * Idempotent 807 */ 808static int 809t4_detach(device_t dev) 810{ 811 struct adapter *sc; 812 struct port_info *pi; 813 int i, rc; 814 815 sc = device_get_softc(dev); 816 817 if (sc->flags & FULL_INIT_DONE) 818 t4_intr_disable(sc); 819 820 if (sc->cdev) { 821 destroy_dev(sc->cdev); 822 sc->cdev = NULL; 823 } 824 825 rc = bus_generic_detach(dev); 826 if (rc) { 827 device_printf(dev, 828 "failed to detach child devices: %d\n", rc); 829 return (rc); 830 } 831 832 for (i = 0; i < sc->intr_count; i++) 833 t4_free_irq(sc, &sc->irq[i]); 834 835 for (i = 0; i < MAX_NPORTS; i++) { 836 pi = sc->port[i]; 837 if (pi) { 838 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 839 if (pi->dev) 840 device_delete_child(dev, pi->dev); 841 842 mtx_destroy(&pi->pi_lock); 843 free(pi, M_CXGBE); 844 } 845 } 846 847 if (sc->flags & FULL_INIT_DONE) 848 adapter_full_uninit(sc); 849 850 if (sc->flags & FW_OK) 851 t4_fw_bye(sc, sc->mbox); 852 853 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 854 pci_release_msi(dev); 855 856 if (sc->regs_res) 857 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 858 sc->regs_res); 859 860 if (sc->udbs_res) 861 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 862 sc->udbs_res); 863 864 if (sc->msix_res) 865 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 866 sc->msix_res); 867 868 if (sc->l2t) 869 t4_free_l2t(sc->l2t); 870 871#ifdef TCP_OFFLOAD 872 free(sc->sge.ofld_rxq, M_CXGBE); 873 free(sc->sge.ofld_txq, M_CXGBE); 874#endif 875 free(sc->irq, M_CXGBE); 876 free(sc->sge.rxq, M_CXGBE); 877 free(sc->sge.txq, M_CXGBE); 878 free(sc->sge.ctrlq, M_CXGBE); 879 free(sc->sge.iqmap, M_CXGBE); 880 free(sc->sge.eqmap, M_CXGBE); 881 free(sc->tids.ftid_tab, M_CXGBE); 882 t4_destroy_dma_tag(sc); 883 if (mtx_initialized(&sc->sc_lock)) { 884 mtx_lock(&t4_list_lock); 885 SLIST_REMOVE(&t4_list, sc, adapter, link); 886 mtx_unlock(&t4_list_lock); 887 mtx_destroy(&sc->sc_lock); 888 } 889 890 if (mtx_initialized(&sc->tids.ftid_lock)) 891 mtx_destroy(&sc->tids.ftid_lock); 892 if (mtx_initialized(&sc->sfl_lock)) 893 mtx_destroy(&sc->sfl_lock); 894 895 bzero(sc, sizeof(*sc)); 896 897 return (0); 898} 899 900 901static int 902cxgbe_probe(device_t dev) 903{ 904 char buf[128]; 905 struct port_info *pi = device_get_softc(dev); 906 907 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 908 device_set_desc_copy(dev, buf); 909 910 return (BUS_PROBE_DEFAULT); 911} 912 913#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 914 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 915 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 916#define T4_CAP_ENABLE (T4_CAP) 917 918static int 919cxgbe_attach(device_t dev) 920{ 921 struct port_info *pi = device_get_softc(dev); 922 struct ifnet *ifp; 923 924 /* Allocate an ifnet and set it up */ 925 ifp = if_alloc(IFT_ETHER); 926 if (ifp == NULL) { 927 device_printf(dev, "Cannot allocate ifnet\n"); 928 return (ENOMEM); 929 } 930 pi->ifp = ifp; 931 ifp->if_softc = pi; 932 933 callout_init(&pi->tick, CALLOUT_MPSAFE); 934 935 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 936 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 937 938 ifp->if_init = cxgbe_init; 939 ifp->if_ioctl = cxgbe_ioctl; 940 ifp->if_transmit = cxgbe_transmit; 941 ifp->if_qflush = cxgbe_qflush; 942 943 ifp->if_capabilities = T4_CAP; 944#ifdef TCP_OFFLOAD 945 if (is_offload(pi->adapter)) 946 ifp->if_capabilities |= IFCAP_TOE; 947#endif 948 ifp->if_capenable = T4_CAP_ENABLE; 949 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 950 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 951 952 /* Initialize ifmedia for this port */ 953 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 954 cxgbe_media_status); 955 build_medialist(pi); 956 957 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 958 EVENTHANDLER_PRI_ANY); 959 960 ether_ifattach(ifp, pi->hw_addr); 961 962#ifdef TCP_OFFLOAD 963 if (is_offload(pi->adapter)) { 964 device_printf(dev, 965 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 966 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 967 } else 968#endif 969 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 970 971 cxgbe_sysctls(pi); 972 973 return (0); 974} 975 976static int 977cxgbe_detach(device_t dev) 978{ 979 struct port_info *pi = device_get_softc(dev); 980 struct adapter *sc = pi->adapter; 981 struct ifnet *ifp = pi->ifp; 982 983 /* Tell if_ioctl and if_init that the port is going away */ 984 ADAPTER_LOCK(sc); 985 SET_DOOMED(pi); 986 wakeup(&sc->flags); 987 while (IS_BUSY(sc)) 988 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 989 SET_BUSY(sc); 990#ifdef INVARIANTS 991 sc->last_op = "t4detach"; 992 sc->last_op_thr = curthread; 993#endif 994 ADAPTER_UNLOCK(sc); 995 996 if (pi->vlan_c) 997 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 998 999 PORT_LOCK(pi); 1000 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1001 callout_stop(&pi->tick); 1002 PORT_UNLOCK(pi); 1003 callout_drain(&pi->tick); 1004 1005 /* Let detach proceed even if these fail. */ 1006 cxgbe_uninit_synchronized(pi); 1007 port_full_uninit(pi); 1008 1009 ifmedia_removeall(&pi->media); 1010 ether_ifdetach(pi->ifp); 1011 if_free(pi->ifp); 1012 1013 ADAPTER_LOCK(sc); 1014 CLR_BUSY(sc); 1015 wakeup(&sc->flags); 1016 ADAPTER_UNLOCK(sc); 1017 1018 return (0); 1019} 1020 1021static void 1022cxgbe_init(void *arg) 1023{ 1024 struct port_info *pi = arg; 1025 struct adapter *sc = pi->adapter; 1026 1027 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1028 return; 1029 cxgbe_init_synchronized(pi); 1030 end_synchronized_op(sc, 0); 1031} 1032 1033static int 1034cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1035{ 1036 int rc = 0, mtu, flags; 1037 struct port_info *pi = ifp->if_softc; 1038 struct adapter *sc = pi->adapter; 1039 struct ifreq *ifr = (struct ifreq *)data; 1040 uint32_t mask; 1041 1042 switch (cmd) { 1043 case SIOCSIFMTU: 1044 mtu = ifr->ifr_mtu; 1045 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1046 return (EINVAL); 1047 1048 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1049 if (rc) 1050 return (rc); 1051 ifp->if_mtu = mtu; 1052 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1053 t4_update_fl_bufsize(ifp); 1054 rc = update_mac_settings(pi, XGMAC_MTU); 1055 } 1056 end_synchronized_op(sc, 0); 1057 break; 1058 1059 case SIOCSIFFLAGS: 1060 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1061 if (rc) 1062 return (rc); 1063 1064 if (ifp->if_flags & IFF_UP) { 1065 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1066 flags = pi->if_flags; 1067 if ((ifp->if_flags ^ flags) & 1068 (IFF_PROMISC | IFF_ALLMULTI)) { 1069 rc = update_mac_settings(pi, 1070 XGMAC_PROMISC | XGMAC_ALLMULTI); 1071 } 1072 } else 1073 rc = cxgbe_init_synchronized(pi); 1074 pi->if_flags = ifp->if_flags; 1075 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1076 rc = cxgbe_uninit_synchronized(pi); 1077 end_synchronized_op(sc, 0); 1078 break; 1079 1080 case SIOCADDMULTI: 1081 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1082 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1083 if (rc) 1084 return (rc); 1085 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1086 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1087 end_synchronized_op(sc, LOCK_HELD); 1088 break; 1089 1090 case SIOCSIFCAP: 1091 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1092 if (rc) 1093 return (rc); 1094 1095 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1096 if (mask & IFCAP_TXCSUM) { 1097 ifp->if_capenable ^= IFCAP_TXCSUM; 1098 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1099 1100 if (IFCAP_TSO4 & ifp->if_capenable && 1101 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1102 ifp->if_capenable &= ~IFCAP_TSO4; 1103 if_printf(ifp, 1104 "tso4 disabled due to -txcsum.\n"); 1105 } 1106 } 1107 if (mask & IFCAP_TXCSUM_IPV6) { 1108 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1109 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1110 1111 if (IFCAP_TSO6 & ifp->if_capenable && 1112 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1113 ifp->if_capenable &= ~IFCAP_TSO6; 1114 if_printf(ifp, 1115 "tso6 disabled due to -txcsum6.\n"); 1116 } 1117 } 1118 if (mask & IFCAP_RXCSUM) 1119 ifp->if_capenable ^= IFCAP_RXCSUM; 1120 if (mask & IFCAP_RXCSUM_IPV6) 1121 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1122 1123 /* 1124 * Note that we leave CSUM_TSO alone (it is always set). The 1125 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1126 * sending a TSO request our way, so it's sufficient to toggle 1127 * IFCAP_TSOx only. 1128 */ 1129 if (mask & IFCAP_TSO4) { 1130 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1131 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1132 if_printf(ifp, "enable txcsum first.\n"); 1133 rc = EAGAIN; 1134 goto fail; 1135 } 1136 ifp->if_capenable ^= IFCAP_TSO4; 1137 } 1138 if (mask & IFCAP_TSO6) { 1139 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1140 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1141 if_printf(ifp, "enable txcsum6 first.\n"); 1142 rc = EAGAIN; 1143 goto fail; 1144 } 1145 ifp->if_capenable ^= IFCAP_TSO6; 1146 } 1147 if (mask & IFCAP_LRO) { 1148#if defined(INET) || defined(INET6) 1149 int i; 1150 struct sge_rxq *rxq; 1151 1152 ifp->if_capenable ^= IFCAP_LRO; 1153 for_each_rxq(pi, i, rxq) { 1154 if (ifp->if_capenable & IFCAP_LRO) 1155 rxq->iq.flags |= IQ_LRO_ENABLED; 1156 else 1157 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1158 } 1159#endif 1160 } 1161#ifdef TCP_OFFLOAD 1162 if (mask & IFCAP_TOE) { 1163 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1164 1165 rc = toe_capability(pi, enable); 1166 if (rc != 0) 1167 goto fail; 1168 1169 ifp->if_capenable ^= mask; 1170 } 1171#endif 1172 if (mask & IFCAP_VLAN_HWTAGGING) { 1173 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1174 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1175 rc = update_mac_settings(pi, XGMAC_VLANEX); 1176 } 1177 if (mask & IFCAP_VLAN_MTU) { 1178 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1179 1180 /* Need to find out how to disable auto-mtu-inflation */ 1181 } 1182 if (mask & IFCAP_VLAN_HWTSO) 1183 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1184 if (mask & IFCAP_VLAN_HWCSUM) 1185 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1186 1187#ifdef VLAN_CAPABILITIES 1188 VLAN_CAPABILITIES(ifp); 1189#endif 1190fail: 1191 end_synchronized_op(sc, 0); 1192 break; 1193 1194 case SIOCSIFMEDIA: 1195 case SIOCGIFMEDIA: 1196 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1197 break; 1198 1199 default: 1200 rc = ether_ioctl(ifp, cmd, data); 1201 } 1202 1203 return (rc); 1204} 1205 1206static int 1207cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1208{ 1209 struct port_info *pi = ifp->if_softc; 1210 struct adapter *sc = pi->adapter; 1211 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1212 struct buf_ring *br; 1213 int rc; 1214 1215 M_ASSERTPKTHDR(m); 1216 1217 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1218 m_freem(m); 1219 return (ENETDOWN); 1220 } 1221 1222 if (m->m_flags & M_FLOWID) 1223 txq += (m->m_pkthdr.flowid % pi->ntxq); 1224 br = txq->br; 1225 1226 if (TXQ_TRYLOCK(txq) == 0) { 1227 struct sge_eq *eq = &txq->eq; 1228 1229 /* 1230 * It is possible that t4_eth_tx finishes up and releases the 1231 * lock between the TRYLOCK above and the drbr_enqueue here. We 1232 * need to make sure that this mbuf doesn't just sit there in 1233 * the drbr. 1234 */ 1235 1236 rc = drbr_enqueue(ifp, br, m); 1237 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1238 !(eq->flags & EQ_DOOMED)) 1239 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1240 return (rc); 1241 } 1242 1243 /* 1244 * txq->m is the mbuf that is held up due to a temporary shortage of 1245 * resources and it should be put on the wire first. Then what's in 1246 * drbr and finally the mbuf that was just passed in to us. 1247 * 1248 * Return code should indicate the fate of the mbuf that was passed in 1249 * this time. 1250 */ 1251 1252 TXQ_LOCK_ASSERT_OWNED(txq); 1253 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1254 1255 /* Queued for transmission. */ 1256 1257 rc = drbr_enqueue(ifp, br, m); 1258 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1259 (void) t4_eth_tx(ifp, txq, m); 1260 TXQ_UNLOCK(txq); 1261 return (rc); 1262 } 1263 1264 /* Direct transmission. */ 1265 rc = t4_eth_tx(ifp, txq, m); 1266 if (rc != 0 && txq->m) 1267 rc = 0; /* held, will be transmitted soon (hopefully) */ 1268 1269 TXQ_UNLOCK(txq); 1270 return (rc); 1271} 1272 1273static void 1274cxgbe_qflush(struct ifnet *ifp) 1275{ 1276 struct port_info *pi = ifp->if_softc; 1277 struct sge_txq *txq; 1278 int i; 1279 struct mbuf *m; 1280 1281 /* queues do not exist if !PORT_INIT_DONE. */ 1282 if (pi->flags & PORT_INIT_DONE) { 1283 for_each_txq(pi, i, txq) { 1284 TXQ_LOCK(txq); 1285 m_freem(txq->m); 1286 txq->m = NULL; 1287 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1288 m_freem(m); 1289 TXQ_UNLOCK(txq); 1290 } 1291 } 1292 if_qflush(ifp); 1293} 1294 1295static int 1296cxgbe_media_change(struct ifnet *ifp) 1297{ 1298 struct port_info *pi = ifp->if_softc; 1299 1300 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1301 1302 return (EOPNOTSUPP); 1303} 1304 1305static void 1306cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1307{ 1308 struct port_info *pi = ifp->if_softc; 1309 struct ifmedia_entry *cur = pi->media.ifm_cur; 1310 int speed = pi->link_cfg.speed; 1311 int data = (pi->port_type << 8) | pi->mod_type; 1312 1313 if (cur->ifm_data != data) { 1314 build_medialist(pi); 1315 cur = pi->media.ifm_cur; 1316 } 1317 1318 ifmr->ifm_status = IFM_AVALID; 1319 if (!pi->link_cfg.link_ok) 1320 return; 1321 1322 ifmr->ifm_status |= IFM_ACTIVE; 1323 1324 /* active and current will differ iff current media is autoselect. */ 1325 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1326 return; 1327 1328 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1329 if (speed == SPEED_10000) 1330 ifmr->ifm_active |= IFM_10G_T; 1331 else if (speed == SPEED_1000) 1332 ifmr->ifm_active |= IFM_1000_T; 1333 else if (speed == SPEED_100) 1334 ifmr->ifm_active |= IFM_100_TX; 1335 else if (speed == SPEED_10) 1336 ifmr->ifm_active |= IFM_10_T; 1337 else 1338 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1339 speed)); 1340} 1341 1342void 1343t4_fatal_err(struct adapter *sc) 1344{ 1345 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1346 t4_intr_disable(sc); 1347 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1348 device_get_nameunit(sc->dev)); 1349} 1350 1351static int 1352map_bars_0_and_4(struct adapter *sc) 1353{ 1354 sc->regs_rid = PCIR_BAR(0); 1355 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1356 &sc->regs_rid, RF_ACTIVE); 1357 if (sc->regs_res == NULL) { 1358 device_printf(sc->dev, "cannot map registers.\n"); 1359 return (ENXIO); 1360 } 1361 sc->bt = rman_get_bustag(sc->regs_res); 1362 sc->bh = rman_get_bushandle(sc->regs_res); 1363 sc->mmio_len = rman_get_size(sc->regs_res); 1364 setbit(&sc->doorbells, DOORBELL_KDB); 1365 1366 sc->msix_rid = PCIR_BAR(4); 1367 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1368 &sc->msix_rid, RF_ACTIVE); 1369 if (sc->msix_res == NULL) { 1370 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1371 return (ENXIO); 1372 } 1373 1374 return (0); 1375} 1376 1377static int 1378map_bar_2(struct adapter *sc) 1379{ 1380 1381 /* 1382 * T4: only iWARP driver uses the userspace doorbells. There is no need 1383 * to map it if RDMA is disabled. 1384 */ 1385 if (is_t4(sc) && sc->rdmacaps == 0) 1386 return (0); 1387 1388 sc->udbs_rid = PCIR_BAR(2); 1389 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1390 &sc->udbs_rid, RF_ACTIVE); 1391 if (sc->udbs_res == NULL) { 1392 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1393 return (ENXIO); 1394 } 1395 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1396 1397 if (is_t5(sc)) { 1398 setbit(&sc->doorbells, DOORBELL_UDB); 1399#if defined(__i386__) || defined(__amd64__) 1400 if (t5_write_combine) { 1401 int rc; 1402 1403 /* 1404 * Enable write combining on BAR2. This is the 1405 * userspace doorbell BAR and is split into 128B 1406 * (UDBS_SEG_SIZE) doorbell regions, each associated 1407 * with an egress queue. The first 64B has the doorbell 1408 * and the second 64B can be used to submit a tx work 1409 * request with an implicit doorbell. 1410 */ 1411 1412 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1413 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1414 if (rc == 0) { 1415 clrbit(&sc->doorbells, DOORBELL_UDB); 1416 setbit(&sc->doorbells, DOORBELL_WRWC); 1417 setbit(&sc->doorbells, DOORBELL_UDBWC); 1418 } else { 1419 device_printf(sc->dev, 1420 "couldn't enable write combining: %d\n", 1421 rc); 1422 } 1423 1424 t4_write_reg(sc, A_SGE_STAT_CFG, 1425 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1426 } 1427#endif 1428 } 1429 1430 return (0); 1431} 1432 1433static const struct memwin t4_memwin[] = { 1434 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1435 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1436 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1437}; 1438 1439static const struct memwin t5_memwin[] = { 1440 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1441 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1442 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1443}; 1444 1445static void 1446setup_memwin(struct adapter *sc) 1447{ 1448 const struct memwin *mw; 1449 int i, n; 1450 uint32_t bar0; 1451 1452 if (is_t4(sc)) { 1453 /* 1454 * Read low 32b of bar0 indirectly via the hardware backdoor 1455 * mechanism. Works from within PCI passthrough environments 1456 * too, where rman_get_start() can return a different value. We 1457 * need to program the T4 memory window decoders with the actual 1458 * addresses that will be coming across the PCIe link. 1459 */ 1460 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1461 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1462 1463 mw = &t4_memwin[0]; 1464 n = nitems(t4_memwin); 1465 } else { 1466 /* T5 uses the relative offset inside the PCIe BAR */ 1467 bar0 = 0; 1468 1469 mw = &t5_memwin[0]; 1470 n = nitems(t5_memwin); 1471 } 1472 1473 for (i = 0; i < n; i++, mw++) { 1474 t4_write_reg(sc, 1475 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1476 (mw->base + bar0) | V_BIR(0) | 1477 V_WINDOW(ilog2(mw->aperture) - 10)); 1478 } 1479 1480 /* flush */ 1481 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1482} 1483 1484/* 1485 * Verify that the memory range specified by the addr/len pair is valid and lies 1486 * entirely within a single region (EDCx or MCx). 1487 */ 1488static int 1489validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1490{ 1491 uint32_t em, addr_len, maddr, mlen; 1492 1493 /* Memory can only be accessed in naturally aligned 4 byte units */ 1494 if (addr & 3 || len & 3 || len == 0) 1495 return (EINVAL); 1496 1497 /* Enabled memories */ 1498 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1499 if (em & F_EDRAM0_ENABLE) { 1500 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1501 maddr = G_EDRAM0_BASE(addr_len) << 20; 1502 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1503 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1504 addr + len <= maddr + mlen) 1505 return (0); 1506 } 1507 if (em & F_EDRAM1_ENABLE) { 1508 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1509 maddr = G_EDRAM1_BASE(addr_len) << 20; 1510 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1511 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1512 addr + len <= maddr + mlen) 1513 return (0); 1514 } 1515 if (em & F_EXT_MEM_ENABLE) { 1516 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1517 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1518 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1519 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1520 addr + len <= maddr + mlen) 1521 return (0); 1522 } 1523 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1524 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1525 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1526 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1527 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1528 addr + len <= maddr + mlen) 1529 return (0); 1530 } 1531 1532 return (EFAULT); 1533} 1534 1535/* 1536 * Verify that the memory range specified by the memtype/offset/len pair is 1537 * valid and lies entirely within the memtype specified. The global address of 1538 * the start of the range is returned in addr. 1539 */ 1540static int 1541validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1542 uint32_t *addr) 1543{ 1544 uint32_t em, addr_len, maddr, mlen; 1545 1546 /* Memory can only be accessed in naturally aligned 4 byte units */ 1547 if (off & 3 || len & 3 || len == 0) 1548 return (EINVAL); 1549 1550 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1551 switch (mtype) { 1552 case MEM_EDC0: 1553 if (!(em & F_EDRAM0_ENABLE)) 1554 return (EINVAL); 1555 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1556 maddr = G_EDRAM0_BASE(addr_len) << 20; 1557 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1558 break; 1559 case MEM_EDC1: 1560 if (!(em & F_EDRAM1_ENABLE)) 1561 return (EINVAL); 1562 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1563 maddr = G_EDRAM1_BASE(addr_len) << 20; 1564 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1565 break; 1566 case MEM_MC: 1567 if (!(em & F_EXT_MEM_ENABLE)) 1568 return (EINVAL); 1569 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1570 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1571 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1572 break; 1573 case MEM_MC1: 1574 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1575 return (EINVAL); 1576 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1577 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1578 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1579 break; 1580 default: 1581 return (EINVAL); 1582 } 1583 1584 if (mlen > 0 && off < mlen && off + len <= mlen) { 1585 *addr = maddr + off; /* global address */ 1586 return (0); 1587 } 1588 1589 return (EFAULT); 1590} 1591 1592static void 1593memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1594{ 1595 const struct memwin *mw; 1596 1597 if (is_t4(sc)) { 1598 KASSERT(win >= 0 && win < nitems(t4_memwin), 1599 ("%s: incorrect memwin# (%d)", __func__, win)); 1600 mw = &t4_memwin[win]; 1601 } else { 1602 KASSERT(win >= 0 && win < nitems(t5_memwin), 1603 ("%s: incorrect memwin# (%d)", __func__, win)); 1604 mw = &t5_memwin[win]; 1605 } 1606 1607 if (base != NULL) 1608 *base = mw->base; 1609 if (aperture != NULL) 1610 *aperture = mw->aperture; 1611} 1612 1613/* 1614 * Positions the memory window such that it can be used to access the specified 1615 * address in the chip's address space. The return value is the offset of addr 1616 * from the start of the window. 1617 */ 1618static uint32_t 1619position_memwin(struct adapter *sc, int n, uint32_t addr) 1620{ 1621 uint32_t start, pf; 1622 uint32_t reg; 1623 1624 KASSERT(n >= 0 && n <= 3, 1625 ("%s: invalid window %d.", __func__, n)); 1626 KASSERT((addr & 3) == 0, 1627 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1628 1629 if (is_t4(sc)) { 1630 pf = 0; 1631 start = addr & ~0xf; /* start must be 16B aligned */ 1632 } else { 1633 pf = V_PFNUM(sc->pf); 1634 start = addr & ~0x7f; /* start must be 128B aligned */ 1635 } 1636 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1637 1638 t4_write_reg(sc, reg, start | pf); 1639 t4_read_reg(sc, reg); 1640 1641 return (addr - start); 1642} 1643 1644static int 1645cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1646 struct intrs_and_queues *iaq) 1647{ 1648 int rc, itype, navail, nrxq10g, nrxq1g, n; 1649 int nofldrxq10g = 0, nofldrxq1g = 0; 1650 1651 bzero(iaq, sizeof(*iaq)); 1652 1653 iaq->ntxq10g = t4_ntxq10g; 1654 iaq->ntxq1g = t4_ntxq1g; 1655 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1656 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1657#ifdef TCP_OFFLOAD 1658 if (is_offload(sc)) { 1659 iaq->nofldtxq10g = t4_nofldtxq10g; 1660 iaq->nofldtxq1g = t4_nofldtxq1g; 1661 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1662 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1663 } 1664#endif 1665 1666 for (itype = INTR_MSIX; itype; itype >>= 1) { 1667 1668 if ((itype & t4_intr_types) == 0) 1669 continue; /* not allowed */ 1670 1671 if (itype == INTR_MSIX) 1672 navail = pci_msix_count(sc->dev); 1673 else if (itype == INTR_MSI) 1674 navail = pci_msi_count(sc->dev); 1675 else 1676 navail = 1; 1677restart: 1678 if (navail == 0) 1679 continue; 1680 1681 iaq->intr_type = itype; 1682 iaq->intr_flags = 0; 1683 1684 /* 1685 * Best option: an interrupt vector for errors, one for the 1686 * firmware event queue, and one each for each rxq (NIC as well 1687 * as offload). 1688 */ 1689 iaq->nirq = T4_EXTRA_INTR; 1690 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1691 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1692 if (iaq->nirq <= navail && 1693 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1694 iaq->intr_flags |= INTR_DIRECT; 1695 goto allocate; 1696 } 1697 1698 /* 1699 * Second best option: an interrupt vector for errors, one for 1700 * the firmware event queue, and one each for either NIC or 1701 * offload rxq's. 1702 */ 1703 iaq->nirq = T4_EXTRA_INTR; 1704 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1705 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1706 if (iaq->nirq <= navail && 1707 (itype != INTR_MSI || powerof2(iaq->nirq))) 1708 goto allocate; 1709 1710 /* 1711 * Next best option: an interrupt vector for errors, one for the 1712 * firmware event queue, and at least one per port. At this 1713 * point we know we'll have to downsize nrxq or nofldrxq to fit 1714 * what's available to us. 1715 */ 1716 iaq->nirq = T4_EXTRA_INTR; 1717 iaq->nirq += n10g + n1g; 1718 if (iaq->nirq <= navail) { 1719 int leftover = navail - iaq->nirq; 1720 1721 if (n10g > 0) { 1722 int target = max(nrxq10g, nofldrxq10g); 1723 1724 n = 1; 1725 while (n < target && leftover >= n10g) { 1726 leftover -= n10g; 1727 iaq->nirq += n10g; 1728 n++; 1729 } 1730 iaq->nrxq10g = min(n, nrxq10g); 1731#ifdef TCP_OFFLOAD 1732 if (is_offload(sc)) 1733 iaq->nofldrxq10g = min(n, nofldrxq10g); 1734#endif 1735 } 1736 1737 if (n1g > 0) { 1738 int target = max(nrxq1g, nofldrxq1g); 1739 1740 n = 1; 1741 while (n < target && leftover >= n1g) { 1742 leftover -= n1g; 1743 iaq->nirq += n1g; 1744 n++; 1745 } 1746 iaq->nrxq1g = min(n, nrxq1g); 1747#ifdef TCP_OFFLOAD 1748 if (is_offload(sc)) 1749 iaq->nofldrxq1g = min(n, nofldrxq1g); 1750#endif 1751 } 1752 1753 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1754 goto allocate; 1755 } 1756 1757 /* 1758 * Least desirable option: one interrupt vector for everything. 1759 */ 1760 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1761#ifdef TCP_OFFLOAD 1762 if (is_offload(sc)) 1763 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1764#endif 1765 1766allocate: 1767 navail = iaq->nirq; 1768 rc = 0; 1769 if (itype == INTR_MSIX) 1770 rc = pci_alloc_msix(sc->dev, &navail); 1771 else if (itype == INTR_MSI) 1772 rc = pci_alloc_msi(sc->dev, &navail); 1773 1774 if (rc == 0) { 1775 if (navail == iaq->nirq) 1776 return (0); 1777 1778 /* 1779 * Didn't get the number requested. Use whatever number 1780 * the kernel is willing to allocate (it's in navail). 1781 */ 1782 device_printf(sc->dev, "fewer vectors than requested, " 1783 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1784 itype, iaq->nirq, navail); 1785 pci_release_msi(sc->dev); 1786 goto restart; 1787 } 1788 1789 device_printf(sc->dev, 1790 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1791 itype, rc, iaq->nirq, navail); 1792 } 1793 1794 device_printf(sc->dev, 1795 "failed to find a usable interrupt type. " 1796 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1797 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1798 1799 return (ENXIO); 1800} 1801 1802#define FW_VERSION(chip) ( \ 1803 V_FW_HDR_FW_VER_MAJOR(FW_VERSION_MAJOR_##chip) | \ 1804 V_FW_HDR_FW_VER_MINOR(FW_VERSION_MINOR_##chip) | \ 1805 V_FW_HDR_FW_VER_MICRO(FW_VERSION_MICRO_##chip) | \ 1806 V_FW_HDR_FW_VER_BUILD(FW_VERSION_BUILD_##chip)) 1807#define FW_INTFVER(chip, intf) (FW_HDR_INTFVER_##intf) 1808 1809struct fw_info { 1810 uint8_t chip; 1811 char *kld_name; 1812 char *fw_mod_name; 1813 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 1814} fw_info[] = { 1815 { 1816 .chip = CHELSIO_T4, 1817 .kld_name = "t4fw_cfg", 1818 .fw_mod_name = "t4fw", 1819 .fw_hdr = { 1820 .chip = FW_HDR_CHIP_T4, 1821 .fw_ver = htobe32_const(FW_VERSION(T4)), 1822 .intfver_nic = FW_INTFVER(T4, NIC), 1823 .intfver_vnic = FW_INTFVER(T4, VNIC), 1824 .intfver_ofld = FW_INTFVER(T4, OFLD), 1825 .intfver_ri = FW_INTFVER(T4, RI), 1826 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 1827 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 1828 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 1829 .intfver_fcoe = FW_INTFVER(T4, FCOE), 1830 }, 1831 }, { 1832 .chip = CHELSIO_T5, 1833 .kld_name = "t5fw_cfg", 1834 .fw_mod_name = "t5fw", 1835 .fw_hdr = { 1836 .chip = FW_HDR_CHIP_T5, 1837 .fw_ver = htobe32_const(FW_VERSION(T5)), 1838 .intfver_nic = FW_INTFVER(T5, NIC), 1839 .intfver_vnic = FW_INTFVER(T5, VNIC), 1840 .intfver_ofld = FW_INTFVER(T5, OFLD), 1841 .intfver_ri = FW_INTFVER(T5, RI), 1842 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 1843 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 1844 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 1845 .intfver_fcoe = FW_INTFVER(T5, FCOE), 1846 }, 1847 } 1848}; 1849 1850static struct fw_info * 1851find_fw_info(int chip) 1852{ 1853 int i; 1854 1855 for (i = 0; i < nitems(fw_info); i++) { 1856 if (fw_info[i].chip == chip) 1857 return (&fw_info[i]); 1858 } 1859 return (NULL); 1860} 1861 1862/* 1863 * Is the given firmware API compatible with the one the driver was compiled 1864 * with? 1865 */ 1866static int 1867fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 1868{ 1869 1870 /* short circuit if it's the exact same firmware version */ 1871 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 1872 return (1); 1873 1874 /* 1875 * XXX: Is this too conservative? Perhaps I should limit this to the 1876 * features that are supported in the driver. 1877 */ 1878#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 1879 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 1880 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 1881 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 1882 return (1); 1883#undef SAME_INTF 1884 1885 return (0); 1886} 1887 1888/* 1889 * The firmware in the KLD is usable and can be installed. But should it be? 1890 * This routine explains itself in detail if it indicates the KLD firmware 1891 * should be installed. 1892 */ 1893static int 1894should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 1895{ 1896 const char *reason; 1897 1898 KASSERT(t4_fw_install != 0, ("%s: Can't install; shouldn't be asked " 1899 "to evaluate if install is a good idea.", __func__)); 1900 1901 if (!card_fw_usable) { 1902 reason = "incompatible or unusable"; 1903 goto install; 1904 } 1905 1906 if (k > c) { 1907 reason = "older than the version bundled with this driver"; 1908 goto install; 1909 } 1910 1911 if (t4_fw_install == 2 && k != c) { 1912 reason = "different than the version bundled with this driver"; 1913 goto install; 1914 } 1915 1916 return (0); 1917 1918install: 1919 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 1920 "installing firmware %u.%u.%u.%u on card.\n", 1921 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 1922 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 1923 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 1924 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 1925 1926 return (1); 1927} 1928/* 1929 * Establish contact with the firmware and determine if we are the master driver 1930 * or not, and whether we are responsible for chip initialization. 1931 */ 1932static int 1933prep_firmware(struct adapter *sc) 1934{ 1935 const struct firmware *fw = NULL, *default_cfg; 1936 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 1937 enum dev_state state; 1938 struct fw_info *fw_info; 1939 struct fw_hdr *card_fw; /* fw on the card */ 1940 const struct fw_hdr *kld_fw; /* fw in the KLD */ 1941 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 1942 against */ 1943 1944 /* Contact firmware. */ 1945 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1946 if (rc < 0 || state == DEV_STATE_ERR) { 1947 rc = -rc; 1948 device_printf(sc->dev, 1949 "failed to connect to the firmware: %d, %d.\n", rc, state); 1950 return (rc); 1951 } 1952 pf = rc; 1953 if (pf == sc->mbox) 1954 sc->flags |= MASTER_PF; 1955 else if (state == DEV_STATE_UNINIT) { 1956 /* 1957 * We didn't get to be the master so we definitely won't be 1958 * configuring the chip. It's a bug if someone else hasn't 1959 * configured it already. 1960 */ 1961 device_printf(sc->dev, "couldn't be master(%d), " 1962 "device not already initialized either(%d).\n", rc, state); 1963 return (EDOOFUS); 1964 } 1965 1966 /* This is the firmware whose headers the driver was compiled against */ 1967 fw_info = find_fw_info(chip_id(sc)); 1968 if (fw_info == NULL) { 1969 device_printf(sc->dev, 1970 "unable to look up firmware information for chip %d.\n", 1971 chip_id(sc)); 1972 return (EINVAL); 1973 } 1974 drv_fw = &fw_info->fw_hdr; 1975 1976 /* 1977 * The firmware KLD contains many modules. The KLD name is also the 1978 * name of the module that contains the default config file. 1979 */ 1980 default_cfg = firmware_get(fw_info->kld_name); 1981 1982 /* Read the header of the firmware on the card */ 1983 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 1984 rc = -t4_read_flash(sc, FLASH_FW_START, 1985 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 1986 if (rc == 0) 1987 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 1988 else { 1989 device_printf(sc->dev, 1990 "Unable to read card's firmware header: %d\n", rc); 1991 card_fw_usable = 0; 1992 } 1993 1994 /* This is the firmware in the KLD */ 1995 fw = firmware_get(fw_info->fw_mod_name); 1996 if (fw != NULL) { 1997 kld_fw = (const void *)fw->data; 1998 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 1999 } else { 2000 kld_fw = NULL; 2001 kld_fw_usable = 0; 2002 } 2003 2004 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2005 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver || 2006 t4_fw_install == 0)) { 2007 /* 2008 * Common case: the firmware on the card is an exact match and 2009 * the KLD is an exact match too, or the KLD is 2010 * absent/incompatible, or we're prohibited from using it. Note 2011 * that t4_fw_install = 2 is ignored here -- use cxgbetool 2012 * loadfw if you want to reinstall the same firmware as the one 2013 * on the card. 2014 */ 2015 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2016 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2017 be32toh(card_fw->fw_ver))) { 2018 2019 rc = -t4_load_fw(sc, fw->data, fw->datasize); 2020 if (rc != 0) { 2021 device_printf(sc->dev, 2022 "failed to install firmware: %d\n", rc); 2023 goto done; 2024 } 2025 2026 /* Installed successfully, update the cached header too. */ 2027 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2028 card_fw_usable = 1; 2029 need_fw_reset = 0; /* already reset as part of load_fw */ 2030 } 2031 2032 if (!card_fw_usable) { 2033 uint32_t d, c, k; 2034 2035 d = ntohl(drv_fw->fw_ver); 2036 c = ntohl(card_fw->fw_ver); 2037 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2038 2039 device_printf(sc->dev, "Cannot find a usable firmware: " 2040 "fw_install %d, chip state %d, " 2041 "driver compiled with %d.%d.%d.%d, " 2042 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2043 t4_fw_install, state, 2044 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2045 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2046 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2047 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2048 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2049 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2050 rc = EINVAL; 2051 goto done; 2052 } 2053 2054 /* We're using whatever's on the card and it's known to be good. */ 2055 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2056 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2057 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2058 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2059 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2060 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2061 2062 /* Reset device */ 2063 if (need_fw_reset && 2064 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2065 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2066 if (rc != ETIMEDOUT && rc != EIO) 2067 t4_fw_bye(sc, sc->mbox); 2068 goto done; 2069 } 2070 sc->flags |= FW_OK; 2071 2072 rc = get_params__pre_init(sc); 2073 if (rc != 0) 2074 goto done; /* error message displayed already */ 2075 2076 /* Partition adapter resources as specified in the config file. */ 2077 if (state == DEV_STATE_UNINIT) { 2078 2079 KASSERT(sc->flags & MASTER_PF, 2080 ("%s: trying to change chip settings when not master.", 2081 __func__)); 2082 2083 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2084 if (rc != 0) 2085 goto done; /* error message displayed already */ 2086 2087 t4_tweak_chip_settings(sc); 2088 2089 /* get basic stuff going */ 2090 rc = -t4_fw_initialize(sc, sc->mbox); 2091 if (rc != 0) { 2092 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2093 goto done; 2094 } 2095 } else { 2096 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2097 sc->cfcsum = 0; 2098 } 2099 2100done: 2101 free(card_fw, M_CXGBE); 2102 if (fw != NULL) 2103 firmware_put(fw, FIRMWARE_UNLOAD); 2104 if (default_cfg != NULL) 2105 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2106 2107 return (rc); 2108} 2109 2110#define FW_PARAM_DEV(param) \ 2111 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2112 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2113#define FW_PARAM_PFVF(param) \ 2114 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2115 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2116 2117/* 2118 * Partition chip resources for use between various PFs, VFs, etc. 2119 */ 2120static int 2121partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2122 const char *name_prefix) 2123{ 2124 const struct firmware *cfg = NULL; 2125 int rc = 0; 2126 struct fw_caps_config_cmd caps; 2127 uint32_t mtype, moff, finicsum, cfcsum; 2128 2129 /* 2130 * Figure out what configuration file to use. Pick the default config 2131 * file for the card if the user hasn't specified one explicitly. 2132 */ 2133 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2134 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2135 /* Card specific overrides go here. */ 2136 if (pci_get_device(sc->dev) == 0x440a) 2137 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2138 if (is_fpga(sc)) 2139 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2140 } 2141 2142 /* 2143 * We need to load another module if the profile is anything except 2144 * "default" or "flash". 2145 */ 2146 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2147 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2148 char s[32]; 2149 2150 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2151 cfg = firmware_get(s); 2152 if (cfg == NULL) { 2153 if (default_cfg != NULL) { 2154 device_printf(sc->dev, 2155 "unable to load module \"%s\" for " 2156 "configuration profile \"%s\", will use " 2157 "the default config file instead.\n", 2158 s, sc->cfg_file); 2159 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2160 "%s", DEFAULT_CF); 2161 } else { 2162 device_printf(sc->dev, 2163 "unable to load module \"%s\" for " 2164 "configuration profile \"%s\", will use " 2165 "the config file on the card's flash " 2166 "instead.\n", s, sc->cfg_file); 2167 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2168 "%s", FLASH_CF); 2169 } 2170 } 2171 } 2172 2173 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2174 default_cfg == NULL) { 2175 device_printf(sc->dev, 2176 "default config file not available, will use the config " 2177 "file on the card's flash instead.\n"); 2178 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2179 } 2180 2181 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2182 u_int cflen, i, n; 2183 const uint32_t *cfdata; 2184 uint32_t param, val, addr, off, mw_base, mw_aperture; 2185 2186 KASSERT(cfg != NULL || default_cfg != NULL, 2187 ("%s: no config to upload", __func__)); 2188 2189 /* 2190 * Ask the firmware where it wants us to upload the config file. 2191 */ 2192 param = FW_PARAM_DEV(CF); 2193 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2194 if (rc != 0) { 2195 /* No support for config file? Shouldn't happen. */ 2196 device_printf(sc->dev, 2197 "failed to query config file location: %d.\n", rc); 2198 goto done; 2199 } 2200 mtype = G_FW_PARAMS_PARAM_Y(val); 2201 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2202 2203 /* 2204 * XXX: sheer laziness. We deliberately added 4 bytes of 2205 * useless stuffing/comments at the end of the config file so 2206 * it's ok to simply throw away the last remaining bytes when 2207 * the config file is not an exact multiple of 4. This also 2208 * helps with the validate_mt_off_len check. 2209 */ 2210 if (cfg != NULL) { 2211 cflen = cfg->datasize & ~3; 2212 cfdata = cfg->data; 2213 } else { 2214 cflen = default_cfg->datasize & ~3; 2215 cfdata = default_cfg->data; 2216 } 2217 2218 if (cflen > FLASH_CFG_MAX_SIZE) { 2219 device_printf(sc->dev, 2220 "config file too long (%d, max allowed is %d). " 2221 "Will try to use the config on the card, if any.\n", 2222 cflen, FLASH_CFG_MAX_SIZE); 2223 goto use_config_on_flash; 2224 } 2225 2226 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2227 if (rc != 0) { 2228 device_printf(sc->dev, 2229 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2230 "Will try to use the config on the card, if any.\n", 2231 __func__, mtype, moff, cflen, rc); 2232 goto use_config_on_flash; 2233 } 2234 2235 memwin_info(sc, 2, &mw_base, &mw_aperture); 2236 while (cflen) { 2237 off = position_memwin(sc, 2, addr); 2238 n = min(cflen, mw_aperture - off); 2239 for (i = 0; i < n; i += 4) 2240 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2241 cflen -= n; 2242 addr += n; 2243 } 2244 } else { 2245use_config_on_flash: 2246 mtype = FW_MEMTYPE_CF_FLASH; 2247 moff = t4_flash_cfg_addr(sc); 2248 } 2249 2250 bzero(&caps, sizeof(caps)); 2251 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2252 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2253 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2254 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2255 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2256 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2257 if (rc != 0) { 2258 device_printf(sc->dev, 2259 "failed to pre-process config file: %d " 2260 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2261 goto done; 2262 } 2263 2264 finicsum = be32toh(caps.finicsum); 2265 cfcsum = be32toh(caps.cfcsum); 2266 if (finicsum != cfcsum) { 2267 device_printf(sc->dev, 2268 "WARNING: config file checksum mismatch: %08x %08x\n", 2269 finicsum, cfcsum); 2270 } 2271 sc->cfcsum = cfcsum; 2272 2273#define LIMIT_CAPS(x) do { \ 2274 caps.x &= htobe16(t4_##x##_allowed); \ 2275 sc->x = htobe16(caps.x); \ 2276} while (0) 2277 2278 /* 2279 * Let the firmware know what features will (not) be used so it can tune 2280 * things accordingly. 2281 */ 2282 LIMIT_CAPS(linkcaps); 2283 LIMIT_CAPS(niccaps); 2284 LIMIT_CAPS(toecaps); 2285 LIMIT_CAPS(rdmacaps); 2286 LIMIT_CAPS(iscsicaps); 2287 LIMIT_CAPS(fcoecaps); 2288#undef LIMIT_CAPS 2289 2290 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2291 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2292 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2293 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2294 if (rc != 0) { 2295 device_printf(sc->dev, 2296 "failed to process config file: %d.\n", rc); 2297 } 2298done: 2299 if (cfg != NULL) 2300 firmware_put(cfg, FIRMWARE_UNLOAD); 2301 return (rc); 2302} 2303 2304/* 2305 * Retrieve parameters that are needed (or nice to have) very early. 2306 */ 2307static int 2308get_params__pre_init(struct adapter *sc) 2309{ 2310 int rc; 2311 uint32_t param[2], val[2]; 2312 struct fw_devlog_cmd cmd; 2313 struct devlog_params *dlog = &sc->params.devlog; 2314 2315 param[0] = FW_PARAM_DEV(PORTVEC); 2316 param[1] = FW_PARAM_DEV(CCLK); 2317 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2318 if (rc != 0) { 2319 device_printf(sc->dev, 2320 "failed to query parameters (pre_init): %d.\n", rc); 2321 return (rc); 2322 } 2323 2324 sc->params.portvec = val[0]; 2325 sc->params.nports = bitcount32(val[0]); 2326 sc->params.vpd.cclk = val[1]; 2327 2328 /* Read device log parameters. */ 2329 bzero(&cmd, sizeof(cmd)); 2330 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2331 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2332 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2333 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2334 if (rc != 0) { 2335 device_printf(sc->dev, 2336 "failed to get devlog parameters: %d.\n", rc); 2337 bzero(dlog, sizeof (*dlog)); 2338 rc = 0; /* devlog isn't critical for device operation */ 2339 } else { 2340 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2341 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2342 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2343 dlog->size = be32toh(cmd.memsize_devlog); 2344 } 2345 2346 return (rc); 2347} 2348 2349/* 2350 * Retrieve various parameters that are of interest to the driver. The device 2351 * has been initialized by the firmware at this point. 2352 */ 2353static int 2354get_params__post_init(struct adapter *sc) 2355{ 2356 int rc; 2357 uint32_t param[7], val[7]; 2358 struct fw_caps_config_cmd caps; 2359 2360 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2361 param[1] = FW_PARAM_PFVF(EQ_START); 2362 param[2] = FW_PARAM_PFVF(FILTER_START); 2363 param[3] = FW_PARAM_PFVF(FILTER_END); 2364 param[4] = FW_PARAM_PFVF(L2T_START); 2365 param[5] = FW_PARAM_PFVF(L2T_END); 2366 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2367 if (rc != 0) { 2368 device_printf(sc->dev, 2369 "failed to query parameters (post_init): %d.\n", rc); 2370 return (rc); 2371 } 2372 2373 sc->sge.iq_start = val[0]; 2374 sc->sge.eq_start = val[1]; 2375 sc->tids.ftid_base = val[2]; 2376 sc->tids.nftids = val[3] - val[2] + 1; 2377 sc->vres.l2t.start = val[4]; 2378 sc->vres.l2t.size = val[5] - val[4] + 1; 2379 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2380 ("%s: L2 table size (%u) larger than expected (%u)", 2381 __func__, sc->vres.l2t.size, L2T_SIZE)); 2382 2383 /* get capabilites */ 2384 bzero(&caps, sizeof(caps)); 2385 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2386 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2387 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2388 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2389 if (rc != 0) { 2390 device_printf(sc->dev, 2391 "failed to get card capabilities: %d.\n", rc); 2392 return (rc); 2393 } 2394 2395 if (caps.toecaps) { 2396 /* query offload-related parameters */ 2397 param[0] = FW_PARAM_DEV(NTID); 2398 param[1] = FW_PARAM_PFVF(SERVER_START); 2399 param[2] = FW_PARAM_PFVF(SERVER_END); 2400 param[3] = FW_PARAM_PFVF(TDDP_START); 2401 param[4] = FW_PARAM_PFVF(TDDP_END); 2402 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2403 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2404 if (rc != 0) { 2405 device_printf(sc->dev, 2406 "failed to query TOE parameters: %d.\n", rc); 2407 return (rc); 2408 } 2409 sc->tids.ntids = val[0]; 2410 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2411 sc->tids.stid_base = val[1]; 2412 sc->tids.nstids = val[2] - val[1] + 1; 2413 sc->vres.ddp.start = val[3]; 2414 sc->vres.ddp.size = val[4] - val[3] + 1; 2415 sc->params.ofldq_wr_cred = val[5]; 2416 sc->params.offload = 1; 2417 } 2418 if (caps.rdmacaps) { 2419 param[0] = FW_PARAM_PFVF(STAG_START); 2420 param[1] = FW_PARAM_PFVF(STAG_END); 2421 param[2] = FW_PARAM_PFVF(RQ_START); 2422 param[3] = FW_PARAM_PFVF(RQ_END); 2423 param[4] = FW_PARAM_PFVF(PBL_START); 2424 param[5] = FW_PARAM_PFVF(PBL_END); 2425 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2426 if (rc != 0) { 2427 device_printf(sc->dev, 2428 "failed to query RDMA parameters(1): %d.\n", rc); 2429 return (rc); 2430 } 2431 sc->vres.stag.start = val[0]; 2432 sc->vres.stag.size = val[1] - val[0] + 1; 2433 sc->vres.rq.start = val[2]; 2434 sc->vres.rq.size = val[3] - val[2] + 1; 2435 sc->vres.pbl.start = val[4]; 2436 sc->vres.pbl.size = val[5] - val[4] + 1; 2437 2438 param[0] = FW_PARAM_PFVF(SQRQ_START); 2439 param[1] = FW_PARAM_PFVF(SQRQ_END); 2440 param[2] = FW_PARAM_PFVF(CQ_START); 2441 param[3] = FW_PARAM_PFVF(CQ_END); 2442 param[4] = FW_PARAM_PFVF(OCQ_START); 2443 param[5] = FW_PARAM_PFVF(OCQ_END); 2444 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 2445 if (rc != 0) { 2446 device_printf(sc->dev, 2447 "failed to query RDMA parameters(2): %d.\n", rc); 2448 return (rc); 2449 } 2450 sc->vres.qp.start = val[0]; 2451 sc->vres.qp.size = val[1] - val[0] + 1; 2452 sc->vres.cq.start = val[2]; 2453 sc->vres.cq.size = val[3] - val[2] + 1; 2454 sc->vres.ocq.start = val[4]; 2455 sc->vres.ocq.size = val[5] - val[4] + 1; 2456 } 2457 if (caps.iscsicaps) { 2458 param[0] = FW_PARAM_PFVF(ISCSI_START); 2459 param[1] = FW_PARAM_PFVF(ISCSI_END); 2460 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2461 if (rc != 0) { 2462 device_printf(sc->dev, 2463 "failed to query iSCSI parameters: %d.\n", rc); 2464 return (rc); 2465 } 2466 sc->vres.iscsi.start = val[0]; 2467 sc->vres.iscsi.size = val[1] - val[0] + 1; 2468 } 2469 2470 /* 2471 * We've got the params we wanted to query via the firmware. Now grab 2472 * some others directly from the chip. 2473 */ 2474 rc = t4_read_chip_settings(sc); 2475 2476 return (rc); 2477} 2478 2479static int 2480set_params__post_init(struct adapter *sc) 2481{ 2482 uint32_t param, val; 2483 2484 /* ask for encapsulated CPLs */ 2485 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2486 val = 1; 2487 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2488 2489 return (0); 2490} 2491 2492#undef FW_PARAM_PFVF 2493#undef FW_PARAM_DEV 2494 2495static void 2496t4_set_desc(struct adapter *sc) 2497{ 2498 char buf[128]; 2499 struct adapter_params *p = &sc->params; 2500 2501 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 2502 p->vpd.id, is_offload(sc) ? "R" : "", chip_rev(sc), p->vpd.sn, 2503 p->vpd.ec); 2504 2505 device_set_desc_copy(sc->dev, buf); 2506} 2507 2508static void 2509build_medialist(struct port_info *pi) 2510{ 2511 struct ifmedia *media = &pi->media; 2512 int data, m; 2513 2514 PORT_LOCK(pi); 2515 2516 ifmedia_removeall(media); 2517 2518 m = IFM_ETHER | IFM_FDX; 2519 data = (pi->port_type << 8) | pi->mod_type; 2520 2521 switch(pi->port_type) { 2522 case FW_PORT_TYPE_BT_XFI: 2523 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2524 break; 2525 2526 case FW_PORT_TYPE_BT_XAUI: 2527 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2528 /* fall through */ 2529 2530 case FW_PORT_TYPE_BT_SGMII: 2531 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2532 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2533 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2534 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2535 break; 2536 2537 case FW_PORT_TYPE_CX4: 2538 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2539 ifmedia_set(media, m | IFM_10G_CX4); 2540 break; 2541 2542 case FW_PORT_TYPE_SFP: 2543 case FW_PORT_TYPE_FIBER_XFI: 2544 case FW_PORT_TYPE_FIBER_XAUI: 2545 switch (pi->mod_type) { 2546 2547 case FW_PORT_MOD_TYPE_LR: 2548 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2549 ifmedia_set(media, m | IFM_10G_LR); 2550 break; 2551 2552 case FW_PORT_MOD_TYPE_SR: 2553 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2554 ifmedia_set(media, m | IFM_10G_SR); 2555 break; 2556 2557 case FW_PORT_MOD_TYPE_LRM: 2558 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2559 ifmedia_set(media, m | IFM_10G_LRM); 2560 break; 2561 2562 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2563 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2564 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2565 ifmedia_set(media, m | IFM_10G_TWINAX); 2566 break; 2567 2568 case FW_PORT_MOD_TYPE_NONE: 2569 m &= ~IFM_FDX; 2570 ifmedia_add(media, m | IFM_NONE, data, NULL); 2571 ifmedia_set(media, m | IFM_NONE); 2572 break; 2573 2574 case FW_PORT_MOD_TYPE_NA: 2575 case FW_PORT_MOD_TYPE_ER: 2576 default: 2577 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2578 ifmedia_set(media, m | IFM_UNKNOWN); 2579 break; 2580 } 2581 break; 2582 2583 case FW_PORT_TYPE_KX4: 2584 case FW_PORT_TYPE_KX: 2585 case FW_PORT_TYPE_KR: 2586 default: 2587 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2588 ifmedia_set(media, m | IFM_UNKNOWN); 2589 break; 2590 } 2591 2592 PORT_UNLOCK(pi); 2593} 2594 2595#define FW_MAC_EXACT_CHUNK 7 2596 2597/* 2598 * Program the port's XGMAC based on parameters in ifnet. The caller also 2599 * indicates which parameters should be programmed (the rest are left alone). 2600 */ 2601static int 2602update_mac_settings(struct port_info *pi, int flags) 2603{ 2604 int rc; 2605 struct ifnet *ifp = pi->ifp; 2606 struct adapter *sc = pi->adapter; 2607 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2608 2609 ASSERT_SYNCHRONIZED_OP(sc); 2610 KASSERT(flags, ("%s: not told what to update.", __func__)); 2611 2612 if (flags & XGMAC_MTU) 2613 mtu = ifp->if_mtu; 2614 2615 if (flags & XGMAC_PROMISC) 2616 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2617 2618 if (flags & XGMAC_ALLMULTI) 2619 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2620 2621 if (flags & XGMAC_VLANEX) 2622 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2623 2624 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2625 vlanex, false); 2626 if (rc) { 2627 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2628 return (rc); 2629 } 2630 2631 if (flags & XGMAC_UCADDR) { 2632 uint8_t ucaddr[ETHER_ADDR_LEN]; 2633 2634 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2635 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2636 ucaddr, true, true); 2637 if (rc < 0) { 2638 rc = -rc; 2639 if_printf(ifp, "change_mac failed: %d\n", rc); 2640 return (rc); 2641 } else { 2642 pi->xact_addr_filt = rc; 2643 rc = 0; 2644 } 2645 } 2646 2647 if (flags & XGMAC_MCADDRS) { 2648 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2649 int del = 1; 2650 uint64_t hash = 0; 2651 struct ifmultiaddr *ifma; 2652 int i = 0, j; 2653 2654 if_maddr_rlock(ifp); 2655 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2656 if (ifma->ifma_addr->sa_family != AF_LINK) 2657 continue; 2658 mcaddr[i++] = 2659 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2660 2661 if (i == FW_MAC_EXACT_CHUNK) { 2662 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2663 del, i, mcaddr, NULL, &hash, 0); 2664 if (rc < 0) { 2665 rc = -rc; 2666 for (j = 0; j < i; j++) { 2667 if_printf(ifp, 2668 "failed to add mc address" 2669 " %02x:%02x:%02x:" 2670 "%02x:%02x:%02x rc=%d\n", 2671 mcaddr[j][0], mcaddr[j][1], 2672 mcaddr[j][2], mcaddr[j][3], 2673 mcaddr[j][4], mcaddr[j][5], 2674 rc); 2675 } 2676 goto mcfail; 2677 } 2678 del = 0; 2679 i = 0; 2680 } 2681 } 2682 if (i > 0) { 2683 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2684 del, i, mcaddr, NULL, &hash, 0); 2685 if (rc < 0) { 2686 rc = -rc; 2687 for (j = 0; j < i; j++) { 2688 if_printf(ifp, 2689 "failed to add mc address" 2690 " %02x:%02x:%02x:" 2691 "%02x:%02x:%02x rc=%d\n", 2692 mcaddr[j][0], mcaddr[j][1], 2693 mcaddr[j][2], mcaddr[j][3], 2694 mcaddr[j][4], mcaddr[j][5], 2695 rc); 2696 } 2697 goto mcfail; 2698 } 2699 } 2700 2701 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2702 if (rc != 0) 2703 if_printf(ifp, "failed to set mc address hash: %d", rc); 2704mcfail: 2705 if_maddr_runlock(ifp); 2706 } 2707 2708 return (rc); 2709} 2710 2711int 2712begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 2713 char *wmesg) 2714{ 2715 int rc, pri; 2716 2717#ifdef WITNESS 2718 /* the caller thinks it's ok to sleep, but is it really? */ 2719 if (flags & SLEEP_OK) 2720 pause("t4slptst", 1); 2721#endif 2722 2723 if (INTR_OK) 2724 pri = PCATCH; 2725 else 2726 pri = 0; 2727 2728 ADAPTER_LOCK(sc); 2729 for (;;) { 2730 2731 if (pi && IS_DOOMED(pi)) { 2732 rc = ENXIO; 2733 goto done; 2734 } 2735 2736 if (!IS_BUSY(sc)) { 2737 rc = 0; 2738 break; 2739 } 2740 2741 if (!(flags & SLEEP_OK)) { 2742 rc = EBUSY; 2743 goto done; 2744 } 2745 2746 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 2747 rc = EINTR; 2748 goto done; 2749 } 2750 } 2751 2752 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2753 SET_BUSY(sc); 2754#ifdef INVARIANTS 2755 sc->last_op = wmesg; 2756 sc->last_op_thr = curthread; 2757#endif 2758 2759done: 2760 if (!(flags & HOLD_LOCK) || rc) 2761 ADAPTER_UNLOCK(sc); 2762 2763 return (rc); 2764} 2765 2766void 2767end_synchronized_op(struct adapter *sc, int flags) 2768{ 2769 2770 if (flags & LOCK_HELD) 2771 ADAPTER_LOCK_ASSERT_OWNED(sc); 2772 else 2773 ADAPTER_LOCK(sc); 2774 2775 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2776 CLR_BUSY(sc); 2777 wakeup(&sc->flags); 2778 ADAPTER_UNLOCK(sc); 2779} 2780 2781static int 2782cxgbe_init_synchronized(struct port_info *pi) 2783{ 2784 struct adapter *sc = pi->adapter; 2785 struct ifnet *ifp = pi->ifp; 2786 int rc = 0; 2787 2788 ASSERT_SYNCHRONIZED_OP(sc); 2789 2790 if (isset(&sc->open_device_map, pi->port_id)) { 2791 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2792 ("mismatch between open_device_map and if_drv_flags")); 2793 return (0); /* already running */ 2794 } 2795 2796 if (!(sc->flags & FULL_INIT_DONE) && 2797 ((rc = adapter_full_init(sc)) != 0)) 2798 return (rc); /* error message displayed already */ 2799 2800 if (!(pi->flags & PORT_INIT_DONE) && 2801 ((rc = port_full_init(pi)) != 0)) 2802 return (rc); /* error message displayed already */ 2803 2804 rc = update_mac_settings(pi, XGMAC_ALL); 2805 if (rc) 2806 goto done; /* error message displayed already */ 2807 2808 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2809 if (rc != 0) { 2810 if_printf(ifp, "start_link failed: %d\n", rc); 2811 goto done; 2812 } 2813 2814 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2815 if (rc != 0) { 2816 if_printf(ifp, "enable_vi failed: %d\n", rc); 2817 goto done; 2818 } 2819 2820 /* all ok */ 2821 setbit(&sc->open_device_map, pi->port_id); 2822 PORT_LOCK(pi); 2823 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2824 PORT_UNLOCK(pi); 2825 2826 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2827done: 2828 if (rc != 0) 2829 cxgbe_uninit_synchronized(pi); 2830 2831 return (rc); 2832} 2833 2834/* 2835 * Idempotent. 2836 */ 2837static int 2838cxgbe_uninit_synchronized(struct port_info *pi) 2839{ 2840 struct adapter *sc = pi->adapter; 2841 struct ifnet *ifp = pi->ifp; 2842 int rc; 2843 2844 ASSERT_SYNCHRONIZED_OP(sc); 2845 2846 /* 2847 * Disable the VI so that all its data in either direction is discarded 2848 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2849 * tick) intact as the TP can deliver negative advice or data that it's 2850 * holding in its RAM (for an offloaded connection) even after the VI is 2851 * disabled. 2852 */ 2853 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2854 if (rc) { 2855 if_printf(ifp, "disable_vi failed: %d\n", rc); 2856 return (rc); 2857 } 2858 2859 clrbit(&sc->open_device_map, pi->port_id); 2860 PORT_LOCK(pi); 2861 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2862 PORT_UNLOCK(pi); 2863 2864 pi->link_cfg.link_ok = 0; 2865 pi->link_cfg.speed = 0; 2866 t4_os_link_changed(sc, pi->port_id, 0); 2867 2868 return (0); 2869} 2870 2871/* 2872 * It is ok for this function to fail midway and return right away. t4_detach 2873 * will walk the entire sc->irq list and clean up whatever is valid. 2874 */ 2875static int 2876setup_intr_handlers(struct adapter *sc) 2877{ 2878 int rc, rid, p, q; 2879 char s[8]; 2880 struct irq *irq; 2881 struct port_info *pi; 2882 struct sge_rxq *rxq; 2883#ifdef TCP_OFFLOAD 2884 struct sge_ofld_rxq *ofld_rxq; 2885#endif 2886 2887 /* 2888 * Setup interrupts. 2889 */ 2890 irq = &sc->irq[0]; 2891 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2892 if (sc->intr_count == 1) { 2893 KASSERT(!(sc->flags & INTR_DIRECT), 2894 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2895 2896 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 2897 if (rc != 0) 2898 return (rc); 2899 } else { 2900 /* Multiple interrupts. */ 2901 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2902 ("%s: too few intr.", __func__)); 2903 2904 /* The first one is always error intr */ 2905 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 2906 if (rc != 0) 2907 return (rc); 2908 irq++; 2909 rid++; 2910 2911 /* The second one is always the firmware event queue */ 2912 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 2913 "evt"); 2914 if (rc != 0) 2915 return (rc); 2916 irq++; 2917 rid++; 2918 2919 /* 2920 * Note that if INTR_DIRECT is not set then either the NIC rx 2921 * queues or (exclusive or) the TOE rx queueus will be taking 2922 * direct interrupts. 2923 * 2924 * There is no need to check for is_offload(sc) as nofldrxq 2925 * will be 0 if offload is disabled. 2926 */ 2927 for_each_port(sc, p) { 2928 pi = sc->port[p]; 2929 2930#ifdef TCP_OFFLOAD 2931 /* 2932 * Skip over the NIC queues if they aren't taking direct 2933 * interrupts. 2934 */ 2935 if (!(sc->flags & INTR_DIRECT) && 2936 pi->nofldrxq > pi->nrxq) 2937 goto ofld_queues; 2938#endif 2939 rxq = &sc->sge.rxq[pi->first_rxq]; 2940 for (q = 0; q < pi->nrxq; q++, rxq++) { 2941 snprintf(s, sizeof(s), "%d.%d", p, q); 2942 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 2943 s); 2944 if (rc != 0) 2945 return (rc); 2946 irq++; 2947 rid++; 2948 } 2949 2950#ifdef TCP_OFFLOAD 2951 /* 2952 * Skip over the offload queues if they aren't taking 2953 * direct interrupts. 2954 */ 2955 if (!(sc->flags & INTR_DIRECT)) 2956 continue; 2957ofld_queues: 2958 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2959 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2960 snprintf(s, sizeof(s), "%d,%d", p, q); 2961 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 2962 ofld_rxq, s); 2963 if (rc != 0) 2964 return (rc); 2965 irq++; 2966 rid++; 2967 } 2968#endif 2969 } 2970 } 2971 2972 return (0); 2973} 2974 2975static int 2976adapter_full_init(struct adapter *sc) 2977{ 2978 int rc, i; 2979 2980 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2981 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2982 ("%s: FULL_INIT_DONE already", __func__)); 2983 2984 /* 2985 * queues that belong to the adapter (not any particular port). 2986 */ 2987 rc = t4_setup_adapter_queues(sc); 2988 if (rc != 0) 2989 goto done; 2990 2991 for (i = 0; i < nitems(sc->tq); i++) { 2992 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 2993 taskqueue_thread_enqueue, &sc->tq[i]); 2994 if (sc->tq[i] == NULL) { 2995 device_printf(sc->dev, 2996 "failed to allocate task queue %d\n", i); 2997 rc = ENOMEM; 2998 goto done; 2999 } 3000 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3001 device_get_nameunit(sc->dev), i); 3002 } 3003 3004 t4_intr_enable(sc); 3005 sc->flags |= FULL_INIT_DONE; 3006done: 3007 if (rc != 0) 3008 adapter_full_uninit(sc); 3009 3010 return (rc); 3011} 3012 3013static int 3014adapter_full_uninit(struct adapter *sc) 3015{ 3016 int i; 3017 3018 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3019 3020 t4_teardown_adapter_queues(sc); 3021 3022 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3023 taskqueue_free(sc->tq[i]); 3024 sc->tq[i] = NULL; 3025 } 3026 3027 sc->flags &= ~FULL_INIT_DONE; 3028 3029 return (0); 3030} 3031 3032static int 3033port_full_init(struct port_info *pi) 3034{ 3035 struct adapter *sc = pi->adapter; 3036 struct ifnet *ifp = pi->ifp; 3037 uint16_t *rss; 3038 struct sge_rxq *rxq; 3039 int rc, i; 3040 3041 ASSERT_SYNCHRONIZED_OP(sc); 3042 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3043 ("%s: PORT_INIT_DONE already", __func__)); 3044 3045 sysctl_ctx_init(&pi->ctx); 3046 pi->flags |= PORT_SYSCTL_CTX; 3047 3048 /* 3049 * Allocate tx/rx/fl queues for this port. 3050 */ 3051 rc = t4_setup_port_queues(pi); 3052 if (rc != 0) 3053 goto done; /* error message displayed already */ 3054 3055 /* 3056 * Setup RSS for this port. 3057 */ 3058 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 3059 M_ZERO | M_WAITOK); 3060 for_each_rxq(pi, i, rxq) { 3061 rss[i] = rxq->iq.abs_id; 3062 } 3063 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 3064 pi->rss_size, rss, pi->nrxq); 3065 free(rss, M_CXGBE); 3066 if (rc != 0) { 3067 if_printf(ifp, "rss_config failed: %d\n", rc); 3068 goto done; 3069 } 3070 3071 pi->flags |= PORT_INIT_DONE; 3072done: 3073 if (rc != 0) 3074 port_full_uninit(pi); 3075 3076 return (rc); 3077} 3078 3079/* 3080 * Idempotent. 3081 */ 3082static int 3083port_full_uninit(struct port_info *pi) 3084{ 3085 struct adapter *sc = pi->adapter; 3086 int i; 3087 struct sge_rxq *rxq; 3088 struct sge_txq *txq; 3089#ifdef TCP_OFFLOAD 3090 struct sge_ofld_rxq *ofld_rxq; 3091 struct sge_wrq *ofld_txq; 3092#endif 3093 3094 if (pi->flags & PORT_INIT_DONE) { 3095 3096 /* Need to quiesce queues. XXX: ctrl queues? */ 3097 3098 for_each_txq(pi, i, txq) { 3099 quiesce_eq(sc, &txq->eq); 3100 } 3101 3102#ifdef TCP_OFFLOAD 3103 for_each_ofld_txq(pi, i, ofld_txq) { 3104 quiesce_eq(sc, &ofld_txq->eq); 3105 } 3106#endif 3107 3108 for_each_rxq(pi, i, rxq) { 3109 quiesce_iq(sc, &rxq->iq); 3110 quiesce_fl(sc, &rxq->fl); 3111 } 3112 3113#ifdef TCP_OFFLOAD 3114 for_each_ofld_rxq(pi, i, ofld_rxq) { 3115 quiesce_iq(sc, &ofld_rxq->iq); 3116 quiesce_fl(sc, &ofld_rxq->fl); 3117 } 3118#endif 3119 } 3120 3121 t4_teardown_port_queues(pi); 3122 pi->flags &= ~PORT_INIT_DONE; 3123 3124 return (0); 3125} 3126 3127static void 3128quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3129{ 3130 EQ_LOCK(eq); 3131 eq->flags |= EQ_DOOMED; 3132 3133 /* 3134 * Wait for the response to a credit flush if one's 3135 * pending. 3136 */ 3137 while (eq->flags & EQ_CRFLUSHED) 3138 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3139 EQ_UNLOCK(eq); 3140 3141 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3142 pause("callout", 10); /* Still iffy */ 3143 3144 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3145} 3146 3147static void 3148quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3149{ 3150 (void) sc; /* unused */ 3151 3152 /* Synchronize with the interrupt handler */ 3153 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3154 pause("iqfree", 1); 3155} 3156 3157static void 3158quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3159{ 3160 mtx_lock(&sc->sfl_lock); 3161 FL_LOCK(fl); 3162 fl->flags |= FL_DOOMED; 3163 FL_UNLOCK(fl); 3164 mtx_unlock(&sc->sfl_lock); 3165 3166 callout_drain(&sc->sfl_callout); 3167 KASSERT((fl->flags & FL_STARVING) == 0, 3168 ("%s: still starving", __func__)); 3169} 3170 3171static int 3172t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3173 driver_intr_t *handler, void *arg, char *name) 3174{ 3175 int rc; 3176 3177 irq->rid = rid; 3178 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3179 RF_SHAREABLE | RF_ACTIVE); 3180 if (irq->res == NULL) { 3181 device_printf(sc->dev, 3182 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3183 return (ENOMEM); 3184 } 3185 3186 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3187 NULL, handler, arg, &irq->tag); 3188 if (rc != 0) { 3189 device_printf(sc->dev, 3190 "failed to setup interrupt for rid %d, name %s: %d\n", 3191 rid, name, rc); 3192 } else if (name) 3193 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3194 3195 return (rc); 3196} 3197 3198static int 3199t4_free_irq(struct adapter *sc, struct irq *irq) 3200{ 3201 if (irq->tag) 3202 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3203 if (irq->res) 3204 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3205 3206 bzero(irq, sizeof(*irq)); 3207 3208 return (0); 3209} 3210 3211static void 3212reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3213 unsigned int end) 3214{ 3215 uint32_t *p = (uint32_t *)(buf + start); 3216 3217 for ( ; start <= end; start += sizeof(uint32_t)) 3218 *p++ = t4_read_reg(sc, start); 3219} 3220 3221static void 3222t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3223{ 3224 int i, n; 3225 const unsigned int *reg_ranges; 3226 static const unsigned int t4_reg_ranges[] = { 3227 0x1008, 0x1108, 3228 0x1180, 0x11b4, 3229 0x11fc, 0x123c, 3230 0x1300, 0x173c, 3231 0x1800, 0x18fc, 3232 0x3000, 0x30d8, 3233 0x30e0, 0x5924, 3234 0x5960, 0x59d4, 3235 0x5a00, 0x5af8, 3236 0x6000, 0x6098, 3237 0x6100, 0x6150, 3238 0x6200, 0x6208, 3239 0x6240, 0x6248, 3240 0x6280, 0x6338, 3241 0x6370, 0x638c, 3242 0x6400, 0x643c, 3243 0x6500, 0x6524, 3244 0x6a00, 0x6a38, 3245 0x6a60, 0x6a78, 3246 0x6b00, 0x6b84, 3247 0x6bf0, 0x6c84, 3248 0x6cf0, 0x6d84, 3249 0x6df0, 0x6e84, 3250 0x6ef0, 0x6f84, 3251 0x6ff0, 0x7084, 3252 0x70f0, 0x7184, 3253 0x71f0, 0x7284, 3254 0x72f0, 0x7384, 3255 0x73f0, 0x7450, 3256 0x7500, 0x7530, 3257 0x7600, 0x761c, 3258 0x7680, 0x76cc, 3259 0x7700, 0x7798, 3260 0x77c0, 0x77fc, 3261 0x7900, 0x79fc, 3262 0x7b00, 0x7c38, 3263 0x7d00, 0x7efc, 3264 0x8dc0, 0x8e1c, 3265 0x8e30, 0x8e78, 3266 0x8ea0, 0x8f6c, 3267 0x8fc0, 0x9074, 3268 0x90fc, 0x90fc, 3269 0x9400, 0x9458, 3270 0x9600, 0x96bc, 3271 0x9800, 0x9808, 3272 0x9820, 0x983c, 3273 0x9850, 0x9864, 3274 0x9c00, 0x9c6c, 3275 0x9c80, 0x9cec, 3276 0x9d00, 0x9d6c, 3277 0x9d80, 0x9dec, 3278 0x9e00, 0x9e6c, 3279 0x9e80, 0x9eec, 3280 0x9f00, 0x9f6c, 3281 0x9f80, 0x9fec, 3282 0xd004, 0xd03c, 3283 0xdfc0, 0xdfe0, 3284 0xe000, 0xea7c, 3285 0xf000, 0x11190, 3286 0x19040, 0x1906c, 3287 0x19078, 0x19080, 3288 0x1908c, 0x19124, 3289 0x19150, 0x191b0, 3290 0x191d0, 0x191e8, 3291 0x19238, 0x1924c, 3292 0x193f8, 0x19474, 3293 0x19490, 0x194f8, 3294 0x19800, 0x19f30, 3295 0x1a000, 0x1a06c, 3296 0x1a0b0, 0x1a120, 3297 0x1a128, 0x1a138, 3298 0x1a190, 0x1a1c4, 3299 0x1a1fc, 0x1a1fc, 3300 0x1e040, 0x1e04c, 3301 0x1e284, 0x1e28c, 3302 0x1e2c0, 0x1e2c0, 3303 0x1e2e0, 0x1e2e0, 3304 0x1e300, 0x1e384, 3305 0x1e3c0, 0x1e3c8, 3306 0x1e440, 0x1e44c, 3307 0x1e684, 0x1e68c, 3308 0x1e6c0, 0x1e6c0, 3309 0x1e6e0, 0x1e6e0, 3310 0x1e700, 0x1e784, 3311 0x1e7c0, 0x1e7c8, 3312 0x1e840, 0x1e84c, 3313 0x1ea84, 0x1ea8c, 3314 0x1eac0, 0x1eac0, 3315 0x1eae0, 0x1eae0, 3316 0x1eb00, 0x1eb84, 3317 0x1ebc0, 0x1ebc8, 3318 0x1ec40, 0x1ec4c, 3319 0x1ee84, 0x1ee8c, 3320 0x1eec0, 0x1eec0, 3321 0x1eee0, 0x1eee0, 3322 0x1ef00, 0x1ef84, 3323 0x1efc0, 0x1efc8, 3324 0x1f040, 0x1f04c, 3325 0x1f284, 0x1f28c, 3326 0x1f2c0, 0x1f2c0, 3327 0x1f2e0, 0x1f2e0, 3328 0x1f300, 0x1f384, 3329 0x1f3c0, 0x1f3c8, 3330 0x1f440, 0x1f44c, 3331 0x1f684, 0x1f68c, 3332 0x1f6c0, 0x1f6c0, 3333 0x1f6e0, 0x1f6e0, 3334 0x1f700, 0x1f784, 3335 0x1f7c0, 0x1f7c8, 3336 0x1f840, 0x1f84c, 3337 0x1fa84, 0x1fa8c, 3338 0x1fac0, 0x1fac0, 3339 0x1fae0, 0x1fae0, 3340 0x1fb00, 0x1fb84, 3341 0x1fbc0, 0x1fbc8, 3342 0x1fc40, 0x1fc4c, 3343 0x1fe84, 0x1fe8c, 3344 0x1fec0, 0x1fec0, 3345 0x1fee0, 0x1fee0, 3346 0x1ff00, 0x1ff84, 3347 0x1ffc0, 0x1ffc8, 3348 0x20000, 0x2002c, 3349 0x20100, 0x2013c, 3350 0x20190, 0x201c8, 3351 0x20200, 0x20318, 3352 0x20400, 0x20528, 3353 0x20540, 0x20614, 3354 0x21000, 0x21040, 3355 0x2104c, 0x21060, 3356 0x210c0, 0x210ec, 3357 0x21200, 0x21268, 3358 0x21270, 0x21284, 3359 0x212fc, 0x21388, 3360 0x21400, 0x21404, 3361 0x21500, 0x21518, 3362 0x2152c, 0x2153c, 3363 0x21550, 0x21554, 3364 0x21600, 0x21600, 3365 0x21608, 0x21628, 3366 0x21630, 0x2163c, 3367 0x21700, 0x2171c, 3368 0x21780, 0x2178c, 3369 0x21800, 0x21c38, 3370 0x21c80, 0x21d7c, 3371 0x21e00, 0x21e04, 3372 0x22000, 0x2202c, 3373 0x22100, 0x2213c, 3374 0x22190, 0x221c8, 3375 0x22200, 0x22318, 3376 0x22400, 0x22528, 3377 0x22540, 0x22614, 3378 0x23000, 0x23040, 3379 0x2304c, 0x23060, 3380 0x230c0, 0x230ec, 3381 0x23200, 0x23268, 3382 0x23270, 0x23284, 3383 0x232fc, 0x23388, 3384 0x23400, 0x23404, 3385 0x23500, 0x23518, 3386 0x2352c, 0x2353c, 3387 0x23550, 0x23554, 3388 0x23600, 0x23600, 3389 0x23608, 0x23628, 3390 0x23630, 0x2363c, 3391 0x23700, 0x2371c, 3392 0x23780, 0x2378c, 3393 0x23800, 0x23c38, 3394 0x23c80, 0x23d7c, 3395 0x23e00, 0x23e04, 3396 0x24000, 0x2402c, 3397 0x24100, 0x2413c, 3398 0x24190, 0x241c8, 3399 0x24200, 0x24318, 3400 0x24400, 0x24528, 3401 0x24540, 0x24614, 3402 0x25000, 0x25040, 3403 0x2504c, 0x25060, 3404 0x250c0, 0x250ec, 3405 0x25200, 0x25268, 3406 0x25270, 0x25284, 3407 0x252fc, 0x25388, 3408 0x25400, 0x25404, 3409 0x25500, 0x25518, 3410 0x2552c, 0x2553c, 3411 0x25550, 0x25554, 3412 0x25600, 0x25600, 3413 0x25608, 0x25628, 3414 0x25630, 0x2563c, 3415 0x25700, 0x2571c, 3416 0x25780, 0x2578c, 3417 0x25800, 0x25c38, 3418 0x25c80, 0x25d7c, 3419 0x25e00, 0x25e04, 3420 0x26000, 0x2602c, 3421 0x26100, 0x2613c, 3422 0x26190, 0x261c8, 3423 0x26200, 0x26318, 3424 0x26400, 0x26528, 3425 0x26540, 0x26614, 3426 0x27000, 0x27040, 3427 0x2704c, 0x27060, 3428 0x270c0, 0x270ec, 3429 0x27200, 0x27268, 3430 0x27270, 0x27284, 3431 0x272fc, 0x27388, 3432 0x27400, 0x27404, 3433 0x27500, 0x27518, 3434 0x2752c, 0x2753c, 3435 0x27550, 0x27554, 3436 0x27600, 0x27600, 3437 0x27608, 0x27628, 3438 0x27630, 0x2763c, 3439 0x27700, 0x2771c, 3440 0x27780, 0x2778c, 3441 0x27800, 0x27c38, 3442 0x27c80, 0x27d7c, 3443 0x27e00, 0x27e04 3444 }; 3445 static const unsigned int t5_reg_ranges[] = { 3446 0x1008, 0x1148, 3447 0x1180, 0x11b4, 3448 0x11fc, 0x123c, 3449 0x1280, 0x173c, 3450 0x1800, 0x18fc, 3451 0x3000, 0x3028, 3452 0x3060, 0x30d8, 3453 0x30e0, 0x30fc, 3454 0x3140, 0x357c, 3455 0x35a8, 0x35cc, 3456 0x35ec, 0x35ec, 3457 0x3600, 0x5624, 3458 0x56cc, 0x575c, 3459 0x580c, 0x5814, 3460 0x5890, 0x58bc, 3461 0x5940, 0x59dc, 3462 0x59fc, 0x5a18, 3463 0x5a60, 0x5a9c, 3464 0x5b94, 0x5bfc, 3465 0x6000, 0x6040, 3466 0x6058, 0x614c, 3467 0x7700, 0x7798, 3468 0x77c0, 0x78fc, 3469 0x7b00, 0x7c54, 3470 0x7d00, 0x7efc, 3471 0x8dc0, 0x8de0, 3472 0x8df8, 0x8e84, 3473 0x8ea0, 0x8f84, 3474 0x8fc0, 0x90f8, 3475 0x9400, 0x9470, 3476 0x9600, 0x96f4, 3477 0x9800, 0x9808, 3478 0x9820, 0x983c, 3479 0x9850, 0x9864, 3480 0x9c00, 0x9c6c, 3481 0x9c80, 0x9cec, 3482 0x9d00, 0x9d6c, 3483 0x9d80, 0x9dec, 3484 0x9e00, 0x9e6c, 3485 0x9e80, 0x9eec, 3486 0x9f00, 0x9f6c, 3487 0x9f80, 0xa020, 3488 0xd004, 0xd03c, 3489 0xdfc0, 0xdfe0, 3490 0xe000, 0x11088, 3491 0x1109c, 0x1117c, 3492 0x11190, 0x11204, 3493 0x19040, 0x1906c, 3494 0x19078, 0x19080, 3495 0x1908c, 0x19124, 3496 0x19150, 0x191b0, 3497 0x191d0, 0x191e8, 3498 0x19238, 0x19290, 3499 0x193f8, 0x19474, 3500 0x19490, 0x194cc, 3501 0x194f0, 0x194f8, 3502 0x19c00, 0x19c60, 3503 0x19c94, 0x19e10, 3504 0x19e50, 0x19f34, 3505 0x19f40, 0x19f50, 3506 0x19f90, 0x19fe4, 3507 0x1a000, 0x1a06c, 3508 0x1a0b0, 0x1a120, 3509 0x1a128, 0x1a138, 3510 0x1a190, 0x1a1c4, 3511 0x1a1fc, 0x1a1fc, 3512 0x1e008, 0x1e00c, 3513 0x1e040, 0x1e04c, 3514 0x1e284, 0x1e290, 3515 0x1e2c0, 0x1e2c0, 3516 0x1e2e0, 0x1e2e0, 3517 0x1e300, 0x1e384, 3518 0x1e3c0, 0x1e3c8, 3519 0x1e408, 0x1e40c, 3520 0x1e440, 0x1e44c, 3521 0x1e684, 0x1e690, 3522 0x1e6c0, 0x1e6c0, 3523 0x1e6e0, 0x1e6e0, 3524 0x1e700, 0x1e784, 3525 0x1e7c0, 0x1e7c8, 3526 0x1e808, 0x1e80c, 3527 0x1e840, 0x1e84c, 3528 0x1ea84, 0x1ea90, 3529 0x1eac0, 0x1eac0, 3530 0x1eae0, 0x1eae0, 3531 0x1eb00, 0x1eb84, 3532 0x1ebc0, 0x1ebc8, 3533 0x1ec08, 0x1ec0c, 3534 0x1ec40, 0x1ec4c, 3535 0x1ee84, 0x1ee90, 3536 0x1eec0, 0x1eec0, 3537 0x1eee0, 0x1eee0, 3538 0x1ef00, 0x1ef84, 3539 0x1efc0, 0x1efc8, 3540 0x1f008, 0x1f00c, 3541 0x1f040, 0x1f04c, 3542 0x1f284, 0x1f290, 3543 0x1f2c0, 0x1f2c0, 3544 0x1f2e0, 0x1f2e0, 3545 0x1f300, 0x1f384, 3546 0x1f3c0, 0x1f3c8, 3547 0x1f408, 0x1f40c, 3548 0x1f440, 0x1f44c, 3549 0x1f684, 0x1f690, 3550 0x1f6c0, 0x1f6c0, 3551 0x1f6e0, 0x1f6e0, 3552 0x1f700, 0x1f784, 3553 0x1f7c0, 0x1f7c8, 3554 0x1f808, 0x1f80c, 3555 0x1f840, 0x1f84c, 3556 0x1fa84, 0x1fa90, 3557 0x1fac0, 0x1fac0, 3558 0x1fae0, 0x1fae0, 3559 0x1fb00, 0x1fb84, 3560 0x1fbc0, 0x1fbc8, 3561 0x1fc08, 0x1fc0c, 3562 0x1fc40, 0x1fc4c, 3563 0x1fe84, 0x1fe90, 3564 0x1fec0, 0x1fec0, 3565 0x1fee0, 0x1fee0, 3566 0x1ff00, 0x1ff84, 3567 0x1ffc0, 0x1ffc8, 3568 0x30000, 0x30040, 3569 0x30100, 0x30144, 3570 0x30190, 0x301d0, 3571 0x30200, 0x30318, 3572 0x30400, 0x3052c, 3573 0x30540, 0x3061c, 3574 0x30800, 0x30834, 3575 0x308c0, 0x30908, 3576 0x30910, 0x309ac, 3577 0x30a00, 0x30a04, 3578 0x30a0c, 0x30a2c, 3579 0x30a44, 0x30a50, 3580 0x30a74, 0x30c24, 3581 0x30d08, 0x30d14, 3582 0x30d1c, 0x30d20, 3583 0x30d3c, 0x30d50, 3584 0x31200, 0x3120c, 3585 0x31220, 0x31220, 3586 0x31240, 0x31240, 3587 0x31600, 0x31600, 3588 0x31608, 0x3160c, 3589 0x31a00, 0x31a1c, 3590 0x31e04, 0x31e20, 3591 0x31e38, 0x31e3c, 3592 0x31e80, 0x31e80, 3593 0x31e88, 0x31ea8, 3594 0x31eb0, 0x31eb4, 3595 0x31ec8, 0x31ed4, 3596 0x31fb8, 0x32004, 3597 0x32208, 0x3223c, 3598 0x32248, 0x3227c, 3599 0x32288, 0x322bc, 3600 0x322c8, 0x322fc, 3601 0x32600, 0x32630, 3602 0x32a00, 0x32abc, 3603 0x32b00, 0x32b70, 3604 0x33000, 0x33048, 3605 0x33060, 0x3309c, 3606 0x330f0, 0x33148, 3607 0x33160, 0x3319c, 3608 0x331f0, 0x332e4, 3609 0x332f8, 0x333e4, 3610 0x333f8, 0x33448, 3611 0x33460, 0x3349c, 3612 0x334f0, 0x33548, 3613 0x33560, 0x3359c, 3614 0x335f0, 0x336e4, 3615 0x336f8, 0x337e4, 3616 0x337f8, 0x337fc, 3617 0x33814, 0x33814, 3618 0x3382c, 0x3382c, 3619 0x33880, 0x3388c, 3620 0x338e8, 0x338ec, 3621 0x33900, 0x33948, 3622 0x33960, 0x3399c, 3623 0x339f0, 0x33ae4, 3624 0x33af8, 0x33b10, 3625 0x33b28, 0x33b28, 3626 0x33b3c, 0x33b50, 3627 0x33bf0, 0x33c10, 3628 0x33c28, 0x33c28, 3629 0x33c3c, 0x33c50, 3630 0x33cf0, 0x33cfc, 3631 0x34000, 0x34040, 3632 0x34100, 0x34144, 3633 0x34190, 0x341d0, 3634 0x34200, 0x34318, 3635 0x34400, 0x3452c, 3636 0x34540, 0x3461c, 3637 0x34800, 0x34834, 3638 0x348c0, 0x34908, 3639 0x34910, 0x349ac, 3640 0x34a00, 0x34a04, 3641 0x34a0c, 0x34a2c, 3642 0x34a44, 0x34a50, 3643 0x34a74, 0x34c24, 3644 0x34d08, 0x34d14, 3645 0x34d1c, 0x34d20, 3646 0x34d3c, 0x34d50, 3647 0x35200, 0x3520c, 3648 0x35220, 0x35220, 3649 0x35240, 0x35240, 3650 0x35600, 0x35600, 3651 0x35608, 0x3560c, 3652 0x35a00, 0x35a1c, 3653 0x35e04, 0x35e20, 3654 0x35e38, 0x35e3c, 3655 0x35e80, 0x35e80, 3656 0x35e88, 0x35ea8, 3657 0x35eb0, 0x35eb4, 3658 0x35ec8, 0x35ed4, 3659 0x35fb8, 0x36004, 3660 0x36208, 0x3623c, 3661 0x36248, 0x3627c, 3662 0x36288, 0x362bc, 3663 0x362c8, 0x362fc, 3664 0x36600, 0x36630, 3665 0x36a00, 0x36abc, 3666 0x36b00, 0x36b70, 3667 0x37000, 0x37048, 3668 0x37060, 0x3709c, 3669 0x370f0, 0x37148, 3670 0x37160, 0x3719c, 3671 0x371f0, 0x372e4, 3672 0x372f8, 0x373e4, 3673 0x373f8, 0x37448, 3674 0x37460, 0x3749c, 3675 0x374f0, 0x37548, 3676 0x37560, 0x3759c, 3677 0x375f0, 0x376e4, 3678 0x376f8, 0x377e4, 3679 0x377f8, 0x377fc, 3680 0x37814, 0x37814, 3681 0x3782c, 0x3782c, 3682 0x37880, 0x3788c, 3683 0x378e8, 0x378ec, 3684 0x37900, 0x37948, 3685 0x37960, 0x3799c, 3686 0x379f0, 0x37ae4, 3687 0x37af8, 0x37b10, 3688 0x37b28, 0x37b28, 3689 0x37b3c, 0x37b50, 3690 0x37bf0, 0x37c10, 3691 0x37c28, 0x37c28, 3692 0x37c3c, 0x37c50, 3693 0x37cf0, 0x37cfc, 3694 0x38000, 0x38040, 3695 0x38100, 0x38144, 3696 0x38190, 0x381d0, 3697 0x38200, 0x38318, 3698 0x38400, 0x3852c, 3699 0x38540, 0x3861c, 3700 0x38800, 0x38834, 3701 0x388c0, 0x38908, 3702 0x38910, 0x389ac, 3703 0x38a00, 0x38a04, 3704 0x38a0c, 0x38a2c, 3705 0x38a44, 0x38a50, 3706 0x38a74, 0x38c24, 3707 0x38d08, 0x38d14, 3708 0x38d1c, 0x38d20, 3709 0x38d3c, 0x38d50, 3710 0x39200, 0x3920c, 3711 0x39220, 0x39220, 3712 0x39240, 0x39240, 3713 0x39600, 0x39600, 3714 0x39608, 0x3960c, 3715 0x39a00, 0x39a1c, 3716 0x39e04, 0x39e20, 3717 0x39e38, 0x39e3c, 3718 0x39e80, 0x39e80, 3719 0x39e88, 0x39ea8, 3720 0x39eb0, 0x39eb4, 3721 0x39ec8, 0x39ed4, 3722 0x39fb8, 0x3a004, 3723 0x3a208, 0x3a23c, 3724 0x3a248, 0x3a27c, 3725 0x3a288, 0x3a2bc, 3726 0x3a2c8, 0x3a2fc, 3727 0x3a600, 0x3a630, 3728 0x3aa00, 0x3aabc, 3729 0x3ab00, 0x3ab70, 3730 0x3b000, 0x3b048, 3731 0x3b060, 0x3b09c, 3732 0x3b0f0, 0x3b148, 3733 0x3b160, 0x3b19c, 3734 0x3b1f0, 0x3b2e4, 3735 0x3b2f8, 0x3b3e4, 3736 0x3b3f8, 0x3b448, 3737 0x3b460, 0x3b49c, 3738 0x3b4f0, 0x3b548, 3739 0x3b560, 0x3b59c, 3740 0x3b5f0, 0x3b6e4, 3741 0x3b6f8, 0x3b7e4, 3742 0x3b7f8, 0x3b7fc, 3743 0x3b814, 0x3b814, 3744 0x3b82c, 0x3b82c, 3745 0x3b880, 0x3b88c, 3746 0x3b8e8, 0x3b8ec, 3747 0x3b900, 0x3b948, 3748 0x3b960, 0x3b99c, 3749 0x3b9f0, 0x3bae4, 3750 0x3baf8, 0x3bb10, 3751 0x3bb28, 0x3bb28, 3752 0x3bb3c, 0x3bb50, 3753 0x3bbf0, 0x3bc10, 3754 0x3bc28, 0x3bc28, 3755 0x3bc3c, 0x3bc50, 3756 0x3bcf0, 0x3bcfc, 3757 0x3c000, 0x3c040, 3758 0x3c100, 0x3c144, 3759 0x3c190, 0x3c1d0, 3760 0x3c200, 0x3c318, 3761 0x3c400, 0x3c52c, 3762 0x3c540, 0x3c61c, 3763 0x3c800, 0x3c834, 3764 0x3c8c0, 0x3c908, 3765 0x3c910, 0x3c9ac, 3766 0x3ca00, 0x3ca04, 3767 0x3ca0c, 0x3ca2c, 3768 0x3ca44, 0x3ca50, 3769 0x3ca74, 0x3cc24, 3770 0x3cd08, 0x3cd14, 3771 0x3cd1c, 0x3cd20, 3772 0x3cd3c, 0x3cd50, 3773 0x3d200, 0x3d20c, 3774 0x3d220, 0x3d220, 3775 0x3d240, 0x3d240, 3776 0x3d600, 0x3d600, 3777 0x3d608, 0x3d60c, 3778 0x3da00, 0x3da1c, 3779 0x3de04, 0x3de20, 3780 0x3de38, 0x3de3c, 3781 0x3de80, 0x3de80, 3782 0x3de88, 0x3dea8, 3783 0x3deb0, 0x3deb4, 3784 0x3dec8, 0x3ded4, 3785 0x3dfb8, 0x3e004, 3786 0x3e208, 0x3e23c, 3787 0x3e248, 0x3e27c, 3788 0x3e288, 0x3e2bc, 3789 0x3e2c8, 0x3e2fc, 3790 0x3e600, 0x3e630, 3791 0x3ea00, 0x3eabc, 3792 0x3eb00, 0x3eb70, 3793 0x3f000, 0x3f048, 3794 0x3f060, 0x3f09c, 3795 0x3f0f0, 0x3f148, 3796 0x3f160, 0x3f19c, 3797 0x3f1f0, 0x3f2e4, 3798 0x3f2f8, 0x3f3e4, 3799 0x3f3f8, 0x3f448, 3800 0x3f460, 0x3f49c, 3801 0x3f4f0, 0x3f548, 3802 0x3f560, 0x3f59c, 3803 0x3f5f0, 0x3f6e4, 3804 0x3f6f8, 0x3f7e4, 3805 0x3f7f8, 0x3f7fc, 3806 0x3f814, 0x3f814, 3807 0x3f82c, 0x3f82c, 3808 0x3f880, 0x3f88c, 3809 0x3f8e8, 0x3f8ec, 3810 0x3f900, 0x3f948, 3811 0x3f960, 0x3f99c, 3812 0x3f9f0, 0x3fae4, 3813 0x3faf8, 0x3fb10, 3814 0x3fb28, 0x3fb28, 3815 0x3fb3c, 0x3fb50, 3816 0x3fbf0, 0x3fc10, 3817 0x3fc28, 0x3fc28, 3818 0x3fc3c, 0x3fc50, 3819 0x3fcf0, 0x3fcfc, 3820 0x40000, 0x4000c, 3821 0x40040, 0x40068, 3822 0x4007c, 0x40144, 3823 0x40180, 0x4018c, 3824 0x40200, 0x40298, 3825 0x402ac, 0x4033c, 3826 0x403f8, 0x403fc, 3827 0x41300, 0x413c4, 3828 0x41400, 0x4141c, 3829 0x41480, 0x414d0, 3830 0x44000, 0x44078, 3831 0x440c0, 0x44278, 3832 0x442c0, 0x44478, 3833 0x444c0, 0x44678, 3834 0x446c0, 0x44878, 3835 0x448c0, 0x449fc, 3836 0x45000, 0x45068, 3837 0x45080, 0x45084, 3838 0x450a0, 0x450b0, 3839 0x45200, 0x45268, 3840 0x45280, 0x45284, 3841 0x452a0, 0x452b0, 3842 0x460c0, 0x460e4, 3843 0x47000, 0x4708c, 3844 0x47200, 0x47250, 3845 0x47400, 0x47420, 3846 0x47600, 0x47618, 3847 0x47800, 0x47814, 3848 0x48000, 0x4800c, 3849 0x48040, 0x48068, 3850 0x4807c, 0x48144, 3851 0x48180, 0x4818c, 3852 0x48200, 0x48298, 3853 0x482ac, 0x4833c, 3854 0x483f8, 0x483fc, 3855 0x49300, 0x493c4, 3856 0x49400, 0x4941c, 3857 0x49480, 0x494d0, 3858 0x4c000, 0x4c078, 3859 0x4c0c0, 0x4c278, 3860 0x4c2c0, 0x4c478, 3861 0x4c4c0, 0x4c678, 3862 0x4c6c0, 0x4c878, 3863 0x4c8c0, 0x4c9fc, 3864 0x4d000, 0x4d068, 3865 0x4d080, 0x4d084, 3866 0x4d0a0, 0x4d0b0, 3867 0x4d200, 0x4d268, 3868 0x4d280, 0x4d284, 3869 0x4d2a0, 0x4d2b0, 3870 0x4e0c0, 0x4e0e4, 3871 0x4f000, 0x4f08c, 3872 0x4f200, 0x4f250, 3873 0x4f400, 0x4f420, 3874 0x4f600, 0x4f618, 3875 0x4f800, 0x4f814, 3876 0x50000, 0x500cc, 3877 0x50400, 0x50400, 3878 0x50800, 0x508cc, 3879 0x50c00, 0x50c00, 3880 0x51000, 0x5101c, 3881 0x51300, 0x51308, 3882 }; 3883 3884 if (is_t4(sc)) { 3885 reg_ranges = &t4_reg_ranges[0]; 3886 n = nitems(t4_reg_ranges); 3887 } else { 3888 reg_ranges = &t5_reg_ranges[0]; 3889 n = nitems(t5_reg_ranges); 3890 } 3891 3892 regs->version = chip_id(sc) | chip_rev(sc) << 10; 3893 for (i = 0; i < n; i += 2) 3894 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 3895} 3896 3897static void 3898cxgbe_tick(void *arg) 3899{ 3900 struct port_info *pi = arg; 3901 struct ifnet *ifp = pi->ifp; 3902 struct sge_txq *txq; 3903 int i, drops; 3904 struct port_stats *s = &pi->stats; 3905 3906 PORT_LOCK(pi); 3907 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3908 PORT_UNLOCK(pi); 3909 return; /* without scheduling another callout */ 3910 } 3911 3912 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 3913 3914 ifp->if_opackets = s->tx_frames - s->tx_pause; 3915 ifp->if_ipackets = s->rx_frames - s->rx_pause; 3916 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 3917 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 3918 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 3919 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 3920 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 3921 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 3922 s->rx_trunc3; 3923 3924 drops = s->tx_drop; 3925 for_each_txq(pi, i, txq) 3926 drops += txq->br->br_drops; 3927 ifp->if_snd.ifq_drops = drops; 3928 3929 ifp->if_oerrors = s->tx_error_frames; 3930 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 3931 s->rx_fcs_err + s->rx_len_err; 3932 3933 callout_schedule(&pi->tick, hz); 3934 PORT_UNLOCK(pi); 3935} 3936 3937static void 3938cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 3939{ 3940 struct ifnet *vlan; 3941 3942 if (arg != ifp || ifp->if_type != IFT_ETHER) 3943 return; 3944 3945 vlan = VLAN_DEVAT(ifp, vid); 3946 VLAN_SETCOOKIE(vlan, ifp); 3947} 3948 3949static int 3950cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 3951{ 3952 3953#ifdef INVARIANTS 3954 panic("%s: opcode 0x%02x on iq %p with payload %p", 3955 __func__, rss->opcode, iq, m); 3956#else 3957 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 3958 __func__, rss->opcode, iq, m); 3959 m_freem(m); 3960#endif 3961 return (EDOOFUS); 3962} 3963 3964int 3965t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3966{ 3967 uintptr_t *loc, new; 3968 3969 if (opcode >= nitems(sc->cpl_handler)) 3970 return (EINVAL); 3971 3972 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3973 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3974 atomic_store_rel_ptr(loc, new); 3975 3976 return (0); 3977} 3978 3979static int 3980an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 3981{ 3982 3983#ifdef INVARIANTS 3984 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 3985#else 3986 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 3987 __func__, iq, ctrl); 3988#endif 3989 return (EDOOFUS); 3990} 3991 3992int 3993t4_register_an_handler(struct adapter *sc, an_handler_t h) 3994{ 3995 uintptr_t *loc, new; 3996 3997 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 3998 loc = (uintptr_t *) &sc->an_handler; 3999 atomic_store_rel_ptr(loc, new); 4000 4001 return (0); 4002} 4003 4004static int 4005fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4006{ 4007 const struct cpl_fw6_msg *cpl = 4008 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4009 4010#ifdef INVARIANTS 4011 panic("%s: fw_msg type %d", __func__, cpl->type); 4012#else 4013 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4014#endif 4015 return (EDOOFUS); 4016} 4017 4018int 4019t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4020{ 4021 uintptr_t *loc, new; 4022 4023 if (type >= nitems(sc->fw_msg_handler)) 4024 return (EINVAL); 4025 4026 /* 4027 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4028 * handler dispatch table. Reject any attempt to install a handler for 4029 * this subtype. 4030 */ 4031 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4032 return (EINVAL); 4033 4034 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4035 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4036 atomic_store_rel_ptr(loc, new); 4037 4038 return (0); 4039} 4040 4041static int 4042t4_sysctls(struct adapter *sc) 4043{ 4044 struct sysctl_ctx_list *ctx; 4045 struct sysctl_oid *oid; 4046 struct sysctl_oid_list *children, *c0; 4047 static char *caps[] = { 4048 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4049 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 4050 "\20\1TOE", /* caps[2] toecaps */ 4051 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4052 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4053 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4054 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4055 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4056 }; 4057 static char *doorbells = {"\20\1UDB\2WRWC\3UDBWC\4KDB"}; 4058 4059 ctx = device_get_sysctl_ctx(sc->dev); 4060 4061 /* 4062 * dev.t4nex.X. 4063 */ 4064 oid = device_get_sysctl_tree(sc->dev); 4065 c0 = children = SYSCTL_CHILDREN(oid); 4066 4067 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4068 sc->params.nports, "# of ports"); 4069 4070 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4071 NULL, chip_rev(sc), "chip hardware revision"); 4072 4073 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4074 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4075 4076 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4077 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4078 4079 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4080 sc->cfcsum, "config file checksum"); 4081 4082 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4083 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4084 sysctl_bitfield, "A", "available doorbells"); 4085 4086 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4087 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4088 sysctl_bitfield, "A", "available link capabilities"); 4089 4090 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4091 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4092 sysctl_bitfield, "A", "available NIC capabilities"); 4093 4094 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4095 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4096 sysctl_bitfield, "A", "available TCP offload capabilities"); 4097 4098 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4099 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4100 sysctl_bitfield, "A", "available RDMA capabilities"); 4101 4102 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4103 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4104 sysctl_bitfield, "A", "available iSCSI capabilities"); 4105 4106 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4107 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4108 sysctl_bitfield, "A", "available FCoE capabilities"); 4109 4110 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4111 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4112 4113 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4114 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4115 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4116 "interrupt holdoff timer values (us)"); 4117 4118 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4119 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4120 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4121 "interrupt holdoff packet counter values"); 4122 4123#ifdef SBUF_DRAIN 4124 /* 4125 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4126 */ 4127 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4128 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4129 "logs and miscellaneous information"); 4130 children = SYSCTL_CHILDREN(oid); 4131 4132 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4133 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4134 sysctl_cctrl, "A", "congestion control"); 4135 4136 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4137 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4138 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4139 4140 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4141 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4142 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4143 4144 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4145 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4146 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4147 4148 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4149 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4150 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4151 4152 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4153 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4154 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4155 4156 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4157 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4158 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4159 4160 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4161 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4162 sysctl_cim_la, "A", "CIM logic analyzer"); 4163 4164 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4165 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4166 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4167 4168 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4169 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4170 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4171 4172 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4173 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4174 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4175 4176 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4177 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4178 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4179 4180 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4181 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4182 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4183 4184 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4185 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4186 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4187 4188 if (is_t5(sc)) { 4189 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4190 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4191 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4192 4193 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4194 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4195 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4196 } 4197 4198 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4199 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4200 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4201 4202 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4203 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4204 sysctl_cpl_stats, "A", "CPL statistics"); 4205 4206 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4207 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4208 sysctl_ddp_stats, "A", "DDP statistics"); 4209 4210 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4211 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4212 sysctl_devlog, "A", "firmware's device log"); 4213 4214 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4215 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4216 sysctl_fcoe_stats, "A", "FCoE statistics"); 4217 4218 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4219 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4220 sysctl_hw_sched, "A", "hardware scheduler "); 4221 4222 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4223 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4224 sysctl_l2t, "A", "hardware L2 table"); 4225 4226 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4227 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4228 sysctl_lb_stats, "A", "loopback statistics"); 4229 4230 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4231 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4232 sysctl_meminfo, "A", "memory regions"); 4233 4234 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4235 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4236 sysctl_path_mtus, "A", "path MTUs"); 4237 4238 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4239 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4240 sysctl_pm_stats, "A", "PM statistics"); 4241 4242 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4243 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4244 sysctl_rdma_stats, "A", "RDMA statistics"); 4245 4246 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4247 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4248 sysctl_tcp_stats, "A", "TCP statistics"); 4249 4250 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4251 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4252 sysctl_tids, "A", "TID information"); 4253 4254 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4255 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4256 sysctl_tp_err_stats, "A", "TP error statistics"); 4257 4258 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4259 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4260 sysctl_tx_rate, "A", "Tx rate"); 4261 4262 if (is_t5(sc)) { 4263 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wrwc_stats", 4264 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4265 sysctl_wrwc_stats, "A", "work request (WC) statistics"); 4266 } 4267#endif 4268 4269#ifdef TCP_OFFLOAD 4270 if (is_offload(sc)) { 4271 /* 4272 * dev.t4nex.X.toe. 4273 */ 4274 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4275 NULL, "TOE parameters"); 4276 children = SYSCTL_CHILDREN(oid); 4277 4278 sc->tt.sndbuf = 256 * 1024; 4279 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4280 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4281 4282 sc->tt.ddp = 0; 4283 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4284 &sc->tt.ddp, 0, "DDP allowed"); 4285 4286 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4287 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4288 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4289 4290 sc->tt.ddp_thres = 4291 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4292 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4293 &sc->tt.ddp_thres, 0, "DDP threshold"); 4294 } 4295#endif 4296 4297 4298 return (0); 4299} 4300 4301static int 4302cxgbe_sysctls(struct port_info *pi) 4303{ 4304 struct sysctl_ctx_list *ctx; 4305 struct sysctl_oid *oid; 4306 struct sysctl_oid_list *children; 4307 4308 ctx = device_get_sysctl_ctx(pi->dev); 4309 4310 /* 4311 * dev.cxgbe.X. 4312 */ 4313 oid = device_get_sysctl_tree(pi->dev); 4314 children = SYSCTL_CHILDREN(oid); 4315 4316 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4317 &pi->nrxq, 0, "# of rx queues"); 4318 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4319 &pi->ntxq, 0, "# of tx queues"); 4320 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4321 &pi->first_rxq, 0, "index of first rx queue"); 4322 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4323 &pi->first_txq, 0, "index of first tx queue"); 4324 4325#ifdef TCP_OFFLOAD 4326 if (is_offload(pi->adapter)) { 4327 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4328 &pi->nofldrxq, 0, 4329 "# of rx queues for offloaded TCP connections"); 4330 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4331 &pi->nofldtxq, 0, 4332 "# of tx queues for offloaded TCP connections"); 4333 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4334 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4335 "index of first TOE rx queue"); 4336 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4337 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4338 "index of first TOE tx queue"); 4339 } 4340#endif 4341 4342 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4343 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4344 "holdoff timer index"); 4345 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4346 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4347 "holdoff packet counter index"); 4348 4349 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4350 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4351 "rx queue size"); 4352 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4353 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4354 "tx queue size"); 4355 4356 /* 4357 * dev.cxgbe.X.stats. 4358 */ 4359 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4360 NULL, "port statistics"); 4361 children = SYSCTL_CHILDREN(oid); 4362 4363#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4364 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4365 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 4366 sysctl_handle_t4_reg64, "QU", desc) 4367 4368 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4369 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4370 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4371 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4372 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4373 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4374 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4375 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4376 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4377 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4378 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4379 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4380 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4381 "# of tx frames in this range", 4382 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4383 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4384 "# of tx frames in this range", 4385 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4386 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4387 "# of tx frames in this range", 4388 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4389 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4390 "# of tx frames in this range", 4391 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4392 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4393 "# of tx frames in this range", 4394 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4395 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4396 "# of tx frames in this range", 4397 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4398 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4399 "# of tx frames in this range", 4400 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4401 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4402 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4403 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4404 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4405 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4406 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4407 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4408 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4409 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4410 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4411 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4412 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4413 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4414 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4415 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4416 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4417 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4418 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4419 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4420 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4421 4422 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4423 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4424 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4425 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4426 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4427 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4428 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4429 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4430 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4431 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4432 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4433 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4434 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4435 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4436 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4437 "# of frames received with bad FCS", 4438 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4439 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4440 "# of frames received with length error", 4441 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4442 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4443 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4444 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4445 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4446 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4447 "# of rx frames in this range", 4448 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4449 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4450 "# of rx frames in this range", 4451 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4452 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4453 "# of rx frames in this range", 4454 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4455 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4456 "# of rx frames in this range", 4457 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4458 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4459 "# of rx frames in this range", 4460 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4461 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4462 "# of rx frames in this range", 4463 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4464 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4465 "# of rx frames in this range", 4466 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4467 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4468 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4469 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4470 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4471 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4472 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4473 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4474 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4475 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4476 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4477 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4478 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4479 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4480 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4481 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4482 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4483 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4484 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4485 4486#undef SYSCTL_ADD_T4_REG64 4487 4488#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4489 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4490 &pi->stats.name, desc) 4491 4492 /* We get these from port_stats and they may be stale by upto 1s */ 4493 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4494 "# drops due to buffer-group 0 overflows"); 4495 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4496 "# drops due to buffer-group 1 overflows"); 4497 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4498 "# drops due to buffer-group 2 overflows"); 4499 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4500 "# drops due to buffer-group 3 overflows"); 4501 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4502 "# of buffer-group 0 truncated packets"); 4503 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4504 "# of buffer-group 1 truncated packets"); 4505 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4506 "# of buffer-group 2 truncated packets"); 4507 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4508 "# of buffer-group 3 truncated packets"); 4509 4510#undef SYSCTL_ADD_T4_PORTSTAT 4511 4512 return (0); 4513} 4514 4515static int 4516sysctl_int_array(SYSCTL_HANDLER_ARGS) 4517{ 4518 int rc, *i; 4519 struct sbuf sb; 4520 4521 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4522 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4523 sbuf_printf(&sb, "%d ", *i); 4524 sbuf_trim(&sb); 4525 sbuf_finish(&sb); 4526 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4527 sbuf_delete(&sb); 4528 return (rc); 4529} 4530 4531static int 4532sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4533{ 4534 int rc; 4535 struct sbuf *sb; 4536 4537 rc = sysctl_wire_old_buffer(req, 0); 4538 if (rc != 0) 4539 return(rc); 4540 4541 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4542 if (sb == NULL) 4543 return (ENOMEM); 4544 4545 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4546 rc = sbuf_finish(sb); 4547 sbuf_delete(sb); 4548 4549 return (rc); 4550} 4551 4552static int 4553sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4554{ 4555 struct port_info *pi = arg1; 4556 struct adapter *sc = pi->adapter; 4557 int idx, rc, i; 4558 struct sge_rxq *rxq; 4559 uint8_t v; 4560 4561 idx = pi->tmr_idx; 4562 4563 rc = sysctl_handle_int(oidp, &idx, 0, req); 4564 if (rc != 0 || req->newptr == NULL) 4565 return (rc); 4566 4567 if (idx < 0 || idx >= SGE_NTIMERS) 4568 return (EINVAL); 4569 4570 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4571 "t4tmr"); 4572 if (rc) 4573 return (rc); 4574 4575 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 4576 for_each_rxq(pi, i, rxq) { 4577#ifdef atomic_store_rel_8 4578 atomic_store_rel_8(&rxq->iq.intr_params, v); 4579#else 4580 rxq->iq.intr_params = v; 4581#endif 4582 } 4583 pi->tmr_idx = idx; 4584 4585 end_synchronized_op(sc, LOCK_HELD); 4586 return (0); 4587} 4588 4589static int 4590sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 4591{ 4592 struct port_info *pi = arg1; 4593 struct adapter *sc = pi->adapter; 4594 int idx, rc; 4595 4596 idx = pi->pktc_idx; 4597 4598 rc = sysctl_handle_int(oidp, &idx, 0, req); 4599 if (rc != 0 || req->newptr == NULL) 4600 return (rc); 4601 4602 if (idx < -1 || idx >= SGE_NCOUNTERS) 4603 return (EINVAL); 4604 4605 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4606 "t4pktc"); 4607 if (rc) 4608 return (rc); 4609 4610 if (pi->flags & PORT_INIT_DONE) 4611 rc = EBUSY; /* cannot be changed once the queues are created */ 4612 else 4613 pi->pktc_idx = idx; 4614 4615 end_synchronized_op(sc, LOCK_HELD); 4616 return (rc); 4617} 4618 4619static int 4620sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 4621{ 4622 struct port_info *pi = arg1; 4623 struct adapter *sc = pi->adapter; 4624 int qsize, rc; 4625 4626 qsize = pi->qsize_rxq; 4627 4628 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4629 if (rc != 0 || req->newptr == NULL) 4630 return (rc); 4631 4632 if (qsize < 128 || (qsize & 7)) 4633 return (EINVAL); 4634 4635 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4636 "t4rxqs"); 4637 if (rc) 4638 return (rc); 4639 4640 if (pi->flags & PORT_INIT_DONE) 4641 rc = EBUSY; /* cannot be changed once the queues are created */ 4642 else 4643 pi->qsize_rxq = qsize; 4644 4645 end_synchronized_op(sc, LOCK_HELD); 4646 return (rc); 4647} 4648 4649static int 4650sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 4651{ 4652 struct port_info *pi = arg1; 4653 struct adapter *sc = pi->adapter; 4654 int qsize, rc; 4655 4656 qsize = pi->qsize_txq; 4657 4658 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4659 if (rc != 0 || req->newptr == NULL) 4660 return (rc); 4661 4662 /* bufring size must be powerof2 */ 4663 if (qsize < 128 || !powerof2(qsize)) 4664 return (EINVAL); 4665 4666 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4667 "t4txqs"); 4668 if (rc) 4669 return (rc); 4670 4671 if (pi->flags & PORT_INIT_DONE) 4672 rc = EBUSY; /* cannot be changed once the queues are created */ 4673 else 4674 pi->qsize_txq = qsize; 4675 4676 end_synchronized_op(sc, LOCK_HELD); 4677 return (rc); 4678} 4679 4680static int 4681sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 4682{ 4683 struct adapter *sc = arg1; 4684 int reg = arg2; 4685 uint64_t val; 4686 4687 val = t4_read_reg64(sc, reg); 4688 4689 return (sysctl_handle_64(oidp, &val, 0, req)); 4690} 4691 4692#ifdef SBUF_DRAIN 4693static int 4694sysctl_cctrl(SYSCTL_HANDLER_ARGS) 4695{ 4696 struct adapter *sc = arg1; 4697 struct sbuf *sb; 4698 int rc, i; 4699 uint16_t incr[NMTUS][NCCTRL_WIN]; 4700 static const char *dec_fac[] = { 4701 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 4702 "0.9375" 4703 }; 4704 4705 rc = sysctl_wire_old_buffer(req, 0); 4706 if (rc != 0) 4707 return (rc); 4708 4709 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4710 if (sb == NULL) 4711 return (ENOMEM); 4712 4713 t4_read_cong_tbl(sc, incr); 4714 4715 for (i = 0; i < NCCTRL_WIN; ++i) { 4716 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 4717 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 4718 incr[5][i], incr[6][i], incr[7][i]); 4719 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 4720 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 4721 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 4722 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 4723 } 4724 4725 rc = sbuf_finish(sb); 4726 sbuf_delete(sb); 4727 4728 return (rc); 4729} 4730 4731static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 4732 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 4733 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 4734 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 4735}; 4736 4737static int 4738sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 4739{ 4740 struct adapter *sc = arg1; 4741 struct sbuf *sb; 4742 int rc, i, n, qid = arg2; 4743 uint32_t *buf, *p; 4744 char *qtype; 4745 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 4746 4747 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 4748 ("%s: bad qid %d\n", __func__, qid)); 4749 4750 if (qid < CIM_NUM_IBQ) { 4751 /* inbound queue */ 4752 qtype = "IBQ"; 4753 n = 4 * CIM_IBQ_SIZE; 4754 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4755 rc = t4_read_cim_ibq(sc, qid, buf, n); 4756 } else { 4757 /* outbound queue */ 4758 qtype = "OBQ"; 4759 qid -= CIM_NUM_IBQ; 4760 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 4761 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4762 rc = t4_read_cim_obq(sc, qid, buf, n); 4763 } 4764 4765 if (rc < 0) { 4766 rc = -rc; 4767 goto done; 4768 } 4769 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 4770 4771 rc = sysctl_wire_old_buffer(req, 0); 4772 if (rc != 0) 4773 goto done; 4774 4775 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4776 if (sb == NULL) { 4777 rc = ENOMEM; 4778 goto done; 4779 } 4780 4781 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 4782 for (i = 0, p = buf; i < n; i += 16, p += 4) 4783 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 4784 p[2], p[3]); 4785 4786 rc = sbuf_finish(sb); 4787 sbuf_delete(sb); 4788done: 4789 free(buf, M_CXGBE); 4790 return (rc); 4791} 4792 4793static int 4794sysctl_cim_la(SYSCTL_HANDLER_ARGS) 4795{ 4796 struct adapter *sc = arg1; 4797 u_int cfg; 4798 struct sbuf *sb; 4799 uint32_t *buf, *p; 4800 int rc; 4801 4802 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 4803 if (rc != 0) 4804 return (rc); 4805 4806 rc = sysctl_wire_old_buffer(req, 0); 4807 if (rc != 0) 4808 return (rc); 4809 4810 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4811 if (sb == NULL) 4812 return (ENOMEM); 4813 4814 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 4815 M_ZERO | M_WAITOK); 4816 4817 rc = -t4_cim_read_la(sc, buf, NULL); 4818 if (rc != 0) 4819 goto done; 4820 4821 sbuf_printf(sb, "Status Data PC%s", 4822 cfg & F_UPDBGLACAPTPCONLY ? "" : 4823 " LS0Stat LS0Addr LS0Data"); 4824 4825 KASSERT((sc->params.cim_la_size & 7) == 0, 4826 ("%s: p will walk off the end of buf", __func__)); 4827 4828 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 4829 if (cfg & F_UPDBGLACAPTPCONLY) { 4830 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 4831 p[6], p[7]); 4832 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 4833 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 4834 p[4] & 0xff, p[5] >> 8); 4835 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 4836 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4837 p[1] & 0xf, p[2] >> 4); 4838 } else { 4839 sbuf_printf(sb, 4840 "\n %02x %x%07x %x%07x %08x %08x " 4841 "%08x%08x%08x%08x", 4842 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4843 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 4844 p[6], p[7]); 4845 } 4846 } 4847 4848 rc = sbuf_finish(sb); 4849 sbuf_delete(sb); 4850done: 4851 free(buf, M_CXGBE); 4852 return (rc); 4853} 4854 4855static int 4856sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 4857{ 4858 struct adapter *sc = arg1; 4859 struct sbuf *sb; 4860 int rc, i; 4861 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4862 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4863 uint16_t thres[CIM_NUM_IBQ]; 4864 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 4865 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 4866 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 4867 4868 if (is_t4(sc)) { 4869 cim_num_obq = CIM_NUM_OBQ; 4870 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 4871 obq_rdaddr = A_UP_OBQ_0_REALADDR; 4872 } else { 4873 cim_num_obq = CIM_NUM_OBQ_T5; 4874 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 4875 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 4876 } 4877 nq = CIM_NUM_IBQ + cim_num_obq; 4878 4879 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 4880 if (rc == 0) 4881 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 4882 if (rc != 0) 4883 return (rc); 4884 4885 t4_read_cimq_cfg(sc, base, size, thres); 4886 4887 rc = sysctl_wire_old_buffer(req, 0); 4888 if (rc != 0) 4889 return (rc); 4890 4891 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4892 if (sb == NULL) 4893 return (ENOMEM); 4894 4895 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 4896 4897 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 4898 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 4899 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 4900 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4901 G_QUEREMFLITS(p[2]) * 16); 4902 for ( ; i < nq; i++, p += 4, wr += 2) 4903 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 4904 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 4905 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4906 G_QUEREMFLITS(p[2]) * 16); 4907 4908 rc = sbuf_finish(sb); 4909 sbuf_delete(sb); 4910 4911 return (rc); 4912} 4913 4914static int 4915sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 4916{ 4917 struct adapter *sc = arg1; 4918 struct sbuf *sb; 4919 int rc; 4920 struct tp_cpl_stats stats; 4921 4922 rc = sysctl_wire_old_buffer(req, 0); 4923 if (rc != 0) 4924 return (rc); 4925 4926 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4927 if (sb == NULL) 4928 return (ENOMEM); 4929 4930 t4_tp_get_cpl_stats(sc, &stats); 4931 4932 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4933 "channel 3\n"); 4934 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 4935 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 4936 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 4937 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 4938 4939 rc = sbuf_finish(sb); 4940 sbuf_delete(sb); 4941 4942 return (rc); 4943} 4944 4945static int 4946sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 4947{ 4948 struct adapter *sc = arg1; 4949 struct sbuf *sb; 4950 int rc; 4951 struct tp_usm_stats stats; 4952 4953 rc = sysctl_wire_old_buffer(req, 0); 4954 if (rc != 0) 4955 return(rc); 4956 4957 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4958 if (sb == NULL) 4959 return (ENOMEM); 4960 4961 t4_get_usm_stats(sc, &stats); 4962 4963 sbuf_printf(sb, "Frames: %u\n", stats.frames); 4964 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 4965 sbuf_printf(sb, "Drops: %u", stats.drops); 4966 4967 rc = sbuf_finish(sb); 4968 sbuf_delete(sb); 4969 4970 return (rc); 4971} 4972 4973const char *devlog_level_strings[] = { 4974 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 4975 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 4976 [FW_DEVLOG_LEVEL_ERR] = "ERR", 4977 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 4978 [FW_DEVLOG_LEVEL_INFO] = "INFO", 4979 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 4980}; 4981 4982const char *devlog_facility_strings[] = { 4983 [FW_DEVLOG_FACILITY_CORE] = "CORE", 4984 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 4985 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 4986 [FW_DEVLOG_FACILITY_RES] = "RES", 4987 [FW_DEVLOG_FACILITY_HW] = "HW", 4988 [FW_DEVLOG_FACILITY_FLR] = "FLR", 4989 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 4990 [FW_DEVLOG_FACILITY_PHY] = "PHY", 4991 [FW_DEVLOG_FACILITY_MAC] = "MAC", 4992 [FW_DEVLOG_FACILITY_PORT] = "PORT", 4993 [FW_DEVLOG_FACILITY_VI] = "VI", 4994 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 4995 [FW_DEVLOG_FACILITY_ACL] = "ACL", 4996 [FW_DEVLOG_FACILITY_TM] = "TM", 4997 [FW_DEVLOG_FACILITY_QFC] = "QFC", 4998 [FW_DEVLOG_FACILITY_DCB] = "DCB", 4999 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5000 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5001 [FW_DEVLOG_FACILITY_RI] = "RI", 5002 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5003 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5004 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5005 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5006}; 5007 5008static int 5009sysctl_devlog(SYSCTL_HANDLER_ARGS) 5010{ 5011 struct adapter *sc = arg1; 5012 struct devlog_params *dparams = &sc->params.devlog; 5013 struct fw_devlog_e *buf, *e; 5014 int i, j, rc, nentries, first = 0; 5015 struct sbuf *sb; 5016 uint64_t ftstamp = UINT64_MAX; 5017 5018 if (dparams->start == 0) { 5019 dparams->memtype = 0; 5020 dparams->start = 0x84000; 5021 dparams->size = 32768; 5022 } 5023 5024 nentries = dparams->size / sizeof(struct fw_devlog_e); 5025 5026 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5027 if (buf == NULL) 5028 return (ENOMEM); 5029 5030 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 5031 (void *)buf); 5032 if (rc != 0) 5033 goto done; 5034 5035 for (i = 0; i < nentries; i++) { 5036 e = &buf[i]; 5037 5038 if (e->timestamp == 0) 5039 break; /* end */ 5040 5041 e->timestamp = be64toh(e->timestamp); 5042 e->seqno = be32toh(e->seqno); 5043 for (j = 0; j < 8; j++) 5044 e->params[j] = be32toh(e->params[j]); 5045 5046 if (e->timestamp < ftstamp) { 5047 ftstamp = e->timestamp; 5048 first = i; 5049 } 5050 } 5051 5052 if (buf[first].timestamp == 0) 5053 goto done; /* nothing in the log */ 5054 5055 rc = sysctl_wire_old_buffer(req, 0); 5056 if (rc != 0) 5057 goto done; 5058 5059 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5060 if (sb == NULL) { 5061 rc = ENOMEM; 5062 goto done; 5063 } 5064 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5065 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5066 5067 i = first; 5068 do { 5069 e = &buf[i]; 5070 if (e->timestamp == 0) 5071 break; /* end */ 5072 5073 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5074 e->seqno, e->timestamp, 5075 (e->level < nitems(devlog_level_strings) ? 5076 devlog_level_strings[e->level] : "UNKNOWN"), 5077 (e->facility < nitems(devlog_facility_strings) ? 5078 devlog_facility_strings[e->facility] : "UNKNOWN")); 5079 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5080 e->params[2], e->params[3], e->params[4], 5081 e->params[5], e->params[6], e->params[7]); 5082 5083 if (++i == nentries) 5084 i = 0; 5085 } while (i != first); 5086 5087 rc = sbuf_finish(sb); 5088 sbuf_delete(sb); 5089done: 5090 free(buf, M_CXGBE); 5091 return (rc); 5092} 5093 5094static int 5095sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5096{ 5097 struct adapter *sc = arg1; 5098 struct sbuf *sb; 5099 int rc; 5100 struct tp_fcoe_stats stats[4]; 5101 5102 rc = sysctl_wire_old_buffer(req, 0); 5103 if (rc != 0) 5104 return (rc); 5105 5106 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5107 if (sb == NULL) 5108 return (ENOMEM); 5109 5110 t4_get_fcoe_stats(sc, 0, &stats[0]); 5111 t4_get_fcoe_stats(sc, 1, &stats[1]); 5112 t4_get_fcoe_stats(sc, 2, &stats[2]); 5113 t4_get_fcoe_stats(sc, 3, &stats[3]); 5114 5115 sbuf_printf(sb, " channel 0 channel 1 " 5116 "channel 2 channel 3\n"); 5117 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5118 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5119 stats[3].octetsDDP); 5120 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5121 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5122 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5123 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5124 stats[3].framesDrop); 5125 5126 rc = sbuf_finish(sb); 5127 sbuf_delete(sb); 5128 5129 return (rc); 5130} 5131 5132static int 5133sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5134{ 5135 struct adapter *sc = arg1; 5136 struct sbuf *sb; 5137 int rc, i; 5138 unsigned int map, kbps, ipg, mode; 5139 unsigned int pace_tab[NTX_SCHED]; 5140 5141 rc = sysctl_wire_old_buffer(req, 0); 5142 if (rc != 0) 5143 return (rc); 5144 5145 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5146 if (sb == NULL) 5147 return (ENOMEM); 5148 5149 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5150 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5151 t4_read_pace_tbl(sc, pace_tab); 5152 5153 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5154 "Class IPG (0.1 ns) Flow IPG (us)"); 5155 5156 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5157 t4_get_tx_sched(sc, i, &kbps, &ipg); 5158 sbuf_printf(sb, "\n %u %-5s %u ", i, 5159 (mode & (1 << i)) ? "flow" : "class", map & 3); 5160 if (kbps) 5161 sbuf_printf(sb, "%9u ", kbps); 5162 else 5163 sbuf_printf(sb, " disabled "); 5164 5165 if (ipg) 5166 sbuf_printf(sb, "%13u ", ipg); 5167 else 5168 sbuf_printf(sb, " disabled "); 5169 5170 if (pace_tab[i]) 5171 sbuf_printf(sb, "%10u", pace_tab[i]); 5172 else 5173 sbuf_printf(sb, " disabled"); 5174 } 5175 5176 rc = sbuf_finish(sb); 5177 sbuf_delete(sb); 5178 5179 return (rc); 5180} 5181 5182static int 5183sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5184{ 5185 struct adapter *sc = arg1; 5186 struct sbuf *sb; 5187 int rc, i, j; 5188 uint64_t *p0, *p1; 5189 struct lb_port_stats s[2]; 5190 static const char *stat_name[] = { 5191 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5192 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5193 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5194 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5195 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5196 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5197 "BG2FramesTrunc:", "BG3FramesTrunc:" 5198 }; 5199 5200 rc = sysctl_wire_old_buffer(req, 0); 5201 if (rc != 0) 5202 return (rc); 5203 5204 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5205 if (sb == NULL) 5206 return (ENOMEM); 5207 5208 memset(s, 0, sizeof(s)); 5209 5210 for (i = 0; i < 4; i += 2) { 5211 t4_get_lb_stats(sc, i, &s[0]); 5212 t4_get_lb_stats(sc, i + 1, &s[1]); 5213 5214 p0 = &s[0].octets; 5215 p1 = &s[1].octets; 5216 sbuf_printf(sb, "%s Loopback %u" 5217 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5218 5219 for (j = 0; j < nitems(stat_name); j++) 5220 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5221 *p0++, *p1++); 5222 } 5223 5224 rc = sbuf_finish(sb); 5225 sbuf_delete(sb); 5226 5227 return (rc); 5228} 5229 5230struct mem_desc { 5231 unsigned int base; 5232 unsigned int limit; 5233 unsigned int idx; 5234}; 5235 5236static int 5237mem_desc_cmp(const void *a, const void *b) 5238{ 5239 return ((const struct mem_desc *)a)->base - 5240 ((const struct mem_desc *)b)->base; 5241} 5242 5243static void 5244mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5245 unsigned int to) 5246{ 5247 unsigned int size; 5248 5249 size = to - from + 1; 5250 if (size == 0) 5251 return; 5252 5253 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5254 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5255} 5256 5257static int 5258sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5259{ 5260 struct adapter *sc = arg1; 5261 struct sbuf *sb; 5262 int rc, i, n; 5263 uint32_t lo, hi, used, alloc; 5264 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5265 static const char *region[] = { 5266 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5267 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5268 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5269 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5270 "RQUDP region:", "PBL region:", "TXPBL region:", 5271 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5272 "On-chip queues:" 5273 }; 5274 struct mem_desc avail[4]; 5275 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5276 struct mem_desc *md = mem; 5277 5278 rc = sysctl_wire_old_buffer(req, 0); 5279 if (rc != 0) 5280 return (rc); 5281 5282 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5283 if (sb == NULL) 5284 return (ENOMEM); 5285 5286 for (i = 0; i < nitems(mem); i++) { 5287 mem[i].limit = 0; 5288 mem[i].idx = i; 5289 } 5290 5291 /* Find and sort the populated memory ranges */ 5292 i = 0; 5293 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5294 if (lo & F_EDRAM0_ENABLE) { 5295 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5296 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5297 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5298 avail[i].idx = 0; 5299 i++; 5300 } 5301 if (lo & F_EDRAM1_ENABLE) { 5302 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5303 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5304 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5305 avail[i].idx = 1; 5306 i++; 5307 } 5308 if (lo & F_EXT_MEM_ENABLE) { 5309 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5310 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5311 avail[i].limit = avail[i].base + 5312 (G_EXT_MEM_SIZE(hi) << 20); 5313 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5314 i++; 5315 } 5316 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5317 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5318 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5319 avail[i].limit = avail[i].base + 5320 (G_EXT_MEM1_SIZE(hi) << 20); 5321 avail[i].idx = 4; 5322 i++; 5323 } 5324 if (!i) /* no memory available */ 5325 return 0; 5326 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5327 5328 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5329 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5330 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5331 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5332 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5333 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5334 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5335 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5336 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5337 5338 /* the next few have explicit upper bounds */ 5339 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5340 md->limit = md->base - 1 + 5341 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5342 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5343 md++; 5344 5345 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5346 md->limit = md->base - 1 + 5347 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5348 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5349 md++; 5350 5351 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5352 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5353 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5354 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5355 } else { 5356 md->base = 0; 5357 md->idx = nitems(region); /* hide it */ 5358 } 5359 md++; 5360 5361#define ulp_region(reg) \ 5362 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5363 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5364 5365 ulp_region(RX_ISCSI); 5366 ulp_region(RX_TDDP); 5367 ulp_region(TX_TPT); 5368 ulp_region(RX_STAG); 5369 ulp_region(RX_RQ); 5370 ulp_region(RX_RQUDP); 5371 ulp_region(RX_PBL); 5372 ulp_region(TX_PBL); 5373#undef ulp_region 5374 5375 md->base = 0; 5376 md->idx = nitems(region); 5377 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5378 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5379 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5380 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5381 } 5382 md++; 5383 5384 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5385 md->limit = md->base + sc->tids.ntids - 1; 5386 md++; 5387 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5388 md->limit = md->base + sc->tids.ntids - 1; 5389 md++; 5390 5391 md->base = sc->vres.ocq.start; 5392 if (sc->vres.ocq.size) 5393 md->limit = md->base + sc->vres.ocq.size - 1; 5394 else 5395 md->idx = nitems(region); /* hide it */ 5396 md++; 5397 5398 /* add any address-space holes, there can be up to 3 */ 5399 for (n = 0; n < i - 1; n++) 5400 if (avail[n].limit < avail[n + 1].base) 5401 (md++)->base = avail[n].limit; 5402 if (avail[n].limit) 5403 (md++)->base = avail[n].limit; 5404 5405 n = md - mem; 5406 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 5407 5408 for (lo = 0; lo < i; lo++) 5409 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 5410 avail[lo].limit - 1); 5411 5412 sbuf_printf(sb, "\n"); 5413 for (i = 0; i < n; i++) { 5414 if (mem[i].idx >= nitems(region)) 5415 continue; /* skip holes */ 5416 if (!mem[i].limit) 5417 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 5418 mem_region_show(sb, region[mem[i].idx], mem[i].base, 5419 mem[i].limit); 5420 } 5421 5422 sbuf_printf(sb, "\n"); 5423 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 5424 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 5425 mem_region_show(sb, "uP RAM:", lo, hi); 5426 5427 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 5428 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 5429 mem_region_show(sb, "uP Extmem2:", lo, hi); 5430 5431 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 5432 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 5433 G_PMRXMAXPAGE(lo), 5434 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 5435 (lo & F_PMRXNUMCHN) ? 2 : 1); 5436 5437 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 5438 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 5439 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 5440 G_PMTXMAXPAGE(lo), 5441 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 5442 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 5443 sbuf_printf(sb, "%u p-structs\n", 5444 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 5445 5446 for (i = 0; i < 4; i++) { 5447 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 5448 if (is_t4(sc)) { 5449 used = G_USED(lo); 5450 alloc = G_ALLOC(lo); 5451 } else { 5452 used = G_T5_USED(lo); 5453 alloc = G_T5_ALLOC(lo); 5454 } 5455 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 5456 i, used, alloc); 5457 } 5458 for (i = 0; i < 4; i++) { 5459 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 5460 if (is_t4(sc)) { 5461 used = G_USED(lo); 5462 alloc = G_ALLOC(lo); 5463 } else { 5464 used = G_T5_USED(lo); 5465 alloc = G_T5_ALLOC(lo); 5466 } 5467 sbuf_printf(sb, 5468 "\nLoopback %d using %u pages out of %u allocated", 5469 i, used, alloc); 5470 } 5471 5472 rc = sbuf_finish(sb); 5473 sbuf_delete(sb); 5474 5475 return (rc); 5476} 5477 5478static int 5479sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 5480{ 5481 struct adapter *sc = arg1; 5482 struct sbuf *sb; 5483 int rc; 5484 uint16_t mtus[NMTUS]; 5485 5486 rc = sysctl_wire_old_buffer(req, 0); 5487 if (rc != 0) 5488 return (rc); 5489 5490 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5491 if (sb == NULL) 5492 return (ENOMEM); 5493 5494 t4_read_mtu_tbl(sc, mtus, NULL); 5495 5496 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 5497 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 5498 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 5499 mtus[14], mtus[15]); 5500 5501 rc = sbuf_finish(sb); 5502 sbuf_delete(sb); 5503 5504 return (rc); 5505} 5506 5507static int 5508sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 5509{ 5510 struct adapter *sc = arg1; 5511 struct sbuf *sb; 5512 int rc, i; 5513 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 5514 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 5515 static const char *pm_stats[] = { 5516 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 5517 }; 5518 5519 rc = sysctl_wire_old_buffer(req, 0); 5520 if (rc != 0) 5521 return (rc); 5522 5523 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5524 if (sb == NULL) 5525 return (ENOMEM); 5526 5527 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 5528 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 5529 5530 sbuf_printf(sb, " Tx count Tx cycles " 5531 "Rx count Rx cycles"); 5532 for (i = 0; i < PM_NSTATS; i++) 5533 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 5534 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 5535 5536 rc = sbuf_finish(sb); 5537 sbuf_delete(sb); 5538 5539 return (rc); 5540} 5541 5542static int 5543sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 5544{ 5545 struct adapter *sc = arg1; 5546 struct sbuf *sb; 5547 int rc; 5548 struct tp_rdma_stats stats; 5549 5550 rc = sysctl_wire_old_buffer(req, 0); 5551 if (rc != 0) 5552 return (rc); 5553 5554 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5555 if (sb == NULL) 5556 return (ENOMEM); 5557 5558 t4_tp_get_rdma_stats(sc, &stats); 5559 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 5560 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 5561 5562 rc = sbuf_finish(sb); 5563 sbuf_delete(sb); 5564 5565 return (rc); 5566} 5567 5568static int 5569sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 5570{ 5571 struct adapter *sc = arg1; 5572 struct sbuf *sb; 5573 int rc; 5574 struct tp_tcp_stats v4, v6; 5575 5576 rc = sysctl_wire_old_buffer(req, 0); 5577 if (rc != 0) 5578 return (rc); 5579 5580 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5581 if (sb == NULL) 5582 return (ENOMEM); 5583 5584 t4_tp_get_tcp_stats(sc, &v4, &v6); 5585 sbuf_printf(sb, 5586 " IP IPv6\n"); 5587 sbuf_printf(sb, "OutRsts: %20u %20u\n", 5588 v4.tcpOutRsts, v6.tcpOutRsts); 5589 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 5590 v4.tcpInSegs, v6.tcpInSegs); 5591 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 5592 v4.tcpOutSegs, v6.tcpOutSegs); 5593 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 5594 v4.tcpRetransSegs, v6.tcpRetransSegs); 5595 5596 rc = sbuf_finish(sb); 5597 sbuf_delete(sb); 5598 5599 return (rc); 5600} 5601 5602static int 5603sysctl_tids(SYSCTL_HANDLER_ARGS) 5604{ 5605 struct adapter *sc = arg1; 5606 struct sbuf *sb; 5607 int rc; 5608 struct tid_info *t = &sc->tids; 5609 5610 rc = sysctl_wire_old_buffer(req, 0); 5611 if (rc != 0) 5612 return (rc); 5613 5614 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5615 if (sb == NULL) 5616 return (ENOMEM); 5617 5618 if (t->natids) { 5619 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 5620 t->atids_in_use); 5621 } 5622 5623 if (t->ntids) { 5624 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5625 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 5626 5627 if (b) { 5628 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 5629 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5630 t->ntids - 1); 5631 } else { 5632 sbuf_printf(sb, "TID range: %u-%u", 5633 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5634 t->ntids - 1); 5635 } 5636 } else 5637 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 5638 sbuf_printf(sb, ", in use: %u\n", 5639 atomic_load_acq_int(&t->tids_in_use)); 5640 } 5641 5642 if (t->nstids) { 5643 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 5644 t->stid_base + t->nstids - 1, t->stids_in_use); 5645 } 5646 5647 if (t->nftids) { 5648 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 5649 t->ftid_base + t->nftids - 1); 5650 } 5651 5652 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 5653 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 5654 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 5655 5656 rc = sbuf_finish(sb); 5657 sbuf_delete(sb); 5658 5659 return (rc); 5660} 5661 5662static int 5663sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 5664{ 5665 struct adapter *sc = arg1; 5666 struct sbuf *sb; 5667 int rc; 5668 struct tp_err_stats stats; 5669 5670 rc = sysctl_wire_old_buffer(req, 0); 5671 if (rc != 0) 5672 return (rc); 5673 5674 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5675 if (sb == NULL) 5676 return (ENOMEM); 5677 5678 t4_tp_get_err_stats(sc, &stats); 5679 5680 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5681 "channel 3\n"); 5682 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 5683 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 5684 stats.macInErrs[3]); 5685 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 5686 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 5687 stats.hdrInErrs[3]); 5688 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 5689 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 5690 stats.tcpInErrs[3]); 5691 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 5692 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 5693 stats.tcp6InErrs[3]); 5694 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 5695 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 5696 stats.tnlCongDrops[3]); 5697 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 5698 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 5699 stats.tnlTxDrops[3]); 5700 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 5701 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 5702 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 5703 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 5704 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 5705 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 5706 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 5707 stats.ofldNoNeigh, stats.ofldCongDefer); 5708 5709 rc = sbuf_finish(sb); 5710 sbuf_delete(sb); 5711 5712 return (rc); 5713} 5714 5715static int 5716sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 5717{ 5718 struct adapter *sc = arg1; 5719 struct sbuf *sb; 5720 int rc; 5721 u64 nrate[NCHAN], orate[NCHAN]; 5722 5723 rc = sysctl_wire_old_buffer(req, 0); 5724 if (rc != 0) 5725 return (rc); 5726 5727 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5728 if (sb == NULL) 5729 return (ENOMEM); 5730 5731 t4_get_chan_txrate(sc, nrate, orate); 5732 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5733 "channel 3\n"); 5734 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 5735 nrate[0], nrate[1], nrate[2], nrate[3]); 5736 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 5737 orate[0], orate[1], orate[2], orate[3]); 5738 5739 rc = sbuf_finish(sb); 5740 sbuf_delete(sb); 5741 5742 return (rc); 5743} 5744 5745static int 5746sysctl_wrwc_stats(SYSCTL_HANDLER_ARGS) 5747{ 5748 struct adapter *sc = arg1; 5749 struct sbuf *sb; 5750 int rc, v; 5751 5752 rc = sysctl_wire_old_buffer(req, 0); 5753 if (rc != 0) 5754 return (rc); 5755 5756 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5757 if (sb == NULL) 5758 return (ENOMEM); 5759 5760 v = t4_read_reg(sc, A_SGE_STAT_CFG); 5761 if (G_STATSOURCE_T5(v) == 7) { 5762 if (G_STATMODE(v) == 0) { 5763 sbuf_printf(sb, "total %d, incomplete %d", 5764 t4_read_reg(sc, A_SGE_STAT_TOTAL), 5765 t4_read_reg(sc, A_SGE_STAT_MATCH)); 5766 } else if (G_STATMODE(v) == 1) { 5767 sbuf_printf(sb, "total %d, data overflow %d", 5768 t4_read_reg(sc, A_SGE_STAT_TOTAL), 5769 t4_read_reg(sc, A_SGE_STAT_MATCH)); 5770 } 5771 } 5772 rc = sbuf_finish(sb); 5773 sbuf_delete(sb); 5774 5775 return (rc); 5776} 5777#endif 5778 5779static inline void 5780txq_start(struct ifnet *ifp, struct sge_txq *txq) 5781{ 5782 struct buf_ring *br; 5783 struct mbuf *m; 5784 5785 TXQ_LOCK_ASSERT_OWNED(txq); 5786 5787 br = txq->br; 5788 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 5789 if (m) 5790 t4_eth_tx(ifp, txq, m); 5791} 5792 5793void 5794t4_tx_callout(void *arg) 5795{ 5796 struct sge_eq *eq = arg; 5797 struct adapter *sc; 5798 5799 if (EQ_TRYLOCK(eq) == 0) 5800 goto reschedule; 5801 5802 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 5803 EQ_UNLOCK(eq); 5804reschedule: 5805 if (__predict_true(!(eq->flags && EQ_DOOMED))) 5806 callout_schedule(&eq->tx_callout, 1); 5807 return; 5808 } 5809 5810 EQ_LOCK_ASSERT_OWNED(eq); 5811 5812 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 5813 5814 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5815 struct sge_txq *txq = arg; 5816 struct port_info *pi = txq->ifp->if_softc; 5817 5818 sc = pi->adapter; 5819 } else { 5820 struct sge_wrq *wrq = arg; 5821 5822 sc = wrq->adapter; 5823 } 5824 5825 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 5826 } 5827 5828 EQ_UNLOCK(eq); 5829} 5830 5831void 5832t4_tx_task(void *arg, int count) 5833{ 5834 struct sge_eq *eq = arg; 5835 5836 EQ_LOCK(eq); 5837 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5838 struct sge_txq *txq = arg; 5839 txq_start(txq->ifp, txq); 5840 } else { 5841 struct sge_wrq *wrq = arg; 5842 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 5843 } 5844 EQ_UNLOCK(eq); 5845} 5846 5847static uint32_t 5848fconf_to_mode(uint32_t fconf) 5849{ 5850 uint32_t mode; 5851 5852 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 5853 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 5854 5855 if (fconf & F_FRAGMENTATION) 5856 mode |= T4_FILTER_IP_FRAGMENT; 5857 5858 if (fconf & F_MPSHITTYPE) 5859 mode |= T4_FILTER_MPS_HIT_TYPE; 5860 5861 if (fconf & F_MACMATCH) 5862 mode |= T4_FILTER_MAC_IDX; 5863 5864 if (fconf & F_ETHERTYPE) 5865 mode |= T4_FILTER_ETH_TYPE; 5866 5867 if (fconf & F_PROTOCOL) 5868 mode |= T4_FILTER_IP_PROTO; 5869 5870 if (fconf & F_TOS) 5871 mode |= T4_FILTER_IP_TOS; 5872 5873 if (fconf & F_VLAN) 5874 mode |= T4_FILTER_VLAN; 5875 5876 if (fconf & F_VNIC_ID) 5877 mode |= T4_FILTER_VNIC; 5878 5879 if (fconf & F_PORT) 5880 mode |= T4_FILTER_PORT; 5881 5882 if (fconf & F_FCOE) 5883 mode |= T4_FILTER_FCoE; 5884 5885 return (mode); 5886} 5887 5888static uint32_t 5889mode_to_fconf(uint32_t mode) 5890{ 5891 uint32_t fconf = 0; 5892 5893 if (mode & T4_FILTER_IP_FRAGMENT) 5894 fconf |= F_FRAGMENTATION; 5895 5896 if (mode & T4_FILTER_MPS_HIT_TYPE) 5897 fconf |= F_MPSHITTYPE; 5898 5899 if (mode & T4_FILTER_MAC_IDX) 5900 fconf |= F_MACMATCH; 5901 5902 if (mode & T4_FILTER_ETH_TYPE) 5903 fconf |= F_ETHERTYPE; 5904 5905 if (mode & T4_FILTER_IP_PROTO) 5906 fconf |= F_PROTOCOL; 5907 5908 if (mode & T4_FILTER_IP_TOS) 5909 fconf |= F_TOS; 5910 5911 if (mode & T4_FILTER_VLAN) 5912 fconf |= F_VLAN; 5913 5914 if (mode & T4_FILTER_VNIC) 5915 fconf |= F_VNIC_ID; 5916 5917 if (mode & T4_FILTER_PORT) 5918 fconf |= F_PORT; 5919 5920 if (mode & T4_FILTER_FCoE) 5921 fconf |= F_FCOE; 5922 5923 return (fconf); 5924} 5925 5926static uint32_t 5927fspec_to_fconf(struct t4_filter_specification *fs) 5928{ 5929 uint32_t fconf = 0; 5930 5931 if (fs->val.frag || fs->mask.frag) 5932 fconf |= F_FRAGMENTATION; 5933 5934 if (fs->val.matchtype || fs->mask.matchtype) 5935 fconf |= F_MPSHITTYPE; 5936 5937 if (fs->val.macidx || fs->mask.macidx) 5938 fconf |= F_MACMATCH; 5939 5940 if (fs->val.ethtype || fs->mask.ethtype) 5941 fconf |= F_ETHERTYPE; 5942 5943 if (fs->val.proto || fs->mask.proto) 5944 fconf |= F_PROTOCOL; 5945 5946 if (fs->val.tos || fs->mask.tos) 5947 fconf |= F_TOS; 5948 5949 if (fs->val.vlan_vld || fs->mask.vlan_vld) 5950 fconf |= F_VLAN; 5951 5952 if (fs->val.vnic_vld || fs->mask.vnic_vld) 5953 fconf |= F_VNIC_ID; 5954 5955 if (fs->val.iport || fs->mask.iport) 5956 fconf |= F_PORT; 5957 5958 if (fs->val.fcoe || fs->mask.fcoe) 5959 fconf |= F_FCOE; 5960 5961 return (fconf); 5962} 5963 5964static int 5965get_filter_mode(struct adapter *sc, uint32_t *mode) 5966{ 5967 int rc; 5968 uint32_t fconf; 5969 5970 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 5971 "t4getfm"); 5972 if (rc) 5973 return (rc); 5974 5975 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 5976 A_TP_VLAN_PRI_MAP); 5977 5978 if (sc->filter_mode != fconf) { 5979 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 5980 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 5981 sc->filter_mode = fconf; 5982 } 5983 5984 *mode = fconf_to_mode(sc->filter_mode); 5985 5986 end_synchronized_op(sc, LOCK_HELD); 5987 return (0); 5988} 5989 5990static int 5991set_filter_mode(struct adapter *sc, uint32_t mode) 5992{ 5993 uint32_t fconf; 5994 int rc; 5995 5996 fconf = mode_to_fconf(mode); 5997 5998 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 5999 "t4setfm"); 6000 if (rc) 6001 return (rc); 6002 6003 if (sc->tids.ftids_in_use > 0) { 6004 rc = EBUSY; 6005 goto done; 6006 } 6007 6008#ifdef TCP_OFFLOAD 6009 if (sc->offload_map) { 6010 rc = EBUSY; 6011 goto done; 6012 } 6013#endif 6014 6015#ifdef notyet 6016 rc = -t4_set_filter_mode(sc, fconf); 6017 if (rc == 0) 6018 sc->filter_mode = fconf; 6019#else 6020 rc = ENOTSUP; 6021#endif 6022 6023done: 6024 end_synchronized_op(sc, LOCK_HELD); 6025 return (rc); 6026} 6027 6028static inline uint64_t 6029get_filter_hits(struct adapter *sc, uint32_t fid) 6030{ 6031 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6032 uint64_t hits; 6033 6034 memwin_info(sc, 0, &mw_base, NULL); 6035 off = position_memwin(sc, 0, 6036 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 6037 hits = t4_read_reg64(sc, mw_base + off + 16); 6038 6039 return (be64toh(hits)); 6040} 6041 6042static int 6043get_filter(struct adapter *sc, struct t4_filter *t) 6044{ 6045 int i, rc, nfilters = sc->tids.nftids; 6046 struct filter_entry *f; 6047 6048 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6049 "t4getf"); 6050 if (rc) 6051 return (rc); 6052 6053 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 6054 t->idx >= nfilters) { 6055 t->idx = 0xffffffff; 6056 goto done; 6057 } 6058 6059 f = &sc->tids.ftid_tab[t->idx]; 6060 for (i = t->idx; i < nfilters; i++, f++) { 6061 if (f->valid) { 6062 t->idx = i; 6063 t->l2tidx = f->l2t ? f->l2t->idx : 0; 6064 t->smtidx = f->smtidx; 6065 if (f->fs.hitcnts) 6066 t->hits = get_filter_hits(sc, t->idx); 6067 else 6068 t->hits = UINT64_MAX; 6069 t->fs = f->fs; 6070 6071 goto done; 6072 } 6073 } 6074 6075 t->idx = 0xffffffff; 6076done: 6077 end_synchronized_op(sc, LOCK_HELD); 6078 return (0); 6079} 6080 6081static int 6082set_filter(struct adapter *sc, struct t4_filter *t) 6083{ 6084 unsigned int nfilters, nports; 6085 struct filter_entry *f; 6086 int i, rc; 6087 6088 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 6089 if (rc) 6090 return (rc); 6091 6092 nfilters = sc->tids.nftids; 6093 nports = sc->params.nports; 6094 6095 if (nfilters == 0) { 6096 rc = ENOTSUP; 6097 goto done; 6098 } 6099 6100 if (!(sc->flags & FULL_INIT_DONE)) { 6101 rc = EAGAIN; 6102 goto done; 6103 } 6104 6105 if (t->idx >= nfilters) { 6106 rc = EINVAL; 6107 goto done; 6108 } 6109 6110 /* Validate against the global filter mode */ 6111 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) { 6112 rc = E2BIG; 6113 goto done; 6114 } 6115 6116 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 6117 rc = EINVAL; 6118 goto done; 6119 } 6120 6121 if (t->fs.val.iport >= nports) { 6122 rc = EINVAL; 6123 goto done; 6124 } 6125 6126 /* Can't specify an iq if not steering to it */ 6127 if (!t->fs.dirsteer && t->fs.iq) { 6128 rc = EINVAL; 6129 goto done; 6130 } 6131 6132 /* IPv6 filter idx must be 4 aligned */ 6133 if (t->fs.type == 1 && 6134 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 6135 rc = EINVAL; 6136 goto done; 6137 } 6138 6139 if (sc->tids.ftid_tab == NULL) { 6140 KASSERT(sc->tids.ftids_in_use == 0, 6141 ("%s: no memory allocated but filters_in_use > 0", 6142 __func__)); 6143 6144 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 6145 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 6146 if (sc->tids.ftid_tab == NULL) { 6147 rc = ENOMEM; 6148 goto done; 6149 } 6150 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 6151 } 6152 6153 for (i = 0; i < 4; i++) { 6154 f = &sc->tids.ftid_tab[t->idx + i]; 6155 6156 if (f->pending || f->valid) { 6157 rc = EBUSY; 6158 goto done; 6159 } 6160 if (f->locked) { 6161 rc = EPERM; 6162 goto done; 6163 } 6164 6165 if (t->fs.type == 0) 6166 break; 6167 } 6168 6169 f = &sc->tids.ftid_tab[t->idx]; 6170 f->fs = t->fs; 6171 6172 rc = set_filter_wr(sc, t->idx); 6173done: 6174 end_synchronized_op(sc, 0); 6175 6176 if (rc == 0) { 6177 mtx_lock(&sc->tids.ftid_lock); 6178 for (;;) { 6179 if (f->pending == 0) { 6180 rc = f->valid ? 0 : EIO; 6181 break; 6182 } 6183 6184 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6185 PCATCH, "t4setfw", 0)) { 6186 rc = EINPROGRESS; 6187 break; 6188 } 6189 } 6190 mtx_unlock(&sc->tids.ftid_lock); 6191 } 6192 return (rc); 6193} 6194 6195static int 6196del_filter(struct adapter *sc, struct t4_filter *t) 6197{ 6198 unsigned int nfilters; 6199 struct filter_entry *f; 6200 int rc; 6201 6202 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 6203 if (rc) 6204 return (rc); 6205 6206 nfilters = sc->tids.nftids; 6207 6208 if (nfilters == 0) { 6209 rc = ENOTSUP; 6210 goto done; 6211 } 6212 6213 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 6214 t->idx >= nfilters) { 6215 rc = EINVAL; 6216 goto done; 6217 } 6218 6219 if (!(sc->flags & FULL_INIT_DONE)) { 6220 rc = EAGAIN; 6221 goto done; 6222 } 6223 6224 f = &sc->tids.ftid_tab[t->idx]; 6225 6226 if (f->pending) { 6227 rc = EBUSY; 6228 goto done; 6229 } 6230 if (f->locked) { 6231 rc = EPERM; 6232 goto done; 6233 } 6234 6235 if (f->valid) { 6236 t->fs = f->fs; /* extra info for the caller */ 6237 rc = del_filter_wr(sc, t->idx); 6238 } 6239 6240done: 6241 end_synchronized_op(sc, 0); 6242 6243 if (rc == 0) { 6244 mtx_lock(&sc->tids.ftid_lock); 6245 for (;;) { 6246 if (f->pending == 0) { 6247 rc = f->valid ? EIO : 0; 6248 break; 6249 } 6250 6251 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6252 PCATCH, "t4delfw", 0)) { 6253 rc = EINPROGRESS; 6254 break; 6255 } 6256 } 6257 mtx_unlock(&sc->tids.ftid_lock); 6258 } 6259 6260 return (rc); 6261} 6262 6263static void 6264clear_filter(struct filter_entry *f) 6265{ 6266 if (f->l2t) 6267 t4_l2t_release(f->l2t); 6268 6269 bzero(f, sizeof (*f)); 6270} 6271 6272static int 6273set_filter_wr(struct adapter *sc, int fidx) 6274{ 6275 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6276 struct wrqe *wr; 6277 struct fw_filter_wr *fwr; 6278 unsigned int ftid; 6279 6280 ASSERT_SYNCHRONIZED_OP(sc); 6281 6282 if (f->fs.newdmac || f->fs.newvlan) { 6283 /* This filter needs an L2T entry; allocate one. */ 6284 f->l2t = t4_l2t_alloc_switching(sc->l2t); 6285 if (f->l2t == NULL) 6286 return (EAGAIN); 6287 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 6288 f->fs.dmac)) { 6289 t4_l2t_release(f->l2t); 6290 f->l2t = NULL; 6291 return (ENOMEM); 6292 } 6293 } 6294 6295 ftid = sc->tids.ftid_base + fidx; 6296 6297 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6298 if (wr == NULL) 6299 return (ENOMEM); 6300 6301 fwr = wrtod(wr); 6302 bzero(fwr, sizeof (*fwr)); 6303 6304 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 6305 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 6306 fwr->tid_to_iq = 6307 htobe32(V_FW_FILTER_WR_TID(ftid) | 6308 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 6309 V_FW_FILTER_WR_NOREPLY(0) | 6310 V_FW_FILTER_WR_IQ(f->fs.iq)); 6311 fwr->del_filter_to_l2tix = 6312 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 6313 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 6314 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 6315 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 6316 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 6317 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 6318 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 6319 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 6320 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 6321 f->fs.newvlan == VLAN_REWRITE) | 6322 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 6323 f->fs.newvlan == VLAN_REWRITE) | 6324 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 6325 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 6326 V_FW_FILTER_WR_PRIO(f->fs.prio) | 6327 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 6328 fwr->ethtype = htobe16(f->fs.val.ethtype); 6329 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 6330 fwr->frag_to_ovlan_vldm = 6331 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 6332 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 6333 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 6334 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 6335 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 6336 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 6337 fwr->smac_sel = 0; 6338 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 6339 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 6340 fwr->maci_to_matchtypem = 6341 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 6342 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 6343 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 6344 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 6345 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 6346 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 6347 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 6348 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 6349 fwr->ptcl = f->fs.val.proto; 6350 fwr->ptclm = f->fs.mask.proto; 6351 fwr->ttyp = f->fs.val.tos; 6352 fwr->ttypm = f->fs.mask.tos; 6353 fwr->ivlan = htobe16(f->fs.val.vlan); 6354 fwr->ivlanm = htobe16(f->fs.mask.vlan); 6355 fwr->ovlan = htobe16(f->fs.val.vnic); 6356 fwr->ovlanm = htobe16(f->fs.mask.vnic); 6357 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 6358 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 6359 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 6360 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 6361 fwr->lp = htobe16(f->fs.val.dport); 6362 fwr->lpm = htobe16(f->fs.mask.dport); 6363 fwr->fp = htobe16(f->fs.val.sport); 6364 fwr->fpm = htobe16(f->fs.mask.sport); 6365 if (f->fs.newsmac) 6366 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 6367 6368 f->pending = 1; 6369 sc->tids.ftids_in_use++; 6370 6371 t4_wrq_tx(sc, wr); 6372 return (0); 6373} 6374 6375static int 6376del_filter_wr(struct adapter *sc, int fidx) 6377{ 6378 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6379 struct wrqe *wr; 6380 struct fw_filter_wr *fwr; 6381 unsigned int ftid; 6382 6383 ftid = sc->tids.ftid_base + fidx; 6384 6385 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6386 if (wr == NULL) 6387 return (ENOMEM); 6388 fwr = wrtod(wr); 6389 bzero(fwr, sizeof (*fwr)); 6390 6391 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 6392 6393 f->pending = 1; 6394 t4_wrq_tx(sc, wr); 6395 return (0); 6396} 6397 6398int 6399t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 6400{ 6401 struct adapter *sc = iq->adapter; 6402 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 6403 unsigned int idx = GET_TID(rpl); 6404 6405 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6406 rss->opcode)); 6407 6408 if (idx >= sc->tids.ftid_base && 6409 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 6410 unsigned int rc = G_COOKIE(rpl->cookie); 6411 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 6412 6413 mtx_lock(&sc->tids.ftid_lock); 6414 if (rc == FW_FILTER_WR_FLT_ADDED) { 6415 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 6416 __func__, idx)); 6417 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 6418 f->pending = 0; /* asynchronous setup completed */ 6419 f->valid = 1; 6420 } else { 6421 if (rc != FW_FILTER_WR_FLT_DELETED) { 6422 /* Add or delete failed, display an error */ 6423 log(LOG_ERR, 6424 "filter %u setup failed with error %u\n", 6425 idx, rc); 6426 } 6427 6428 clear_filter(f); 6429 sc->tids.ftids_in_use--; 6430 } 6431 wakeup(&sc->tids.ftid_tab); 6432 mtx_unlock(&sc->tids.ftid_lock); 6433 } 6434 6435 return (0); 6436} 6437 6438static int 6439get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 6440{ 6441 int rc; 6442 6443 if (cntxt->cid > M_CTXTQID) 6444 return (EINVAL); 6445 6446 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 6447 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 6448 return (EINVAL); 6449 6450 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 6451 if (rc) 6452 return (rc); 6453 6454 if (sc->flags & FW_OK) { 6455 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 6456 &cntxt->data[0]); 6457 if (rc == 0) 6458 goto done; 6459 } 6460 6461 /* 6462 * Read via firmware failed or wasn't even attempted. Read directly via 6463 * the backdoor. 6464 */ 6465 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 6466done: 6467 end_synchronized_op(sc, 0); 6468 return (rc); 6469} 6470 6471static int 6472load_fw(struct adapter *sc, struct t4_data *fw) 6473{ 6474 int rc; 6475 uint8_t *fw_data; 6476 6477 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 6478 if (rc) 6479 return (rc); 6480 6481 if (sc->flags & FULL_INIT_DONE) { 6482 rc = EBUSY; 6483 goto done; 6484 } 6485 6486 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 6487 if (fw_data == NULL) { 6488 rc = ENOMEM; 6489 goto done; 6490 } 6491 6492 rc = copyin(fw->data, fw_data, fw->len); 6493 if (rc == 0) 6494 rc = -t4_load_fw(sc, fw_data, fw->len); 6495 6496 free(fw_data, M_CXGBE); 6497done: 6498 end_synchronized_op(sc, 0); 6499 return (rc); 6500} 6501 6502static int 6503read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 6504{ 6505 uint32_t addr, off, remaining, i, n; 6506 uint32_t *buf, *b; 6507 uint32_t mw_base, mw_aperture; 6508 int rc; 6509 uint8_t *dst; 6510 6511 rc = validate_mem_range(sc, mr->addr, mr->len); 6512 if (rc != 0) 6513 return (rc); 6514 6515 memwin_info(sc, win, &mw_base, &mw_aperture); 6516 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 6517 addr = mr->addr; 6518 remaining = mr->len; 6519 dst = (void *)mr->data; 6520 6521 while (remaining) { 6522 off = position_memwin(sc, win, addr); 6523 6524 /* number of bytes that we'll copy in the inner loop */ 6525 n = min(remaining, mw_aperture - off); 6526 for (i = 0; i < n; i += 4) 6527 *b++ = t4_read_reg(sc, mw_base + off + i); 6528 6529 rc = copyout(buf, dst, n); 6530 if (rc != 0) 6531 break; 6532 6533 b = buf; 6534 dst += n; 6535 remaining -= n; 6536 addr += n; 6537 } 6538 6539 free(buf, M_CXGBE); 6540 return (rc); 6541} 6542 6543static int 6544read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 6545{ 6546 int rc; 6547 6548 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 6549 return (EINVAL); 6550 6551 if (i2cd->len > 1) { 6552 /* XXX: need fw support for longer reads in one go */ 6553 return (ENOTSUP); 6554 } 6555 6556 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 6557 if (rc) 6558 return (rc); 6559 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 6560 i2cd->offset, &i2cd->data[0]); 6561 end_synchronized_op(sc, 0); 6562 6563 return (rc); 6564} 6565 6566int 6567t4_os_find_pci_capability(struct adapter *sc, int cap) 6568{ 6569 int i; 6570 6571 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 6572} 6573 6574int 6575t4_os_pci_save_state(struct adapter *sc) 6576{ 6577 device_t dev; 6578 struct pci_devinfo *dinfo; 6579 6580 dev = sc->dev; 6581 dinfo = device_get_ivars(dev); 6582 6583 pci_cfg_save(dev, dinfo, 0); 6584 return (0); 6585} 6586 6587int 6588t4_os_pci_restore_state(struct adapter *sc) 6589{ 6590 device_t dev; 6591 struct pci_devinfo *dinfo; 6592 6593 dev = sc->dev; 6594 dinfo = device_get_ivars(dev); 6595 6596 pci_cfg_restore(dev, dinfo); 6597 return (0); 6598} 6599 6600void 6601t4_os_portmod_changed(const struct adapter *sc, int idx) 6602{ 6603 struct port_info *pi = sc->port[idx]; 6604 static const char *mod_str[] = { 6605 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 6606 }; 6607 6608 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 6609 if_printf(pi->ifp, "transceiver unplugged.\n"); 6610 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 6611 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 6612 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 6613 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 6614 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 6615 if_printf(pi->ifp, "%s transceiver inserted.\n", 6616 mod_str[pi->mod_type]); 6617 } else { 6618 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 6619 pi->mod_type); 6620 } 6621} 6622 6623void 6624t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 6625{ 6626 struct port_info *pi = sc->port[idx]; 6627 struct ifnet *ifp = pi->ifp; 6628 6629 if (link_stat) { 6630 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 6631 if_link_state_change(ifp, LINK_STATE_UP); 6632 } else 6633 if_link_state_change(ifp, LINK_STATE_DOWN); 6634} 6635 6636void 6637t4_iterate(void (*func)(struct adapter *, void *), void *arg) 6638{ 6639 struct adapter *sc; 6640 6641 mtx_lock(&t4_list_lock); 6642 SLIST_FOREACH(sc, &t4_list, link) { 6643 /* 6644 * func should not make any assumptions about what state sc is 6645 * in - the only guarantee is that sc->sc_lock is a valid lock. 6646 */ 6647 func(sc, arg); 6648 } 6649 mtx_unlock(&t4_list_lock); 6650} 6651 6652static int 6653t4_open(struct cdev *dev, int flags, int type, struct thread *td) 6654{ 6655 return (0); 6656} 6657 6658static int 6659t4_close(struct cdev *dev, int flags, int type, struct thread *td) 6660{ 6661 return (0); 6662} 6663 6664static int 6665t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 6666 struct thread *td) 6667{ 6668 int rc; 6669 struct adapter *sc = dev->si_drv1; 6670 6671 rc = priv_check(td, PRIV_DRIVER); 6672 if (rc != 0) 6673 return (rc); 6674 6675 switch (cmd) { 6676 case CHELSIO_T4_GETREG: { 6677 struct t4_reg *edata = (struct t4_reg *)data; 6678 6679 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6680 return (EFAULT); 6681 6682 if (edata->size == 4) 6683 edata->val = t4_read_reg(sc, edata->addr); 6684 else if (edata->size == 8) 6685 edata->val = t4_read_reg64(sc, edata->addr); 6686 else 6687 return (EINVAL); 6688 6689 break; 6690 } 6691 case CHELSIO_T4_SETREG: { 6692 struct t4_reg *edata = (struct t4_reg *)data; 6693 6694 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6695 return (EFAULT); 6696 6697 if (edata->size == 4) { 6698 if (edata->val & 0xffffffff00000000) 6699 return (EINVAL); 6700 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 6701 } else if (edata->size == 8) 6702 t4_write_reg64(sc, edata->addr, edata->val); 6703 else 6704 return (EINVAL); 6705 break; 6706 } 6707 case CHELSIO_T4_REGDUMP: { 6708 struct t4_regdump *regs = (struct t4_regdump *)data; 6709 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 6710 uint8_t *buf; 6711 6712 if (regs->len < reglen) { 6713 regs->len = reglen; /* hint to the caller */ 6714 return (ENOBUFS); 6715 } 6716 6717 regs->len = reglen; 6718 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 6719 t4_get_regs(sc, regs, buf); 6720 rc = copyout(buf, regs->data, reglen); 6721 free(buf, M_CXGBE); 6722 break; 6723 } 6724 case CHELSIO_T4_GET_FILTER_MODE: 6725 rc = get_filter_mode(sc, (uint32_t *)data); 6726 break; 6727 case CHELSIO_T4_SET_FILTER_MODE: 6728 rc = set_filter_mode(sc, *(uint32_t *)data); 6729 break; 6730 case CHELSIO_T4_GET_FILTER: 6731 rc = get_filter(sc, (struct t4_filter *)data); 6732 break; 6733 case CHELSIO_T4_SET_FILTER: 6734 rc = set_filter(sc, (struct t4_filter *)data); 6735 break; 6736 case CHELSIO_T4_DEL_FILTER: 6737 rc = del_filter(sc, (struct t4_filter *)data); 6738 break; 6739 case CHELSIO_T4_GET_SGE_CONTEXT: 6740 rc = get_sge_context(sc, (struct t4_sge_context *)data); 6741 break; 6742 case CHELSIO_T4_LOAD_FW: 6743 rc = load_fw(sc, (struct t4_data *)data); 6744 break; 6745 case CHELSIO_T4_GET_MEM: 6746 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 6747 break; 6748 case CHELSIO_T4_GET_I2C: 6749 rc = read_i2c(sc, (struct t4_i2c_data *)data); 6750 break; 6751 case CHELSIO_T4_CLEAR_STATS: { 6752 int i; 6753 u_int port_id = *(uint32_t *)data; 6754 struct port_info *pi; 6755 6756 if (port_id >= sc->params.nports) 6757 return (EINVAL); 6758 6759 /* MAC stats */ 6760 t4_clr_port_stats(sc, port_id); 6761 6762 pi = sc->port[port_id]; 6763 if (pi->flags & PORT_INIT_DONE) { 6764 struct sge_rxq *rxq; 6765 struct sge_txq *txq; 6766 struct sge_wrq *wrq; 6767 6768 for_each_rxq(pi, i, rxq) { 6769#if defined(INET) || defined(INET6) 6770 rxq->lro.lro_queued = 0; 6771 rxq->lro.lro_flushed = 0; 6772#endif 6773 rxq->rxcsum = 0; 6774 rxq->vlan_extraction = 0; 6775 } 6776 6777 for_each_txq(pi, i, txq) { 6778 txq->txcsum = 0; 6779 txq->tso_wrs = 0; 6780 txq->vlan_insertion = 0; 6781 txq->imm_wrs = 0; 6782 txq->sgl_wrs = 0; 6783 txq->txpkt_wrs = 0; 6784 txq->txpkts_wrs = 0; 6785 txq->txpkts_pkts = 0; 6786 txq->br->br_drops = 0; 6787 txq->no_dmamap = 0; 6788 txq->no_desc = 0; 6789 } 6790 6791#ifdef TCP_OFFLOAD 6792 /* nothing to clear for each ofld_rxq */ 6793 6794 for_each_ofld_txq(pi, i, wrq) { 6795 wrq->tx_wrs = 0; 6796 wrq->no_desc = 0; 6797 } 6798#endif 6799 wrq = &sc->sge.ctrlq[pi->port_id]; 6800 wrq->tx_wrs = 0; 6801 wrq->no_desc = 0; 6802 } 6803 break; 6804 } 6805 default: 6806 rc = EINVAL; 6807 } 6808 6809 return (rc); 6810} 6811 6812#ifdef TCP_OFFLOAD 6813static int 6814toe_capability(struct port_info *pi, int enable) 6815{ 6816 int rc; 6817 struct adapter *sc = pi->adapter; 6818 6819 ASSERT_SYNCHRONIZED_OP(sc); 6820 6821 if (!is_offload(sc)) 6822 return (ENODEV); 6823 6824 if (enable) { 6825 if (!(sc->flags & FULL_INIT_DONE)) { 6826 rc = cxgbe_init_synchronized(pi); 6827 if (rc) 6828 return (rc); 6829 } 6830 6831 if (isset(&sc->offload_map, pi->port_id)) 6832 return (0); 6833 6834 if (!(sc->flags & TOM_INIT_DONE)) { 6835 rc = t4_activate_uld(sc, ULD_TOM); 6836 if (rc == EAGAIN) { 6837 log(LOG_WARNING, 6838 "You must kldload t4_tom.ko before trying " 6839 "to enable TOE on a cxgbe interface.\n"); 6840 } 6841 if (rc != 0) 6842 return (rc); 6843 KASSERT(sc->tom_softc != NULL, 6844 ("%s: TOM activated but softc NULL", __func__)); 6845 KASSERT(sc->flags & TOM_INIT_DONE, 6846 ("%s: TOM activated but flag not set", __func__)); 6847 } 6848 6849 setbit(&sc->offload_map, pi->port_id); 6850 } else { 6851 if (!isset(&sc->offload_map, pi->port_id)) 6852 return (0); 6853 6854 KASSERT(sc->flags & TOM_INIT_DONE, 6855 ("%s: TOM never initialized?", __func__)); 6856 clrbit(&sc->offload_map, pi->port_id); 6857 } 6858 6859 return (0); 6860} 6861 6862/* 6863 * Add an upper layer driver to the global list. 6864 */ 6865int 6866t4_register_uld(struct uld_info *ui) 6867{ 6868 int rc = 0; 6869 struct uld_info *u; 6870 6871 mtx_lock(&t4_uld_list_lock); 6872 SLIST_FOREACH(u, &t4_uld_list, link) { 6873 if (u->uld_id == ui->uld_id) { 6874 rc = EEXIST; 6875 goto done; 6876 } 6877 } 6878 6879 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 6880 ui->refcount = 0; 6881done: 6882 mtx_unlock(&t4_uld_list_lock); 6883 return (rc); 6884} 6885 6886int 6887t4_unregister_uld(struct uld_info *ui) 6888{ 6889 int rc = EINVAL; 6890 struct uld_info *u; 6891 6892 mtx_lock(&t4_uld_list_lock); 6893 6894 SLIST_FOREACH(u, &t4_uld_list, link) { 6895 if (u == ui) { 6896 if (ui->refcount > 0) { 6897 rc = EBUSY; 6898 goto done; 6899 } 6900 6901 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 6902 rc = 0; 6903 goto done; 6904 } 6905 } 6906done: 6907 mtx_unlock(&t4_uld_list_lock); 6908 return (rc); 6909} 6910 6911int 6912t4_activate_uld(struct adapter *sc, int id) 6913{ 6914 int rc = EAGAIN; 6915 struct uld_info *ui; 6916 6917 ASSERT_SYNCHRONIZED_OP(sc); 6918 6919 mtx_lock(&t4_uld_list_lock); 6920 6921 SLIST_FOREACH(ui, &t4_uld_list, link) { 6922 if (ui->uld_id == id) { 6923 rc = ui->activate(sc); 6924 if (rc == 0) 6925 ui->refcount++; 6926 goto done; 6927 } 6928 } 6929done: 6930 mtx_unlock(&t4_uld_list_lock); 6931 6932 return (rc); 6933} 6934 6935int 6936t4_deactivate_uld(struct adapter *sc, int id) 6937{ 6938 int rc = EINVAL; 6939 struct uld_info *ui; 6940 6941 ASSERT_SYNCHRONIZED_OP(sc); 6942 6943 mtx_lock(&t4_uld_list_lock); 6944 6945 SLIST_FOREACH(ui, &t4_uld_list, link) { 6946 if (ui->uld_id == id) { 6947 rc = ui->deactivate(sc); 6948 if (rc == 0) 6949 ui->refcount--; 6950 goto done; 6951 } 6952 } 6953done: 6954 mtx_unlock(&t4_uld_list_lock); 6955 6956 return (rc); 6957} 6958#endif 6959 6960/* 6961 * Come up with reasonable defaults for some of the tunables, provided they're 6962 * not set by the user (in which case we'll use the values as is). 6963 */ 6964static void 6965tweak_tunables(void) 6966{ 6967 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 6968 6969 if (t4_ntxq10g < 1) 6970 t4_ntxq10g = min(nc, NTXQ_10G); 6971 6972 if (t4_ntxq1g < 1) 6973 t4_ntxq1g = min(nc, NTXQ_1G); 6974 6975 if (t4_nrxq10g < 1) 6976 t4_nrxq10g = min(nc, NRXQ_10G); 6977 6978 if (t4_nrxq1g < 1) 6979 t4_nrxq1g = min(nc, NRXQ_1G); 6980 6981#ifdef TCP_OFFLOAD 6982 if (t4_nofldtxq10g < 1) 6983 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 6984 6985 if (t4_nofldtxq1g < 1) 6986 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 6987 6988 if (t4_nofldrxq10g < 1) 6989 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 6990 6991 if (t4_nofldrxq1g < 1) 6992 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 6993 6994 if (t4_toecaps_allowed == -1) 6995 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 6996#else 6997 if (t4_toecaps_allowed == -1) 6998 t4_toecaps_allowed = 0; 6999#endif 7000 7001 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 7002 t4_tmr_idx_10g = TMR_IDX_10G; 7003 7004 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 7005 t4_pktc_idx_10g = PKTC_IDX_10G; 7006 7007 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 7008 t4_tmr_idx_1g = TMR_IDX_1G; 7009 7010 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 7011 t4_pktc_idx_1g = PKTC_IDX_1G; 7012 7013 if (t4_qsize_txq < 128) 7014 t4_qsize_txq = 128; 7015 7016 if (t4_qsize_rxq < 128) 7017 t4_qsize_rxq = 128; 7018 while (t4_qsize_rxq & 7) 7019 t4_qsize_rxq++; 7020 7021 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 7022} 7023 7024static int 7025mod_event(module_t mod, int cmd, void *arg) 7026{ 7027 int rc = 0; 7028 static int loaded = 0; 7029 7030 switch (cmd) { 7031 case MOD_LOAD: 7032 if (atomic_fetchadd_int(&loaded, 1)) 7033 break; 7034 t4_sge_modload(); 7035 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 7036 SLIST_INIT(&t4_list); 7037#ifdef TCP_OFFLOAD 7038 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 7039 SLIST_INIT(&t4_uld_list); 7040#endif 7041 tweak_tunables(); 7042 break; 7043 7044 case MOD_UNLOAD: 7045 if (atomic_fetchadd_int(&loaded, -1) > 1) 7046 break; 7047#ifdef TCP_OFFLOAD 7048 mtx_lock(&t4_uld_list_lock); 7049 if (!SLIST_EMPTY(&t4_uld_list)) { 7050 rc = EBUSY; 7051 mtx_unlock(&t4_uld_list_lock); 7052 break; 7053 } 7054 mtx_unlock(&t4_uld_list_lock); 7055 mtx_destroy(&t4_uld_list_lock); 7056#endif 7057 mtx_lock(&t4_list_lock); 7058 if (!SLIST_EMPTY(&t4_list)) { 7059 rc = EBUSY; 7060 mtx_unlock(&t4_list_lock); 7061 break; 7062 } 7063 mtx_unlock(&t4_list_lock); 7064 mtx_destroy(&t4_list_lock); 7065 break; 7066 } 7067 7068 return (rc); 7069} 7070 7071static devclass_t t4_devclass, t5_devclass; 7072static devclass_t cxgbe_devclass, cxl_devclass; 7073 7074DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 7075MODULE_VERSION(t4nex, 1); 7076 7077DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 7078MODULE_VERSION(t5nex, 1); 7079 7080DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 7081MODULE_VERSION(cxgbe, 1); 7082 7083DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 7084MODULE_VERSION(cxl, 1); 7085