t4_main.c revision 250614
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_main.c 250614 2013-05-13 20:00:03Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58#if defined(__i386__) || defined(__amd64__) 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" 69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80}; 81static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85}; 86 87 88/* T4 port (cxgbe) interface */ 89static int cxgbe_probe(device_t); 90static int cxgbe_attach(device_t); 91static int cxgbe_detach(device_t); 92static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97}; 98static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102}; 103 104static d_ioctl_t t4_ioctl; 105static d_open_t t4_open; 106static d_close_t t4_close; 107 108static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115}; 116 117/* T5 bus driver interface */ 118static int t5_probe(device_t); 119static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125}; 126static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130}; 131 132 133/* T5 port (cxl) interface */ 134static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138}; 139 140static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147}; 148 149/* ifnet + media interface */ 150static void cxgbe_init(void *); 151static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153static void cxgbe_qflush(struct ifnet *); 154static int cxgbe_media_change(struct ifnet *); 155static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159/* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163static struct mtx t4_list_lock; 164static SLIST_HEAD(, adapter) t4_list; 165#ifdef TCP_OFFLOAD 166static struct mtx t4_uld_list_lock; 167static SLIST_HEAD(, uld_info) t4_uld_list; 168#endif 169 170/* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181/* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184#define NTXQ_10G 16 185static int t4_ntxq10g = -1; 186TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188#define NRXQ_10G 8 189static int t4_nrxq10g = -1; 190TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192#define NTXQ_1G 4 193static int t4_ntxq1g = -1; 194TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196#define NRXQ_1G 2 197static int t4_nrxq1g = -1; 198TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200#ifdef TCP_OFFLOAD 201#define NOFLDTXQ_10G 8 202static int t4_nofldtxq10g = -1; 203TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 204 205#define NOFLDRXQ_10G 2 206static int t4_nofldrxq10g = -1; 207TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 208 209#define NOFLDTXQ_1G 2 210static int t4_nofldtxq1g = -1; 211TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 212 213#define NOFLDRXQ_1G 1 214static int t4_nofldrxq1g = -1; 215TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 216#endif 217 218/* 219 * Holdoff parameters for 10G and 1G ports. 220 */ 221#define TMR_IDX_10G 1 222static int t4_tmr_idx_10g = TMR_IDX_10G; 223TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 224 225#define PKTC_IDX_10G (-1) 226static int t4_pktc_idx_10g = PKTC_IDX_10G; 227TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 228 229#define TMR_IDX_1G 1 230static int t4_tmr_idx_1g = TMR_IDX_1G; 231TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 232 233#define PKTC_IDX_1G (-1) 234static int t4_pktc_idx_1g = PKTC_IDX_1G; 235TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 236 237/* 238 * Size (# of entries) of each tx and rx queue. 239 */ 240static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 241TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 242 243static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 244TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 245 246/* 247 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 248 */ 249static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 250TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 251 252/* 253 * Configuration file. 254 */ 255#define DEFAULT_CF "default" 256#define FLASH_CF "flash" 257#define UWIRE_CF "uwire" 258#define FPGA_CF "fpga" 259static char t4_cfg_file[32] = DEFAULT_CF; 260TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 261 262/* 263 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 264 * encouraged respectively). 265 */ 266static unsigned int t4_fw_install = 1; 267TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 268 269/* 270 * ASIC features that will be used. Disable the ones you don't want so that the 271 * chip resources aren't wasted on features that will not be used. 272 */ 273static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 274TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 275 276static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 277TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 278 279static int t4_toecaps_allowed = -1; 280TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 281 282static int t4_rdmacaps_allowed = 0; 283TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 284 285static int t4_iscsicaps_allowed = 0; 286TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 287 288static int t4_fcoecaps_allowed = 0; 289TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 290 291static int t5_write_combine = 0; 292TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 293 294struct intrs_and_queues { 295 int intr_type; /* INTx, MSI, or MSI-X */ 296 int nirq; /* Number of vectors */ 297 int intr_flags; 298 int ntxq10g; /* # of NIC txq's for each 10G port */ 299 int nrxq10g; /* # of NIC rxq's for each 10G port */ 300 int ntxq1g; /* # of NIC txq's for each 1G port */ 301 int nrxq1g; /* # of NIC rxq's for each 1G port */ 302#ifdef TCP_OFFLOAD 303 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 304 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 305 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 306 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 307#endif 308}; 309 310struct filter_entry { 311 uint32_t valid:1; /* filter allocated and valid */ 312 uint32_t locked:1; /* filter is administratively locked */ 313 uint32_t pending:1; /* filter action is pending firmware reply */ 314 uint32_t smtidx:8; /* Source MAC Table index for smac */ 315 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 316 317 struct t4_filter_specification fs; 318}; 319 320enum { 321 XGMAC_MTU = (1 << 0), 322 XGMAC_PROMISC = (1 << 1), 323 XGMAC_ALLMULTI = (1 << 2), 324 XGMAC_VLANEX = (1 << 3), 325 XGMAC_UCADDR = (1 << 4), 326 XGMAC_MCADDRS = (1 << 5), 327 328 XGMAC_ALL = 0xffff 329}; 330 331static int map_bars_0_and_4(struct adapter *); 332static int map_bar_2(struct adapter *); 333static void setup_memwin(struct adapter *); 334static int validate_mem_range(struct adapter *, uint32_t, int); 335static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 336 uint32_t *); 337static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 338static uint32_t position_memwin(struct adapter *, int, uint32_t); 339static int cfg_itype_and_nqueues(struct adapter *, int, int, 340 struct intrs_and_queues *); 341static int prep_firmware(struct adapter *); 342static int partition_resources(struct adapter *, const struct firmware *, 343 const char *); 344static int get_params__pre_init(struct adapter *); 345static int get_params__post_init(struct adapter *); 346static int set_params__post_init(struct adapter *); 347static void t4_set_desc(struct adapter *); 348static void build_medialist(struct port_info *); 349static int update_mac_settings(struct port_info *, int); 350static int cxgbe_init_synchronized(struct port_info *); 351static int cxgbe_uninit_synchronized(struct port_info *); 352static int setup_intr_handlers(struct adapter *); 353static int adapter_full_init(struct adapter *); 354static int adapter_full_uninit(struct adapter *); 355static int port_full_init(struct port_info *); 356static int port_full_uninit(struct port_info *); 357static void quiesce_eq(struct adapter *, struct sge_eq *); 358static void quiesce_iq(struct adapter *, struct sge_iq *); 359static void quiesce_fl(struct adapter *, struct sge_fl *); 360static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 361 driver_intr_t *, void *, char *); 362static int t4_free_irq(struct adapter *, struct irq *); 363static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 364 unsigned int); 365static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 366static void cxgbe_tick(void *); 367static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 368static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 369 struct mbuf *); 370static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 371static int fw_msg_not_handled(struct adapter *, const __be64 *); 372static int t4_sysctls(struct adapter *); 373static int cxgbe_sysctls(struct port_info *); 374static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 375static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 376static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 377static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 378static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 379static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 380static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 381#ifdef SBUF_DRAIN 382static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 383static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 384static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 385static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 386static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 387static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 388static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 389static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 390static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 391static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 392static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 393static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 394static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 395static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 396static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 397static int sysctl_tids(SYSCTL_HANDLER_ARGS); 398static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 399static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 400static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 401#endif 402static inline void txq_start(struct ifnet *, struct sge_txq *); 403static uint32_t fconf_to_mode(uint32_t); 404static uint32_t mode_to_fconf(uint32_t); 405static uint32_t fspec_to_fconf(struct t4_filter_specification *); 406static int get_filter_mode(struct adapter *, uint32_t *); 407static int set_filter_mode(struct adapter *, uint32_t); 408static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 409static int get_filter(struct adapter *, struct t4_filter *); 410static int set_filter(struct adapter *, struct t4_filter *); 411static int del_filter(struct adapter *, struct t4_filter *); 412static void clear_filter(struct filter_entry *); 413static int set_filter_wr(struct adapter *, int); 414static int del_filter_wr(struct adapter *, int); 415static int get_sge_context(struct adapter *, struct t4_sge_context *); 416static int load_fw(struct adapter *, struct t4_data *); 417static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 418static int read_i2c(struct adapter *, struct t4_i2c_data *); 419#ifdef TCP_OFFLOAD 420static int toe_capability(struct port_info *, int); 421#endif 422static int mod_event(module_t, int, void *); 423 424struct { 425 uint16_t device; 426 char *desc; 427} t4_pciids[] = { 428 {0xa000, "Chelsio Terminator 4 FPGA"}, 429 {0x4400, "Chelsio T440-dbg"}, 430 {0x4401, "Chelsio T420-CR"}, 431 {0x4402, "Chelsio T422-CR"}, 432 {0x4403, "Chelsio T440-CR"}, 433 {0x4404, "Chelsio T420-BCH"}, 434 {0x4405, "Chelsio T440-BCH"}, 435 {0x4406, "Chelsio T440-CH"}, 436 {0x4407, "Chelsio T420-SO"}, 437 {0x4408, "Chelsio T420-CX"}, 438 {0x4409, "Chelsio T420-BT"}, 439 {0x440a, "Chelsio T404-BT"}, 440 {0x440e, "Chelsio T440-LP-CR"}, 441}, t5_pciids[] = { 442 {0xb000, "Chelsio Terminator 5 FPGA"}, 443 {0x5400, "Chelsio T580-dbg"}, 444 {0x5401, "Chelsio T520-CR"}, 445 {0x5407, "Chelsio T520-SO"}, 446 {0x5408, "Chelsio T520-CX"}, 447 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 448 {0x5411, "Chelsio T520-LL-CR"}, 449#ifdef notyet 450 {0x5402, "Chelsio T522-CR"}, 451 {0x5403, "Chelsio T540-CR"}, 452 {0x5404, "Chelsio T520-BCH"}, 453 {0x5405, "Chelsio T540-BCH"}, 454 {0x5406, "Chelsio T540-CH"}, 455 {0x5409, "Chelsio T520-BT"}, 456 {0x540a, "Chelsio T504-BT"}, 457 {0x540b, "Chelsio B520-SR"}, 458 {0x540c, "Chelsio B504-BT"}, 459 {0x540d, "Chelsio T580-CR"}, 460 {0x540e, "Chelsio T540-LP-CR"}, 461 {0x540f, "Chelsio Amsterdam"}, 462 {0x5412, "Chelsio T560-CR"}, 463 {0x5413, "Chelsio T580-CR"}, 464#endif 465}; 466 467#ifdef TCP_OFFLOAD 468/* 469 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 470 * exactly the same for both rxq and ofld_rxq. 471 */ 472CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 473CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 474#endif 475 476/* No easy way to include t4_msg.h before adapter.h so we check this way */ 477CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 478CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 479 480static int 481t4_probe(device_t dev) 482{ 483 int i; 484 uint16_t v = pci_get_vendor(dev); 485 uint16_t d = pci_get_device(dev); 486 uint8_t f = pci_get_function(dev); 487 488 if (v != PCI_VENDOR_ID_CHELSIO) 489 return (ENXIO); 490 491 /* Attach only to PF0 of the FPGA */ 492 if (d == 0xa000 && f != 0) 493 return (ENXIO); 494 495 for (i = 0; i < nitems(t4_pciids); i++) { 496 if (d == t4_pciids[i].device) { 497 device_set_desc(dev, t4_pciids[i].desc); 498 return (BUS_PROBE_DEFAULT); 499 } 500 } 501 502 return (ENXIO); 503} 504 505static int 506t5_probe(device_t dev) 507{ 508 int i; 509 uint16_t v = pci_get_vendor(dev); 510 uint16_t d = pci_get_device(dev); 511 uint8_t f = pci_get_function(dev); 512 513 if (v != PCI_VENDOR_ID_CHELSIO) 514 return (ENXIO); 515 516 /* Attach only to PF0 of the FPGA */ 517 if (d == 0xb000 && f != 0) 518 return (ENXIO); 519 520 for (i = 0; i < nitems(t5_pciids); i++) { 521 if (d == t5_pciids[i].device) { 522 device_set_desc(dev, t5_pciids[i].desc); 523 return (BUS_PROBE_DEFAULT); 524 } 525 } 526 527 return (ENXIO); 528} 529 530static int 531t4_attach(device_t dev) 532{ 533 struct adapter *sc; 534 int rc = 0, i, n10g, n1g, rqidx, tqidx; 535 struct intrs_and_queues iaq; 536 struct sge *s; 537#ifdef TCP_OFFLOAD 538 int ofld_rqidx, ofld_tqidx; 539#endif 540 541 sc = device_get_softc(dev); 542 sc->dev = dev; 543 544 pci_enable_busmaster(dev); 545 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 546 uint32_t v; 547 548 pci_set_max_read_req(dev, 4096); 549 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 550 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 551 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 552 } 553 554 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 555 device_get_nameunit(dev)); 556 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 557 mtx_lock(&t4_list_lock); 558 SLIST_INSERT_HEAD(&t4_list, sc, link); 559 mtx_unlock(&t4_list_lock); 560 561 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 562 TAILQ_INIT(&sc->sfl); 563 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 564 565 rc = map_bars_0_and_4(sc); 566 if (rc != 0) 567 goto done; /* error message displayed already */ 568 569 /* 570 * This is the real PF# to which we're attaching. Works from within PCI 571 * passthrough environments too, where pci_get_function() could return a 572 * different PF# depending on the passthrough configuration. We need to 573 * use the real PF# in all our communication with the firmware. 574 */ 575 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 576 sc->mbox = sc->pf; 577 578 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 579 sc->an_handler = an_not_handled; 580 for (i = 0; i < nitems(sc->cpl_handler); i++) 581 sc->cpl_handler[i] = cpl_not_handled; 582 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 583 sc->fw_msg_handler[i] = fw_msg_not_handled; 584 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 585 t4_init_sge_cpl_handlers(sc); 586 587 /* Prepare the adapter for operation */ 588 rc = -t4_prep_adapter(sc); 589 if (rc != 0) { 590 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 591 goto done; 592 } 593 594 /* 595 * Do this really early, with the memory windows set up even before the 596 * character device. The userland tool's register i/o and mem read 597 * will work even in "recovery mode". 598 */ 599 setup_memwin(sc); 600 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 601 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 602 device_get_nameunit(dev)); 603 if (sc->cdev == NULL) 604 device_printf(dev, "failed to create nexus char device.\n"); 605 else 606 sc->cdev->si_drv1 = sc; 607 608 /* Go no further if recovery mode has been requested. */ 609 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 610 device_printf(dev, "recovery mode.\n"); 611 goto done; 612 } 613 614 /* Prepare the firmware for operation */ 615 rc = prep_firmware(sc); 616 if (rc != 0) 617 goto done; /* error message displayed already */ 618 619 rc = get_params__post_init(sc); 620 if (rc != 0) 621 goto done; /* error message displayed already */ 622 623 rc = set_params__post_init(sc); 624 if (rc != 0) 625 goto done; /* error message displayed already */ 626 627 rc = map_bar_2(sc); 628 if (rc != 0) 629 goto done; /* error message displayed already */ 630 631 for (i = 0; i < NCHAN; i++) 632 sc->params.tp.tx_modq[i] = i; 633 634 rc = t4_create_dma_tag(sc); 635 if (rc != 0) 636 goto done; /* error message displayed already */ 637 638 /* 639 * First pass over all the ports - allocate VIs and initialize some 640 * basic parameters like mac address, port type, etc. We also figure 641 * out whether a port is 10G or 1G and use that information when 642 * calculating how many interrupts to attempt to allocate. 643 */ 644 n10g = n1g = 0; 645 for_each_port(sc, i) { 646 struct port_info *pi; 647 648 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 649 sc->port[i] = pi; 650 651 /* These must be set before t4_port_init */ 652 pi->adapter = sc; 653 pi->port_id = i; 654 655 /* Allocate the vi and initialize parameters like mac addr */ 656 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 657 if (rc != 0) { 658 device_printf(dev, "unable to initialize port %d: %d\n", 659 i, rc); 660 free(pi, M_CXGBE); 661 sc->port[i] = NULL; 662 goto done; 663 } 664 665 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 666 device_get_nameunit(dev), i); 667 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 668 669 if (is_10G_port(pi) || is_40G_port(pi)) { 670 n10g++; 671 pi->tmr_idx = t4_tmr_idx_10g; 672 pi->pktc_idx = t4_pktc_idx_10g; 673 } else { 674 n1g++; 675 pi->tmr_idx = t4_tmr_idx_1g; 676 pi->pktc_idx = t4_pktc_idx_1g; 677 } 678 679 pi->xact_addr_filt = -1; 680 681 pi->qsize_rxq = t4_qsize_rxq; 682 pi->qsize_txq = t4_qsize_txq; 683 684 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 685 if (pi->dev == NULL) { 686 device_printf(dev, 687 "failed to add device for port %d.\n", i); 688 rc = ENXIO; 689 goto done; 690 } 691 device_set_softc(pi->dev, pi); 692 } 693 694 /* 695 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 696 */ 697 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 698 if (rc != 0) 699 goto done; /* error message displayed already */ 700 701 sc->intr_type = iaq.intr_type; 702 sc->intr_count = iaq.nirq; 703 sc->flags |= iaq.intr_flags; 704 705 s = &sc->sge; 706 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 707 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 708 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 709 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 710 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 711 712#ifdef TCP_OFFLOAD 713 if (is_offload(sc)) { 714 715 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 716 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 717 s->neq += s->nofldtxq + s->nofldrxq; 718 s->niq += s->nofldrxq; 719 720 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 721 M_CXGBE, M_ZERO | M_WAITOK); 722 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 723 M_CXGBE, M_ZERO | M_WAITOK); 724 } 725#endif 726 727 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 728 M_ZERO | M_WAITOK); 729 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 730 M_ZERO | M_WAITOK); 731 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 732 M_ZERO | M_WAITOK); 733 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 734 M_ZERO | M_WAITOK); 735 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 736 M_ZERO | M_WAITOK); 737 738 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 739 M_ZERO | M_WAITOK); 740 741 t4_init_l2t(sc, M_WAITOK); 742 743 /* 744 * Second pass over the ports. This time we know the number of rx and 745 * tx queues that each port should get. 746 */ 747 rqidx = tqidx = 0; 748#ifdef TCP_OFFLOAD 749 ofld_rqidx = ofld_tqidx = 0; 750#endif 751 for_each_port(sc, i) { 752 struct port_info *pi = sc->port[i]; 753 754 if (pi == NULL) 755 continue; 756 757 pi->first_rxq = rqidx; 758 pi->first_txq = tqidx; 759 if (is_10G_port(pi) || is_40G_port(pi)) { 760 pi->nrxq = iaq.nrxq10g; 761 pi->ntxq = iaq.ntxq10g; 762 } else { 763 pi->nrxq = iaq.nrxq1g; 764 pi->ntxq = iaq.ntxq1g; 765 } 766 767 rqidx += pi->nrxq; 768 tqidx += pi->ntxq; 769 770#ifdef TCP_OFFLOAD 771 if (is_offload(sc)) { 772 pi->first_ofld_rxq = ofld_rqidx; 773 pi->first_ofld_txq = ofld_tqidx; 774 if (is_10G_port(pi) || is_40G_port(pi)) { 775 pi->nofldrxq = iaq.nofldrxq10g; 776 pi->nofldtxq = iaq.nofldtxq10g; 777 } else { 778 pi->nofldrxq = iaq.nofldrxq1g; 779 pi->nofldtxq = iaq.nofldtxq1g; 780 } 781 ofld_rqidx += pi->nofldrxq; 782 ofld_tqidx += pi->nofldtxq; 783 } 784#endif 785 } 786 787 rc = setup_intr_handlers(sc); 788 if (rc != 0) { 789 device_printf(dev, 790 "failed to setup interrupt handlers: %d\n", rc); 791 goto done; 792 } 793 794 rc = bus_generic_attach(dev); 795 if (rc != 0) { 796 device_printf(dev, 797 "failed to attach all child ports: %d\n", rc); 798 goto done; 799 } 800 801 device_printf(dev, 802 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 803 sc->params.pci.width, sc->params.nports, sc->intr_count, 804 sc->intr_type == INTR_MSIX ? "MSI-X" : 805 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 806 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 807 808 t4_set_desc(sc); 809 810done: 811 if (rc != 0 && sc->cdev) { 812 /* cdev was created and so cxgbetool works; recover that way. */ 813 device_printf(dev, 814 "error during attach, adapter is now in recovery mode.\n"); 815 rc = 0; 816 } 817 818 if (rc != 0) 819 t4_detach(dev); 820 else 821 t4_sysctls(sc); 822 823 return (rc); 824} 825 826/* 827 * Idempotent 828 */ 829static int 830t4_detach(device_t dev) 831{ 832 struct adapter *sc; 833 struct port_info *pi; 834 int i, rc; 835 836 sc = device_get_softc(dev); 837 838 if (sc->flags & FULL_INIT_DONE) 839 t4_intr_disable(sc); 840 841 if (sc->cdev) { 842 destroy_dev(sc->cdev); 843 sc->cdev = NULL; 844 } 845 846 rc = bus_generic_detach(dev); 847 if (rc) { 848 device_printf(dev, 849 "failed to detach child devices: %d\n", rc); 850 return (rc); 851 } 852 853 for (i = 0; i < sc->intr_count; i++) 854 t4_free_irq(sc, &sc->irq[i]); 855 856 for (i = 0; i < MAX_NPORTS; i++) { 857 pi = sc->port[i]; 858 if (pi) { 859 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 860 if (pi->dev) 861 device_delete_child(dev, pi->dev); 862 863 mtx_destroy(&pi->pi_lock); 864 free(pi, M_CXGBE); 865 } 866 } 867 868 if (sc->flags & FULL_INIT_DONE) 869 adapter_full_uninit(sc); 870 871 if (sc->flags & FW_OK) 872 t4_fw_bye(sc, sc->mbox); 873 874 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 875 pci_release_msi(dev); 876 877 if (sc->regs_res) 878 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 879 sc->regs_res); 880 881 if (sc->udbs_res) 882 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 883 sc->udbs_res); 884 885 if (sc->msix_res) 886 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 887 sc->msix_res); 888 889 if (sc->l2t) 890 t4_free_l2t(sc->l2t); 891 892#ifdef TCP_OFFLOAD 893 free(sc->sge.ofld_rxq, M_CXGBE); 894 free(sc->sge.ofld_txq, M_CXGBE); 895#endif 896 free(sc->irq, M_CXGBE); 897 free(sc->sge.rxq, M_CXGBE); 898 free(sc->sge.txq, M_CXGBE); 899 free(sc->sge.ctrlq, M_CXGBE); 900 free(sc->sge.iqmap, M_CXGBE); 901 free(sc->sge.eqmap, M_CXGBE); 902 free(sc->tids.ftid_tab, M_CXGBE); 903 t4_destroy_dma_tag(sc); 904 if (mtx_initialized(&sc->sc_lock)) { 905 mtx_lock(&t4_list_lock); 906 SLIST_REMOVE(&t4_list, sc, adapter, link); 907 mtx_unlock(&t4_list_lock); 908 mtx_destroy(&sc->sc_lock); 909 } 910 911 if (mtx_initialized(&sc->tids.ftid_lock)) 912 mtx_destroy(&sc->tids.ftid_lock); 913 if (mtx_initialized(&sc->sfl_lock)) 914 mtx_destroy(&sc->sfl_lock); 915 916 bzero(sc, sizeof(*sc)); 917 918 return (0); 919} 920 921 922static int 923cxgbe_probe(device_t dev) 924{ 925 char buf[128]; 926 struct port_info *pi = device_get_softc(dev); 927 928 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 929 device_set_desc_copy(dev, buf); 930 931 return (BUS_PROBE_DEFAULT); 932} 933 934#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 935 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 936 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 937#define T4_CAP_ENABLE (T4_CAP) 938 939static int 940cxgbe_attach(device_t dev) 941{ 942 struct port_info *pi = device_get_softc(dev); 943 struct ifnet *ifp; 944 945 /* Allocate an ifnet and set it up */ 946 ifp = if_alloc(IFT_ETHER); 947 if (ifp == NULL) { 948 device_printf(dev, "Cannot allocate ifnet\n"); 949 return (ENOMEM); 950 } 951 pi->ifp = ifp; 952 ifp->if_softc = pi; 953 954 callout_init(&pi->tick, CALLOUT_MPSAFE); 955 956 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 957 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 958 959 ifp->if_init = cxgbe_init; 960 ifp->if_ioctl = cxgbe_ioctl; 961 ifp->if_transmit = cxgbe_transmit; 962 ifp->if_qflush = cxgbe_qflush; 963 964 ifp->if_capabilities = T4_CAP; 965#ifdef TCP_OFFLOAD 966 if (is_offload(pi->adapter)) 967 ifp->if_capabilities |= IFCAP_TOE; 968#endif 969 ifp->if_capenable = T4_CAP_ENABLE; 970 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 971 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 972 973 /* Initialize ifmedia for this port */ 974 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 975 cxgbe_media_status); 976 build_medialist(pi); 977 978 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 979 EVENTHANDLER_PRI_ANY); 980 981 ether_ifattach(ifp, pi->hw_addr); 982 983#ifdef TCP_OFFLOAD 984 if (is_offload(pi->adapter)) { 985 device_printf(dev, 986 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 987 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 988 } else 989#endif 990 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 991 992 cxgbe_sysctls(pi); 993 994 return (0); 995} 996 997static int 998cxgbe_detach(device_t dev) 999{ 1000 struct port_info *pi = device_get_softc(dev); 1001 struct adapter *sc = pi->adapter; 1002 struct ifnet *ifp = pi->ifp; 1003 1004 /* Tell if_ioctl and if_init that the port is going away */ 1005 ADAPTER_LOCK(sc); 1006 SET_DOOMED(pi); 1007 wakeup(&sc->flags); 1008 while (IS_BUSY(sc)) 1009 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1010 SET_BUSY(sc); 1011#ifdef INVARIANTS 1012 sc->last_op = "t4detach"; 1013 sc->last_op_thr = curthread; 1014#endif 1015 ADAPTER_UNLOCK(sc); 1016 1017 if (pi->vlan_c) 1018 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1019 1020 PORT_LOCK(pi); 1021 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1022 callout_stop(&pi->tick); 1023 PORT_UNLOCK(pi); 1024 callout_drain(&pi->tick); 1025 1026 /* Let detach proceed even if these fail. */ 1027 cxgbe_uninit_synchronized(pi); 1028 port_full_uninit(pi); 1029 1030 ifmedia_removeall(&pi->media); 1031 ether_ifdetach(pi->ifp); 1032 if_free(pi->ifp); 1033 1034 ADAPTER_LOCK(sc); 1035 CLR_BUSY(sc); 1036 wakeup(&sc->flags); 1037 ADAPTER_UNLOCK(sc); 1038 1039 return (0); 1040} 1041 1042static void 1043cxgbe_init(void *arg) 1044{ 1045 struct port_info *pi = arg; 1046 struct adapter *sc = pi->adapter; 1047 1048 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1049 return; 1050 cxgbe_init_synchronized(pi); 1051 end_synchronized_op(sc, 0); 1052} 1053 1054static int 1055cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1056{ 1057 int rc = 0, mtu, flags; 1058 struct port_info *pi = ifp->if_softc; 1059 struct adapter *sc = pi->adapter; 1060 struct ifreq *ifr = (struct ifreq *)data; 1061 uint32_t mask; 1062 1063 switch (cmd) { 1064 case SIOCSIFMTU: 1065 mtu = ifr->ifr_mtu; 1066 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1067 return (EINVAL); 1068 1069 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1070 if (rc) 1071 return (rc); 1072 ifp->if_mtu = mtu; 1073 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1074 t4_update_fl_bufsize(ifp); 1075 rc = update_mac_settings(pi, XGMAC_MTU); 1076 } 1077 end_synchronized_op(sc, 0); 1078 break; 1079 1080 case SIOCSIFFLAGS: 1081 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1082 if (rc) 1083 return (rc); 1084 1085 if (ifp->if_flags & IFF_UP) { 1086 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1087 flags = pi->if_flags; 1088 if ((ifp->if_flags ^ flags) & 1089 (IFF_PROMISC | IFF_ALLMULTI)) { 1090 rc = update_mac_settings(pi, 1091 XGMAC_PROMISC | XGMAC_ALLMULTI); 1092 } 1093 } else 1094 rc = cxgbe_init_synchronized(pi); 1095 pi->if_flags = ifp->if_flags; 1096 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1097 rc = cxgbe_uninit_synchronized(pi); 1098 end_synchronized_op(sc, 0); 1099 break; 1100 1101 case SIOCADDMULTI: 1102 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1103 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1104 if (rc) 1105 return (rc); 1106 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1107 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1108 end_synchronized_op(sc, LOCK_HELD); 1109 break; 1110 1111 case SIOCSIFCAP: 1112 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1113 if (rc) 1114 return (rc); 1115 1116 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1117 if (mask & IFCAP_TXCSUM) { 1118 ifp->if_capenable ^= IFCAP_TXCSUM; 1119 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1120 1121 if (IFCAP_TSO4 & ifp->if_capenable && 1122 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1123 ifp->if_capenable &= ~IFCAP_TSO4; 1124 if_printf(ifp, 1125 "tso4 disabled due to -txcsum.\n"); 1126 } 1127 } 1128 if (mask & IFCAP_TXCSUM_IPV6) { 1129 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1130 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1131 1132 if (IFCAP_TSO6 & ifp->if_capenable && 1133 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1134 ifp->if_capenable &= ~IFCAP_TSO6; 1135 if_printf(ifp, 1136 "tso6 disabled due to -txcsum6.\n"); 1137 } 1138 } 1139 if (mask & IFCAP_RXCSUM) 1140 ifp->if_capenable ^= IFCAP_RXCSUM; 1141 if (mask & IFCAP_RXCSUM_IPV6) 1142 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1143 1144 /* 1145 * Note that we leave CSUM_TSO alone (it is always set). The 1146 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1147 * sending a TSO request our way, so it's sufficient to toggle 1148 * IFCAP_TSOx only. 1149 */ 1150 if (mask & IFCAP_TSO4) { 1151 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1152 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1153 if_printf(ifp, "enable txcsum first.\n"); 1154 rc = EAGAIN; 1155 goto fail; 1156 } 1157 ifp->if_capenable ^= IFCAP_TSO4; 1158 } 1159 if (mask & IFCAP_TSO6) { 1160 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1161 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1162 if_printf(ifp, "enable txcsum6 first.\n"); 1163 rc = EAGAIN; 1164 goto fail; 1165 } 1166 ifp->if_capenable ^= IFCAP_TSO6; 1167 } 1168 if (mask & IFCAP_LRO) { 1169#if defined(INET) || defined(INET6) 1170 int i; 1171 struct sge_rxq *rxq; 1172 1173 ifp->if_capenable ^= IFCAP_LRO; 1174 for_each_rxq(pi, i, rxq) { 1175 if (ifp->if_capenable & IFCAP_LRO) 1176 rxq->iq.flags |= IQ_LRO_ENABLED; 1177 else 1178 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1179 } 1180#endif 1181 } 1182#ifdef TCP_OFFLOAD 1183 if (mask & IFCAP_TOE) { 1184 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1185 1186 rc = toe_capability(pi, enable); 1187 if (rc != 0) 1188 goto fail; 1189 1190 ifp->if_capenable ^= mask; 1191 } 1192#endif 1193 if (mask & IFCAP_VLAN_HWTAGGING) { 1194 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1195 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1196 rc = update_mac_settings(pi, XGMAC_VLANEX); 1197 } 1198 if (mask & IFCAP_VLAN_MTU) { 1199 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1200 1201 /* Need to find out how to disable auto-mtu-inflation */ 1202 } 1203 if (mask & IFCAP_VLAN_HWTSO) 1204 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1205 if (mask & IFCAP_VLAN_HWCSUM) 1206 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1207 1208#ifdef VLAN_CAPABILITIES 1209 VLAN_CAPABILITIES(ifp); 1210#endif 1211fail: 1212 end_synchronized_op(sc, 0); 1213 break; 1214 1215 case SIOCSIFMEDIA: 1216 case SIOCGIFMEDIA: 1217 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1218 break; 1219 1220 default: 1221 rc = ether_ioctl(ifp, cmd, data); 1222 } 1223 1224 return (rc); 1225} 1226 1227static int 1228cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1229{ 1230 struct port_info *pi = ifp->if_softc; 1231 struct adapter *sc = pi->adapter; 1232 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1233 struct buf_ring *br; 1234 int rc; 1235 1236 M_ASSERTPKTHDR(m); 1237 1238 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1239 m_freem(m); 1240 return (ENETDOWN); 1241 } 1242 1243 if (m->m_flags & M_FLOWID) 1244 txq += (m->m_pkthdr.flowid % pi->ntxq); 1245 br = txq->br; 1246 1247 if (TXQ_TRYLOCK(txq) == 0) { 1248 struct sge_eq *eq = &txq->eq; 1249 1250 /* 1251 * It is possible that t4_eth_tx finishes up and releases the 1252 * lock between the TRYLOCK above and the drbr_enqueue here. We 1253 * need to make sure that this mbuf doesn't just sit there in 1254 * the drbr. 1255 */ 1256 1257 rc = drbr_enqueue(ifp, br, m); 1258 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1259 !(eq->flags & EQ_DOOMED)) 1260 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1261 return (rc); 1262 } 1263 1264 /* 1265 * txq->m is the mbuf that is held up due to a temporary shortage of 1266 * resources and it should be put on the wire first. Then what's in 1267 * drbr and finally the mbuf that was just passed in to us. 1268 * 1269 * Return code should indicate the fate of the mbuf that was passed in 1270 * this time. 1271 */ 1272 1273 TXQ_LOCK_ASSERT_OWNED(txq); 1274 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1275 1276 /* Queued for transmission. */ 1277 1278 rc = drbr_enqueue(ifp, br, m); 1279 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1280 (void) t4_eth_tx(ifp, txq, m); 1281 TXQ_UNLOCK(txq); 1282 return (rc); 1283 } 1284 1285 /* Direct transmission. */ 1286 rc = t4_eth_tx(ifp, txq, m); 1287 if (rc != 0 && txq->m) 1288 rc = 0; /* held, will be transmitted soon (hopefully) */ 1289 1290 TXQ_UNLOCK(txq); 1291 return (rc); 1292} 1293 1294static void 1295cxgbe_qflush(struct ifnet *ifp) 1296{ 1297 struct port_info *pi = ifp->if_softc; 1298 struct sge_txq *txq; 1299 int i; 1300 struct mbuf *m; 1301 1302 /* queues do not exist if !PORT_INIT_DONE. */ 1303 if (pi->flags & PORT_INIT_DONE) { 1304 for_each_txq(pi, i, txq) { 1305 TXQ_LOCK(txq); 1306 m_freem(txq->m); 1307 txq->m = NULL; 1308 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1309 m_freem(m); 1310 TXQ_UNLOCK(txq); 1311 } 1312 } 1313 if_qflush(ifp); 1314} 1315 1316static int 1317cxgbe_media_change(struct ifnet *ifp) 1318{ 1319 struct port_info *pi = ifp->if_softc; 1320 1321 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1322 1323 return (EOPNOTSUPP); 1324} 1325 1326static void 1327cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1328{ 1329 struct port_info *pi = ifp->if_softc; 1330 struct ifmedia_entry *cur = pi->media.ifm_cur; 1331 int speed = pi->link_cfg.speed; 1332 int data = (pi->port_type << 8) | pi->mod_type; 1333 1334 if (cur->ifm_data != data) { 1335 build_medialist(pi); 1336 cur = pi->media.ifm_cur; 1337 } 1338 1339 ifmr->ifm_status = IFM_AVALID; 1340 if (!pi->link_cfg.link_ok) 1341 return; 1342 1343 ifmr->ifm_status |= IFM_ACTIVE; 1344 1345 /* active and current will differ iff current media is autoselect. */ 1346 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1347 return; 1348 1349 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1350 if (speed == SPEED_10000) 1351 ifmr->ifm_active |= IFM_10G_T; 1352 else if (speed == SPEED_1000) 1353 ifmr->ifm_active |= IFM_1000_T; 1354 else if (speed == SPEED_100) 1355 ifmr->ifm_active |= IFM_100_TX; 1356 else if (speed == SPEED_10) 1357 ifmr->ifm_active |= IFM_10_T; 1358 else 1359 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1360 speed)); 1361} 1362 1363void 1364t4_fatal_err(struct adapter *sc) 1365{ 1366 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1367 t4_intr_disable(sc); 1368 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1369 device_get_nameunit(sc->dev)); 1370} 1371 1372static int 1373map_bars_0_and_4(struct adapter *sc) 1374{ 1375 sc->regs_rid = PCIR_BAR(0); 1376 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1377 &sc->regs_rid, RF_ACTIVE); 1378 if (sc->regs_res == NULL) { 1379 device_printf(sc->dev, "cannot map registers.\n"); 1380 return (ENXIO); 1381 } 1382 sc->bt = rman_get_bustag(sc->regs_res); 1383 sc->bh = rman_get_bushandle(sc->regs_res); 1384 sc->mmio_len = rman_get_size(sc->regs_res); 1385 setbit(&sc->doorbells, DOORBELL_KDB); 1386 1387 sc->msix_rid = PCIR_BAR(4); 1388 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1389 &sc->msix_rid, RF_ACTIVE); 1390 if (sc->msix_res == NULL) { 1391 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1392 return (ENXIO); 1393 } 1394 1395 return (0); 1396} 1397 1398static int 1399map_bar_2(struct adapter *sc) 1400{ 1401 1402 /* 1403 * T4: only iWARP driver uses the userspace doorbells. There is no need 1404 * to map it if RDMA is disabled. 1405 */ 1406 if (is_t4(sc) && sc->rdmacaps == 0) 1407 return (0); 1408 1409 sc->udbs_rid = PCIR_BAR(2); 1410 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1411 &sc->udbs_rid, RF_ACTIVE); 1412 if (sc->udbs_res == NULL) { 1413 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1414 return (ENXIO); 1415 } 1416 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1417 1418 if (is_t5(sc)) { 1419 setbit(&sc->doorbells, DOORBELL_UDB); 1420#if defined(__i386__) || defined(__amd64__) 1421 if (t5_write_combine) { 1422 int rc; 1423 1424 /* 1425 * Enable write combining on BAR2. This is the 1426 * userspace doorbell BAR and is split into 128B 1427 * (UDBS_SEG_SIZE) doorbell regions, each associated 1428 * with an egress queue. The first 64B has the doorbell 1429 * and the second 64B can be used to submit a tx work 1430 * request with an implicit doorbell. 1431 */ 1432 1433 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1434 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1435 if (rc == 0) { 1436 clrbit(&sc->doorbells, DOORBELL_UDB); 1437 setbit(&sc->doorbells, DOORBELL_WCWR); 1438 setbit(&sc->doorbells, DOORBELL_UDBWC); 1439 } else { 1440 device_printf(sc->dev, 1441 "couldn't enable write combining: %d\n", 1442 rc); 1443 } 1444 1445 t4_write_reg(sc, A_SGE_STAT_CFG, 1446 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1447 } 1448#endif 1449 } 1450 1451 return (0); 1452} 1453 1454static const struct memwin t4_memwin[] = { 1455 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1456 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1457 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1458}; 1459 1460static const struct memwin t5_memwin[] = { 1461 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1462 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1463 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1464}; 1465 1466static void 1467setup_memwin(struct adapter *sc) 1468{ 1469 const struct memwin *mw; 1470 int i, n; 1471 uint32_t bar0; 1472 1473 if (is_t4(sc)) { 1474 /* 1475 * Read low 32b of bar0 indirectly via the hardware backdoor 1476 * mechanism. Works from within PCI passthrough environments 1477 * too, where rman_get_start() can return a different value. We 1478 * need to program the T4 memory window decoders with the actual 1479 * addresses that will be coming across the PCIe link. 1480 */ 1481 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1482 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1483 1484 mw = &t4_memwin[0]; 1485 n = nitems(t4_memwin); 1486 } else { 1487 /* T5 uses the relative offset inside the PCIe BAR */ 1488 bar0 = 0; 1489 1490 mw = &t5_memwin[0]; 1491 n = nitems(t5_memwin); 1492 } 1493 1494 for (i = 0; i < n; i++, mw++) { 1495 t4_write_reg(sc, 1496 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1497 (mw->base + bar0) | V_BIR(0) | 1498 V_WINDOW(ilog2(mw->aperture) - 10)); 1499 } 1500 1501 /* flush */ 1502 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1503} 1504 1505/* 1506 * Verify that the memory range specified by the addr/len pair is valid and lies 1507 * entirely within a single region (EDCx or MCx). 1508 */ 1509static int 1510validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1511{ 1512 uint32_t em, addr_len, maddr, mlen; 1513 1514 /* Memory can only be accessed in naturally aligned 4 byte units */ 1515 if (addr & 3 || len & 3 || len == 0) 1516 return (EINVAL); 1517 1518 /* Enabled memories */ 1519 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1520 if (em & F_EDRAM0_ENABLE) { 1521 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1522 maddr = G_EDRAM0_BASE(addr_len) << 20; 1523 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1524 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1525 addr + len <= maddr + mlen) 1526 return (0); 1527 } 1528 if (em & F_EDRAM1_ENABLE) { 1529 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1530 maddr = G_EDRAM1_BASE(addr_len) << 20; 1531 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1532 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1533 addr + len <= maddr + mlen) 1534 return (0); 1535 } 1536 if (em & F_EXT_MEM_ENABLE) { 1537 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1538 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1539 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1540 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1541 addr + len <= maddr + mlen) 1542 return (0); 1543 } 1544 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1545 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1546 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1547 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1548 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1549 addr + len <= maddr + mlen) 1550 return (0); 1551 } 1552 1553 return (EFAULT); 1554} 1555 1556/* 1557 * Verify that the memory range specified by the memtype/offset/len pair is 1558 * valid and lies entirely within the memtype specified. The global address of 1559 * the start of the range is returned in addr. 1560 */ 1561static int 1562validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1563 uint32_t *addr) 1564{ 1565 uint32_t em, addr_len, maddr, mlen; 1566 1567 /* Memory can only be accessed in naturally aligned 4 byte units */ 1568 if (off & 3 || len & 3 || len == 0) 1569 return (EINVAL); 1570 1571 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1572 switch (mtype) { 1573 case MEM_EDC0: 1574 if (!(em & F_EDRAM0_ENABLE)) 1575 return (EINVAL); 1576 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1577 maddr = G_EDRAM0_BASE(addr_len) << 20; 1578 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1579 break; 1580 case MEM_EDC1: 1581 if (!(em & F_EDRAM1_ENABLE)) 1582 return (EINVAL); 1583 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1584 maddr = G_EDRAM1_BASE(addr_len) << 20; 1585 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1586 break; 1587 case MEM_MC: 1588 if (!(em & F_EXT_MEM_ENABLE)) 1589 return (EINVAL); 1590 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1591 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1592 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1593 break; 1594 case MEM_MC1: 1595 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1596 return (EINVAL); 1597 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1598 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1599 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1600 break; 1601 default: 1602 return (EINVAL); 1603 } 1604 1605 if (mlen > 0 && off < mlen && off + len <= mlen) { 1606 *addr = maddr + off; /* global address */ 1607 return (0); 1608 } 1609 1610 return (EFAULT); 1611} 1612 1613static void 1614memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1615{ 1616 const struct memwin *mw; 1617 1618 if (is_t4(sc)) { 1619 KASSERT(win >= 0 && win < nitems(t4_memwin), 1620 ("%s: incorrect memwin# (%d)", __func__, win)); 1621 mw = &t4_memwin[win]; 1622 } else { 1623 KASSERT(win >= 0 && win < nitems(t5_memwin), 1624 ("%s: incorrect memwin# (%d)", __func__, win)); 1625 mw = &t5_memwin[win]; 1626 } 1627 1628 if (base != NULL) 1629 *base = mw->base; 1630 if (aperture != NULL) 1631 *aperture = mw->aperture; 1632} 1633 1634/* 1635 * Positions the memory window such that it can be used to access the specified 1636 * address in the chip's address space. The return value is the offset of addr 1637 * from the start of the window. 1638 */ 1639static uint32_t 1640position_memwin(struct adapter *sc, int n, uint32_t addr) 1641{ 1642 uint32_t start, pf; 1643 uint32_t reg; 1644 1645 KASSERT(n >= 0 && n <= 3, 1646 ("%s: invalid window %d.", __func__, n)); 1647 KASSERT((addr & 3) == 0, 1648 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1649 1650 if (is_t4(sc)) { 1651 pf = 0; 1652 start = addr & ~0xf; /* start must be 16B aligned */ 1653 } else { 1654 pf = V_PFNUM(sc->pf); 1655 start = addr & ~0x7f; /* start must be 128B aligned */ 1656 } 1657 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1658 1659 t4_write_reg(sc, reg, start | pf); 1660 t4_read_reg(sc, reg); 1661 1662 return (addr - start); 1663} 1664 1665static int 1666cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1667 struct intrs_and_queues *iaq) 1668{ 1669 int rc, itype, navail, nrxq10g, nrxq1g, n; 1670 int nofldrxq10g = 0, nofldrxq1g = 0; 1671 1672 bzero(iaq, sizeof(*iaq)); 1673 1674 iaq->ntxq10g = t4_ntxq10g; 1675 iaq->ntxq1g = t4_ntxq1g; 1676 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1677 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1678#ifdef TCP_OFFLOAD 1679 if (is_offload(sc)) { 1680 iaq->nofldtxq10g = t4_nofldtxq10g; 1681 iaq->nofldtxq1g = t4_nofldtxq1g; 1682 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1683 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1684 } 1685#endif 1686 1687 for (itype = INTR_MSIX; itype; itype >>= 1) { 1688 1689 if ((itype & t4_intr_types) == 0) 1690 continue; /* not allowed */ 1691 1692 if (itype == INTR_MSIX) 1693 navail = pci_msix_count(sc->dev); 1694 else if (itype == INTR_MSI) 1695 navail = pci_msi_count(sc->dev); 1696 else 1697 navail = 1; 1698restart: 1699 if (navail == 0) 1700 continue; 1701 1702 iaq->intr_type = itype; 1703 iaq->intr_flags = 0; 1704 1705 /* 1706 * Best option: an interrupt vector for errors, one for the 1707 * firmware event queue, and one each for each rxq (NIC as well 1708 * as offload). 1709 */ 1710 iaq->nirq = T4_EXTRA_INTR; 1711 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1712 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1713 if (iaq->nirq <= navail && 1714 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1715 iaq->intr_flags |= INTR_DIRECT; 1716 goto allocate; 1717 } 1718 1719 /* 1720 * Second best option: an interrupt vector for errors, one for 1721 * the firmware event queue, and one each for either NIC or 1722 * offload rxq's. 1723 */ 1724 iaq->nirq = T4_EXTRA_INTR; 1725 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1726 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1727 if (iaq->nirq <= navail && 1728 (itype != INTR_MSI || powerof2(iaq->nirq))) 1729 goto allocate; 1730 1731 /* 1732 * Next best option: an interrupt vector for errors, one for the 1733 * firmware event queue, and at least one per port. At this 1734 * point we know we'll have to downsize nrxq or nofldrxq to fit 1735 * what's available to us. 1736 */ 1737 iaq->nirq = T4_EXTRA_INTR; 1738 iaq->nirq += n10g + n1g; 1739 if (iaq->nirq <= navail) { 1740 int leftover = navail - iaq->nirq; 1741 1742 if (n10g > 0) { 1743 int target = max(nrxq10g, nofldrxq10g); 1744 1745 n = 1; 1746 while (n < target && leftover >= n10g) { 1747 leftover -= n10g; 1748 iaq->nirq += n10g; 1749 n++; 1750 } 1751 iaq->nrxq10g = min(n, nrxq10g); 1752#ifdef TCP_OFFLOAD 1753 if (is_offload(sc)) 1754 iaq->nofldrxq10g = min(n, nofldrxq10g); 1755#endif 1756 } 1757 1758 if (n1g > 0) { 1759 int target = max(nrxq1g, nofldrxq1g); 1760 1761 n = 1; 1762 while (n < target && leftover >= n1g) { 1763 leftover -= n1g; 1764 iaq->nirq += n1g; 1765 n++; 1766 } 1767 iaq->nrxq1g = min(n, nrxq1g); 1768#ifdef TCP_OFFLOAD 1769 if (is_offload(sc)) 1770 iaq->nofldrxq1g = min(n, nofldrxq1g); 1771#endif 1772 } 1773 1774 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1775 goto allocate; 1776 } 1777 1778 /* 1779 * Least desirable option: one interrupt vector for everything. 1780 */ 1781 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1782#ifdef TCP_OFFLOAD 1783 if (is_offload(sc)) 1784 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1785#endif 1786 1787allocate: 1788 navail = iaq->nirq; 1789 rc = 0; 1790 if (itype == INTR_MSIX) 1791 rc = pci_alloc_msix(sc->dev, &navail); 1792 else if (itype == INTR_MSI) 1793 rc = pci_alloc_msi(sc->dev, &navail); 1794 1795 if (rc == 0) { 1796 if (navail == iaq->nirq) 1797 return (0); 1798 1799 /* 1800 * Didn't get the number requested. Use whatever number 1801 * the kernel is willing to allocate (it's in navail). 1802 */ 1803 device_printf(sc->dev, "fewer vectors than requested, " 1804 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1805 itype, iaq->nirq, navail); 1806 pci_release_msi(sc->dev); 1807 goto restart; 1808 } 1809 1810 device_printf(sc->dev, 1811 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1812 itype, rc, iaq->nirq, navail); 1813 } 1814 1815 device_printf(sc->dev, 1816 "failed to find a usable interrupt type. " 1817 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1818 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1819 1820 return (ENXIO); 1821} 1822 1823#define FW_VERSION(chip) ( \ 1824 V_FW_HDR_FW_VER_MAJOR(FW_VERSION_MAJOR_##chip) | \ 1825 V_FW_HDR_FW_VER_MINOR(FW_VERSION_MINOR_##chip) | \ 1826 V_FW_HDR_FW_VER_MICRO(FW_VERSION_MICRO_##chip) | \ 1827 V_FW_HDR_FW_VER_BUILD(FW_VERSION_BUILD_##chip)) 1828#define FW_INTFVER(chip, intf) (FW_HDR_INTFVER_##intf) 1829 1830struct fw_info { 1831 uint8_t chip; 1832 char *kld_name; 1833 char *fw_mod_name; 1834 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 1835} fw_info[] = { 1836 { 1837 .chip = CHELSIO_T4, 1838 .kld_name = "t4fw_cfg", 1839 .fw_mod_name = "t4fw", 1840 .fw_hdr = { 1841 .chip = FW_HDR_CHIP_T4, 1842 .fw_ver = htobe32_const(FW_VERSION(T4)), 1843 .intfver_nic = FW_INTFVER(T4, NIC), 1844 .intfver_vnic = FW_INTFVER(T4, VNIC), 1845 .intfver_ofld = FW_INTFVER(T4, OFLD), 1846 .intfver_ri = FW_INTFVER(T4, RI), 1847 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 1848 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 1849 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 1850 .intfver_fcoe = FW_INTFVER(T4, FCOE), 1851 }, 1852 }, { 1853 .chip = CHELSIO_T5, 1854 .kld_name = "t5fw_cfg", 1855 .fw_mod_name = "t5fw", 1856 .fw_hdr = { 1857 .chip = FW_HDR_CHIP_T5, 1858 .fw_ver = htobe32_const(FW_VERSION(T5)), 1859 .intfver_nic = FW_INTFVER(T5, NIC), 1860 .intfver_vnic = FW_INTFVER(T5, VNIC), 1861 .intfver_ofld = FW_INTFVER(T5, OFLD), 1862 .intfver_ri = FW_INTFVER(T5, RI), 1863 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 1864 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 1865 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 1866 .intfver_fcoe = FW_INTFVER(T5, FCOE), 1867 }, 1868 } 1869}; 1870 1871static struct fw_info * 1872find_fw_info(int chip) 1873{ 1874 int i; 1875 1876 for (i = 0; i < nitems(fw_info); i++) { 1877 if (fw_info[i].chip == chip) 1878 return (&fw_info[i]); 1879 } 1880 return (NULL); 1881} 1882 1883/* 1884 * Is the given firmware API compatible with the one the driver was compiled 1885 * with? 1886 */ 1887static int 1888fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 1889{ 1890 1891 /* short circuit if it's the exact same firmware version */ 1892 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 1893 return (1); 1894 1895 /* 1896 * XXX: Is this too conservative? Perhaps I should limit this to the 1897 * features that are supported in the driver. 1898 */ 1899#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 1900 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 1901 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 1902 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 1903 return (1); 1904#undef SAME_INTF 1905 1906 return (0); 1907} 1908 1909/* 1910 * The firmware in the KLD is usable and can be installed. But should it be? 1911 * This routine explains itself in detail if it indicates the KLD firmware 1912 * should be installed. 1913 */ 1914static int 1915should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 1916{ 1917 const char *reason; 1918 1919 KASSERT(t4_fw_install != 0, ("%s: Can't install; shouldn't be asked " 1920 "to evaluate if install is a good idea.", __func__)); 1921 1922 if (!card_fw_usable) { 1923 reason = "incompatible or unusable"; 1924 goto install; 1925 } 1926 1927 if (k > c) { 1928 reason = "older than the version bundled with this driver"; 1929 goto install; 1930 } 1931 1932 if (t4_fw_install == 2 && k != c) { 1933 reason = "different than the version bundled with this driver"; 1934 goto install; 1935 } 1936 1937 return (0); 1938 1939install: 1940 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 1941 "installing firmware %u.%u.%u.%u on card.\n", 1942 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 1943 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 1944 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 1945 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 1946 1947 return (1); 1948} 1949/* 1950 * Establish contact with the firmware and determine if we are the master driver 1951 * or not, and whether we are responsible for chip initialization. 1952 */ 1953static int 1954prep_firmware(struct adapter *sc) 1955{ 1956 const struct firmware *fw = NULL, *default_cfg; 1957 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 1958 enum dev_state state; 1959 struct fw_info *fw_info; 1960 struct fw_hdr *card_fw; /* fw on the card */ 1961 const struct fw_hdr *kld_fw; /* fw in the KLD */ 1962 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 1963 against */ 1964 1965 /* Contact firmware. */ 1966 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1967 if (rc < 0 || state == DEV_STATE_ERR) { 1968 rc = -rc; 1969 device_printf(sc->dev, 1970 "failed to connect to the firmware: %d, %d.\n", rc, state); 1971 return (rc); 1972 } 1973 pf = rc; 1974 if (pf == sc->mbox) 1975 sc->flags |= MASTER_PF; 1976 else if (state == DEV_STATE_UNINIT) { 1977 /* 1978 * We didn't get to be the master so we definitely won't be 1979 * configuring the chip. It's a bug if someone else hasn't 1980 * configured it already. 1981 */ 1982 device_printf(sc->dev, "couldn't be master(%d), " 1983 "device not already initialized either(%d).\n", rc, state); 1984 return (EDOOFUS); 1985 } 1986 1987 /* This is the firmware whose headers the driver was compiled against */ 1988 fw_info = find_fw_info(chip_id(sc)); 1989 if (fw_info == NULL) { 1990 device_printf(sc->dev, 1991 "unable to look up firmware information for chip %d.\n", 1992 chip_id(sc)); 1993 return (EINVAL); 1994 } 1995 drv_fw = &fw_info->fw_hdr; 1996 1997 /* 1998 * The firmware KLD contains many modules. The KLD name is also the 1999 * name of the module that contains the default config file. 2000 */ 2001 default_cfg = firmware_get(fw_info->kld_name); 2002 2003 /* Read the header of the firmware on the card */ 2004 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2005 rc = -t4_read_flash(sc, FLASH_FW_START, 2006 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2007 if (rc == 0) 2008 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2009 else { 2010 device_printf(sc->dev, 2011 "Unable to read card's firmware header: %d\n", rc); 2012 card_fw_usable = 0; 2013 } 2014 2015 /* This is the firmware in the KLD */ 2016 fw = firmware_get(fw_info->fw_mod_name); 2017 if (fw != NULL) { 2018 kld_fw = (const void *)fw->data; 2019 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2020 } else { 2021 kld_fw = NULL; 2022 kld_fw_usable = 0; 2023 } 2024 2025 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2026 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver || 2027 t4_fw_install == 0)) { 2028 /* 2029 * Common case: the firmware on the card is an exact match and 2030 * the KLD is an exact match too, or the KLD is 2031 * absent/incompatible, or we're prohibited from using it. Note 2032 * that t4_fw_install = 2 is ignored here -- use cxgbetool 2033 * loadfw if you want to reinstall the same firmware as the one 2034 * on the card. 2035 */ 2036 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2037 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2038 be32toh(card_fw->fw_ver))) { 2039 2040 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2041 if (rc != 0) { 2042 device_printf(sc->dev, 2043 "failed to install firmware: %d\n", rc); 2044 goto done; 2045 } 2046 2047 /* Installed successfully, update the cached header too. */ 2048 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2049 card_fw_usable = 1; 2050 need_fw_reset = 0; /* already reset as part of load_fw */ 2051 } 2052 2053 if (!card_fw_usable) { 2054 uint32_t d, c, k; 2055 2056 d = ntohl(drv_fw->fw_ver); 2057 c = ntohl(card_fw->fw_ver); 2058 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2059 2060 device_printf(sc->dev, "Cannot find a usable firmware: " 2061 "fw_install %d, chip state %d, " 2062 "driver compiled with %d.%d.%d.%d, " 2063 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2064 t4_fw_install, state, 2065 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2066 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2067 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2068 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2069 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2070 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2071 rc = EINVAL; 2072 goto done; 2073 } 2074 2075 /* We're using whatever's on the card and it's known to be good. */ 2076 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2077 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2078 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2079 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2080 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2081 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2082 2083 /* Reset device */ 2084 if (need_fw_reset && 2085 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2086 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2087 if (rc != ETIMEDOUT && rc != EIO) 2088 t4_fw_bye(sc, sc->mbox); 2089 goto done; 2090 } 2091 sc->flags |= FW_OK; 2092 2093 rc = get_params__pre_init(sc); 2094 if (rc != 0) 2095 goto done; /* error message displayed already */ 2096 2097 /* Partition adapter resources as specified in the config file. */ 2098 if (state == DEV_STATE_UNINIT) { 2099 2100 KASSERT(sc->flags & MASTER_PF, 2101 ("%s: trying to change chip settings when not master.", 2102 __func__)); 2103 2104 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2105 if (rc != 0) 2106 goto done; /* error message displayed already */ 2107 2108 t4_tweak_chip_settings(sc); 2109 2110 /* get basic stuff going */ 2111 rc = -t4_fw_initialize(sc, sc->mbox); 2112 if (rc != 0) { 2113 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2114 goto done; 2115 } 2116 } else { 2117 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2118 sc->cfcsum = 0; 2119 } 2120 2121done: 2122 free(card_fw, M_CXGBE); 2123 if (fw != NULL) 2124 firmware_put(fw, FIRMWARE_UNLOAD); 2125 if (default_cfg != NULL) 2126 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2127 2128 return (rc); 2129} 2130 2131#define FW_PARAM_DEV(param) \ 2132 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2133 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2134#define FW_PARAM_PFVF(param) \ 2135 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2136 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2137 2138/* 2139 * Partition chip resources for use between various PFs, VFs, etc. 2140 */ 2141static int 2142partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2143 const char *name_prefix) 2144{ 2145 const struct firmware *cfg = NULL; 2146 int rc = 0; 2147 struct fw_caps_config_cmd caps; 2148 uint32_t mtype, moff, finicsum, cfcsum; 2149 2150 /* 2151 * Figure out what configuration file to use. Pick the default config 2152 * file for the card if the user hasn't specified one explicitly. 2153 */ 2154 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2155 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2156 /* Card specific overrides go here. */ 2157 if (pci_get_device(sc->dev) == 0x440a) 2158 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2159 if (is_fpga(sc)) 2160 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2161 } 2162 2163 /* 2164 * We need to load another module if the profile is anything except 2165 * "default" or "flash". 2166 */ 2167 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2168 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2169 char s[32]; 2170 2171 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2172 cfg = firmware_get(s); 2173 if (cfg == NULL) { 2174 if (default_cfg != NULL) { 2175 device_printf(sc->dev, 2176 "unable to load module \"%s\" for " 2177 "configuration profile \"%s\", will use " 2178 "the default config file instead.\n", 2179 s, sc->cfg_file); 2180 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2181 "%s", DEFAULT_CF); 2182 } else { 2183 device_printf(sc->dev, 2184 "unable to load module \"%s\" for " 2185 "configuration profile \"%s\", will use " 2186 "the config file on the card's flash " 2187 "instead.\n", s, sc->cfg_file); 2188 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2189 "%s", FLASH_CF); 2190 } 2191 } 2192 } 2193 2194 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2195 default_cfg == NULL) { 2196 device_printf(sc->dev, 2197 "default config file not available, will use the config " 2198 "file on the card's flash instead.\n"); 2199 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2200 } 2201 2202 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2203 u_int cflen, i, n; 2204 const uint32_t *cfdata; 2205 uint32_t param, val, addr, off, mw_base, mw_aperture; 2206 2207 KASSERT(cfg != NULL || default_cfg != NULL, 2208 ("%s: no config to upload", __func__)); 2209 2210 /* 2211 * Ask the firmware where it wants us to upload the config file. 2212 */ 2213 param = FW_PARAM_DEV(CF); 2214 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2215 if (rc != 0) { 2216 /* No support for config file? Shouldn't happen. */ 2217 device_printf(sc->dev, 2218 "failed to query config file location: %d.\n", rc); 2219 goto done; 2220 } 2221 mtype = G_FW_PARAMS_PARAM_Y(val); 2222 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2223 2224 /* 2225 * XXX: sheer laziness. We deliberately added 4 bytes of 2226 * useless stuffing/comments at the end of the config file so 2227 * it's ok to simply throw away the last remaining bytes when 2228 * the config file is not an exact multiple of 4. This also 2229 * helps with the validate_mt_off_len check. 2230 */ 2231 if (cfg != NULL) { 2232 cflen = cfg->datasize & ~3; 2233 cfdata = cfg->data; 2234 } else { 2235 cflen = default_cfg->datasize & ~3; 2236 cfdata = default_cfg->data; 2237 } 2238 2239 if (cflen > FLASH_CFG_MAX_SIZE) { 2240 device_printf(sc->dev, 2241 "config file too long (%d, max allowed is %d). " 2242 "Will try to use the config on the card, if any.\n", 2243 cflen, FLASH_CFG_MAX_SIZE); 2244 goto use_config_on_flash; 2245 } 2246 2247 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2248 if (rc != 0) { 2249 device_printf(sc->dev, 2250 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2251 "Will try to use the config on the card, if any.\n", 2252 __func__, mtype, moff, cflen, rc); 2253 goto use_config_on_flash; 2254 } 2255 2256 memwin_info(sc, 2, &mw_base, &mw_aperture); 2257 while (cflen) { 2258 off = position_memwin(sc, 2, addr); 2259 n = min(cflen, mw_aperture - off); 2260 for (i = 0; i < n; i += 4) 2261 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2262 cflen -= n; 2263 addr += n; 2264 } 2265 } else { 2266use_config_on_flash: 2267 mtype = FW_MEMTYPE_CF_FLASH; 2268 moff = t4_flash_cfg_addr(sc); 2269 } 2270 2271 bzero(&caps, sizeof(caps)); 2272 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2273 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2274 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2275 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2276 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2277 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2278 if (rc != 0) { 2279 device_printf(sc->dev, 2280 "failed to pre-process config file: %d " 2281 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2282 goto done; 2283 } 2284 2285 finicsum = be32toh(caps.finicsum); 2286 cfcsum = be32toh(caps.cfcsum); 2287 if (finicsum != cfcsum) { 2288 device_printf(sc->dev, 2289 "WARNING: config file checksum mismatch: %08x %08x\n", 2290 finicsum, cfcsum); 2291 } 2292 sc->cfcsum = cfcsum; 2293 2294#define LIMIT_CAPS(x) do { \ 2295 caps.x &= htobe16(t4_##x##_allowed); \ 2296 sc->x = htobe16(caps.x); \ 2297} while (0) 2298 2299 /* 2300 * Let the firmware know what features will (not) be used so it can tune 2301 * things accordingly. 2302 */ 2303 LIMIT_CAPS(linkcaps); 2304 LIMIT_CAPS(niccaps); 2305 LIMIT_CAPS(toecaps); 2306 LIMIT_CAPS(rdmacaps); 2307 LIMIT_CAPS(iscsicaps); 2308 LIMIT_CAPS(fcoecaps); 2309#undef LIMIT_CAPS 2310 2311 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2312 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2313 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2314 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2315 if (rc != 0) { 2316 device_printf(sc->dev, 2317 "failed to process config file: %d.\n", rc); 2318 } 2319done: 2320 if (cfg != NULL) 2321 firmware_put(cfg, FIRMWARE_UNLOAD); 2322 return (rc); 2323} 2324 2325/* 2326 * Retrieve parameters that are needed (or nice to have) very early. 2327 */ 2328static int 2329get_params__pre_init(struct adapter *sc) 2330{ 2331 int rc; 2332 uint32_t param[2], val[2]; 2333 struct fw_devlog_cmd cmd; 2334 struct devlog_params *dlog = &sc->params.devlog; 2335 2336 param[0] = FW_PARAM_DEV(PORTVEC); 2337 param[1] = FW_PARAM_DEV(CCLK); 2338 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2339 if (rc != 0) { 2340 device_printf(sc->dev, 2341 "failed to query parameters (pre_init): %d.\n", rc); 2342 return (rc); 2343 } 2344 2345 sc->params.portvec = val[0]; 2346 sc->params.nports = bitcount32(val[0]); 2347 sc->params.vpd.cclk = val[1]; 2348 2349 /* Read device log parameters. */ 2350 bzero(&cmd, sizeof(cmd)); 2351 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2352 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2353 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2354 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2355 if (rc != 0) { 2356 device_printf(sc->dev, 2357 "failed to get devlog parameters: %d.\n", rc); 2358 bzero(dlog, sizeof (*dlog)); 2359 rc = 0; /* devlog isn't critical for device operation */ 2360 } else { 2361 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2362 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2363 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2364 dlog->size = be32toh(cmd.memsize_devlog); 2365 } 2366 2367 return (rc); 2368} 2369 2370/* 2371 * Retrieve various parameters that are of interest to the driver. The device 2372 * has been initialized by the firmware at this point. 2373 */ 2374static int 2375get_params__post_init(struct adapter *sc) 2376{ 2377 int rc; 2378 uint32_t param[7], val[7]; 2379 struct fw_caps_config_cmd caps; 2380 2381 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2382 param[1] = FW_PARAM_PFVF(EQ_START); 2383 param[2] = FW_PARAM_PFVF(FILTER_START); 2384 param[3] = FW_PARAM_PFVF(FILTER_END); 2385 param[4] = FW_PARAM_PFVF(L2T_START); 2386 param[5] = FW_PARAM_PFVF(L2T_END); 2387 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2388 if (rc != 0) { 2389 device_printf(sc->dev, 2390 "failed to query parameters (post_init): %d.\n", rc); 2391 return (rc); 2392 } 2393 2394 sc->sge.iq_start = val[0]; 2395 sc->sge.eq_start = val[1]; 2396 sc->tids.ftid_base = val[2]; 2397 sc->tids.nftids = val[3] - val[2] + 1; 2398 sc->vres.l2t.start = val[4]; 2399 sc->vres.l2t.size = val[5] - val[4] + 1; 2400 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2401 ("%s: L2 table size (%u) larger than expected (%u)", 2402 __func__, sc->vres.l2t.size, L2T_SIZE)); 2403 2404 /* get capabilites */ 2405 bzero(&caps, sizeof(caps)); 2406 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2407 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2408 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2409 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2410 if (rc != 0) { 2411 device_printf(sc->dev, 2412 "failed to get card capabilities: %d.\n", rc); 2413 return (rc); 2414 } 2415 2416 if (caps.toecaps) { 2417 /* query offload-related parameters */ 2418 param[0] = FW_PARAM_DEV(NTID); 2419 param[1] = FW_PARAM_PFVF(SERVER_START); 2420 param[2] = FW_PARAM_PFVF(SERVER_END); 2421 param[3] = FW_PARAM_PFVF(TDDP_START); 2422 param[4] = FW_PARAM_PFVF(TDDP_END); 2423 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2424 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2425 if (rc != 0) { 2426 device_printf(sc->dev, 2427 "failed to query TOE parameters: %d.\n", rc); 2428 return (rc); 2429 } 2430 sc->tids.ntids = val[0]; 2431 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2432 sc->tids.stid_base = val[1]; 2433 sc->tids.nstids = val[2] - val[1] + 1; 2434 sc->vres.ddp.start = val[3]; 2435 sc->vres.ddp.size = val[4] - val[3] + 1; 2436 sc->params.ofldq_wr_cred = val[5]; 2437 sc->params.offload = 1; 2438 } 2439 if (caps.rdmacaps) { 2440 param[0] = FW_PARAM_PFVF(STAG_START); 2441 param[1] = FW_PARAM_PFVF(STAG_END); 2442 param[2] = FW_PARAM_PFVF(RQ_START); 2443 param[3] = FW_PARAM_PFVF(RQ_END); 2444 param[4] = FW_PARAM_PFVF(PBL_START); 2445 param[5] = FW_PARAM_PFVF(PBL_END); 2446 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2447 if (rc != 0) { 2448 device_printf(sc->dev, 2449 "failed to query RDMA parameters(1): %d.\n", rc); 2450 return (rc); 2451 } 2452 sc->vres.stag.start = val[0]; 2453 sc->vres.stag.size = val[1] - val[0] + 1; 2454 sc->vres.rq.start = val[2]; 2455 sc->vres.rq.size = val[3] - val[2] + 1; 2456 sc->vres.pbl.start = val[4]; 2457 sc->vres.pbl.size = val[5] - val[4] + 1; 2458 2459 param[0] = FW_PARAM_PFVF(SQRQ_START); 2460 param[1] = FW_PARAM_PFVF(SQRQ_END); 2461 param[2] = FW_PARAM_PFVF(CQ_START); 2462 param[3] = FW_PARAM_PFVF(CQ_END); 2463 param[4] = FW_PARAM_PFVF(OCQ_START); 2464 param[5] = FW_PARAM_PFVF(OCQ_END); 2465 rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 2466 if (rc != 0) { 2467 device_printf(sc->dev, 2468 "failed to query RDMA parameters(2): %d.\n", rc); 2469 return (rc); 2470 } 2471 sc->vres.qp.start = val[0]; 2472 sc->vres.qp.size = val[1] - val[0] + 1; 2473 sc->vres.cq.start = val[2]; 2474 sc->vres.cq.size = val[3] - val[2] + 1; 2475 sc->vres.ocq.start = val[4]; 2476 sc->vres.ocq.size = val[5] - val[4] + 1; 2477 } 2478 if (caps.iscsicaps) { 2479 param[0] = FW_PARAM_PFVF(ISCSI_START); 2480 param[1] = FW_PARAM_PFVF(ISCSI_END); 2481 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2482 if (rc != 0) { 2483 device_printf(sc->dev, 2484 "failed to query iSCSI parameters: %d.\n", rc); 2485 return (rc); 2486 } 2487 sc->vres.iscsi.start = val[0]; 2488 sc->vres.iscsi.size = val[1] - val[0] + 1; 2489 } 2490 2491 /* 2492 * We've got the params we wanted to query via the firmware. Now grab 2493 * some others directly from the chip. 2494 */ 2495 rc = t4_read_chip_settings(sc); 2496 2497 return (rc); 2498} 2499 2500static int 2501set_params__post_init(struct adapter *sc) 2502{ 2503 uint32_t param, val; 2504 2505 /* ask for encapsulated CPLs */ 2506 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2507 val = 1; 2508 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2509 2510 return (0); 2511} 2512 2513#undef FW_PARAM_PFVF 2514#undef FW_PARAM_DEV 2515 2516static void 2517t4_set_desc(struct adapter *sc) 2518{ 2519 char buf[128]; 2520 struct adapter_params *p = &sc->params; 2521 2522 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 2523 p->vpd.id, is_offload(sc) ? "R" : "", chip_rev(sc), p->vpd.sn, 2524 p->vpd.ec); 2525 2526 device_set_desc_copy(sc->dev, buf); 2527} 2528 2529static void 2530build_medialist(struct port_info *pi) 2531{ 2532 struct ifmedia *media = &pi->media; 2533 int data, m; 2534 2535 PORT_LOCK(pi); 2536 2537 ifmedia_removeall(media); 2538 2539 m = IFM_ETHER | IFM_FDX; 2540 data = (pi->port_type << 8) | pi->mod_type; 2541 2542 switch(pi->port_type) { 2543 case FW_PORT_TYPE_BT_XFI: 2544 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2545 break; 2546 2547 case FW_PORT_TYPE_BT_XAUI: 2548 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2549 /* fall through */ 2550 2551 case FW_PORT_TYPE_BT_SGMII: 2552 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2553 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2554 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2555 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2556 break; 2557 2558 case FW_PORT_TYPE_CX4: 2559 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2560 ifmedia_set(media, m | IFM_10G_CX4); 2561 break; 2562 2563 case FW_PORT_TYPE_SFP: 2564 case FW_PORT_TYPE_FIBER_XFI: 2565 case FW_PORT_TYPE_FIBER_XAUI: 2566 switch (pi->mod_type) { 2567 2568 case FW_PORT_MOD_TYPE_LR: 2569 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2570 ifmedia_set(media, m | IFM_10G_LR); 2571 break; 2572 2573 case FW_PORT_MOD_TYPE_SR: 2574 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2575 ifmedia_set(media, m | IFM_10G_SR); 2576 break; 2577 2578 case FW_PORT_MOD_TYPE_LRM: 2579 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2580 ifmedia_set(media, m | IFM_10G_LRM); 2581 break; 2582 2583 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2584 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2585 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2586 ifmedia_set(media, m | IFM_10G_TWINAX); 2587 break; 2588 2589 case FW_PORT_MOD_TYPE_NONE: 2590 m &= ~IFM_FDX; 2591 ifmedia_add(media, m | IFM_NONE, data, NULL); 2592 ifmedia_set(media, m | IFM_NONE); 2593 break; 2594 2595 case FW_PORT_MOD_TYPE_NA: 2596 case FW_PORT_MOD_TYPE_ER: 2597 default: 2598 device_printf(pi->dev, 2599 "unknown port_type (%d), mod_type (%d)\n", 2600 pi->port_type, pi->mod_type); 2601 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2602 ifmedia_set(media, m | IFM_UNKNOWN); 2603 break; 2604 } 2605 break; 2606 2607 case FW_PORT_TYPE_QSFP: 2608 switch (pi->mod_type) { 2609 2610 case FW_PORT_MOD_TYPE_LR: 2611 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2612 ifmedia_set(media, m | IFM_40G_LR4); 2613 break; 2614 2615 case FW_PORT_MOD_TYPE_SR: 2616 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2617 ifmedia_set(media, m | IFM_40G_SR4); 2618 break; 2619 2620 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2621 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2622 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2623 ifmedia_set(media, m | IFM_40G_CR4); 2624 break; 2625 2626 case FW_PORT_MOD_TYPE_NONE: 2627 m &= ~IFM_FDX; 2628 ifmedia_add(media, m | IFM_NONE, data, NULL); 2629 ifmedia_set(media, m | IFM_NONE); 2630 break; 2631 2632 default: 2633 device_printf(pi->dev, 2634 "unknown port_type (%d), mod_type (%d)\n", 2635 pi->port_type, pi->mod_type); 2636 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2637 ifmedia_set(media, m | IFM_UNKNOWN); 2638 break; 2639 } 2640 break; 2641 2642 default: 2643 device_printf(pi->dev, 2644 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2645 pi->mod_type); 2646 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2647 ifmedia_set(media, m | IFM_UNKNOWN); 2648 break; 2649 } 2650 2651 PORT_UNLOCK(pi); 2652} 2653 2654#define FW_MAC_EXACT_CHUNK 7 2655 2656/* 2657 * Program the port's XGMAC based on parameters in ifnet. The caller also 2658 * indicates which parameters should be programmed (the rest are left alone). 2659 */ 2660static int 2661update_mac_settings(struct port_info *pi, int flags) 2662{ 2663 int rc; 2664 struct ifnet *ifp = pi->ifp; 2665 struct adapter *sc = pi->adapter; 2666 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2667 2668 ASSERT_SYNCHRONIZED_OP(sc); 2669 KASSERT(flags, ("%s: not told what to update.", __func__)); 2670 2671 if (flags & XGMAC_MTU) 2672 mtu = ifp->if_mtu; 2673 2674 if (flags & XGMAC_PROMISC) 2675 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2676 2677 if (flags & XGMAC_ALLMULTI) 2678 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2679 2680 if (flags & XGMAC_VLANEX) 2681 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2682 2683 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2684 vlanex, false); 2685 if (rc) { 2686 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2687 return (rc); 2688 } 2689 2690 if (flags & XGMAC_UCADDR) { 2691 uint8_t ucaddr[ETHER_ADDR_LEN]; 2692 2693 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2694 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2695 ucaddr, true, true); 2696 if (rc < 0) { 2697 rc = -rc; 2698 if_printf(ifp, "change_mac failed: %d\n", rc); 2699 return (rc); 2700 } else { 2701 pi->xact_addr_filt = rc; 2702 rc = 0; 2703 } 2704 } 2705 2706 if (flags & XGMAC_MCADDRS) { 2707 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2708 int del = 1; 2709 uint64_t hash = 0; 2710 struct ifmultiaddr *ifma; 2711 int i = 0, j; 2712 2713 if_maddr_rlock(ifp); 2714 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2715 if (ifma->ifma_addr->sa_family != AF_LINK) 2716 continue; 2717 mcaddr[i++] = 2718 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2719 2720 if (i == FW_MAC_EXACT_CHUNK) { 2721 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2722 del, i, mcaddr, NULL, &hash, 0); 2723 if (rc < 0) { 2724 rc = -rc; 2725 for (j = 0; j < i; j++) { 2726 if_printf(ifp, 2727 "failed to add mc address" 2728 " %02x:%02x:%02x:" 2729 "%02x:%02x:%02x rc=%d\n", 2730 mcaddr[j][0], mcaddr[j][1], 2731 mcaddr[j][2], mcaddr[j][3], 2732 mcaddr[j][4], mcaddr[j][5], 2733 rc); 2734 } 2735 goto mcfail; 2736 } 2737 del = 0; 2738 i = 0; 2739 } 2740 } 2741 if (i > 0) { 2742 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2743 del, i, mcaddr, NULL, &hash, 0); 2744 if (rc < 0) { 2745 rc = -rc; 2746 for (j = 0; j < i; j++) { 2747 if_printf(ifp, 2748 "failed to add mc address" 2749 " %02x:%02x:%02x:" 2750 "%02x:%02x:%02x rc=%d\n", 2751 mcaddr[j][0], mcaddr[j][1], 2752 mcaddr[j][2], mcaddr[j][3], 2753 mcaddr[j][4], mcaddr[j][5], 2754 rc); 2755 } 2756 goto mcfail; 2757 } 2758 } 2759 2760 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2761 if (rc != 0) 2762 if_printf(ifp, "failed to set mc address hash: %d", rc); 2763mcfail: 2764 if_maddr_runlock(ifp); 2765 } 2766 2767 return (rc); 2768} 2769 2770int 2771begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 2772 char *wmesg) 2773{ 2774 int rc, pri; 2775 2776#ifdef WITNESS 2777 /* the caller thinks it's ok to sleep, but is it really? */ 2778 if (flags & SLEEP_OK) 2779 pause("t4slptst", 1); 2780#endif 2781 2782 if (INTR_OK) 2783 pri = PCATCH; 2784 else 2785 pri = 0; 2786 2787 ADAPTER_LOCK(sc); 2788 for (;;) { 2789 2790 if (pi && IS_DOOMED(pi)) { 2791 rc = ENXIO; 2792 goto done; 2793 } 2794 2795 if (!IS_BUSY(sc)) { 2796 rc = 0; 2797 break; 2798 } 2799 2800 if (!(flags & SLEEP_OK)) { 2801 rc = EBUSY; 2802 goto done; 2803 } 2804 2805 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 2806 rc = EINTR; 2807 goto done; 2808 } 2809 } 2810 2811 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2812 SET_BUSY(sc); 2813#ifdef INVARIANTS 2814 sc->last_op = wmesg; 2815 sc->last_op_thr = curthread; 2816#endif 2817 2818done: 2819 if (!(flags & HOLD_LOCK) || rc) 2820 ADAPTER_UNLOCK(sc); 2821 2822 return (rc); 2823} 2824 2825void 2826end_synchronized_op(struct adapter *sc, int flags) 2827{ 2828 2829 if (flags & LOCK_HELD) 2830 ADAPTER_LOCK_ASSERT_OWNED(sc); 2831 else 2832 ADAPTER_LOCK(sc); 2833 2834 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2835 CLR_BUSY(sc); 2836 wakeup(&sc->flags); 2837 ADAPTER_UNLOCK(sc); 2838} 2839 2840static int 2841cxgbe_init_synchronized(struct port_info *pi) 2842{ 2843 struct adapter *sc = pi->adapter; 2844 struct ifnet *ifp = pi->ifp; 2845 int rc = 0; 2846 2847 ASSERT_SYNCHRONIZED_OP(sc); 2848 2849 if (isset(&sc->open_device_map, pi->port_id)) { 2850 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2851 ("mismatch between open_device_map and if_drv_flags")); 2852 return (0); /* already running */ 2853 } 2854 2855 if (!(sc->flags & FULL_INIT_DONE) && 2856 ((rc = adapter_full_init(sc)) != 0)) 2857 return (rc); /* error message displayed already */ 2858 2859 if (!(pi->flags & PORT_INIT_DONE) && 2860 ((rc = port_full_init(pi)) != 0)) 2861 return (rc); /* error message displayed already */ 2862 2863 rc = update_mac_settings(pi, XGMAC_ALL); 2864 if (rc) 2865 goto done; /* error message displayed already */ 2866 2867 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2868 if (rc != 0) { 2869 if_printf(ifp, "start_link failed: %d\n", rc); 2870 goto done; 2871 } 2872 2873 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2874 if (rc != 0) { 2875 if_printf(ifp, "enable_vi failed: %d\n", rc); 2876 goto done; 2877 } 2878 2879 /* all ok */ 2880 setbit(&sc->open_device_map, pi->port_id); 2881 PORT_LOCK(pi); 2882 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2883 PORT_UNLOCK(pi); 2884 2885 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2886done: 2887 if (rc != 0) 2888 cxgbe_uninit_synchronized(pi); 2889 2890 return (rc); 2891} 2892 2893/* 2894 * Idempotent. 2895 */ 2896static int 2897cxgbe_uninit_synchronized(struct port_info *pi) 2898{ 2899 struct adapter *sc = pi->adapter; 2900 struct ifnet *ifp = pi->ifp; 2901 int rc; 2902 2903 ASSERT_SYNCHRONIZED_OP(sc); 2904 2905 /* 2906 * Disable the VI so that all its data in either direction is discarded 2907 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2908 * tick) intact as the TP can deliver negative advice or data that it's 2909 * holding in its RAM (for an offloaded connection) even after the VI is 2910 * disabled. 2911 */ 2912 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2913 if (rc) { 2914 if_printf(ifp, "disable_vi failed: %d\n", rc); 2915 return (rc); 2916 } 2917 2918 clrbit(&sc->open_device_map, pi->port_id); 2919 PORT_LOCK(pi); 2920 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2921 PORT_UNLOCK(pi); 2922 2923 pi->link_cfg.link_ok = 0; 2924 pi->link_cfg.speed = 0; 2925 t4_os_link_changed(sc, pi->port_id, 0); 2926 2927 return (0); 2928} 2929 2930/* 2931 * It is ok for this function to fail midway and return right away. t4_detach 2932 * will walk the entire sc->irq list and clean up whatever is valid. 2933 */ 2934static int 2935setup_intr_handlers(struct adapter *sc) 2936{ 2937 int rc, rid, p, q; 2938 char s[8]; 2939 struct irq *irq; 2940 struct port_info *pi; 2941 struct sge_rxq *rxq; 2942#ifdef TCP_OFFLOAD 2943 struct sge_ofld_rxq *ofld_rxq; 2944#endif 2945 2946 /* 2947 * Setup interrupts. 2948 */ 2949 irq = &sc->irq[0]; 2950 rid = sc->intr_type == INTR_INTX ? 0 : 1; 2951 if (sc->intr_count == 1) { 2952 KASSERT(!(sc->flags & INTR_DIRECT), 2953 ("%s: single interrupt && INTR_DIRECT?", __func__)); 2954 2955 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 2956 if (rc != 0) 2957 return (rc); 2958 } else { 2959 /* Multiple interrupts. */ 2960 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2961 ("%s: too few intr.", __func__)); 2962 2963 /* The first one is always error intr */ 2964 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 2965 if (rc != 0) 2966 return (rc); 2967 irq++; 2968 rid++; 2969 2970 /* The second one is always the firmware event queue */ 2971 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 2972 "evt"); 2973 if (rc != 0) 2974 return (rc); 2975 irq++; 2976 rid++; 2977 2978 /* 2979 * Note that if INTR_DIRECT is not set then either the NIC rx 2980 * queues or (exclusive or) the TOE rx queueus will be taking 2981 * direct interrupts. 2982 * 2983 * There is no need to check for is_offload(sc) as nofldrxq 2984 * will be 0 if offload is disabled. 2985 */ 2986 for_each_port(sc, p) { 2987 pi = sc->port[p]; 2988 2989#ifdef TCP_OFFLOAD 2990 /* 2991 * Skip over the NIC queues if they aren't taking direct 2992 * interrupts. 2993 */ 2994 if (!(sc->flags & INTR_DIRECT) && 2995 pi->nofldrxq > pi->nrxq) 2996 goto ofld_queues; 2997#endif 2998 rxq = &sc->sge.rxq[pi->first_rxq]; 2999 for (q = 0; q < pi->nrxq; q++, rxq++) { 3000 snprintf(s, sizeof(s), "%d.%d", p, q); 3001 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3002 s); 3003 if (rc != 0) 3004 return (rc); 3005 irq++; 3006 rid++; 3007 } 3008 3009#ifdef TCP_OFFLOAD 3010 /* 3011 * Skip over the offload queues if they aren't taking 3012 * direct interrupts. 3013 */ 3014 if (!(sc->flags & INTR_DIRECT)) 3015 continue; 3016ofld_queues: 3017 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 3018 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 3019 snprintf(s, sizeof(s), "%d,%d", p, q); 3020 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3021 ofld_rxq, s); 3022 if (rc != 0) 3023 return (rc); 3024 irq++; 3025 rid++; 3026 } 3027#endif 3028 } 3029 } 3030 3031 return (0); 3032} 3033 3034static int 3035adapter_full_init(struct adapter *sc) 3036{ 3037 int rc, i; 3038 3039 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3040 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3041 ("%s: FULL_INIT_DONE already", __func__)); 3042 3043 /* 3044 * queues that belong to the adapter (not any particular port). 3045 */ 3046 rc = t4_setup_adapter_queues(sc); 3047 if (rc != 0) 3048 goto done; 3049 3050 for (i = 0; i < nitems(sc->tq); i++) { 3051 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3052 taskqueue_thread_enqueue, &sc->tq[i]); 3053 if (sc->tq[i] == NULL) { 3054 device_printf(sc->dev, 3055 "failed to allocate task queue %d\n", i); 3056 rc = ENOMEM; 3057 goto done; 3058 } 3059 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3060 device_get_nameunit(sc->dev), i); 3061 } 3062 3063 t4_intr_enable(sc); 3064 sc->flags |= FULL_INIT_DONE; 3065done: 3066 if (rc != 0) 3067 adapter_full_uninit(sc); 3068 3069 return (rc); 3070} 3071 3072static int 3073adapter_full_uninit(struct adapter *sc) 3074{ 3075 int i; 3076 3077 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3078 3079 t4_teardown_adapter_queues(sc); 3080 3081 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3082 taskqueue_free(sc->tq[i]); 3083 sc->tq[i] = NULL; 3084 } 3085 3086 sc->flags &= ~FULL_INIT_DONE; 3087 3088 return (0); 3089} 3090 3091static int 3092port_full_init(struct port_info *pi) 3093{ 3094 struct adapter *sc = pi->adapter; 3095 struct ifnet *ifp = pi->ifp; 3096 uint16_t *rss; 3097 struct sge_rxq *rxq; 3098 int rc, i; 3099 3100 ASSERT_SYNCHRONIZED_OP(sc); 3101 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3102 ("%s: PORT_INIT_DONE already", __func__)); 3103 3104 sysctl_ctx_init(&pi->ctx); 3105 pi->flags |= PORT_SYSCTL_CTX; 3106 3107 /* 3108 * Allocate tx/rx/fl queues for this port. 3109 */ 3110 rc = t4_setup_port_queues(pi); 3111 if (rc != 0) 3112 goto done; /* error message displayed already */ 3113 3114 /* 3115 * Setup RSS for this port. 3116 */ 3117 rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 3118 M_ZERO | M_WAITOK); 3119 for_each_rxq(pi, i, rxq) { 3120 rss[i] = rxq->iq.abs_id; 3121 } 3122 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 3123 pi->rss_size, rss, pi->nrxq); 3124 free(rss, M_CXGBE); 3125 if (rc != 0) { 3126 if_printf(ifp, "rss_config failed: %d\n", rc); 3127 goto done; 3128 } 3129 3130 pi->flags |= PORT_INIT_DONE; 3131done: 3132 if (rc != 0) 3133 port_full_uninit(pi); 3134 3135 return (rc); 3136} 3137 3138/* 3139 * Idempotent. 3140 */ 3141static int 3142port_full_uninit(struct port_info *pi) 3143{ 3144 struct adapter *sc = pi->adapter; 3145 int i; 3146 struct sge_rxq *rxq; 3147 struct sge_txq *txq; 3148#ifdef TCP_OFFLOAD 3149 struct sge_ofld_rxq *ofld_rxq; 3150 struct sge_wrq *ofld_txq; 3151#endif 3152 3153 if (pi->flags & PORT_INIT_DONE) { 3154 3155 /* Need to quiesce queues. XXX: ctrl queues? */ 3156 3157 for_each_txq(pi, i, txq) { 3158 quiesce_eq(sc, &txq->eq); 3159 } 3160 3161#ifdef TCP_OFFLOAD 3162 for_each_ofld_txq(pi, i, ofld_txq) { 3163 quiesce_eq(sc, &ofld_txq->eq); 3164 } 3165#endif 3166 3167 for_each_rxq(pi, i, rxq) { 3168 quiesce_iq(sc, &rxq->iq); 3169 quiesce_fl(sc, &rxq->fl); 3170 } 3171 3172#ifdef TCP_OFFLOAD 3173 for_each_ofld_rxq(pi, i, ofld_rxq) { 3174 quiesce_iq(sc, &ofld_rxq->iq); 3175 quiesce_fl(sc, &ofld_rxq->fl); 3176 } 3177#endif 3178 } 3179 3180 t4_teardown_port_queues(pi); 3181 pi->flags &= ~PORT_INIT_DONE; 3182 3183 return (0); 3184} 3185 3186static void 3187quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3188{ 3189 EQ_LOCK(eq); 3190 eq->flags |= EQ_DOOMED; 3191 3192 /* 3193 * Wait for the response to a credit flush if one's 3194 * pending. 3195 */ 3196 while (eq->flags & EQ_CRFLUSHED) 3197 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3198 EQ_UNLOCK(eq); 3199 3200 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3201 pause("callout", 10); /* Still iffy */ 3202 3203 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3204} 3205 3206static void 3207quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3208{ 3209 (void) sc; /* unused */ 3210 3211 /* Synchronize with the interrupt handler */ 3212 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3213 pause("iqfree", 1); 3214} 3215 3216static void 3217quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3218{ 3219 mtx_lock(&sc->sfl_lock); 3220 FL_LOCK(fl); 3221 fl->flags |= FL_DOOMED; 3222 FL_UNLOCK(fl); 3223 mtx_unlock(&sc->sfl_lock); 3224 3225 callout_drain(&sc->sfl_callout); 3226 KASSERT((fl->flags & FL_STARVING) == 0, 3227 ("%s: still starving", __func__)); 3228} 3229 3230static int 3231t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3232 driver_intr_t *handler, void *arg, char *name) 3233{ 3234 int rc; 3235 3236 irq->rid = rid; 3237 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3238 RF_SHAREABLE | RF_ACTIVE); 3239 if (irq->res == NULL) { 3240 device_printf(sc->dev, 3241 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3242 return (ENOMEM); 3243 } 3244 3245 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3246 NULL, handler, arg, &irq->tag); 3247 if (rc != 0) { 3248 device_printf(sc->dev, 3249 "failed to setup interrupt for rid %d, name %s: %d\n", 3250 rid, name, rc); 3251 } else if (name) 3252 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3253 3254 return (rc); 3255} 3256 3257static int 3258t4_free_irq(struct adapter *sc, struct irq *irq) 3259{ 3260 if (irq->tag) 3261 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3262 if (irq->res) 3263 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3264 3265 bzero(irq, sizeof(*irq)); 3266 3267 return (0); 3268} 3269 3270static void 3271reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3272 unsigned int end) 3273{ 3274 uint32_t *p = (uint32_t *)(buf + start); 3275 3276 for ( ; start <= end; start += sizeof(uint32_t)) 3277 *p++ = t4_read_reg(sc, start); 3278} 3279 3280static void 3281t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3282{ 3283 int i, n; 3284 const unsigned int *reg_ranges; 3285 static const unsigned int t4_reg_ranges[] = { 3286 0x1008, 0x1108, 3287 0x1180, 0x11b4, 3288 0x11fc, 0x123c, 3289 0x1300, 0x173c, 3290 0x1800, 0x18fc, 3291 0x3000, 0x30d8, 3292 0x30e0, 0x5924, 3293 0x5960, 0x59d4, 3294 0x5a00, 0x5af8, 3295 0x6000, 0x6098, 3296 0x6100, 0x6150, 3297 0x6200, 0x6208, 3298 0x6240, 0x6248, 3299 0x6280, 0x6338, 3300 0x6370, 0x638c, 3301 0x6400, 0x643c, 3302 0x6500, 0x6524, 3303 0x6a00, 0x6a38, 3304 0x6a60, 0x6a78, 3305 0x6b00, 0x6b84, 3306 0x6bf0, 0x6c84, 3307 0x6cf0, 0x6d84, 3308 0x6df0, 0x6e84, 3309 0x6ef0, 0x6f84, 3310 0x6ff0, 0x7084, 3311 0x70f0, 0x7184, 3312 0x71f0, 0x7284, 3313 0x72f0, 0x7384, 3314 0x73f0, 0x7450, 3315 0x7500, 0x7530, 3316 0x7600, 0x761c, 3317 0x7680, 0x76cc, 3318 0x7700, 0x7798, 3319 0x77c0, 0x77fc, 3320 0x7900, 0x79fc, 3321 0x7b00, 0x7c38, 3322 0x7d00, 0x7efc, 3323 0x8dc0, 0x8e1c, 3324 0x8e30, 0x8e78, 3325 0x8ea0, 0x8f6c, 3326 0x8fc0, 0x9074, 3327 0x90fc, 0x90fc, 3328 0x9400, 0x9458, 3329 0x9600, 0x96bc, 3330 0x9800, 0x9808, 3331 0x9820, 0x983c, 3332 0x9850, 0x9864, 3333 0x9c00, 0x9c6c, 3334 0x9c80, 0x9cec, 3335 0x9d00, 0x9d6c, 3336 0x9d80, 0x9dec, 3337 0x9e00, 0x9e6c, 3338 0x9e80, 0x9eec, 3339 0x9f00, 0x9f6c, 3340 0x9f80, 0x9fec, 3341 0xd004, 0xd03c, 3342 0xdfc0, 0xdfe0, 3343 0xe000, 0xea7c, 3344 0xf000, 0x11190, 3345 0x19040, 0x1906c, 3346 0x19078, 0x19080, 3347 0x1908c, 0x19124, 3348 0x19150, 0x191b0, 3349 0x191d0, 0x191e8, 3350 0x19238, 0x1924c, 3351 0x193f8, 0x19474, 3352 0x19490, 0x194f8, 3353 0x19800, 0x19f30, 3354 0x1a000, 0x1a06c, 3355 0x1a0b0, 0x1a120, 3356 0x1a128, 0x1a138, 3357 0x1a190, 0x1a1c4, 3358 0x1a1fc, 0x1a1fc, 3359 0x1e040, 0x1e04c, 3360 0x1e284, 0x1e28c, 3361 0x1e2c0, 0x1e2c0, 3362 0x1e2e0, 0x1e2e0, 3363 0x1e300, 0x1e384, 3364 0x1e3c0, 0x1e3c8, 3365 0x1e440, 0x1e44c, 3366 0x1e684, 0x1e68c, 3367 0x1e6c0, 0x1e6c0, 3368 0x1e6e0, 0x1e6e0, 3369 0x1e700, 0x1e784, 3370 0x1e7c0, 0x1e7c8, 3371 0x1e840, 0x1e84c, 3372 0x1ea84, 0x1ea8c, 3373 0x1eac0, 0x1eac0, 3374 0x1eae0, 0x1eae0, 3375 0x1eb00, 0x1eb84, 3376 0x1ebc0, 0x1ebc8, 3377 0x1ec40, 0x1ec4c, 3378 0x1ee84, 0x1ee8c, 3379 0x1eec0, 0x1eec0, 3380 0x1eee0, 0x1eee0, 3381 0x1ef00, 0x1ef84, 3382 0x1efc0, 0x1efc8, 3383 0x1f040, 0x1f04c, 3384 0x1f284, 0x1f28c, 3385 0x1f2c0, 0x1f2c0, 3386 0x1f2e0, 0x1f2e0, 3387 0x1f300, 0x1f384, 3388 0x1f3c0, 0x1f3c8, 3389 0x1f440, 0x1f44c, 3390 0x1f684, 0x1f68c, 3391 0x1f6c0, 0x1f6c0, 3392 0x1f6e0, 0x1f6e0, 3393 0x1f700, 0x1f784, 3394 0x1f7c0, 0x1f7c8, 3395 0x1f840, 0x1f84c, 3396 0x1fa84, 0x1fa8c, 3397 0x1fac0, 0x1fac0, 3398 0x1fae0, 0x1fae0, 3399 0x1fb00, 0x1fb84, 3400 0x1fbc0, 0x1fbc8, 3401 0x1fc40, 0x1fc4c, 3402 0x1fe84, 0x1fe8c, 3403 0x1fec0, 0x1fec0, 3404 0x1fee0, 0x1fee0, 3405 0x1ff00, 0x1ff84, 3406 0x1ffc0, 0x1ffc8, 3407 0x20000, 0x2002c, 3408 0x20100, 0x2013c, 3409 0x20190, 0x201c8, 3410 0x20200, 0x20318, 3411 0x20400, 0x20528, 3412 0x20540, 0x20614, 3413 0x21000, 0x21040, 3414 0x2104c, 0x21060, 3415 0x210c0, 0x210ec, 3416 0x21200, 0x21268, 3417 0x21270, 0x21284, 3418 0x212fc, 0x21388, 3419 0x21400, 0x21404, 3420 0x21500, 0x21518, 3421 0x2152c, 0x2153c, 3422 0x21550, 0x21554, 3423 0x21600, 0x21600, 3424 0x21608, 0x21628, 3425 0x21630, 0x2163c, 3426 0x21700, 0x2171c, 3427 0x21780, 0x2178c, 3428 0x21800, 0x21c38, 3429 0x21c80, 0x21d7c, 3430 0x21e00, 0x21e04, 3431 0x22000, 0x2202c, 3432 0x22100, 0x2213c, 3433 0x22190, 0x221c8, 3434 0x22200, 0x22318, 3435 0x22400, 0x22528, 3436 0x22540, 0x22614, 3437 0x23000, 0x23040, 3438 0x2304c, 0x23060, 3439 0x230c0, 0x230ec, 3440 0x23200, 0x23268, 3441 0x23270, 0x23284, 3442 0x232fc, 0x23388, 3443 0x23400, 0x23404, 3444 0x23500, 0x23518, 3445 0x2352c, 0x2353c, 3446 0x23550, 0x23554, 3447 0x23600, 0x23600, 3448 0x23608, 0x23628, 3449 0x23630, 0x2363c, 3450 0x23700, 0x2371c, 3451 0x23780, 0x2378c, 3452 0x23800, 0x23c38, 3453 0x23c80, 0x23d7c, 3454 0x23e00, 0x23e04, 3455 0x24000, 0x2402c, 3456 0x24100, 0x2413c, 3457 0x24190, 0x241c8, 3458 0x24200, 0x24318, 3459 0x24400, 0x24528, 3460 0x24540, 0x24614, 3461 0x25000, 0x25040, 3462 0x2504c, 0x25060, 3463 0x250c0, 0x250ec, 3464 0x25200, 0x25268, 3465 0x25270, 0x25284, 3466 0x252fc, 0x25388, 3467 0x25400, 0x25404, 3468 0x25500, 0x25518, 3469 0x2552c, 0x2553c, 3470 0x25550, 0x25554, 3471 0x25600, 0x25600, 3472 0x25608, 0x25628, 3473 0x25630, 0x2563c, 3474 0x25700, 0x2571c, 3475 0x25780, 0x2578c, 3476 0x25800, 0x25c38, 3477 0x25c80, 0x25d7c, 3478 0x25e00, 0x25e04, 3479 0x26000, 0x2602c, 3480 0x26100, 0x2613c, 3481 0x26190, 0x261c8, 3482 0x26200, 0x26318, 3483 0x26400, 0x26528, 3484 0x26540, 0x26614, 3485 0x27000, 0x27040, 3486 0x2704c, 0x27060, 3487 0x270c0, 0x270ec, 3488 0x27200, 0x27268, 3489 0x27270, 0x27284, 3490 0x272fc, 0x27388, 3491 0x27400, 0x27404, 3492 0x27500, 0x27518, 3493 0x2752c, 0x2753c, 3494 0x27550, 0x27554, 3495 0x27600, 0x27600, 3496 0x27608, 0x27628, 3497 0x27630, 0x2763c, 3498 0x27700, 0x2771c, 3499 0x27780, 0x2778c, 3500 0x27800, 0x27c38, 3501 0x27c80, 0x27d7c, 3502 0x27e00, 0x27e04 3503 }; 3504 static const unsigned int t5_reg_ranges[] = { 3505 0x1008, 0x1148, 3506 0x1180, 0x11b4, 3507 0x11fc, 0x123c, 3508 0x1280, 0x173c, 3509 0x1800, 0x18fc, 3510 0x3000, 0x3028, 3511 0x3060, 0x30d8, 3512 0x30e0, 0x30fc, 3513 0x3140, 0x357c, 3514 0x35a8, 0x35cc, 3515 0x35ec, 0x35ec, 3516 0x3600, 0x5624, 3517 0x56cc, 0x575c, 3518 0x580c, 0x5814, 3519 0x5890, 0x58bc, 3520 0x5940, 0x59dc, 3521 0x59fc, 0x5a18, 3522 0x5a60, 0x5a9c, 3523 0x5b94, 0x5bfc, 3524 0x6000, 0x6040, 3525 0x6058, 0x614c, 3526 0x7700, 0x7798, 3527 0x77c0, 0x78fc, 3528 0x7b00, 0x7c54, 3529 0x7d00, 0x7efc, 3530 0x8dc0, 0x8de0, 3531 0x8df8, 0x8e84, 3532 0x8ea0, 0x8f84, 3533 0x8fc0, 0x90f8, 3534 0x9400, 0x9470, 3535 0x9600, 0x96f4, 3536 0x9800, 0x9808, 3537 0x9820, 0x983c, 3538 0x9850, 0x9864, 3539 0x9c00, 0x9c6c, 3540 0x9c80, 0x9cec, 3541 0x9d00, 0x9d6c, 3542 0x9d80, 0x9dec, 3543 0x9e00, 0x9e6c, 3544 0x9e80, 0x9eec, 3545 0x9f00, 0x9f6c, 3546 0x9f80, 0xa020, 3547 0xd004, 0xd03c, 3548 0xdfc0, 0xdfe0, 3549 0xe000, 0x11088, 3550 0x1109c, 0x1117c, 3551 0x11190, 0x11204, 3552 0x19040, 0x1906c, 3553 0x19078, 0x19080, 3554 0x1908c, 0x19124, 3555 0x19150, 0x191b0, 3556 0x191d0, 0x191e8, 3557 0x19238, 0x19290, 3558 0x193f8, 0x19474, 3559 0x19490, 0x194cc, 3560 0x194f0, 0x194f8, 3561 0x19c00, 0x19c60, 3562 0x19c94, 0x19e10, 3563 0x19e50, 0x19f34, 3564 0x19f40, 0x19f50, 3565 0x19f90, 0x19fe4, 3566 0x1a000, 0x1a06c, 3567 0x1a0b0, 0x1a120, 3568 0x1a128, 0x1a138, 3569 0x1a190, 0x1a1c4, 3570 0x1a1fc, 0x1a1fc, 3571 0x1e008, 0x1e00c, 3572 0x1e040, 0x1e04c, 3573 0x1e284, 0x1e290, 3574 0x1e2c0, 0x1e2c0, 3575 0x1e2e0, 0x1e2e0, 3576 0x1e300, 0x1e384, 3577 0x1e3c0, 0x1e3c8, 3578 0x1e408, 0x1e40c, 3579 0x1e440, 0x1e44c, 3580 0x1e684, 0x1e690, 3581 0x1e6c0, 0x1e6c0, 3582 0x1e6e0, 0x1e6e0, 3583 0x1e700, 0x1e784, 3584 0x1e7c0, 0x1e7c8, 3585 0x1e808, 0x1e80c, 3586 0x1e840, 0x1e84c, 3587 0x1ea84, 0x1ea90, 3588 0x1eac0, 0x1eac0, 3589 0x1eae0, 0x1eae0, 3590 0x1eb00, 0x1eb84, 3591 0x1ebc0, 0x1ebc8, 3592 0x1ec08, 0x1ec0c, 3593 0x1ec40, 0x1ec4c, 3594 0x1ee84, 0x1ee90, 3595 0x1eec0, 0x1eec0, 3596 0x1eee0, 0x1eee0, 3597 0x1ef00, 0x1ef84, 3598 0x1efc0, 0x1efc8, 3599 0x1f008, 0x1f00c, 3600 0x1f040, 0x1f04c, 3601 0x1f284, 0x1f290, 3602 0x1f2c0, 0x1f2c0, 3603 0x1f2e0, 0x1f2e0, 3604 0x1f300, 0x1f384, 3605 0x1f3c0, 0x1f3c8, 3606 0x1f408, 0x1f40c, 3607 0x1f440, 0x1f44c, 3608 0x1f684, 0x1f690, 3609 0x1f6c0, 0x1f6c0, 3610 0x1f6e0, 0x1f6e0, 3611 0x1f700, 0x1f784, 3612 0x1f7c0, 0x1f7c8, 3613 0x1f808, 0x1f80c, 3614 0x1f840, 0x1f84c, 3615 0x1fa84, 0x1fa90, 3616 0x1fac0, 0x1fac0, 3617 0x1fae0, 0x1fae0, 3618 0x1fb00, 0x1fb84, 3619 0x1fbc0, 0x1fbc8, 3620 0x1fc08, 0x1fc0c, 3621 0x1fc40, 0x1fc4c, 3622 0x1fe84, 0x1fe90, 3623 0x1fec0, 0x1fec0, 3624 0x1fee0, 0x1fee0, 3625 0x1ff00, 0x1ff84, 3626 0x1ffc0, 0x1ffc8, 3627 0x30000, 0x30040, 3628 0x30100, 0x30144, 3629 0x30190, 0x301d0, 3630 0x30200, 0x30318, 3631 0x30400, 0x3052c, 3632 0x30540, 0x3061c, 3633 0x30800, 0x30834, 3634 0x308c0, 0x30908, 3635 0x30910, 0x309ac, 3636 0x30a00, 0x30a04, 3637 0x30a0c, 0x30a2c, 3638 0x30a44, 0x30a50, 3639 0x30a74, 0x30c24, 3640 0x30d08, 0x30d14, 3641 0x30d1c, 0x30d20, 3642 0x30d3c, 0x30d50, 3643 0x31200, 0x3120c, 3644 0x31220, 0x31220, 3645 0x31240, 0x31240, 3646 0x31600, 0x31600, 3647 0x31608, 0x3160c, 3648 0x31a00, 0x31a1c, 3649 0x31e04, 0x31e20, 3650 0x31e38, 0x31e3c, 3651 0x31e80, 0x31e80, 3652 0x31e88, 0x31ea8, 3653 0x31eb0, 0x31eb4, 3654 0x31ec8, 0x31ed4, 3655 0x31fb8, 0x32004, 3656 0x32208, 0x3223c, 3657 0x32248, 0x3227c, 3658 0x32288, 0x322bc, 3659 0x322c8, 0x322fc, 3660 0x32600, 0x32630, 3661 0x32a00, 0x32abc, 3662 0x32b00, 0x32b70, 3663 0x33000, 0x33048, 3664 0x33060, 0x3309c, 3665 0x330f0, 0x33148, 3666 0x33160, 0x3319c, 3667 0x331f0, 0x332e4, 3668 0x332f8, 0x333e4, 3669 0x333f8, 0x33448, 3670 0x33460, 0x3349c, 3671 0x334f0, 0x33548, 3672 0x33560, 0x3359c, 3673 0x335f0, 0x336e4, 3674 0x336f8, 0x337e4, 3675 0x337f8, 0x337fc, 3676 0x33814, 0x33814, 3677 0x3382c, 0x3382c, 3678 0x33880, 0x3388c, 3679 0x338e8, 0x338ec, 3680 0x33900, 0x33948, 3681 0x33960, 0x3399c, 3682 0x339f0, 0x33ae4, 3683 0x33af8, 0x33b10, 3684 0x33b28, 0x33b28, 3685 0x33b3c, 0x33b50, 3686 0x33bf0, 0x33c10, 3687 0x33c28, 0x33c28, 3688 0x33c3c, 0x33c50, 3689 0x33cf0, 0x33cfc, 3690 0x34000, 0x34040, 3691 0x34100, 0x34144, 3692 0x34190, 0x341d0, 3693 0x34200, 0x34318, 3694 0x34400, 0x3452c, 3695 0x34540, 0x3461c, 3696 0x34800, 0x34834, 3697 0x348c0, 0x34908, 3698 0x34910, 0x349ac, 3699 0x34a00, 0x34a04, 3700 0x34a0c, 0x34a2c, 3701 0x34a44, 0x34a50, 3702 0x34a74, 0x34c24, 3703 0x34d08, 0x34d14, 3704 0x34d1c, 0x34d20, 3705 0x34d3c, 0x34d50, 3706 0x35200, 0x3520c, 3707 0x35220, 0x35220, 3708 0x35240, 0x35240, 3709 0x35600, 0x35600, 3710 0x35608, 0x3560c, 3711 0x35a00, 0x35a1c, 3712 0x35e04, 0x35e20, 3713 0x35e38, 0x35e3c, 3714 0x35e80, 0x35e80, 3715 0x35e88, 0x35ea8, 3716 0x35eb0, 0x35eb4, 3717 0x35ec8, 0x35ed4, 3718 0x35fb8, 0x36004, 3719 0x36208, 0x3623c, 3720 0x36248, 0x3627c, 3721 0x36288, 0x362bc, 3722 0x362c8, 0x362fc, 3723 0x36600, 0x36630, 3724 0x36a00, 0x36abc, 3725 0x36b00, 0x36b70, 3726 0x37000, 0x37048, 3727 0x37060, 0x3709c, 3728 0x370f0, 0x37148, 3729 0x37160, 0x3719c, 3730 0x371f0, 0x372e4, 3731 0x372f8, 0x373e4, 3732 0x373f8, 0x37448, 3733 0x37460, 0x3749c, 3734 0x374f0, 0x37548, 3735 0x37560, 0x3759c, 3736 0x375f0, 0x376e4, 3737 0x376f8, 0x377e4, 3738 0x377f8, 0x377fc, 3739 0x37814, 0x37814, 3740 0x3782c, 0x3782c, 3741 0x37880, 0x3788c, 3742 0x378e8, 0x378ec, 3743 0x37900, 0x37948, 3744 0x37960, 0x3799c, 3745 0x379f0, 0x37ae4, 3746 0x37af8, 0x37b10, 3747 0x37b28, 0x37b28, 3748 0x37b3c, 0x37b50, 3749 0x37bf0, 0x37c10, 3750 0x37c28, 0x37c28, 3751 0x37c3c, 0x37c50, 3752 0x37cf0, 0x37cfc, 3753 0x38000, 0x38040, 3754 0x38100, 0x38144, 3755 0x38190, 0x381d0, 3756 0x38200, 0x38318, 3757 0x38400, 0x3852c, 3758 0x38540, 0x3861c, 3759 0x38800, 0x38834, 3760 0x388c0, 0x38908, 3761 0x38910, 0x389ac, 3762 0x38a00, 0x38a04, 3763 0x38a0c, 0x38a2c, 3764 0x38a44, 0x38a50, 3765 0x38a74, 0x38c24, 3766 0x38d08, 0x38d14, 3767 0x38d1c, 0x38d20, 3768 0x38d3c, 0x38d50, 3769 0x39200, 0x3920c, 3770 0x39220, 0x39220, 3771 0x39240, 0x39240, 3772 0x39600, 0x39600, 3773 0x39608, 0x3960c, 3774 0x39a00, 0x39a1c, 3775 0x39e04, 0x39e20, 3776 0x39e38, 0x39e3c, 3777 0x39e80, 0x39e80, 3778 0x39e88, 0x39ea8, 3779 0x39eb0, 0x39eb4, 3780 0x39ec8, 0x39ed4, 3781 0x39fb8, 0x3a004, 3782 0x3a208, 0x3a23c, 3783 0x3a248, 0x3a27c, 3784 0x3a288, 0x3a2bc, 3785 0x3a2c8, 0x3a2fc, 3786 0x3a600, 0x3a630, 3787 0x3aa00, 0x3aabc, 3788 0x3ab00, 0x3ab70, 3789 0x3b000, 0x3b048, 3790 0x3b060, 0x3b09c, 3791 0x3b0f0, 0x3b148, 3792 0x3b160, 0x3b19c, 3793 0x3b1f0, 0x3b2e4, 3794 0x3b2f8, 0x3b3e4, 3795 0x3b3f8, 0x3b448, 3796 0x3b460, 0x3b49c, 3797 0x3b4f0, 0x3b548, 3798 0x3b560, 0x3b59c, 3799 0x3b5f0, 0x3b6e4, 3800 0x3b6f8, 0x3b7e4, 3801 0x3b7f8, 0x3b7fc, 3802 0x3b814, 0x3b814, 3803 0x3b82c, 0x3b82c, 3804 0x3b880, 0x3b88c, 3805 0x3b8e8, 0x3b8ec, 3806 0x3b900, 0x3b948, 3807 0x3b960, 0x3b99c, 3808 0x3b9f0, 0x3bae4, 3809 0x3baf8, 0x3bb10, 3810 0x3bb28, 0x3bb28, 3811 0x3bb3c, 0x3bb50, 3812 0x3bbf0, 0x3bc10, 3813 0x3bc28, 0x3bc28, 3814 0x3bc3c, 0x3bc50, 3815 0x3bcf0, 0x3bcfc, 3816 0x3c000, 0x3c040, 3817 0x3c100, 0x3c144, 3818 0x3c190, 0x3c1d0, 3819 0x3c200, 0x3c318, 3820 0x3c400, 0x3c52c, 3821 0x3c540, 0x3c61c, 3822 0x3c800, 0x3c834, 3823 0x3c8c0, 0x3c908, 3824 0x3c910, 0x3c9ac, 3825 0x3ca00, 0x3ca04, 3826 0x3ca0c, 0x3ca2c, 3827 0x3ca44, 0x3ca50, 3828 0x3ca74, 0x3cc24, 3829 0x3cd08, 0x3cd14, 3830 0x3cd1c, 0x3cd20, 3831 0x3cd3c, 0x3cd50, 3832 0x3d200, 0x3d20c, 3833 0x3d220, 0x3d220, 3834 0x3d240, 0x3d240, 3835 0x3d600, 0x3d600, 3836 0x3d608, 0x3d60c, 3837 0x3da00, 0x3da1c, 3838 0x3de04, 0x3de20, 3839 0x3de38, 0x3de3c, 3840 0x3de80, 0x3de80, 3841 0x3de88, 0x3dea8, 3842 0x3deb0, 0x3deb4, 3843 0x3dec8, 0x3ded4, 3844 0x3dfb8, 0x3e004, 3845 0x3e208, 0x3e23c, 3846 0x3e248, 0x3e27c, 3847 0x3e288, 0x3e2bc, 3848 0x3e2c8, 0x3e2fc, 3849 0x3e600, 0x3e630, 3850 0x3ea00, 0x3eabc, 3851 0x3eb00, 0x3eb70, 3852 0x3f000, 0x3f048, 3853 0x3f060, 0x3f09c, 3854 0x3f0f0, 0x3f148, 3855 0x3f160, 0x3f19c, 3856 0x3f1f0, 0x3f2e4, 3857 0x3f2f8, 0x3f3e4, 3858 0x3f3f8, 0x3f448, 3859 0x3f460, 0x3f49c, 3860 0x3f4f0, 0x3f548, 3861 0x3f560, 0x3f59c, 3862 0x3f5f0, 0x3f6e4, 3863 0x3f6f8, 0x3f7e4, 3864 0x3f7f8, 0x3f7fc, 3865 0x3f814, 0x3f814, 3866 0x3f82c, 0x3f82c, 3867 0x3f880, 0x3f88c, 3868 0x3f8e8, 0x3f8ec, 3869 0x3f900, 0x3f948, 3870 0x3f960, 0x3f99c, 3871 0x3f9f0, 0x3fae4, 3872 0x3faf8, 0x3fb10, 3873 0x3fb28, 0x3fb28, 3874 0x3fb3c, 0x3fb50, 3875 0x3fbf0, 0x3fc10, 3876 0x3fc28, 0x3fc28, 3877 0x3fc3c, 0x3fc50, 3878 0x3fcf0, 0x3fcfc, 3879 0x40000, 0x4000c, 3880 0x40040, 0x40068, 3881 0x4007c, 0x40144, 3882 0x40180, 0x4018c, 3883 0x40200, 0x40298, 3884 0x402ac, 0x4033c, 3885 0x403f8, 0x403fc, 3886 0x41300, 0x413c4, 3887 0x41400, 0x4141c, 3888 0x41480, 0x414d0, 3889 0x44000, 0x44078, 3890 0x440c0, 0x44278, 3891 0x442c0, 0x44478, 3892 0x444c0, 0x44678, 3893 0x446c0, 0x44878, 3894 0x448c0, 0x449fc, 3895 0x45000, 0x45068, 3896 0x45080, 0x45084, 3897 0x450a0, 0x450b0, 3898 0x45200, 0x45268, 3899 0x45280, 0x45284, 3900 0x452a0, 0x452b0, 3901 0x460c0, 0x460e4, 3902 0x47000, 0x4708c, 3903 0x47200, 0x47250, 3904 0x47400, 0x47420, 3905 0x47600, 0x47618, 3906 0x47800, 0x47814, 3907 0x48000, 0x4800c, 3908 0x48040, 0x48068, 3909 0x4807c, 0x48144, 3910 0x48180, 0x4818c, 3911 0x48200, 0x48298, 3912 0x482ac, 0x4833c, 3913 0x483f8, 0x483fc, 3914 0x49300, 0x493c4, 3915 0x49400, 0x4941c, 3916 0x49480, 0x494d0, 3917 0x4c000, 0x4c078, 3918 0x4c0c0, 0x4c278, 3919 0x4c2c0, 0x4c478, 3920 0x4c4c0, 0x4c678, 3921 0x4c6c0, 0x4c878, 3922 0x4c8c0, 0x4c9fc, 3923 0x4d000, 0x4d068, 3924 0x4d080, 0x4d084, 3925 0x4d0a0, 0x4d0b0, 3926 0x4d200, 0x4d268, 3927 0x4d280, 0x4d284, 3928 0x4d2a0, 0x4d2b0, 3929 0x4e0c0, 0x4e0e4, 3930 0x4f000, 0x4f08c, 3931 0x4f200, 0x4f250, 3932 0x4f400, 0x4f420, 3933 0x4f600, 0x4f618, 3934 0x4f800, 0x4f814, 3935 0x50000, 0x500cc, 3936 0x50400, 0x50400, 3937 0x50800, 0x508cc, 3938 0x50c00, 0x50c00, 3939 0x51000, 0x5101c, 3940 0x51300, 0x51308, 3941 }; 3942 3943 if (is_t4(sc)) { 3944 reg_ranges = &t4_reg_ranges[0]; 3945 n = nitems(t4_reg_ranges); 3946 } else { 3947 reg_ranges = &t5_reg_ranges[0]; 3948 n = nitems(t5_reg_ranges); 3949 } 3950 3951 regs->version = chip_id(sc) | chip_rev(sc) << 10; 3952 for (i = 0; i < n; i += 2) 3953 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 3954} 3955 3956static void 3957cxgbe_tick(void *arg) 3958{ 3959 struct port_info *pi = arg; 3960 struct ifnet *ifp = pi->ifp; 3961 struct sge_txq *txq; 3962 int i, drops; 3963 struct port_stats *s = &pi->stats; 3964 3965 PORT_LOCK(pi); 3966 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3967 PORT_UNLOCK(pi); 3968 return; /* without scheduling another callout */ 3969 } 3970 3971 t4_get_port_stats(pi->adapter, pi->tx_chan, s); 3972 3973 ifp->if_opackets = s->tx_frames - s->tx_pause; 3974 ifp->if_ipackets = s->rx_frames - s->rx_pause; 3975 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 3976 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 3977 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 3978 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 3979 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 3980 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 3981 s->rx_trunc3; 3982 3983 drops = s->tx_drop; 3984 for_each_txq(pi, i, txq) 3985 drops += txq->br->br_drops; 3986 ifp->if_snd.ifq_drops = drops; 3987 3988 ifp->if_oerrors = s->tx_error_frames; 3989 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 3990 s->rx_fcs_err + s->rx_len_err; 3991 3992 callout_schedule(&pi->tick, hz); 3993 PORT_UNLOCK(pi); 3994} 3995 3996static void 3997cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 3998{ 3999 struct ifnet *vlan; 4000 4001 if (arg != ifp || ifp->if_type != IFT_ETHER) 4002 return; 4003 4004 vlan = VLAN_DEVAT(ifp, vid); 4005 VLAN_SETCOOKIE(vlan, ifp); 4006} 4007 4008static int 4009cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4010{ 4011 4012#ifdef INVARIANTS 4013 panic("%s: opcode 0x%02x on iq %p with payload %p", 4014 __func__, rss->opcode, iq, m); 4015#else 4016 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4017 __func__, rss->opcode, iq, m); 4018 m_freem(m); 4019#endif 4020 return (EDOOFUS); 4021} 4022 4023int 4024t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4025{ 4026 uintptr_t *loc, new; 4027 4028 if (opcode >= nitems(sc->cpl_handler)) 4029 return (EINVAL); 4030 4031 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4032 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4033 atomic_store_rel_ptr(loc, new); 4034 4035 return (0); 4036} 4037 4038static int 4039an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4040{ 4041 4042#ifdef INVARIANTS 4043 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4044#else 4045 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4046 __func__, iq, ctrl); 4047#endif 4048 return (EDOOFUS); 4049} 4050 4051int 4052t4_register_an_handler(struct adapter *sc, an_handler_t h) 4053{ 4054 uintptr_t *loc, new; 4055 4056 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4057 loc = (uintptr_t *) &sc->an_handler; 4058 atomic_store_rel_ptr(loc, new); 4059 4060 return (0); 4061} 4062 4063static int 4064fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4065{ 4066 const struct cpl_fw6_msg *cpl = 4067 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4068 4069#ifdef INVARIANTS 4070 panic("%s: fw_msg type %d", __func__, cpl->type); 4071#else 4072 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4073#endif 4074 return (EDOOFUS); 4075} 4076 4077int 4078t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4079{ 4080 uintptr_t *loc, new; 4081 4082 if (type >= nitems(sc->fw_msg_handler)) 4083 return (EINVAL); 4084 4085 /* 4086 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4087 * handler dispatch table. Reject any attempt to install a handler for 4088 * this subtype. 4089 */ 4090 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4091 return (EINVAL); 4092 4093 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4094 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4095 atomic_store_rel_ptr(loc, new); 4096 4097 return (0); 4098} 4099 4100static int 4101t4_sysctls(struct adapter *sc) 4102{ 4103 struct sysctl_ctx_list *ctx; 4104 struct sysctl_oid *oid; 4105 struct sysctl_oid_list *children, *c0; 4106 static char *caps[] = { 4107 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4108 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 4109 "\20\1TOE", /* caps[2] toecaps */ 4110 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4111 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4112 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4113 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4114 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4115 }; 4116 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4117 4118 ctx = device_get_sysctl_ctx(sc->dev); 4119 4120 /* 4121 * dev.t4nex.X. 4122 */ 4123 oid = device_get_sysctl_tree(sc->dev); 4124 c0 = children = SYSCTL_CHILDREN(oid); 4125 4126 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4127 sc->params.nports, "# of ports"); 4128 4129 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4130 NULL, chip_rev(sc), "chip hardware revision"); 4131 4132 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4133 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4134 4135 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4136 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4137 4138 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4139 sc->cfcsum, "config file checksum"); 4140 4141 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4142 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4143 sysctl_bitfield, "A", "available doorbells"); 4144 4145 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4146 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4147 sysctl_bitfield, "A", "available link capabilities"); 4148 4149 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4150 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4151 sysctl_bitfield, "A", "available NIC capabilities"); 4152 4153 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4154 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4155 sysctl_bitfield, "A", "available TCP offload capabilities"); 4156 4157 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4158 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4159 sysctl_bitfield, "A", "available RDMA capabilities"); 4160 4161 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4162 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4163 sysctl_bitfield, "A", "available iSCSI capabilities"); 4164 4165 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4166 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4167 sysctl_bitfield, "A", "available FCoE capabilities"); 4168 4169 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4170 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4171 4172 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4173 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4174 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4175 "interrupt holdoff timer values (us)"); 4176 4177 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4178 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4179 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4180 "interrupt holdoff packet counter values"); 4181 4182#ifdef SBUF_DRAIN 4183 /* 4184 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4185 */ 4186 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4187 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4188 "logs and miscellaneous information"); 4189 children = SYSCTL_CHILDREN(oid); 4190 4191 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4192 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4193 sysctl_cctrl, "A", "congestion control"); 4194 4195 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4196 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4197 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4198 4199 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4200 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4201 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4202 4203 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4204 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4205 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4206 4207 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4208 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4209 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4210 4211 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4212 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4213 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4214 4215 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4216 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4217 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4218 4219 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4220 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4221 sysctl_cim_la, "A", "CIM logic analyzer"); 4222 4223 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4224 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4225 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4226 4227 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4228 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4229 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4230 4231 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4232 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4233 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4234 4235 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4236 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4237 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4238 4239 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4240 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4241 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4242 4243 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4244 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4245 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4246 4247 if (is_t5(sc)) { 4248 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4249 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4250 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4251 4252 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4253 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4254 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4255 } 4256 4257 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4258 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4259 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4260 4261 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4262 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4263 sysctl_cpl_stats, "A", "CPL statistics"); 4264 4265 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4266 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4267 sysctl_ddp_stats, "A", "DDP statistics"); 4268 4269 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4270 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4271 sysctl_devlog, "A", "firmware's device log"); 4272 4273 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4274 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4275 sysctl_fcoe_stats, "A", "FCoE statistics"); 4276 4277 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4278 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4279 sysctl_hw_sched, "A", "hardware scheduler "); 4280 4281 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4282 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4283 sysctl_l2t, "A", "hardware L2 table"); 4284 4285 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4286 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4287 sysctl_lb_stats, "A", "loopback statistics"); 4288 4289 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4290 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4291 sysctl_meminfo, "A", "memory regions"); 4292 4293 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4294 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4295 sysctl_path_mtus, "A", "path MTUs"); 4296 4297 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4298 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4299 sysctl_pm_stats, "A", "PM statistics"); 4300 4301 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4302 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4303 sysctl_rdma_stats, "A", "RDMA statistics"); 4304 4305 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4306 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4307 sysctl_tcp_stats, "A", "TCP statistics"); 4308 4309 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4310 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4311 sysctl_tids, "A", "TID information"); 4312 4313 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4314 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4315 sysctl_tp_err_stats, "A", "TP error statistics"); 4316 4317 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4318 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4319 sysctl_tx_rate, "A", "Tx rate"); 4320 4321 if (is_t5(sc)) { 4322 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4323 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4324 sysctl_wcwr_stats, "A", "write combined work requests"); 4325 } 4326#endif 4327 4328#ifdef TCP_OFFLOAD 4329 if (is_offload(sc)) { 4330 /* 4331 * dev.t4nex.X.toe. 4332 */ 4333 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4334 NULL, "TOE parameters"); 4335 children = SYSCTL_CHILDREN(oid); 4336 4337 sc->tt.sndbuf = 256 * 1024; 4338 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4339 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4340 4341 sc->tt.ddp = 0; 4342 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4343 &sc->tt.ddp, 0, "DDP allowed"); 4344 4345 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4346 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4347 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4348 4349 sc->tt.ddp_thres = 4350 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4351 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4352 &sc->tt.ddp_thres, 0, "DDP threshold"); 4353 } 4354#endif 4355 4356 4357 return (0); 4358} 4359 4360static int 4361cxgbe_sysctls(struct port_info *pi) 4362{ 4363 struct sysctl_ctx_list *ctx; 4364 struct sysctl_oid *oid; 4365 struct sysctl_oid_list *children; 4366 4367 ctx = device_get_sysctl_ctx(pi->dev); 4368 4369 /* 4370 * dev.cxgbe.X. 4371 */ 4372 oid = device_get_sysctl_tree(pi->dev); 4373 children = SYSCTL_CHILDREN(oid); 4374 4375 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4376 &pi->nrxq, 0, "# of rx queues"); 4377 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4378 &pi->ntxq, 0, "# of tx queues"); 4379 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4380 &pi->first_rxq, 0, "index of first rx queue"); 4381 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4382 &pi->first_txq, 0, "index of first tx queue"); 4383 4384#ifdef TCP_OFFLOAD 4385 if (is_offload(pi->adapter)) { 4386 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4387 &pi->nofldrxq, 0, 4388 "# of rx queues for offloaded TCP connections"); 4389 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4390 &pi->nofldtxq, 0, 4391 "# of tx queues for offloaded TCP connections"); 4392 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4393 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4394 "index of first TOE rx queue"); 4395 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4396 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4397 "index of first TOE tx queue"); 4398 } 4399#endif 4400 4401 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4402 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4403 "holdoff timer index"); 4404 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4405 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4406 "holdoff packet counter index"); 4407 4408 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4409 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4410 "rx queue size"); 4411 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4412 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4413 "tx queue size"); 4414 4415 /* 4416 * dev.cxgbe.X.stats. 4417 */ 4418 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4419 NULL, "port statistics"); 4420 children = SYSCTL_CHILDREN(oid); 4421 4422#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4423 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4424 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 4425 sysctl_handle_t4_reg64, "QU", desc) 4426 4427 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4428 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4429 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4430 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4431 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4432 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4433 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4434 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4435 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4436 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4437 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4438 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4439 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4440 "# of tx frames in this range", 4441 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4442 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4443 "# of tx frames in this range", 4444 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4445 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4446 "# of tx frames in this range", 4447 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4448 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4449 "# of tx frames in this range", 4450 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4451 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4452 "# of tx frames in this range", 4453 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4454 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4455 "# of tx frames in this range", 4456 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4457 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4458 "# of tx frames in this range", 4459 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4460 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4461 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4462 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4463 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4464 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4465 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4466 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4467 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4468 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4469 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4470 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4471 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4472 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4473 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4474 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4475 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4476 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4477 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4478 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4479 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4480 4481 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4482 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4483 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4484 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4485 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4486 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4487 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4488 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4489 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4490 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4491 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4492 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4493 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4494 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4495 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4496 "# of frames received with bad FCS", 4497 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4498 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4499 "# of frames received with length error", 4500 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4501 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4502 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4503 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4504 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4505 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4506 "# of rx frames in this range", 4507 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4508 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4509 "# of rx frames in this range", 4510 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4511 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4512 "# of rx frames in this range", 4513 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4514 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4515 "# of rx frames in this range", 4516 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4517 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4518 "# of rx frames in this range", 4519 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4520 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4521 "# of rx frames in this range", 4522 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4523 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4524 "# of rx frames in this range", 4525 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4526 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4527 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4528 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4529 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4530 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4531 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4532 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4533 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4534 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4535 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4536 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4537 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4538 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4539 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4540 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4541 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4542 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4543 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4544 4545#undef SYSCTL_ADD_T4_REG64 4546 4547#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4548 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4549 &pi->stats.name, desc) 4550 4551 /* We get these from port_stats and they may be stale by upto 1s */ 4552 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4553 "# drops due to buffer-group 0 overflows"); 4554 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4555 "# drops due to buffer-group 1 overflows"); 4556 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4557 "# drops due to buffer-group 2 overflows"); 4558 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4559 "# drops due to buffer-group 3 overflows"); 4560 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4561 "# of buffer-group 0 truncated packets"); 4562 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4563 "# of buffer-group 1 truncated packets"); 4564 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4565 "# of buffer-group 2 truncated packets"); 4566 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4567 "# of buffer-group 3 truncated packets"); 4568 4569#undef SYSCTL_ADD_T4_PORTSTAT 4570 4571 return (0); 4572} 4573 4574static int 4575sysctl_int_array(SYSCTL_HANDLER_ARGS) 4576{ 4577 int rc, *i; 4578 struct sbuf sb; 4579 4580 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4581 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4582 sbuf_printf(&sb, "%d ", *i); 4583 sbuf_trim(&sb); 4584 sbuf_finish(&sb); 4585 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4586 sbuf_delete(&sb); 4587 return (rc); 4588} 4589 4590static int 4591sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4592{ 4593 int rc; 4594 struct sbuf *sb; 4595 4596 rc = sysctl_wire_old_buffer(req, 0); 4597 if (rc != 0) 4598 return(rc); 4599 4600 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4601 if (sb == NULL) 4602 return (ENOMEM); 4603 4604 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4605 rc = sbuf_finish(sb); 4606 sbuf_delete(sb); 4607 4608 return (rc); 4609} 4610 4611static int 4612sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4613{ 4614 struct port_info *pi = arg1; 4615 struct adapter *sc = pi->adapter; 4616 int idx, rc, i; 4617 struct sge_rxq *rxq; 4618 uint8_t v; 4619 4620 idx = pi->tmr_idx; 4621 4622 rc = sysctl_handle_int(oidp, &idx, 0, req); 4623 if (rc != 0 || req->newptr == NULL) 4624 return (rc); 4625 4626 if (idx < 0 || idx >= SGE_NTIMERS) 4627 return (EINVAL); 4628 4629 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4630 "t4tmr"); 4631 if (rc) 4632 return (rc); 4633 4634 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 4635 for_each_rxq(pi, i, rxq) { 4636#ifdef atomic_store_rel_8 4637 atomic_store_rel_8(&rxq->iq.intr_params, v); 4638#else 4639 rxq->iq.intr_params = v; 4640#endif 4641 } 4642 pi->tmr_idx = idx; 4643 4644 end_synchronized_op(sc, LOCK_HELD); 4645 return (0); 4646} 4647 4648static int 4649sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 4650{ 4651 struct port_info *pi = arg1; 4652 struct adapter *sc = pi->adapter; 4653 int idx, rc; 4654 4655 idx = pi->pktc_idx; 4656 4657 rc = sysctl_handle_int(oidp, &idx, 0, req); 4658 if (rc != 0 || req->newptr == NULL) 4659 return (rc); 4660 4661 if (idx < -1 || idx >= SGE_NCOUNTERS) 4662 return (EINVAL); 4663 4664 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4665 "t4pktc"); 4666 if (rc) 4667 return (rc); 4668 4669 if (pi->flags & PORT_INIT_DONE) 4670 rc = EBUSY; /* cannot be changed once the queues are created */ 4671 else 4672 pi->pktc_idx = idx; 4673 4674 end_synchronized_op(sc, LOCK_HELD); 4675 return (rc); 4676} 4677 4678static int 4679sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 4680{ 4681 struct port_info *pi = arg1; 4682 struct adapter *sc = pi->adapter; 4683 int qsize, rc; 4684 4685 qsize = pi->qsize_rxq; 4686 4687 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4688 if (rc != 0 || req->newptr == NULL) 4689 return (rc); 4690 4691 if (qsize < 128 || (qsize & 7)) 4692 return (EINVAL); 4693 4694 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4695 "t4rxqs"); 4696 if (rc) 4697 return (rc); 4698 4699 if (pi->flags & PORT_INIT_DONE) 4700 rc = EBUSY; /* cannot be changed once the queues are created */ 4701 else 4702 pi->qsize_rxq = qsize; 4703 4704 end_synchronized_op(sc, LOCK_HELD); 4705 return (rc); 4706} 4707 4708static int 4709sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 4710{ 4711 struct port_info *pi = arg1; 4712 struct adapter *sc = pi->adapter; 4713 int qsize, rc; 4714 4715 qsize = pi->qsize_txq; 4716 4717 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4718 if (rc != 0 || req->newptr == NULL) 4719 return (rc); 4720 4721 /* bufring size must be powerof2 */ 4722 if (qsize < 128 || !powerof2(qsize)) 4723 return (EINVAL); 4724 4725 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4726 "t4txqs"); 4727 if (rc) 4728 return (rc); 4729 4730 if (pi->flags & PORT_INIT_DONE) 4731 rc = EBUSY; /* cannot be changed once the queues are created */ 4732 else 4733 pi->qsize_txq = qsize; 4734 4735 end_synchronized_op(sc, LOCK_HELD); 4736 return (rc); 4737} 4738 4739static int 4740sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 4741{ 4742 struct adapter *sc = arg1; 4743 int reg = arg2; 4744 uint64_t val; 4745 4746 val = t4_read_reg64(sc, reg); 4747 4748 return (sysctl_handle_64(oidp, &val, 0, req)); 4749} 4750 4751#ifdef SBUF_DRAIN 4752static int 4753sysctl_cctrl(SYSCTL_HANDLER_ARGS) 4754{ 4755 struct adapter *sc = arg1; 4756 struct sbuf *sb; 4757 int rc, i; 4758 uint16_t incr[NMTUS][NCCTRL_WIN]; 4759 static const char *dec_fac[] = { 4760 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 4761 "0.9375" 4762 }; 4763 4764 rc = sysctl_wire_old_buffer(req, 0); 4765 if (rc != 0) 4766 return (rc); 4767 4768 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4769 if (sb == NULL) 4770 return (ENOMEM); 4771 4772 t4_read_cong_tbl(sc, incr); 4773 4774 for (i = 0; i < NCCTRL_WIN; ++i) { 4775 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 4776 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 4777 incr[5][i], incr[6][i], incr[7][i]); 4778 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 4779 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 4780 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 4781 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 4782 } 4783 4784 rc = sbuf_finish(sb); 4785 sbuf_delete(sb); 4786 4787 return (rc); 4788} 4789 4790static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 4791 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 4792 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 4793 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 4794}; 4795 4796static int 4797sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 4798{ 4799 struct adapter *sc = arg1; 4800 struct sbuf *sb; 4801 int rc, i, n, qid = arg2; 4802 uint32_t *buf, *p; 4803 char *qtype; 4804 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 4805 4806 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 4807 ("%s: bad qid %d\n", __func__, qid)); 4808 4809 if (qid < CIM_NUM_IBQ) { 4810 /* inbound queue */ 4811 qtype = "IBQ"; 4812 n = 4 * CIM_IBQ_SIZE; 4813 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4814 rc = t4_read_cim_ibq(sc, qid, buf, n); 4815 } else { 4816 /* outbound queue */ 4817 qtype = "OBQ"; 4818 qid -= CIM_NUM_IBQ; 4819 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 4820 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4821 rc = t4_read_cim_obq(sc, qid, buf, n); 4822 } 4823 4824 if (rc < 0) { 4825 rc = -rc; 4826 goto done; 4827 } 4828 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 4829 4830 rc = sysctl_wire_old_buffer(req, 0); 4831 if (rc != 0) 4832 goto done; 4833 4834 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4835 if (sb == NULL) { 4836 rc = ENOMEM; 4837 goto done; 4838 } 4839 4840 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 4841 for (i = 0, p = buf; i < n; i += 16, p += 4) 4842 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 4843 p[2], p[3]); 4844 4845 rc = sbuf_finish(sb); 4846 sbuf_delete(sb); 4847done: 4848 free(buf, M_CXGBE); 4849 return (rc); 4850} 4851 4852static int 4853sysctl_cim_la(SYSCTL_HANDLER_ARGS) 4854{ 4855 struct adapter *sc = arg1; 4856 u_int cfg; 4857 struct sbuf *sb; 4858 uint32_t *buf, *p; 4859 int rc; 4860 4861 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 4862 if (rc != 0) 4863 return (rc); 4864 4865 rc = sysctl_wire_old_buffer(req, 0); 4866 if (rc != 0) 4867 return (rc); 4868 4869 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4870 if (sb == NULL) 4871 return (ENOMEM); 4872 4873 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 4874 M_ZERO | M_WAITOK); 4875 4876 rc = -t4_cim_read_la(sc, buf, NULL); 4877 if (rc != 0) 4878 goto done; 4879 4880 sbuf_printf(sb, "Status Data PC%s", 4881 cfg & F_UPDBGLACAPTPCONLY ? "" : 4882 " LS0Stat LS0Addr LS0Data"); 4883 4884 KASSERT((sc->params.cim_la_size & 7) == 0, 4885 ("%s: p will walk off the end of buf", __func__)); 4886 4887 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 4888 if (cfg & F_UPDBGLACAPTPCONLY) { 4889 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 4890 p[6], p[7]); 4891 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 4892 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 4893 p[4] & 0xff, p[5] >> 8); 4894 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 4895 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4896 p[1] & 0xf, p[2] >> 4); 4897 } else { 4898 sbuf_printf(sb, 4899 "\n %02x %x%07x %x%07x %08x %08x " 4900 "%08x%08x%08x%08x", 4901 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4902 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 4903 p[6], p[7]); 4904 } 4905 } 4906 4907 rc = sbuf_finish(sb); 4908 sbuf_delete(sb); 4909done: 4910 free(buf, M_CXGBE); 4911 return (rc); 4912} 4913 4914static int 4915sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 4916{ 4917 struct adapter *sc = arg1; 4918 struct sbuf *sb; 4919 int rc, i; 4920 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4921 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4922 uint16_t thres[CIM_NUM_IBQ]; 4923 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 4924 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 4925 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 4926 4927 if (is_t4(sc)) { 4928 cim_num_obq = CIM_NUM_OBQ; 4929 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 4930 obq_rdaddr = A_UP_OBQ_0_REALADDR; 4931 } else { 4932 cim_num_obq = CIM_NUM_OBQ_T5; 4933 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 4934 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 4935 } 4936 nq = CIM_NUM_IBQ + cim_num_obq; 4937 4938 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 4939 if (rc == 0) 4940 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 4941 if (rc != 0) 4942 return (rc); 4943 4944 t4_read_cimq_cfg(sc, base, size, thres); 4945 4946 rc = sysctl_wire_old_buffer(req, 0); 4947 if (rc != 0) 4948 return (rc); 4949 4950 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4951 if (sb == NULL) 4952 return (ENOMEM); 4953 4954 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 4955 4956 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 4957 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 4958 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 4959 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4960 G_QUEREMFLITS(p[2]) * 16); 4961 for ( ; i < nq; i++, p += 4, wr += 2) 4962 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 4963 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 4964 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4965 G_QUEREMFLITS(p[2]) * 16); 4966 4967 rc = sbuf_finish(sb); 4968 sbuf_delete(sb); 4969 4970 return (rc); 4971} 4972 4973static int 4974sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 4975{ 4976 struct adapter *sc = arg1; 4977 struct sbuf *sb; 4978 int rc; 4979 struct tp_cpl_stats stats; 4980 4981 rc = sysctl_wire_old_buffer(req, 0); 4982 if (rc != 0) 4983 return (rc); 4984 4985 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4986 if (sb == NULL) 4987 return (ENOMEM); 4988 4989 t4_tp_get_cpl_stats(sc, &stats); 4990 4991 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4992 "channel 3\n"); 4993 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 4994 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 4995 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 4996 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 4997 4998 rc = sbuf_finish(sb); 4999 sbuf_delete(sb); 5000 5001 return (rc); 5002} 5003 5004static int 5005sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5006{ 5007 struct adapter *sc = arg1; 5008 struct sbuf *sb; 5009 int rc; 5010 struct tp_usm_stats stats; 5011 5012 rc = sysctl_wire_old_buffer(req, 0); 5013 if (rc != 0) 5014 return(rc); 5015 5016 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5017 if (sb == NULL) 5018 return (ENOMEM); 5019 5020 t4_get_usm_stats(sc, &stats); 5021 5022 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5023 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5024 sbuf_printf(sb, "Drops: %u", stats.drops); 5025 5026 rc = sbuf_finish(sb); 5027 sbuf_delete(sb); 5028 5029 return (rc); 5030} 5031 5032const char *devlog_level_strings[] = { 5033 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5034 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5035 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5036 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5037 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5038 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5039}; 5040 5041const char *devlog_facility_strings[] = { 5042 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5043 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5044 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5045 [FW_DEVLOG_FACILITY_RES] = "RES", 5046 [FW_DEVLOG_FACILITY_HW] = "HW", 5047 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5048 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5049 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5050 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5051 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5052 [FW_DEVLOG_FACILITY_VI] = "VI", 5053 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5054 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5055 [FW_DEVLOG_FACILITY_TM] = "TM", 5056 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5057 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5058 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5059 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5060 [FW_DEVLOG_FACILITY_RI] = "RI", 5061 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5062 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5063 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5064 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5065}; 5066 5067static int 5068sysctl_devlog(SYSCTL_HANDLER_ARGS) 5069{ 5070 struct adapter *sc = arg1; 5071 struct devlog_params *dparams = &sc->params.devlog; 5072 struct fw_devlog_e *buf, *e; 5073 int i, j, rc, nentries, first = 0; 5074 struct sbuf *sb; 5075 uint64_t ftstamp = UINT64_MAX; 5076 5077 if (dparams->start == 0) { 5078 dparams->memtype = 0; 5079 dparams->start = 0x84000; 5080 dparams->size = 32768; 5081 } 5082 5083 nentries = dparams->size / sizeof(struct fw_devlog_e); 5084 5085 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5086 if (buf == NULL) 5087 return (ENOMEM); 5088 5089 rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 5090 (void *)buf); 5091 if (rc != 0) 5092 goto done; 5093 5094 for (i = 0; i < nentries; i++) { 5095 e = &buf[i]; 5096 5097 if (e->timestamp == 0) 5098 break; /* end */ 5099 5100 e->timestamp = be64toh(e->timestamp); 5101 e->seqno = be32toh(e->seqno); 5102 for (j = 0; j < 8; j++) 5103 e->params[j] = be32toh(e->params[j]); 5104 5105 if (e->timestamp < ftstamp) { 5106 ftstamp = e->timestamp; 5107 first = i; 5108 } 5109 } 5110 5111 if (buf[first].timestamp == 0) 5112 goto done; /* nothing in the log */ 5113 5114 rc = sysctl_wire_old_buffer(req, 0); 5115 if (rc != 0) 5116 goto done; 5117 5118 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5119 if (sb == NULL) { 5120 rc = ENOMEM; 5121 goto done; 5122 } 5123 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5124 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5125 5126 i = first; 5127 do { 5128 e = &buf[i]; 5129 if (e->timestamp == 0) 5130 break; /* end */ 5131 5132 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5133 e->seqno, e->timestamp, 5134 (e->level < nitems(devlog_level_strings) ? 5135 devlog_level_strings[e->level] : "UNKNOWN"), 5136 (e->facility < nitems(devlog_facility_strings) ? 5137 devlog_facility_strings[e->facility] : "UNKNOWN")); 5138 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5139 e->params[2], e->params[3], e->params[4], 5140 e->params[5], e->params[6], e->params[7]); 5141 5142 if (++i == nentries) 5143 i = 0; 5144 } while (i != first); 5145 5146 rc = sbuf_finish(sb); 5147 sbuf_delete(sb); 5148done: 5149 free(buf, M_CXGBE); 5150 return (rc); 5151} 5152 5153static int 5154sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5155{ 5156 struct adapter *sc = arg1; 5157 struct sbuf *sb; 5158 int rc; 5159 struct tp_fcoe_stats stats[4]; 5160 5161 rc = sysctl_wire_old_buffer(req, 0); 5162 if (rc != 0) 5163 return (rc); 5164 5165 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5166 if (sb == NULL) 5167 return (ENOMEM); 5168 5169 t4_get_fcoe_stats(sc, 0, &stats[0]); 5170 t4_get_fcoe_stats(sc, 1, &stats[1]); 5171 t4_get_fcoe_stats(sc, 2, &stats[2]); 5172 t4_get_fcoe_stats(sc, 3, &stats[3]); 5173 5174 sbuf_printf(sb, " channel 0 channel 1 " 5175 "channel 2 channel 3\n"); 5176 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5177 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5178 stats[3].octetsDDP); 5179 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5180 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5181 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5182 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5183 stats[3].framesDrop); 5184 5185 rc = sbuf_finish(sb); 5186 sbuf_delete(sb); 5187 5188 return (rc); 5189} 5190 5191static int 5192sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5193{ 5194 struct adapter *sc = arg1; 5195 struct sbuf *sb; 5196 int rc, i; 5197 unsigned int map, kbps, ipg, mode; 5198 unsigned int pace_tab[NTX_SCHED]; 5199 5200 rc = sysctl_wire_old_buffer(req, 0); 5201 if (rc != 0) 5202 return (rc); 5203 5204 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5205 if (sb == NULL) 5206 return (ENOMEM); 5207 5208 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5209 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5210 t4_read_pace_tbl(sc, pace_tab); 5211 5212 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5213 "Class IPG (0.1 ns) Flow IPG (us)"); 5214 5215 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5216 t4_get_tx_sched(sc, i, &kbps, &ipg); 5217 sbuf_printf(sb, "\n %u %-5s %u ", i, 5218 (mode & (1 << i)) ? "flow" : "class", map & 3); 5219 if (kbps) 5220 sbuf_printf(sb, "%9u ", kbps); 5221 else 5222 sbuf_printf(sb, " disabled "); 5223 5224 if (ipg) 5225 sbuf_printf(sb, "%13u ", ipg); 5226 else 5227 sbuf_printf(sb, " disabled "); 5228 5229 if (pace_tab[i]) 5230 sbuf_printf(sb, "%10u", pace_tab[i]); 5231 else 5232 sbuf_printf(sb, " disabled"); 5233 } 5234 5235 rc = sbuf_finish(sb); 5236 sbuf_delete(sb); 5237 5238 return (rc); 5239} 5240 5241static int 5242sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5243{ 5244 struct adapter *sc = arg1; 5245 struct sbuf *sb; 5246 int rc, i, j; 5247 uint64_t *p0, *p1; 5248 struct lb_port_stats s[2]; 5249 static const char *stat_name[] = { 5250 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5251 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5252 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5253 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5254 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5255 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5256 "BG2FramesTrunc:", "BG3FramesTrunc:" 5257 }; 5258 5259 rc = sysctl_wire_old_buffer(req, 0); 5260 if (rc != 0) 5261 return (rc); 5262 5263 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5264 if (sb == NULL) 5265 return (ENOMEM); 5266 5267 memset(s, 0, sizeof(s)); 5268 5269 for (i = 0; i < 4; i += 2) { 5270 t4_get_lb_stats(sc, i, &s[0]); 5271 t4_get_lb_stats(sc, i + 1, &s[1]); 5272 5273 p0 = &s[0].octets; 5274 p1 = &s[1].octets; 5275 sbuf_printf(sb, "%s Loopback %u" 5276 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5277 5278 for (j = 0; j < nitems(stat_name); j++) 5279 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5280 *p0++, *p1++); 5281 } 5282 5283 rc = sbuf_finish(sb); 5284 sbuf_delete(sb); 5285 5286 return (rc); 5287} 5288 5289struct mem_desc { 5290 unsigned int base; 5291 unsigned int limit; 5292 unsigned int idx; 5293}; 5294 5295static int 5296mem_desc_cmp(const void *a, const void *b) 5297{ 5298 return ((const struct mem_desc *)a)->base - 5299 ((const struct mem_desc *)b)->base; 5300} 5301 5302static void 5303mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5304 unsigned int to) 5305{ 5306 unsigned int size; 5307 5308 size = to - from + 1; 5309 if (size == 0) 5310 return; 5311 5312 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5313 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5314} 5315 5316static int 5317sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5318{ 5319 struct adapter *sc = arg1; 5320 struct sbuf *sb; 5321 int rc, i, n; 5322 uint32_t lo, hi, used, alloc; 5323 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5324 static const char *region[] = { 5325 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5326 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5327 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5328 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5329 "RQUDP region:", "PBL region:", "TXPBL region:", 5330 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5331 "On-chip queues:" 5332 }; 5333 struct mem_desc avail[4]; 5334 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5335 struct mem_desc *md = mem; 5336 5337 rc = sysctl_wire_old_buffer(req, 0); 5338 if (rc != 0) 5339 return (rc); 5340 5341 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5342 if (sb == NULL) 5343 return (ENOMEM); 5344 5345 for (i = 0; i < nitems(mem); i++) { 5346 mem[i].limit = 0; 5347 mem[i].idx = i; 5348 } 5349 5350 /* Find and sort the populated memory ranges */ 5351 i = 0; 5352 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5353 if (lo & F_EDRAM0_ENABLE) { 5354 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5355 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5356 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5357 avail[i].idx = 0; 5358 i++; 5359 } 5360 if (lo & F_EDRAM1_ENABLE) { 5361 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5362 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5363 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5364 avail[i].idx = 1; 5365 i++; 5366 } 5367 if (lo & F_EXT_MEM_ENABLE) { 5368 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5369 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5370 avail[i].limit = avail[i].base + 5371 (G_EXT_MEM_SIZE(hi) << 20); 5372 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5373 i++; 5374 } 5375 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5376 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5377 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5378 avail[i].limit = avail[i].base + 5379 (G_EXT_MEM1_SIZE(hi) << 20); 5380 avail[i].idx = 4; 5381 i++; 5382 } 5383 if (!i) /* no memory available */ 5384 return 0; 5385 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5386 5387 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5388 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5389 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5390 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5391 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5392 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5393 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5394 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5395 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5396 5397 /* the next few have explicit upper bounds */ 5398 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5399 md->limit = md->base - 1 + 5400 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5401 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5402 md++; 5403 5404 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5405 md->limit = md->base - 1 + 5406 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5407 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5408 md++; 5409 5410 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5411 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5412 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5413 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5414 } else { 5415 md->base = 0; 5416 md->idx = nitems(region); /* hide it */ 5417 } 5418 md++; 5419 5420#define ulp_region(reg) \ 5421 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5422 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5423 5424 ulp_region(RX_ISCSI); 5425 ulp_region(RX_TDDP); 5426 ulp_region(TX_TPT); 5427 ulp_region(RX_STAG); 5428 ulp_region(RX_RQ); 5429 ulp_region(RX_RQUDP); 5430 ulp_region(RX_PBL); 5431 ulp_region(TX_PBL); 5432#undef ulp_region 5433 5434 md->base = 0; 5435 md->idx = nitems(region); 5436 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5437 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5438 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5439 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5440 } 5441 md++; 5442 5443 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5444 md->limit = md->base + sc->tids.ntids - 1; 5445 md++; 5446 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5447 md->limit = md->base + sc->tids.ntids - 1; 5448 md++; 5449 5450 md->base = sc->vres.ocq.start; 5451 if (sc->vres.ocq.size) 5452 md->limit = md->base + sc->vres.ocq.size - 1; 5453 else 5454 md->idx = nitems(region); /* hide it */ 5455 md++; 5456 5457 /* add any address-space holes, there can be up to 3 */ 5458 for (n = 0; n < i - 1; n++) 5459 if (avail[n].limit < avail[n + 1].base) 5460 (md++)->base = avail[n].limit; 5461 if (avail[n].limit) 5462 (md++)->base = avail[n].limit; 5463 5464 n = md - mem; 5465 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 5466 5467 for (lo = 0; lo < i; lo++) 5468 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 5469 avail[lo].limit - 1); 5470 5471 sbuf_printf(sb, "\n"); 5472 for (i = 0; i < n; i++) { 5473 if (mem[i].idx >= nitems(region)) 5474 continue; /* skip holes */ 5475 if (!mem[i].limit) 5476 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 5477 mem_region_show(sb, region[mem[i].idx], mem[i].base, 5478 mem[i].limit); 5479 } 5480 5481 sbuf_printf(sb, "\n"); 5482 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 5483 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 5484 mem_region_show(sb, "uP RAM:", lo, hi); 5485 5486 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 5487 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 5488 mem_region_show(sb, "uP Extmem2:", lo, hi); 5489 5490 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 5491 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 5492 G_PMRXMAXPAGE(lo), 5493 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 5494 (lo & F_PMRXNUMCHN) ? 2 : 1); 5495 5496 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 5497 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 5498 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 5499 G_PMTXMAXPAGE(lo), 5500 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 5501 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 5502 sbuf_printf(sb, "%u p-structs\n", 5503 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 5504 5505 for (i = 0; i < 4; i++) { 5506 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 5507 if (is_t4(sc)) { 5508 used = G_USED(lo); 5509 alloc = G_ALLOC(lo); 5510 } else { 5511 used = G_T5_USED(lo); 5512 alloc = G_T5_ALLOC(lo); 5513 } 5514 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 5515 i, used, alloc); 5516 } 5517 for (i = 0; i < 4; i++) { 5518 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 5519 if (is_t4(sc)) { 5520 used = G_USED(lo); 5521 alloc = G_ALLOC(lo); 5522 } else { 5523 used = G_T5_USED(lo); 5524 alloc = G_T5_ALLOC(lo); 5525 } 5526 sbuf_printf(sb, 5527 "\nLoopback %d using %u pages out of %u allocated", 5528 i, used, alloc); 5529 } 5530 5531 rc = sbuf_finish(sb); 5532 sbuf_delete(sb); 5533 5534 return (rc); 5535} 5536 5537static int 5538sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 5539{ 5540 struct adapter *sc = arg1; 5541 struct sbuf *sb; 5542 int rc; 5543 uint16_t mtus[NMTUS]; 5544 5545 rc = sysctl_wire_old_buffer(req, 0); 5546 if (rc != 0) 5547 return (rc); 5548 5549 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5550 if (sb == NULL) 5551 return (ENOMEM); 5552 5553 t4_read_mtu_tbl(sc, mtus, NULL); 5554 5555 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 5556 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 5557 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 5558 mtus[14], mtus[15]); 5559 5560 rc = sbuf_finish(sb); 5561 sbuf_delete(sb); 5562 5563 return (rc); 5564} 5565 5566static int 5567sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 5568{ 5569 struct adapter *sc = arg1; 5570 struct sbuf *sb; 5571 int rc, i; 5572 uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 5573 uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 5574 static const char *pm_stats[] = { 5575 "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 5576 }; 5577 5578 rc = sysctl_wire_old_buffer(req, 0); 5579 if (rc != 0) 5580 return (rc); 5581 5582 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5583 if (sb == NULL) 5584 return (ENOMEM); 5585 5586 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 5587 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 5588 5589 sbuf_printf(sb, " Tx count Tx cycles " 5590 "Rx count Rx cycles"); 5591 for (i = 0; i < PM_NSTATS; i++) 5592 sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 5593 pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 5594 5595 rc = sbuf_finish(sb); 5596 sbuf_delete(sb); 5597 5598 return (rc); 5599} 5600 5601static int 5602sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 5603{ 5604 struct adapter *sc = arg1; 5605 struct sbuf *sb; 5606 int rc; 5607 struct tp_rdma_stats stats; 5608 5609 rc = sysctl_wire_old_buffer(req, 0); 5610 if (rc != 0) 5611 return (rc); 5612 5613 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5614 if (sb == NULL) 5615 return (ENOMEM); 5616 5617 t4_tp_get_rdma_stats(sc, &stats); 5618 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 5619 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 5620 5621 rc = sbuf_finish(sb); 5622 sbuf_delete(sb); 5623 5624 return (rc); 5625} 5626 5627static int 5628sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 5629{ 5630 struct adapter *sc = arg1; 5631 struct sbuf *sb; 5632 int rc; 5633 struct tp_tcp_stats v4, v6; 5634 5635 rc = sysctl_wire_old_buffer(req, 0); 5636 if (rc != 0) 5637 return (rc); 5638 5639 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5640 if (sb == NULL) 5641 return (ENOMEM); 5642 5643 t4_tp_get_tcp_stats(sc, &v4, &v6); 5644 sbuf_printf(sb, 5645 " IP IPv6\n"); 5646 sbuf_printf(sb, "OutRsts: %20u %20u\n", 5647 v4.tcpOutRsts, v6.tcpOutRsts); 5648 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 5649 v4.tcpInSegs, v6.tcpInSegs); 5650 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 5651 v4.tcpOutSegs, v6.tcpOutSegs); 5652 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 5653 v4.tcpRetransSegs, v6.tcpRetransSegs); 5654 5655 rc = sbuf_finish(sb); 5656 sbuf_delete(sb); 5657 5658 return (rc); 5659} 5660 5661static int 5662sysctl_tids(SYSCTL_HANDLER_ARGS) 5663{ 5664 struct adapter *sc = arg1; 5665 struct sbuf *sb; 5666 int rc; 5667 struct tid_info *t = &sc->tids; 5668 5669 rc = sysctl_wire_old_buffer(req, 0); 5670 if (rc != 0) 5671 return (rc); 5672 5673 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5674 if (sb == NULL) 5675 return (ENOMEM); 5676 5677 if (t->natids) { 5678 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 5679 t->atids_in_use); 5680 } 5681 5682 if (t->ntids) { 5683 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5684 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 5685 5686 if (b) { 5687 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 5688 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5689 t->ntids - 1); 5690 } else { 5691 sbuf_printf(sb, "TID range: %u-%u", 5692 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5693 t->ntids - 1); 5694 } 5695 } else 5696 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 5697 sbuf_printf(sb, ", in use: %u\n", 5698 atomic_load_acq_int(&t->tids_in_use)); 5699 } 5700 5701 if (t->nstids) { 5702 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 5703 t->stid_base + t->nstids - 1, t->stids_in_use); 5704 } 5705 5706 if (t->nftids) { 5707 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 5708 t->ftid_base + t->nftids - 1); 5709 } 5710 5711 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 5712 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 5713 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 5714 5715 rc = sbuf_finish(sb); 5716 sbuf_delete(sb); 5717 5718 return (rc); 5719} 5720 5721static int 5722sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 5723{ 5724 struct adapter *sc = arg1; 5725 struct sbuf *sb; 5726 int rc; 5727 struct tp_err_stats stats; 5728 5729 rc = sysctl_wire_old_buffer(req, 0); 5730 if (rc != 0) 5731 return (rc); 5732 5733 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5734 if (sb == NULL) 5735 return (ENOMEM); 5736 5737 t4_tp_get_err_stats(sc, &stats); 5738 5739 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5740 "channel 3\n"); 5741 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 5742 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 5743 stats.macInErrs[3]); 5744 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 5745 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 5746 stats.hdrInErrs[3]); 5747 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 5748 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 5749 stats.tcpInErrs[3]); 5750 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 5751 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 5752 stats.tcp6InErrs[3]); 5753 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 5754 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 5755 stats.tnlCongDrops[3]); 5756 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 5757 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 5758 stats.tnlTxDrops[3]); 5759 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 5760 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 5761 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 5762 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 5763 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 5764 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 5765 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 5766 stats.ofldNoNeigh, stats.ofldCongDefer); 5767 5768 rc = sbuf_finish(sb); 5769 sbuf_delete(sb); 5770 5771 return (rc); 5772} 5773 5774static int 5775sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 5776{ 5777 struct adapter *sc = arg1; 5778 struct sbuf *sb; 5779 int rc; 5780 u64 nrate[NCHAN], orate[NCHAN]; 5781 5782 rc = sysctl_wire_old_buffer(req, 0); 5783 if (rc != 0) 5784 return (rc); 5785 5786 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5787 if (sb == NULL) 5788 return (ENOMEM); 5789 5790 t4_get_chan_txrate(sc, nrate, orate); 5791 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5792 "channel 3\n"); 5793 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 5794 nrate[0], nrate[1], nrate[2], nrate[3]); 5795 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 5796 orate[0], orate[1], orate[2], orate[3]); 5797 5798 rc = sbuf_finish(sb); 5799 sbuf_delete(sb); 5800 5801 return (rc); 5802} 5803 5804static int 5805sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 5806{ 5807 struct adapter *sc = arg1; 5808 struct sbuf *sb; 5809 int rc, v; 5810 5811 rc = sysctl_wire_old_buffer(req, 0); 5812 if (rc != 0) 5813 return (rc); 5814 5815 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5816 if (sb == NULL) 5817 return (ENOMEM); 5818 5819 v = t4_read_reg(sc, A_SGE_STAT_CFG); 5820 if (G_STATSOURCE_T5(v) == 7) { 5821 if (G_STATMODE(v) == 0) { 5822 sbuf_printf(sb, "total %d, incomplete %d", 5823 t4_read_reg(sc, A_SGE_STAT_TOTAL), 5824 t4_read_reg(sc, A_SGE_STAT_MATCH)); 5825 } else if (G_STATMODE(v) == 1) { 5826 sbuf_printf(sb, "total %d, data overflow %d", 5827 t4_read_reg(sc, A_SGE_STAT_TOTAL), 5828 t4_read_reg(sc, A_SGE_STAT_MATCH)); 5829 } 5830 } 5831 rc = sbuf_finish(sb); 5832 sbuf_delete(sb); 5833 5834 return (rc); 5835} 5836#endif 5837 5838static inline void 5839txq_start(struct ifnet *ifp, struct sge_txq *txq) 5840{ 5841 struct buf_ring *br; 5842 struct mbuf *m; 5843 5844 TXQ_LOCK_ASSERT_OWNED(txq); 5845 5846 br = txq->br; 5847 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 5848 if (m) 5849 t4_eth_tx(ifp, txq, m); 5850} 5851 5852void 5853t4_tx_callout(void *arg) 5854{ 5855 struct sge_eq *eq = arg; 5856 struct adapter *sc; 5857 5858 if (EQ_TRYLOCK(eq) == 0) 5859 goto reschedule; 5860 5861 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 5862 EQ_UNLOCK(eq); 5863reschedule: 5864 if (__predict_true(!(eq->flags && EQ_DOOMED))) 5865 callout_schedule(&eq->tx_callout, 1); 5866 return; 5867 } 5868 5869 EQ_LOCK_ASSERT_OWNED(eq); 5870 5871 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 5872 5873 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5874 struct sge_txq *txq = arg; 5875 struct port_info *pi = txq->ifp->if_softc; 5876 5877 sc = pi->adapter; 5878 } else { 5879 struct sge_wrq *wrq = arg; 5880 5881 sc = wrq->adapter; 5882 } 5883 5884 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 5885 } 5886 5887 EQ_UNLOCK(eq); 5888} 5889 5890void 5891t4_tx_task(void *arg, int count) 5892{ 5893 struct sge_eq *eq = arg; 5894 5895 EQ_LOCK(eq); 5896 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5897 struct sge_txq *txq = arg; 5898 txq_start(txq->ifp, txq); 5899 } else { 5900 struct sge_wrq *wrq = arg; 5901 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 5902 } 5903 EQ_UNLOCK(eq); 5904} 5905 5906static uint32_t 5907fconf_to_mode(uint32_t fconf) 5908{ 5909 uint32_t mode; 5910 5911 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 5912 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 5913 5914 if (fconf & F_FRAGMENTATION) 5915 mode |= T4_FILTER_IP_FRAGMENT; 5916 5917 if (fconf & F_MPSHITTYPE) 5918 mode |= T4_FILTER_MPS_HIT_TYPE; 5919 5920 if (fconf & F_MACMATCH) 5921 mode |= T4_FILTER_MAC_IDX; 5922 5923 if (fconf & F_ETHERTYPE) 5924 mode |= T4_FILTER_ETH_TYPE; 5925 5926 if (fconf & F_PROTOCOL) 5927 mode |= T4_FILTER_IP_PROTO; 5928 5929 if (fconf & F_TOS) 5930 mode |= T4_FILTER_IP_TOS; 5931 5932 if (fconf & F_VLAN) 5933 mode |= T4_FILTER_VLAN; 5934 5935 if (fconf & F_VNIC_ID) 5936 mode |= T4_FILTER_VNIC; 5937 5938 if (fconf & F_PORT) 5939 mode |= T4_FILTER_PORT; 5940 5941 if (fconf & F_FCOE) 5942 mode |= T4_FILTER_FCoE; 5943 5944 return (mode); 5945} 5946 5947static uint32_t 5948mode_to_fconf(uint32_t mode) 5949{ 5950 uint32_t fconf = 0; 5951 5952 if (mode & T4_FILTER_IP_FRAGMENT) 5953 fconf |= F_FRAGMENTATION; 5954 5955 if (mode & T4_FILTER_MPS_HIT_TYPE) 5956 fconf |= F_MPSHITTYPE; 5957 5958 if (mode & T4_FILTER_MAC_IDX) 5959 fconf |= F_MACMATCH; 5960 5961 if (mode & T4_FILTER_ETH_TYPE) 5962 fconf |= F_ETHERTYPE; 5963 5964 if (mode & T4_FILTER_IP_PROTO) 5965 fconf |= F_PROTOCOL; 5966 5967 if (mode & T4_FILTER_IP_TOS) 5968 fconf |= F_TOS; 5969 5970 if (mode & T4_FILTER_VLAN) 5971 fconf |= F_VLAN; 5972 5973 if (mode & T4_FILTER_VNIC) 5974 fconf |= F_VNIC_ID; 5975 5976 if (mode & T4_FILTER_PORT) 5977 fconf |= F_PORT; 5978 5979 if (mode & T4_FILTER_FCoE) 5980 fconf |= F_FCOE; 5981 5982 return (fconf); 5983} 5984 5985static uint32_t 5986fspec_to_fconf(struct t4_filter_specification *fs) 5987{ 5988 uint32_t fconf = 0; 5989 5990 if (fs->val.frag || fs->mask.frag) 5991 fconf |= F_FRAGMENTATION; 5992 5993 if (fs->val.matchtype || fs->mask.matchtype) 5994 fconf |= F_MPSHITTYPE; 5995 5996 if (fs->val.macidx || fs->mask.macidx) 5997 fconf |= F_MACMATCH; 5998 5999 if (fs->val.ethtype || fs->mask.ethtype) 6000 fconf |= F_ETHERTYPE; 6001 6002 if (fs->val.proto || fs->mask.proto) 6003 fconf |= F_PROTOCOL; 6004 6005 if (fs->val.tos || fs->mask.tos) 6006 fconf |= F_TOS; 6007 6008 if (fs->val.vlan_vld || fs->mask.vlan_vld) 6009 fconf |= F_VLAN; 6010 6011 if (fs->val.vnic_vld || fs->mask.vnic_vld) 6012 fconf |= F_VNIC_ID; 6013 6014 if (fs->val.iport || fs->mask.iport) 6015 fconf |= F_PORT; 6016 6017 if (fs->val.fcoe || fs->mask.fcoe) 6018 fconf |= F_FCOE; 6019 6020 return (fconf); 6021} 6022 6023static int 6024get_filter_mode(struct adapter *sc, uint32_t *mode) 6025{ 6026 int rc; 6027 uint32_t fconf; 6028 6029 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6030 "t4getfm"); 6031 if (rc) 6032 return (rc); 6033 6034 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 6035 A_TP_VLAN_PRI_MAP); 6036 6037 if (sc->filter_mode != fconf) { 6038 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 6039 device_get_nameunit(sc->dev), sc->filter_mode, fconf); 6040 sc->filter_mode = fconf; 6041 } 6042 6043 *mode = fconf_to_mode(sc->filter_mode); 6044 6045 end_synchronized_op(sc, LOCK_HELD); 6046 return (0); 6047} 6048 6049static int 6050set_filter_mode(struct adapter *sc, uint32_t mode) 6051{ 6052 uint32_t fconf; 6053 int rc; 6054 6055 fconf = mode_to_fconf(mode); 6056 6057 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6058 "t4setfm"); 6059 if (rc) 6060 return (rc); 6061 6062 if (sc->tids.ftids_in_use > 0) { 6063 rc = EBUSY; 6064 goto done; 6065 } 6066 6067#ifdef TCP_OFFLOAD 6068 if (sc->offload_map) { 6069 rc = EBUSY; 6070 goto done; 6071 } 6072#endif 6073 6074#ifdef notyet 6075 rc = -t4_set_filter_mode(sc, fconf); 6076 if (rc == 0) 6077 sc->filter_mode = fconf; 6078#else 6079 rc = ENOTSUP; 6080#endif 6081 6082done: 6083 end_synchronized_op(sc, LOCK_HELD); 6084 return (rc); 6085} 6086 6087static inline uint64_t 6088get_filter_hits(struct adapter *sc, uint32_t fid) 6089{ 6090 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6091 uint64_t hits; 6092 6093 memwin_info(sc, 0, &mw_base, NULL); 6094 off = position_memwin(sc, 0, 6095 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 6096 hits = t4_read_reg64(sc, mw_base + off + 16); 6097 6098 return (be64toh(hits)); 6099} 6100 6101static int 6102get_filter(struct adapter *sc, struct t4_filter *t) 6103{ 6104 int i, rc, nfilters = sc->tids.nftids; 6105 struct filter_entry *f; 6106 6107 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6108 "t4getf"); 6109 if (rc) 6110 return (rc); 6111 6112 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 6113 t->idx >= nfilters) { 6114 t->idx = 0xffffffff; 6115 goto done; 6116 } 6117 6118 f = &sc->tids.ftid_tab[t->idx]; 6119 for (i = t->idx; i < nfilters; i++, f++) { 6120 if (f->valid) { 6121 t->idx = i; 6122 t->l2tidx = f->l2t ? f->l2t->idx : 0; 6123 t->smtidx = f->smtidx; 6124 if (f->fs.hitcnts) 6125 t->hits = get_filter_hits(sc, t->idx); 6126 else 6127 t->hits = UINT64_MAX; 6128 t->fs = f->fs; 6129 6130 goto done; 6131 } 6132 } 6133 6134 t->idx = 0xffffffff; 6135done: 6136 end_synchronized_op(sc, LOCK_HELD); 6137 return (0); 6138} 6139 6140static int 6141set_filter(struct adapter *sc, struct t4_filter *t) 6142{ 6143 unsigned int nfilters, nports; 6144 struct filter_entry *f; 6145 int i, rc; 6146 6147 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 6148 if (rc) 6149 return (rc); 6150 6151 nfilters = sc->tids.nftids; 6152 nports = sc->params.nports; 6153 6154 if (nfilters == 0) { 6155 rc = ENOTSUP; 6156 goto done; 6157 } 6158 6159 if (!(sc->flags & FULL_INIT_DONE)) { 6160 rc = EAGAIN; 6161 goto done; 6162 } 6163 6164 if (t->idx >= nfilters) { 6165 rc = EINVAL; 6166 goto done; 6167 } 6168 6169 /* Validate against the global filter mode */ 6170 if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) { 6171 rc = E2BIG; 6172 goto done; 6173 } 6174 6175 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 6176 rc = EINVAL; 6177 goto done; 6178 } 6179 6180 if (t->fs.val.iport >= nports) { 6181 rc = EINVAL; 6182 goto done; 6183 } 6184 6185 /* Can't specify an iq if not steering to it */ 6186 if (!t->fs.dirsteer && t->fs.iq) { 6187 rc = EINVAL; 6188 goto done; 6189 } 6190 6191 /* IPv6 filter idx must be 4 aligned */ 6192 if (t->fs.type == 1 && 6193 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 6194 rc = EINVAL; 6195 goto done; 6196 } 6197 6198 if (sc->tids.ftid_tab == NULL) { 6199 KASSERT(sc->tids.ftids_in_use == 0, 6200 ("%s: no memory allocated but filters_in_use > 0", 6201 __func__)); 6202 6203 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 6204 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 6205 if (sc->tids.ftid_tab == NULL) { 6206 rc = ENOMEM; 6207 goto done; 6208 } 6209 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 6210 } 6211 6212 for (i = 0; i < 4; i++) { 6213 f = &sc->tids.ftid_tab[t->idx + i]; 6214 6215 if (f->pending || f->valid) { 6216 rc = EBUSY; 6217 goto done; 6218 } 6219 if (f->locked) { 6220 rc = EPERM; 6221 goto done; 6222 } 6223 6224 if (t->fs.type == 0) 6225 break; 6226 } 6227 6228 f = &sc->tids.ftid_tab[t->idx]; 6229 f->fs = t->fs; 6230 6231 rc = set_filter_wr(sc, t->idx); 6232done: 6233 end_synchronized_op(sc, 0); 6234 6235 if (rc == 0) { 6236 mtx_lock(&sc->tids.ftid_lock); 6237 for (;;) { 6238 if (f->pending == 0) { 6239 rc = f->valid ? 0 : EIO; 6240 break; 6241 } 6242 6243 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6244 PCATCH, "t4setfw", 0)) { 6245 rc = EINPROGRESS; 6246 break; 6247 } 6248 } 6249 mtx_unlock(&sc->tids.ftid_lock); 6250 } 6251 return (rc); 6252} 6253 6254static int 6255del_filter(struct adapter *sc, struct t4_filter *t) 6256{ 6257 unsigned int nfilters; 6258 struct filter_entry *f; 6259 int rc; 6260 6261 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 6262 if (rc) 6263 return (rc); 6264 6265 nfilters = sc->tids.nftids; 6266 6267 if (nfilters == 0) { 6268 rc = ENOTSUP; 6269 goto done; 6270 } 6271 6272 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 6273 t->idx >= nfilters) { 6274 rc = EINVAL; 6275 goto done; 6276 } 6277 6278 if (!(sc->flags & FULL_INIT_DONE)) { 6279 rc = EAGAIN; 6280 goto done; 6281 } 6282 6283 f = &sc->tids.ftid_tab[t->idx]; 6284 6285 if (f->pending) { 6286 rc = EBUSY; 6287 goto done; 6288 } 6289 if (f->locked) { 6290 rc = EPERM; 6291 goto done; 6292 } 6293 6294 if (f->valid) { 6295 t->fs = f->fs; /* extra info for the caller */ 6296 rc = del_filter_wr(sc, t->idx); 6297 } 6298 6299done: 6300 end_synchronized_op(sc, 0); 6301 6302 if (rc == 0) { 6303 mtx_lock(&sc->tids.ftid_lock); 6304 for (;;) { 6305 if (f->pending == 0) { 6306 rc = f->valid ? EIO : 0; 6307 break; 6308 } 6309 6310 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6311 PCATCH, "t4delfw", 0)) { 6312 rc = EINPROGRESS; 6313 break; 6314 } 6315 } 6316 mtx_unlock(&sc->tids.ftid_lock); 6317 } 6318 6319 return (rc); 6320} 6321 6322static void 6323clear_filter(struct filter_entry *f) 6324{ 6325 if (f->l2t) 6326 t4_l2t_release(f->l2t); 6327 6328 bzero(f, sizeof (*f)); 6329} 6330 6331static int 6332set_filter_wr(struct adapter *sc, int fidx) 6333{ 6334 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6335 struct wrqe *wr; 6336 struct fw_filter_wr *fwr; 6337 unsigned int ftid; 6338 6339 ASSERT_SYNCHRONIZED_OP(sc); 6340 6341 if (f->fs.newdmac || f->fs.newvlan) { 6342 /* This filter needs an L2T entry; allocate one. */ 6343 f->l2t = t4_l2t_alloc_switching(sc->l2t); 6344 if (f->l2t == NULL) 6345 return (EAGAIN); 6346 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 6347 f->fs.dmac)) { 6348 t4_l2t_release(f->l2t); 6349 f->l2t = NULL; 6350 return (ENOMEM); 6351 } 6352 } 6353 6354 ftid = sc->tids.ftid_base + fidx; 6355 6356 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6357 if (wr == NULL) 6358 return (ENOMEM); 6359 6360 fwr = wrtod(wr); 6361 bzero(fwr, sizeof (*fwr)); 6362 6363 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 6364 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 6365 fwr->tid_to_iq = 6366 htobe32(V_FW_FILTER_WR_TID(ftid) | 6367 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 6368 V_FW_FILTER_WR_NOREPLY(0) | 6369 V_FW_FILTER_WR_IQ(f->fs.iq)); 6370 fwr->del_filter_to_l2tix = 6371 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 6372 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 6373 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 6374 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 6375 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 6376 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 6377 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 6378 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 6379 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 6380 f->fs.newvlan == VLAN_REWRITE) | 6381 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 6382 f->fs.newvlan == VLAN_REWRITE) | 6383 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 6384 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 6385 V_FW_FILTER_WR_PRIO(f->fs.prio) | 6386 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 6387 fwr->ethtype = htobe16(f->fs.val.ethtype); 6388 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 6389 fwr->frag_to_ovlan_vldm = 6390 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 6391 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 6392 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 6393 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 6394 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 6395 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 6396 fwr->smac_sel = 0; 6397 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 6398 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 6399 fwr->maci_to_matchtypem = 6400 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 6401 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 6402 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 6403 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 6404 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 6405 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 6406 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 6407 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 6408 fwr->ptcl = f->fs.val.proto; 6409 fwr->ptclm = f->fs.mask.proto; 6410 fwr->ttyp = f->fs.val.tos; 6411 fwr->ttypm = f->fs.mask.tos; 6412 fwr->ivlan = htobe16(f->fs.val.vlan); 6413 fwr->ivlanm = htobe16(f->fs.mask.vlan); 6414 fwr->ovlan = htobe16(f->fs.val.vnic); 6415 fwr->ovlanm = htobe16(f->fs.mask.vnic); 6416 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 6417 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 6418 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 6419 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 6420 fwr->lp = htobe16(f->fs.val.dport); 6421 fwr->lpm = htobe16(f->fs.mask.dport); 6422 fwr->fp = htobe16(f->fs.val.sport); 6423 fwr->fpm = htobe16(f->fs.mask.sport); 6424 if (f->fs.newsmac) 6425 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 6426 6427 f->pending = 1; 6428 sc->tids.ftids_in_use++; 6429 6430 t4_wrq_tx(sc, wr); 6431 return (0); 6432} 6433 6434static int 6435del_filter_wr(struct adapter *sc, int fidx) 6436{ 6437 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6438 struct wrqe *wr; 6439 struct fw_filter_wr *fwr; 6440 unsigned int ftid; 6441 6442 ftid = sc->tids.ftid_base + fidx; 6443 6444 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6445 if (wr == NULL) 6446 return (ENOMEM); 6447 fwr = wrtod(wr); 6448 bzero(fwr, sizeof (*fwr)); 6449 6450 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 6451 6452 f->pending = 1; 6453 t4_wrq_tx(sc, wr); 6454 return (0); 6455} 6456 6457int 6458t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 6459{ 6460 struct adapter *sc = iq->adapter; 6461 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 6462 unsigned int idx = GET_TID(rpl); 6463 6464 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6465 rss->opcode)); 6466 6467 if (idx >= sc->tids.ftid_base && 6468 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 6469 unsigned int rc = G_COOKIE(rpl->cookie); 6470 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 6471 6472 mtx_lock(&sc->tids.ftid_lock); 6473 if (rc == FW_FILTER_WR_FLT_ADDED) { 6474 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 6475 __func__, idx)); 6476 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 6477 f->pending = 0; /* asynchronous setup completed */ 6478 f->valid = 1; 6479 } else { 6480 if (rc != FW_FILTER_WR_FLT_DELETED) { 6481 /* Add or delete failed, display an error */ 6482 log(LOG_ERR, 6483 "filter %u setup failed with error %u\n", 6484 idx, rc); 6485 } 6486 6487 clear_filter(f); 6488 sc->tids.ftids_in_use--; 6489 } 6490 wakeup(&sc->tids.ftid_tab); 6491 mtx_unlock(&sc->tids.ftid_lock); 6492 } 6493 6494 return (0); 6495} 6496 6497static int 6498get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 6499{ 6500 int rc; 6501 6502 if (cntxt->cid > M_CTXTQID) 6503 return (EINVAL); 6504 6505 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 6506 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 6507 return (EINVAL); 6508 6509 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 6510 if (rc) 6511 return (rc); 6512 6513 if (sc->flags & FW_OK) { 6514 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 6515 &cntxt->data[0]); 6516 if (rc == 0) 6517 goto done; 6518 } 6519 6520 /* 6521 * Read via firmware failed or wasn't even attempted. Read directly via 6522 * the backdoor. 6523 */ 6524 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 6525done: 6526 end_synchronized_op(sc, 0); 6527 return (rc); 6528} 6529 6530static int 6531load_fw(struct adapter *sc, struct t4_data *fw) 6532{ 6533 int rc; 6534 uint8_t *fw_data; 6535 6536 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 6537 if (rc) 6538 return (rc); 6539 6540 if (sc->flags & FULL_INIT_DONE) { 6541 rc = EBUSY; 6542 goto done; 6543 } 6544 6545 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 6546 if (fw_data == NULL) { 6547 rc = ENOMEM; 6548 goto done; 6549 } 6550 6551 rc = copyin(fw->data, fw_data, fw->len); 6552 if (rc == 0) 6553 rc = -t4_load_fw(sc, fw_data, fw->len); 6554 6555 free(fw_data, M_CXGBE); 6556done: 6557 end_synchronized_op(sc, 0); 6558 return (rc); 6559} 6560 6561static int 6562read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 6563{ 6564 uint32_t addr, off, remaining, i, n; 6565 uint32_t *buf, *b; 6566 uint32_t mw_base, mw_aperture; 6567 int rc; 6568 uint8_t *dst; 6569 6570 rc = validate_mem_range(sc, mr->addr, mr->len); 6571 if (rc != 0) 6572 return (rc); 6573 6574 memwin_info(sc, win, &mw_base, &mw_aperture); 6575 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 6576 addr = mr->addr; 6577 remaining = mr->len; 6578 dst = (void *)mr->data; 6579 6580 while (remaining) { 6581 off = position_memwin(sc, win, addr); 6582 6583 /* number of bytes that we'll copy in the inner loop */ 6584 n = min(remaining, mw_aperture - off); 6585 for (i = 0; i < n; i += 4) 6586 *b++ = t4_read_reg(sc, mw_base + off + i); 6587 6588 rc = copyout(buf, dst, n); 6589 if (rc != 0) 6590 break; 6591 6592 b = buf; 6593 dst += n; 6594 remaining -= n; 6595 addr += n; 6596 } 6597 6598 free(buf, M_CXGBE); 6599 return (rc); 6600} 6601 6602static int 6603read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 6604{ 6605 int rc; 6606 6607 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 6608 return (EINVAL); 6609 6610 if (i2cd->len > 1) { 6611 /* XXX: need fw support for longer reads in one go */ 6612 return (ENOTSUP); 6613 } 6614 6615 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 6616 if (rc) 6617 return (rc); 6618 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 6619 i2cd->offset, &i2cd->data[0]); 6620 end_synchronized_op(sc, 0); 6621 6622 return (rc); 6623} 6624 6625int 6626t4_os_find_pci_capability(struct adapter *sc, int cap) 6627{ 6628 int i; 6629 6630 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 6631} 6632 6633int 6634t4_os_pci_save_state(struct adapter *sc) 6635{ 6636 device_t dev; 6637 struct pci_devinfo *dinfo; 6638 6639 dev = sc->dev; 6640 dinfo = device_get_ivars(dev); 6641 6642 pci_cfg_save(dev, dinfo, 0); 6643 return (0); 6644} 6645 6646int 6647t4_os_pci_restore_state(struct adapter *sc) 6648{ 6649 device_t dev; 6650 struct pci_devinfo *dinfo; 6651 6652 dev = sc->dev; 6653 dinfo = device_get_ivars(dev); 6654 6655 pci_cfg_restore(dev, dinfo); 6656 return (0); 6657} 6658 6659void 6660t4_os_portmod_changed(const struct adapter *sc, int idx) 6661{ 6662 struct port_info *pi = sc->port[idx]; 6663 static const char *mod_str[] = { 6664 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 6665 }; 6666 6667 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 6668 if_printf(pi->ifp, "transceiver unplugged.\n"); 6669 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 6670 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 6671 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 6672 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 6673 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 6674 if_printf(pi->ifp, "%s transceiver inserted.\n", 6675 mod_str[pi->mod_type]); 6676 } else { 6677 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 6678 pi->mod_type); 6679 } 6680} 6681 6682void 6683t4_os_link_changed(struct adapter *sc, int idx, int link_stat) 6684{ 6685 struct port_info *pi = sc->port[idx]; 6686 struct ifnet *ifp = pi->ifp; 6687 6688 if (link_stat) { 6689 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 6690 if_link_state_change(ifp, LINK_STATE_UP); 6691 } else 6692 if_link_state_change(ifp, LINK_STATE_DOWN); 6693} 6694 6695void 6696t4_iterate(void (*func)(struct adapter *, void *), void *arg) 6697{ 6698 struct adapter *sc; 6699 6700 mtx_lock(&t4_list_lock); 6701 SLIST_FOREACH(sc, &t4_list, link) { 6702 /* 6703 * func should not make any assumptions about what state sc is 6704 * in - the only guarantee is that sc->sc_lock is a valid lock. 6705 */ 6706 func(sc, arg); 6707 } 6708 mtx_unlock(&t4_list_lock); 6709} 6710 6711static int 6712t4_open(struct cdev *dev, int flags, int type, struct thread *td) 6713{ 6714 return (0); 6715} 6716 6717static int 6718t4_close(struct cdev *dev, int flags, int type, struct thread *td) 6719{ 6720 return (0); 6721} 6722 6723static int 6724t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 6725 struct thread *td) 6726{ 6727 int rc; 6728 struct adapter *sc = dev->si_drv1; 6729 6730 rc = priv_check(td, PRIV_DRIVER); 6731 if (rc != 0) 6732 return (rc); 6733 6734 switch (cmd) { 6735 case CHELSIO_T4_GETREG: { 6736 struct t4_reg *edata = (struct t4_reg *)data; 6737 6738 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6739 return (EFAULT); 6740 6741 if (edata->size == 4) 6742 edata->val = t4_read_reg(sc, edata->addr); 6743 else if (edata->size == 8) 6744 edata->val = t4_read_reg64(sc, edata->addr); 6745 else 6746 return (EINVAL); 6747 6748 break; 6749 } 6750 case CHELSIO_T4_SETREG: { 6751 struct t4_reg *edata = (struct t4_reg *)data; 6752 6753 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6754 return (EFAULT); 6755 6756 if (edata->size == 4) { 6757 if (edata->val & 0xffffffff00000000) 6758 return (EINVAL); 6759 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 6760 } else if (edata->size == 8) 6761 t4_write_reg64(sc, edata->addr, edata->val); 6762 else 6763 return (EINVAL); 6764 break; 6765 } 6766 case CHELSIO_T4_REGDUMP: { 6767 struct t4_regdump *regs = (struct t4_regdump *)data; 6768 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 6769 uint8_t *buf; 6770 6771 if (regs->len < reglen) { 6772 regs->len = reglen; /* hint to the caller */ 6773 return (ENOBUFS); 6774 } 6775 6776 regs->len = reglen; 6777 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 6778 t4_get_regs(sc, regs, buf); 6779 rc = copyout(buf, regs->data, reglen); 6780 free(buf, M_CXGBE); 6781 break; 6782 } 6783 case CHELSIO_T4_GET_FILTER_MODE: 6784 rc = get_filter_mode(sc, (uint32_t *)data); 6785 break; 6786 case CHELSIO_T4_SET_FILTER_MODE: 6787 rc = set_filter_mode(sc, *(uint32_t *)data); 6788 break; 6789 case CHELSIO_T4_GET_FILTER: 6790 rc = get_filter(sc, (struct t4_filter *)data); 6791 break; 6792 case CHELSIO_T4_SET_FILTER: 6793 rc = set_filter(sc, (struct t4_filter *)data); 6794 break; 6795 case CHELSIO_T4_DEL_FILTER: 6796 rc = del_filter(sc, (struct t4_filter *)data); 6797 break; 6798 case CHELSIO_T4_GET_SGE_CONTEXT: 6799 rc = get_sge_context(sc, (struct t4_sge_context *)data); 6800 break; 6801 case CHELSIO_T4_LOAD_FW: 6802 rc = load_fw(sc, (struct t4_data *)data); 6803 break; 6804 case CHELSIO_T4_GET_MEM: 6805 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 6806 break; 6807 case CHELSIO_T4_GET_I2C: 6808 rc = read_i2c(sc, (struct t4_i2c_data *)data); 6809 break; 6810 case CHELSIO_T4_CLEAR_STATS: { 6811 int i; 6812 u_int port_id = *(uint32_t *)data; 6813 struct port_info *pi; 6814 6815 if (port_id >= sc->params.nports) 6816 return (EINVAL); 6817 6818 /* MAC stats */ 6819 t4_clr_port_stats(sc, port_id); 6820 6821 pi = sc->port[port_id]; 6822 if (pi->flags & PORT_INIT_DONE) { 6823 struct sge_rxq *rxq; 6824 struct sge_txq *txq; 6825 struct sge_wrq *wrq; 6826 6827 for_each_rxq(pi, i, rxq) { 6828#if defined(INET) || defined(INET6) 6829 rxq->lro.lro_queued = 0; 6830 rxq->lro.lro_flushed = 0; 6831#endif 6832 rxq->rxcsum = 0; 6833 rxq->vlan_extraction = 0; 6834 } 6835 6836 for_each_txq(pi, i, txq) { 6837 txq->txcsum = 0; 6838 txq->tso_wrs = 0; 6839 txq->vlan_insertion = 0; 6840 txq->imm_wrs = 0; 6841 txq->sgl_wrs = 0; 6842 txq->txpkt_wrs = 0; 6843 txq->txpkts_wrs = 0; 6844 txq->txpkts_pkts = 0; 6845 txq->br->br_drops = 0; 6846 txq->no_dmamap = 0; 6847 txq->no_desc = 0; 6848 } 6849 6850#ifdef TCP_OFFLOAD 6851 /* nothing to clear for each ofld_rxq */ 6852 6853 for_each_ofld_txq(pi, i, wrq) { 6854 wrq->tx_wrs = 0; 6855 wrq->no_desc = 0; 6856 } 6857#endif 6858 wrq = &sc->sge.ctrlq[pi->port_id]; 6859 wrq->tx_wrs = 0; 6860 wrq->no_desc = 0; 6861 } 6862 break; 6863 } 6864 default: 6865 rc = EINVAL; 6866 } 6867 6868 return (rc); 6869} 6870 6871#ifdef TCP_OFFLOAD 6872static int 6873toe_capability(struct port_info *pi, int enable) 6874{ 6875 int rc; 6876 struct adapter *sc = pi->adapter; 6877 6878 ASSERT_SYNCHRONIZED_OP(sc); 6879 6880 if (!is_offload(sc)) 6881 return (ENODEV); 6882 6883 if (enable) { 6884 if (!(sc->flags & FULL_INIT_DONE)) { 6885 rc = cxgbe_init_synchronized(pi); 6886 if (rc) 6887 return (rc); 6888 } 6889 6890 if (isset(&sc->offload_map, pi->port_id)) 6891 return (0); 6892 6893 if (!(sc->flags & TOM_INIT_DONE)) { 6894 rc = t4_activate_uld(sc, ULD_TOM); 6895 if (rc == EAGAIN) { 6896 log(LOG_WARNING, 6897 "You must kldload t4_tom.ko before trying " 6898 "to enable TOE on a cxgbe interface.\n"); 6899 } 6900 if (rc != 0) 6901 return (rc); 6902 KASSERT(sc->tom_softc != NULL, 6903 ("%s: TOM activated but softc NULL", __func__)); 6904 KASSERT(sc->flags & TOM_INIT_DONE, 6905 ("%s: TOM activated but flag not set", __func__)); 6906 } 6907 6908 setbit(&sc->offload_map, pi->port_id); 6909 } else { 6910 if (!isset(&sc->offload_map, pi->port_id)) 6911 return (0); 6912 6913 KASSERT(sc->flags & TOM_INIT_DONE, 6914 ("%s: TOM never initialized?", __func__)); 6915 clrbit(&sc->offload_map, pi->port_id); 6916 } 6917 6918 return (0); 6919} 6920 6921/* 6922 * Add an upper layer driver to the global list. 6923 */ 6924int 6925t4_register_uld(struct uld_info *ui) 6926{ 6927 int rc = 0; 6928 struct uld_info *u; 6929 6930 mtx_lock(&t4_uld_list_lock); 6931 SLIST_FOREACH(u, &t4_uld_list, link) { 6932 if (u->uld_id == ui->uld_id) { 6933 rc = EEXIST; 6934 goto done; 6935 } 6936 } 6937 6938 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 6939 ui->refcount = 0; 6940done: 6941 mtx_unlock(&t4_uld_list_lock); 6942 return (rc); 6943} 6944 6945int 6946t4_unregister_uld(struct uld_info *ui) 6947{ 6948 int rc = EINVAL; 6949 struct uld_info *u; 6950 6951 mtx_lock(&t4_uld_list_lock); 6952 6953 SLIST_FOREACH(u, &t4_uld_list, link) { 6954 if (u == ui) { 6955 if (ui->refcount > 0) { 6956 rc = EBUSY; 6957 goto done; 6958 } 6959 6960 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 6961 rc = 0; 6962 goto done; 6963 } 6964 } 6965done: 6966 mtx_unlock(&t4_uld_list_lock); 6967 return (rc); 6968} 6969 6970int 6971t4_activate_uld(struct adapter *sc, int id) 6972{ 6973 int rc = EAGAIN; 6974 struct uld_info *ui; 6975 6976 ASSERT_SYNCHRONIZED_OP(sc); 6977 6978 mtx_lock(&t4_uld_list_lock); 6979 6980 SLIST_FOREACH(ui, &t4_uld_list, link) { 6981 if (ui->uld_id == id) { 6982 rc = ui->activate(sc); 6983 if (rc == 0) 6984 ui->refcount++; 6985 goto done; 6986 } 6987 } 6988done: 6989 mtx_unlock(&t4_uld_list_lock); 6990 6991 return (rc); 6992} 6993 6994int 6995t4_deactivate_uld(struct adapter *sc, int id) 6996{ 6997 int rc = EINVAL; 6998 struct uld_info *ui; 6999 7000 ASSERT_SYNCHRONIZED_OP(sc); 7001 7002 mtx_lock(&t4_uld_list_lock); 7003 7004 SLIST_FOREACH(ui, &t4_uld_list, link) { 7005 if (ui->uld_id == id) { 7006 rc = ui->deactivate(sc); 7007 if (rc == 0) 7008 ui->refcount--; 7009 goto done; 7010 } 7011 } 7012done: 7013 mtx_unlock(&t4_uld_list_lock); 7014 7015 return (rc); 7016} 7017#endif 7018 7019/* 7020 * Come up with reasonable defaults for some of the tunables, provided they're 7021 * not set by the user (in which case we'll use the values as is). 7022 */ 7023static void 7024tweak_tunables(void) 7025{ 7026 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 7027 7028 if (t4_ntxq10g < 1) 7029 t4_ntxq10g = min(nc, NTXQ_10G); 7030 7031 if (t4_ntxq1g < 1) 7032 t4_ntxq1g = min(nc, NTXQ_1G); 7033 7034 if (t4_nrxq10g < 1) 7035 t4_nrxq10g = min(nc, NRXQ_10G); 7036 7037 if (t4_nrxq1g < 1) 7038 t4_nrxq1g = min(nc, NRXQ_1G); 7039 7040#ifdef TCP_OFFLOAD 7041 if (t4_nofldtxq10g < 1) 7042 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 7043 7044 if (t4_nofldtxq1g < 1) 7045 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 7046 7047 if (t4_nofldrxq10g < 1) 7048 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 7049 7050 if (t4_nofldrxq1g < 1) 7051 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 7052 7053 if (t4_toecaps_allowed == -1) 7054 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 7055#else 7056 if (t4_toecaps_allowed == -1) 7057 t4_toecaps_allowed = 0; 7058#endif 7059 7060 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 7061 t4_tmr_idx_10g = TMR_IDX_10G; 7062 7063 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 7064 t4_pktc_idx_10g = PKTC_IDX_10G; 7065 7066 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 7067 t4_tmr_idx_1g = TMR_IDX_1G; 7068 7069 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 7070 t4_pktc_idx_1g = PKTC_IDX_1G; 7071 7072 if (t4_qsize_txq < 128) 7073 t4_qsize_txq = 128; 7074 7075 if (t4_qsize_rxq < 128) 7076 t4_qsize_rxq = 128; 7077 while (t4_qsize_rxq & 7) 7078 t4_qsize_rxq++; 7079 7080 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 7081} 7082 7083static int 7084mod_event(module_t mod, int cmd, void *arg) 7085{ 7086 int rc = 0; 7087 static int loaded = 0; 7088 7089 switch (cmd) { 7090 case MOD_LOAD: 7091 if (atomic_fetchadd_int(&loaded, 1)) 7092 break; 7093 t4_sge_modload(); 7094 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 7095 SLIST_INIT(&t4_list); 7096#ifdef TCP_OFFLOAD 7097 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 7098 SLIST_INIT(&t4_uld_list); 7099#endif 7100 tweak_tunables(); 7101 break; 7102 7103 case MOD_UNLOAD: 7104 if (atomic_fetchadd_int(&loaded, -1) > 1) 7105 break; 7106#ifdef TCP_OFFLOAD 7107 mtx_lock(&t4_uld_list_lock); 7108 if (!SLIST_EMPTY(&t4_uld_list)) { 7109 rc = EBUSY; 7110 mtx_unlock(&t4_uld_list_lock); 7111 break; 7112 } 7113 mtx_unlock(&t4_uld_list_lock); 7114 mtx_destroy(&t4_uld_list_lock); 7115#endif 7116 mtx_lock(&t4_list_lock); 7117 if (!SLIST_EMPTY(&t4_list)) { 7118 rc = EBUSY; 7119 mtx_unlock(&t4_list_lock); 7120 break; 7121 } 7122 mtx_unlock(&t4_list_lock); 7123 mtx_destroy(&t4_list_lock); 7124 break; 7125 } 7126 7127 return (rc); 7128} 7129 7130static devclass_t t4_devclass, t5_devclass; 7131static devclass_t cxgbe_devclass, cxl_devclass; 7132 7133DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 7134MODULE_VERSION(t4nex, 1); 7135 7136DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 7137MODULE_VERSION(t5nex, 1); 7138 7139DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 7140MODULE_VERSION(cxgbe, 1); 7141 7142DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 7143MODULE_VERSION(cxl, 1); 7144