1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD$"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58#if defined(__i386__) || defined(__amd64__) 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" 69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80}; 81static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85}; 86 87 88/* T4 port (cxgbe) interface */ 89static int cxgbe_probe(device_t); 90static int cxgbe_attach(device_t); 91static int cxgbe_detach(device_t); 92static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97}; 98static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102}; 103 104static d_ioctl_t t4_ioctl; 105static d_open_t t4_open; 106static d_close_t t4_close; 107 108static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115}; 116 117/* T5 bus driver interface */ 118static int t5_probe(device_t); 119static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125}; 126static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130}; 131 132 133/* T5 port (cxl) interface */ 134static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138}; 139 140static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147}; 148 149/* ifnet + media interface */ 150static void cxgbe_init(void *); 151static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153static void cxgbe_qflush(struct ifnet *); 154static int cxgbe_media_change(struct ifnet *); 155static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159/* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163static struct mtx t4_list_lock; 164static SLIST_HEAD(, adapter) t4_list; 165#ifdef TCP_OFFLOAD 166static struct mtx t4_uld_list_lock; 167static SLIST_HEAD(, uld_info) t4_uld_list; 168#endif 169 170/* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181/* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184#define NTXQ_10G 16 185static int t4_ntxq10g = -1; 186TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188#define NRXQ_10G 8 189static int t4_nrxq10g = -1; 190TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192#define NTXQ_1G 4 193static int t4_ntxq1g = -1; 194TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196#define NRXQ_1G 2 197static int t4_nrxq1g = -1; 198TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200static int t4_rsrv_noflowq = 0; 201TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203#ifdef TCP_OFFLOAD 204#define NOFLDTXQ_10G 8 205static int t4_nofldtxq10g = -1; 206TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208#define NOFLDRXQ_10G 2 209static int t4_nofldrxq10g = -1; 210TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212#define NOFLDTXQ_1G 2 213static int t4_nofldtxq1g = -1; 214TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216#define NOFLDRXQ_1G 1 217static int t4_nofldrxq1g = -1; 218TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219#endif 220 221/* 222 * Holdoff parameters for 10G and 1G ports. 223 */ 224#define TMR_IDX_10G 1 225static int t4_tmr_idx_10g = TMR_IDX_10G; 226TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 227 228#define PKTC_IDX_10G (-1) 229static int t4_pktc_idx_10g = PKTC_IDX_10G; 230TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 231 232#define TMR_IDX_1G 1 233static int t4_tmr_idx_1g = TMR_IDX_1G; 234TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 235 236#define PKTC_IDX_1G (-1) 237static int t4_pktc_idx_1g = PKTC_IDX_1G; 238TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 239 240/* 241 * Size (# of entries) of each tx and rx queue. 242 */ 243static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 244TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 245 246static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 247TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 248 249/* 250 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 251 */ 252static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 253TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 254 255/* 256 * Configuration file. 257 */ 258#define DEFAULT_CF "default" 259#define FLASH_CF "flash" 260#define UWIRE_CF "uwire" 261#define FPGA_CF "fpga" 262static char t4_cfg_file[32] = DEFAULT_CF; 263TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 264 265/* 266 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 267 * encouraged respectively). 268 */ 269static unsigned int t4_fw_install = 1; 270TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 271 272/* 273 * ASIC features that will be used. Disable the ones you don't want so that the 274 * chip resources aren't wasted on features that will not be used. 275 */ 276static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 277TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 278 279static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 280TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 281 282static int t4_toecaps_allowed = -1; 283TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 284 285static int t4_rdmacaps_allowed = 0; 286TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 287 288static int t4_iscsicaps_allowed = 0; 289TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 290 291static int t4_fcoecaps_allowed = 0; 292TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 293 294static int t5_write_combine = 0; 295TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 296 297struct intrs_and_queues { 298 int intr_type; /* INTx, MSI, or MSI-X */ 299 int nirq; /* Number of vectors */ 300 int intr_flags; 301 int ntxq10g; /* # of NIC txq's for each 10G port */ 302 int nrxq10g; /* # of NIC rxq's for each 10G port */ 303 int ntxq1g; /* # of NIC txq's for each 1G port */ 304 int nrxq1g; /* # of NIC rxq's for each 1G port */ 305 int rsrv_noflowq; /* Flag whether to reserve queue 0 */ 306#ifdef TCP_OFFLOAD 307 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 308 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 309 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 310 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 311#endif 312}; 313 314struct filter_entry { 315 uint32_t valid:1; /* filter allocated and valid */ 316 uint32_t locked:1; /* filter is administratively locked */ 317 uint32_t pending:1; /* filter action is pending firmware reply */ 318 uint32_t smtidx:8; /* Source MAC Table index for smac */ 319 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 320 321 struct t4_filter_specification fs; 322}; 323 324enum { 325 XGMAC_MTU = (1 << 0), 326 XGMAC_PROMISC = (1 << 1), 327 XGMAC_ALLMULTI = (1 << 2), 328 XGMAC_VLANEX = (1 << 3), 329 XGMAC_UCADDR = (1 << 4), 330 XGMAC_MCADDRS = (1 << 5), 331 332 XGMAC_ALL = 0xffff 333}; 334 335static int map_bars_0_and_4(struct adapter *); 336static int map_bar_2(struct adapter *); 337static void setup_memwin(struct adapter *); 338static int validate_mem_range(struct adapter *, uint32_t, int); 339static int fwmtype_to_hwmtype(int); 340static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 341 uint32_t *); 342static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 343static uint32_t position_memwin(struct adapter *, int, uint32_t); 344static int cfg_itype_and_nqueues(struct adapter *, int, int, 345 struct intrs_and_queues *); 346static int prep_firmware(struct adapter *); 347static int partition_resources(struct adapter *, const struct firmware *, 348 const char *); 349static int get_params__pre_init(struct adapter *); 350static int get_params__post_init(struct adapter *); 351static int set_params__post_init(struct adapter *); 352static void t4_set_desc(struct adapter *); 353static void build_medialist(struct port_info *); 354static int update_mac_settings(struct port_info *, int); 355static int cxgbe_init_synchronized(struct port_info *); 356static int cxgbe_uninit_synchronized(struct port_info *); 357static int setup_intr_handlers(struct adapter *); 358static int adapter_full_init(struct adapter *); 359static int adapter_full_uninit(struct adapter *); 360static int port_full_init(struct port_info *); 361static int port_full_uninit(struct port_info *); 362static void quiesce_eq(struct adapter *, struct sge_eq *); 363static void quiesce_iq(struct adapter *, struct sge_iq *); 364static void quiesce_fl(struct adapter *, struct sge_fl *); 365static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 366 driver_intr_t *, void *, char *); 367static int t4_free_irq(struct adapter *, struct irq *); 368static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 369 unsigned int); 370static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 371static void cxgbe_tick(void *); 372static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 373static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 374 struct mbuf *); 375static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 376static int fw_msg_not_handled(struct adapter *, const __be64 *); 377static int t4_sysctls(struct adapter *); 378static int cxgbe_sysctls(struct port_info *); 379static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 380static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 381static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 382static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 383static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 384static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 385static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 386static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 387static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 388static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 389#ifdef SBUF_DRAIN 390static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 391static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 392static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 393static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 394static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 395static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 396static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 397static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 398static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 399static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 400static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 401static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 402static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 403static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 404static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 405static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 406static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 407static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 408static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 409static int sysctl_tids(SYSCTL_HANDLER_ARGS); 410static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 411static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 412static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 413static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 414static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 415#endif 416static inline void txq_start(struct ifnet *, struct sge_txq *); 417static uint32_t fconf_to_mode(uint32_t); 418static uint32_t mode_to_fconf(uint32_t); 419static uint32_t fspec_to_fconf(struct t4_filter_specification *); 420static int get_filter_mode(struct adapter *, uint32_t *); 421static int set_filter_mode(struct adapter *, uint32_t); 422static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 423static int get_filter(struct adapter *, struct t4_filter *); 424static int set_filter(struct adapter *, struct t4_filter *); 425static int del_filter(struct adapter *, struct t4_filter *); 426static void clear_filter(struct filter_entry *); 427static int set_filter_wr(struct adapter *, int); 428static int del_filter_wr(struct adapter *, int); 429static int get_sge_context(struct adapter *, struct t4_sge_context *); 430static int load_fw(struct adapter *, struct t4_data *); 431static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 432static int read_i2c(struct adapter *, struct t4_i2c_data *); 433static int set_sched_class(struct adapter *, struct t4_sched_params *); 434static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 435#ifdef TCP_OFFLOAD 436static int toe_capability(struct port_info *, int); 437#endif 438static int mod_event(module_t, int, void *); 439 440struct { 441 uint16_t device; 442 char *desc; 443} t4_pciids[] = { 444 {0xa000, "Chelsio Terminator 4 FPGA"}, 445 {0x4400, "Chelsio T440-dbg"}, 446 {0x4401, "Chelsio T420-CR"}, 447 {0x4402, "Chelsio T422-CR"}, 448 {0x4403, "Chelsio T440-CR"}, 449 {0x4404, "Chelsio T420-BCH"}, 450 {0x4405, "Chelsio T440-BCH"}, 451 {0x4406, "Chelsio T440-CH"}, 452 {0x4407, "Chelsio T420-SO"}, 453 {0x4408, "Chelsio T420-CX"}, 454 {0x4409, "Chelsio T420-BT"}, 455 {0x440a, "Chelsio T404-BT"}, 456 {0x440e, "Chelsio T440-LP-CR"}, 457}, t5_pciids[] = { 458 {0xb000, "Chelsio Terminator 5 FPGA"}, 459 {0x5400, "Chelsio T580-dbg"}, 460 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 461 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 462 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 463 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 464 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 465 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 466 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 467 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 468 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 469 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 470 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 471 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 472#ifdef notyet 473 {0x5404, "Chelsio T520-BCH"}, 474 {0x5405, "Chelsio T540-BCH"}, 475 {0x5406, "Chelsio T540-CH"}, 476 {0x5408, "Chelsio T520-CX"}, 477 {0x540b, "Chelsio B520-SR"}, 478 {0x540c, "Chelsio B504-BT"}, 479 {0x540f, "Chelsio Amsterdam"}, 480 {0x5413, "Chelsio T580-CHR"}, 481#endif 482}; 483 484#ifdef TCP_OFFLOAD 485/* 486 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 487 * exactly the same for both rxq and ofld_rxq. 488 */ 489CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 490CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 491#endif 492 493/* No easy way to include t4_msg.h before adapter.h so we check this way */ 494CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 495CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 496 497CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 498 499static int 500t4_probe(device_t dev) 501{ 502 int i; 503 uint16_t v = pci_get_vendor(dev); 504 uint16_t d = pci_get_device(dev); 505 uint8_t f = pci_get_function(dev); 506 507 if (v != PCI_VENDOR_ID_CHELSIO) 508 return (ENXIO); 509 510 /* Attach only to PF0 of the FPGA */ 511 if (d == 0xa000 && f != 0) 512 return (ENXIO); 513 514 for (i = 0; i < nitems(t4_pciids); i++) { 515 if (d == t4_pciids[i].device) { 516 device_set_desc(dev, t4_pciids[i].desc); 517 return (BUS_PROBE_DEFAULT); 518 } 519 } 520 521 return (ENXIO); 522} 523 524static int 525t5_probe(device_t dev) 526{ 527 int i; 528 uint16_t v = pci_get_vendor(dev); 529 uint16_t d = pci_get_device(dev); 530 uint8_t f = pci_get_function(dev); 531 532 if (v != PCI_VENDOR_ID_CHELSIO) 533 return (ENXIO); 534 535 /* Attach only to PF0 of the FPGA */ 536 if (d == 0xb000 && f != 0) 537 return (ENXIO); 538 539 for (i = 0; i < nitems(t5_pciids); i++) { 540 if (d == t5_pciids[i].device) { 541 device_set_desc(dev, t5_pciids[i].desc); 542 return (BUS_PROBE_DEFAULT); 543 } 544 } 545 546 return (ENXIO); 547} 548 549static int 550t4_attach(device_t dev) 551{ 552 struct adapter *sc; 553 int rc = 0, i, n10g, n1g, rqidx, tqidx; 554 struct intrs_and_queues iaq; 555 struct sge *s; 556#ifdef TCP_OFFLOAD 557 int ofld_rqidx, ofld_tqidx; 558#endif 559 560 sc = device_get_softc(dev); 561 sc->dev = dev; 562 563 pci_enable_busmaster(dev); 564 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 565 uint32_t v; 566 567 pci_set_max_read_req(dev, 4096); 568 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 569 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 570 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 571 } 572 573 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 574 device_get_nameunit(dev)); 575 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 576 mtx_lock(&t4_list_lock); 577 SLIST_INSERT_HEAD(&t4_list, sc, link); 578 mtx_unlock(&t4_list_lock); 579 580 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 581 TAILQ_INIT(&sc->sfl); 582 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 583 584 rc = map_bars_0_and_4(sc); 585 if (rc != 0) 586 goto done; /* error message displayed already */ 587 588 /* 589 * This is the real PF# to which we're attaching. Works from within PCI 590 * passthrough environments too, where pci_get_function() could return a 591 * different PF# depending on the passthrough configuration. We need to 592 * use the real PF# in all our communication with the firmware. 593 */ 594 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 595 sc->mbox = sc->pf; 596 597 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 598 sc->an_handler = an_not_handled; 599 for (i = 0; i < nitems(sc->cpl_handler); i++) 600 sc->cpl_handler[i] = cpl_not_handled; 601 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 602 sc->fw_msg_handler[i] = fw_msg_not_handled; 603 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 604 t4_init_sge_cpl_handlers(sc); 605 606 /* Prepare the adapter for operation */ 607 rc = -t4_prep_adapter(sc); 608 if (rc != 0) { 609 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 610 goto done; 611 } 612 613 /* 614 * Do this really early, with the memory windows set up even before the 615 * character device. The userland tool's register i/o and mem read 616 * will work even in "recovery mode". 617 */ 618 setup_memwin(sc); 619 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 620 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 621 device_get_nameunit(dev)); 622 if (sc->cdev == NULL) 623 device_printf(dev, "failed to create nexus char device.\n"); 624 else 625 sc->cdev->si_drv1 = sc; 626 627 /* Go no further if recovery mode has been requested. */ 628 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 629 device_printf(dev, "recovery mode.\n"); 630 goto done; 631 } 632 633 /* Prepare the firmware for operation */ 634 rc = prep_firmware(sc); 635 if (rc != 0) 636 goto done; /* error message displayed already */ 637 638 rc = get_params__post_init(sc); 639 if (rc != 0) 640 goto done; /* error message displayed already */ 641 642 rc = set_params__post_init(sc); 643 if (rc != 0) 644 goto done; /* error message displayed already */ 645 646 rc = map_bar_2(sc); 647 if (rc != 0) 648 goto done; /* error message displayed already */ 649 650 rc = t4_create_dma_tag(sc); 651 if (rc != 0) 652 goto done; /* error message displayed already */ 653 654 /* 655 * First pass over all the ports - allocate VIs and initialize some 656 * basic parameters like mac address, port type, etc. We also figure 657 * out whether a port is 10G or 1G and use that information when 658 * calculating how many interrupts to attempt to allocate. 659 */ 660 n10g = n1g = 0; 661 for_each_port(sc, i) { 662 struct port_info *pi; 663 664 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 665 sc->port[i] = pi; 666 667 /* These must be set before t4_port_init */ 668 pi->adapter = sc; 669 pi->port_id = i; 670 671 /* Allocate the vi and initialize parameters like mac addr */ 672 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 673 if (rc != 0) { 674 device_printf(dev, "unable to initialize port %d: %d\n", 675 i, rc); 676 free(pi, M_CXGBE); 677 sc->port[i] = NULL; 678 goto done; 679 } 680 681 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 682 device_get_nameunit(dev), i); 683 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 684 685 if (is_10G_port(pi) || is_40G_port(pi)) { 686 n10g++; 687 pi->tmr_idx = t4_tmr_idx_10g; 688 pi->pktc_idx = t4_pktc_idx_10g; 689 } else { 690 n1g++; 691 pi->tmr_idx = t4_tmr_idx_1g; 692 pi->pktc_idx = t4_pktc_idx_1g; 693 } 694 695 pi->xact_addr_filt = -1; 696 pi->linkdnrc = -1; 697 698 pi->qsize_rxq = t4_qsize_rxq; 699 pi->qsize_txq = t4_qsize_txq; 700 701 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 702 if (pi->dev == NULL) { 703 device_printf(dev, 704 "failed to add device for port %d.\n", i); 705 rc = ENXIO; 706 goto done; 707 } 708 device_set_softc(pi->dev, pi); 709 } 710 711 /* 712 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 713 */ 714 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 715 if (rc != 0) 716 goto done; /* error message displayed already */ 717 718 sc->intr_type = iaq.intr_type; 719 sc->intr_count = iaq.nirq; 720 sc->flags |= iaq.intr_flags; 721 722 s = &sc->sge; 723 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 724 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 725 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 726 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 727 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 728 729#ifdef TCP_OFFLOAD 730 if (is_offload(sc)) { 731 732 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 733 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 734 s->neq += s->nofldtxq + s->nofldrxq; 735 s->niq += s->nofldrxq; 736 737 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 738 M_CXGBE, M_ZERO | M_WAITOK); 739 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 740 M_CXGBE, M_ZERO | M_WAITOK); 741 } 742#endif 743 744 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 745 M_ZERO | M_WAITOK); 746 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 747 M_ZERO | M_WAITOK); 748 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 749 M_ZERO | M_WAITOK); 750 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 751 M_ZERO | M_WAITOK); 752 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 753 M_ZERO | M_WAITOK); 754 755 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 756 M_ZERO | M_WAITOK); 757 758 t4_init_l2t(sc, M_WAITOK); 759 760 /* 761 * Second pass over the ports. This time we know the number of rx and 762 * tx queues that each port should get. 763 */ 764 rqidx = tqidx = 0; 765#ifdef TCP_OFFLOAD 766 ofld_rqidx = ofld_tqidx = 0; 767#endif 768 for_each_port(sc, i) { 769 struct port_info *pi = sc->port[i]; 770 771 if (pi == NULL) 772 continue; 773 774 pi->first_rxq = rqidx; 775 pi->first_txq = tqidx; 776 if (is_10G_port(pi) || is_40G_port(pi)) { 777 pi->nrxq = iaq.nrxq10g; 778 pi->ntxq = iaq.ntxq10g; 779 } else { 780 pi->nrxq = iaq.nrxq1g; 781 pi->ntxq = iaq.ntxq1g; 782 } 783 784 if (pi->ntxq > 1) 785 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 786 else 787 pi->rsrv_noflowq = 0; 788 789 rqidx += pi->nrxq; 790 tqidx += pi->ntxq; 791 792#ifdef TCP_OFFLOAD 793 if (is_offload(sc)) { 794 pi->first_ofld_rxq = ofld_rqidx; 795 pi->first_ofld_txq = ofld_tqidx; 796 if (is_10G_port(pi) || is_40G_port(pi)) { 797 pi->nofldrxq = iaq.nofldrxq10g; 798 pi->nofldtxq = iaq.nofldtxq10g; 799 } else { 800 pi->nofldrxq = iaq.nofldrxq1g; 801 pi->nofldtxq = iaq.nofldtxq1g; 802 } 803 ofld_rqidx += pi->nofldrxq; 804 ofld_tqidx += pi->nofldtxq; 805 } 806#endif 807 } 808 809 rc = setup_intr_handlers(sc); 810 if (rc != 0) { 811 device_printf(dev, 812 "failed to setup interrupt handlers: %d\n", rc); 813 goto done; 814 } 815 816 rc = bus_generic_attach(dev); 817 if (rc != 0) { 818 device_printf(dev, 819 "failed to attach all child ports: %d\n", rc); 820 goto done; 821 } 822 823 device_printf(dev, 824 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 825 sc->params.pci.width, sc->params.nports, sc->intr_count, 826 sc->intr_type == INTR_MSIX ? "MSI-X" : 827 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 828 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 829 830 t4_set_desc(sc); 831 832done: 833 if (rc != 0 && sc->cdev) { 834 /* cdev was created and so cxgbetool works; recover that way. */ 835 device_printf(dev, 836 "error during attach, adapter is now in recovery mode.\n"); 837 rc = 0; 838 } 839 840 if (rc != 0) 841 t4_detach(dev); 842 else 843 t4_sysctls(sc); 844 845 return (rc); 846} 847 848/* 849 * Idempotent 850 */ 851static int 852t4_detach(device_t dev) 853{ 854 struct adapter *sc; 855 struct port_info *pi; 856 int i, rc; 857 858 sc = device_get_softc(dev); 859 860 if (sc->flags & FULL_INIT_DONE) 861 t4_intr_disable(sc); 862 863 if (sc->cdev) { 864 destroy_dev(sc->cdev); 865 sc->cdev = NULL; 866 } 867 868 rc = bus_generic_detach(dev); 869 if (rc) { 870 device_printf(dev, 871 "failed to detach child devices: %d\n", rc); 872 return (rc); 873 } 874 875 for (i = 0; i < sc->intr_count; i++) 876 t4_free_irq(sc, &sc->irq[i]); 877 878 for (i = 0; i < MAX_NPORTS; i++) { 879 pi = sc->port[i]; 880 if (pi) { 881 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 882 if (pi->dev) 883 device_delete_child(dev, pi->dev); 884 885 mtx_destroy(&pi->pi_lock); 886 free(pi, M_CXGBE); 887 } 888 } 889 890 if (sc->flags & FULL_INIT_DONE) 891 adapter_full_uninit(sc); 892 893 if (sc->flags & FW_OK) 894 t4_fw_bye(sc, sc->mbox); 895 896 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 897 pci_release_msi(dev); 898 899 if (sc->regs_res) 900 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 901 sc->regs_res); 902 903 if (sc->udbs_res) 904 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 905 sc->udbs_res); 906 907 if (sc->msix_res) 908 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 909 sc->msix_res); 910 911 if (sc->l2t) 912 t4_free_l2t(sc->l2t); 913 914#ifdef TCP_OFFLOAD 915 free(sc->sge.ofld_rxq, M_CXGBE); 916 free(sc->sge.ofld_txq, M_CXGBE); 917#endif 918 free(sc->irq, M_CXGBE); 919 free(sc->sge.rxq, M_CXGBE); 920 free(sc->sge.txq, M_CXGBE); 921 free(sc->sge.ctrlq, M_CXGBE); 922 free(sc->sge.iqmap, M_CXGBE); 923 free(sc->sge.eqmap, M_CXGBE); 924 free(sc->tids.ftid_tab, M_CXGBE); 925 t4_destroy_dma_tag(sc); 926 if (mtx_initialized(&sc->sc_lock)) { 927 mtx_lock(&t4_list_lock); 928 SLIST_REMOVE(&t4_list, sc, adapter, link); 929 mtx_unlock(&t4_list_lock); 930 mtx_destroy(&sc->sc_lock); 931 } 932 933 if (mtx_initialized(&sc->tids.ftid_lock)) 934 mtx_destroy(&sc->tids.ftid_lock); 935 if (mtx_initialized(&sc->sfl_lock)) 936 mtx_destroy(&sc->sfl_lock); 937 938 bzero(sc, sizeof(*sc)); 939 940 return (0); 941} 942 943 944static int 945cxgbe_probe(device_t dev) 946{ 947 char buf[128]; 948 struct port_info *pi = device_get_softc(dev); 949 950 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 951 device_set_desc_copy(dev, buf); 952 953 return (BUS_PROBE_DEFAULT); 954} 955 956#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 957 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 958 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 959#define T4_CAP_ENABLE (T4_CAP) 960 961static int 962cxgbe_attach(device_t dev) 963{ 964 struct port_info *pi = device_get_softc(dev); 965 struct ifnet *ifp; 966 967 /* Allocate an ifnet and set it up */ 968 ifp = if_alloc(IFT_ETHER); 969 if (ifp == NULL) { 970 device_printf(dev, "Cannot allocate ifnet\n"); 971 return (ENOMEM); 972 } 973 pi->ifp = ifp; 974 ifp->if_softc = pi; 975 976 callout_init(&pi->tick, CALLOUT_MPSAFE); 977 978 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 979 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 980 981 ifp->if_init = cxgbe_init; 982 ifp->if_ioctl = cxgbe_ioctl; 983 ifp->if_transmit = cxgbe_transmit; 984 ifp->if_qflush = cxgbe_qflush; 985 986 ifp->if_capabilities = T4_CAP; 987#ifdef TCP_OFFLOAD 988 if (is_offload(pi->adapter)) 989 ifp->if_capabilities |= IFCAP_TOE; 990#endif 991 ifp->if_capenable = T4_CAP_ENABLE; 992 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 993 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 994 995 /* Initialize ifmedia for this port */ 996 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 997 cxgbe_media_status); 998 build_medialist(pi); 999 1000 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1001 EVENTHANDLER_PRI_ANY); 1002 1003 ether_ifattach(ifp, pi->hw_addr); 1004 1005#ifdef TCP_OFFLOAD 1006 if (is_offload(pi->adapter)) { 1007 device_printf(dev, 1008 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 1009 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 1010 } else 1011#endif 1012 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 1013 1014 cxgbe_sysctls(pi); 1015 1016 return (0); 1017} 1018 1019static int 1020cxgbe_detach(device_t dev) 1021{ 1022 struct port_info *pi = device_get_softc(dev); 1023 struct adapter *sc = pi->adapter; 1024 struct ifnet *ifp = pi->ifp; 1025 1026 /* Tell if_ioctl and if_init that the port is going away */ 1027 ADAPTER_LOCK(sc); 1028 SET_DOOMED(pi); 1029 wakeup(&sc->flags); 1030 while (IS_BUSY(sc)) 1031 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1032 SET_BUSY(sc); 1033#ifdef INVARIANTS 1034 sc->last_op = "t4detach"; 1035 sc->last_op_thr = curthread; 1036#endif 1037 ADAPTER_UNLOCK(sc); 1038 1039 if (pi->vlan_c) 1040 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1041 1042 PORT_LOCK(pi); 1043 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1044 callout_stop(&pi->tick); 1045 PORT_UNLOCK(pi); 1046 callout_drain(&pi->tick); 1047 1048 /* Let detach proceed even if these fail. */ 1049 cxgbe_uninit_synchronized(pi); 1050 port_full_uninit(pi); 1051 1052 ifmedia_removeall(&pi->media); 1053 ether_ifdetach(pi->ifp); 1054 if_free(pi->ifp); 1055 1056 ADAPTER_LOCK(sc); 1057 CLR_BUSY(sc); 1058 wakeup(&sc->flags); 1059 ADAPTER_UNLOCK(sc); 1060 1061 return (0); 1062} 1063 1064static void 1065cxgbe_init(void *arg) 1066{ 1067 struct port_info *pi = arg; 1068 struct adapter *sc = pi->adapter; 1069 1070 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1071 return; 1072 cxgbe_init_synchronized(pi); 1073 end_synchronized_op(sc, 0); 1074} 1075 1076static int 1077cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1078{ 1079 int rc = 0, mtu, flags; 1080 struct port_info *pi = ifp->if_softc; 1081 struct adapter *sc = pi->adapter; 1082 struct ifreq *ifr = (struct ifreq *)data; 1083 uint32_t mask; 1084 1085 switch (cmd) { 1086 case SIOCSIFMTU: 1087 mtu = ifr->ifr_mtu; 1088 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1089 return (EINVAL); 1090 1091 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1092 if (rc) 1093 return (rc); 1094 ifp->if_mtu = mtu; 1095 if (pi->flags & PORT_INIT_DONE) { 1096 t4_update_fl_bufsize(ifp); 1097 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1098 rc = update_mac_settings(pi, XGMAC_MTU); 1099 } 1100 end_synchronized_op(sc, 0); 1101 break; 1102 1103 case SIOCSIFFLAGS: 1104 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1105 if (rc) 1106 return (rc); 1107 1108 if (ifp->if_flags & IFF_UP) { 1109 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1110 flags = pi->if_flags; 1111 if ((ifp->if_flags ^ flags) & 1112 (IFF_PROMISC | IFF_ALLMULTI)) { 1113 rc = update_mac_settings(pi, 1114 XGMAC_PROMISC | XGMAC_ALLMULTI); 1115 } 1116 } else 1117 rc = cxgbe_init_synchronized(pi); 1118 pi->if_flags = ifp->if_flags; 1119 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1120 rc = cxgbe_uninit_synchronized(pi); 1121 end_synchronized_op(sc, 0); 1122 break; 1123 1124 case SIOCADDMULTI: 1125 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1126 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1127 if (rc) 1128 return (rc); 1129 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1130 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1131 end_synchronized_op(sc, LOCK_HELD); 1132 break; 1133 1134 case SIOCSIFCAP: 1135 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1136 if (rc) 1137 return (rc); 1138 1139 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1140 if (mask & IFCAP_TXCSUM) { 1141 ifp->if_capenable ^= IFCAP_TXCSUM; 1142 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1143 1144 if (IFCAP_TSO4 & ifp->if_capenable && 1145 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1146 ifp->if_capenable &= ~IFCAP_TSO4; 1147 if_printf(ifp, 1148 "tso4 disabled due to -txcsum.\n"); 1149 } 1150 } 1151 if (mask & IFCAP_TXCSUM_IPV6) { 1152 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1153 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1154 1155 if (IFCAP_TSO6 & ifp->if_capenable && 1156 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1157 ifp->if_capenable &= ~IFCAP_TSO6; 1158 if_printf(ifp, 1159 "tso6 disabled due to -txcsum6.\n"); 1160 } 1161 } 1162 if (mask & IFCAP_RXCSUM) 1163 ifp->if_capenable ^= IFCAP_RXCSUM; 1164 if (mask & IFCAP_RXCSUM_IPV6) 1165 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1166 1167 /* 1168 * Note that we leave CSUM_TSO alone (it is always set). The 1169 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1170 * sending a TSO request our way, so it's sufficient to toggle 1171 * IFCAP_TSOx only. 1172 */ 1173 if (mask & IFCAP_TSO4) { 1174 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1175 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1176 if_printf(ifp, "enable txcsum first.\n"); 1177 rc = EAGAIN; 1178 goto fail; 1179 } 1180 ifp->if_capenable ^= IFCAP_TSO4; 1181 } 1182 if (mask & IFCAP_TSO6) { 1183 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1184 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1185 if_printf(ifp, "enable txcsum6 first.\n"); 1186 rc = EAGAIN; 1187 goto fail; 1188 } 1189 ifp->if_capenable ^= IFCAP_TSO6; 1190 } 1191 if (mask & IFCAP_LRO) { 1192#if defined(INET) || defined(INET6) 1193 int i; 1194 struct sge_rxq *rxq; 1195 1196 ifp->if_capenable ^= IFCAP_LRO; 1197 for_each_rxq(pi, i, rxq) { 1198 if (ifp->if_capenable & IFCAP_LRO) 1199 rxq->iq.flags |= IQ_LRO_ENABLED; 1200 else 1201 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1202 } 1203#endif 1204 } 1205#ifdef TCP_OFFLOAD 1206 if (mask & IFCAP_TOE) { 1207 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1208 1209 rc = toe_capability(pi, enable); 1210 if (rc != 0) 1211 goto fail; 1212 1213 ifp->if_capenable ^= mask; 1214 } 1215#endif 1216 if (mask & IFCAP_VLAN_HWTAGGING) { 1217 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1218 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1219 rc = update_mac_settings(pi, XGMAC_VLANEX); 1220 } 1221 if (mask & IFCAP_VLAN_MTU) { 1222 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1223 1224 /* Need to find out how to disable auto-mtu-inflation */ 1225 } 1226 if (mask & IFCAP_VLAN_HWTSO) 1227 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1228 if (mask & IFCAP_VLAN_HWCSUM) 1229 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1230 1231#ifdef VLAN_CAPABILITIES 1232 VLAN_CAPABILITIES(ifp); 1233#endif 1234fail: 1235 end_synchronized_op(sc, 0); 1236 break; 1237 1238 case SIOCSIFMEDIA: 1239 case SIOCGIFMEDIA: 1240 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1241 break; 1242 1243 default: 1244 rc = ether_ioctl(ifp, cmd, data); 1245 } 1246 1247 return (rc); 1248} 1249 1250static int 1251cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1252{ 1253 struct port_info *pi = ifp->if_softc; 1254 struct adapter *sc = pi->adapter; 1255 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1256 struct buf_ring *br; 1257 int rc; 1258 1259 M_ASSERTPKTHDR(m); 1260 1261 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1262 m_freem(m); 1263 return (ENETDOWN); 1264 } 1265 1266 if (m->m_flags & M_FLOWID) 1267 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1268 + pi->rsrv_noflowq); 1269 br = txq->br; 1270 1271 if (TXQ_TRYLOCK(txq) == 0) { 1272 struct sge_eq *eq = &txq->eq; 1273 1274 /* 1275 * It is possible that t4_eth_tx finishes up and releases the 1276 * lock between the TRYLOCK above and the drbr_enqueue here. We 1277 * need to make sure that this mbuf doesn't just sit there in 1278 * the drbr. 1279 */ 1280 1281 rc = drbr_enqueue(ifp, br, m); 1282 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1283 !(eq->flags & EQ_DOOMED)) 1284 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1285 return (rc); 1286 } 1287 1288 /* 1289 * txq->m is the mbuf that is held up due to a temporary shortage of 1290 * resources and it should be put on the wire first. Then what's in 1291 * drbr and finally the mbuf that was just passed in to us. 1292 * 1293 * Return code should indicate the fate of the mbuf that was passed in 1294 * this time. 1295 */ 1296 1297 TXQ_LOCK_ASSERT_OWNED(txq); 1298 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1299 1300 /* Queued for transmission. */ 1301 1302 rc = drbr_enqueue(ifp, br, m); 1303 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1304 (void) t4_eth_tx(ifp, txq, m); 1305 TXQ_UNLOCK(txq); 1306 return (rc); 1307 } 1308 1309 /* Direct transmission. */ 1310 rc = t4_eth_tx(ifp, txq, m); 1311 if (rc != 0 && txq->m) 1312 rc = 0; /* held, will be transmitted soon (hopefully) */ 1313 1314 TXQ_UNLOCK(txq); 1315 return (rc); 1316} 1317 1318static void 1319cxgbe_qflush(struct ifnet *ifp) 1320{ 1321 struct port_info *pi = ifp->if_softc; 1322 struct sge_txq *txq; 1323 int i; 1324 struct mbuf *m; 1325 1326 /* queues do not exist if !PORT_INIT_DONE. */ 1327 if (pi->flags & PORT_INIT_DONE) { 1328 for_each_txq(pi, i, txq) { 1329 TXQ_LOCK(txq); 1330 m_freem(txq->m); 1331 txq->m = NULL; 1332 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1333 m_freem(m); 1334 TXQ_UNLOCK(txq); 1335 } 1336 } 1337 if_qflush(ifp); 1338} 1339 1340static int 1341cxgbe_media_change(struct ifnet *ifp) 1342{ 1343 struct port_info *pi = ifp->if_softc; 1344 1345 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1346 1347 return (EOPNOTSUPP); 1348} 1349 1350static void 1351cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1352{ 1353 struct port_info *pi = ifp->if_softc; 1354 struct ifmedia_entry *cur = pi->media.ifm_cur; 1355 int speed = pi->link_cfg.speed; 1356 int data = (pi->port_type << 8) | pi->mod_type; 1357 1358 if (cur->ifm_data != data) { 1359 build_medialist(pi); 1360 cur = pi->media.ifm_cur; 1361 } 1362 1363 ifmr->ifm_status = IFM_AVALID; 1364 if (!pi->link_cfg.link_ok) 1365 return; 1366 1367 ifmr->ifm_status |= IFM_ACTIVE; 1368 1369 /* active and current will differ iff current media is autoselect. */ 1370 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1371 return; 1372 1373 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1374 if (speed == SPEED_10000) 1375 ifmr->ifm_active |= IFM_10G_T; 1376 else if (speed == SPEED_1000) 1377 ifmr->ifm_active |= IFM_1000_T; 1378 else if (speed == SPEED_100) 1379 ifmr->ifm_active |= IFM_100_TX; 1380 else if (speed == SPEED_10) 1381 ifmr->ifm_active |= IFM_10_T; 1382 else 1383 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1384 speed)); 1385} 1386 1387void 1388t4_fatal_err(struct adapter *sc) 1389{ 1390 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1391 t4_intr_disable(sc); 1392 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1393 device_get_nameunit(sc->dev)); 1394} 1395 1396static int 1397map_bars_0_and_4(struct adapter *sc) 1398{ 1399 sc->regs_rid = PCIR_BAR(0); 1400 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1401 &sc->regs_rid, RF_ACTIVE); 1402 if (sc->regs_res == NULL) { 1403 device_printf(sc->dev, "cannot map registers.\n"); 1404 return (ENXIO); 1405 } 1406 sc->bt = rman_get_bustag(sc->regs_res); 1407 sc->bh = rman_get_bushandle(sc->regs_res); 1408 sc->mmio_len = rman_get_size(sc->regs_res); 1409 setbit(&sc->doorbells, DOORBELL_KDB); 1410 1411 sc->msix_rid = PCIR_BAR(4); 1412 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1413 &sc->msix_rid, RF_ACTIVE); 1414 if (sc->msix_res == NULL) { 1415 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1416 return (ENXIO); 1417 } 1418 1419 return (0); 1420} 1421 1422static int 1423map_bar_2(struct adapter *sc) 1424{ 1425 1426 /* 1427 * T4: only iWARP driver uses the userspace doorbells. There is no need 1428 * to map it if RDMA is disabled. 1429 */ 1430 if (is_t4(sc) && sc->rdmacaps == 0) 1431 return (0); 1432 1433 sc->udbs_rid = PCIR_BAR(2); 1434 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1435 &sc->udbs_rid, RF_ACTIVE); 1436 if (sc->udbs_res == NULL) { 1437 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1438 return (ENXIO); 1439 } 1440 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1441 1442 if (is_t5(sc)) { 1443 setbit(&sc->doorbells, DOORBELL_UDB); 1444#if defined(__i386__) || defined(__amd64__) 1445 if (t5_write_combine) { 1446 int rc; 1447 1448 /* 1449 * Enable write combining on BAR2. This is the 1450 * userspace doorbell BAR and is split into 128B 1451 * (UDBS_SEG_SIZE) doorbell regions, each associated 1452 * with an egress queue. The first 64B has the doorbell 1453 * and the second 64B can be used to submit a tx work 1454 * request with an implicit doorbell. 1455 */ 1456 1457 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1458 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1459 if (rc == 0) { 1460 clrbit(&sc->doorbells, DOORBELL_UDB); 1461 setbit(&sc->doorbells, DOORBELL_WCWR); 1462 setbit(&sc->doorbells, DOORBELL_UDBWC); 1463 } else { 1464 device_printf(sc->dev, 1465 "couldn't enable write combining: %d\n", 1466 rc); 1467 } 1468 1469 t4_write_reg(sc, A_SGE_STAT_CFG, 1470 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1471 } 1472#endif 1473 } 1474 1475 return (0); 1476} 1477 1478static const struct memwin t4_memwin[] = { 1479 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1480 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1481 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1482}; 1483 1484static const struct memwin t5_memwin[] = { 1485 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1486 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1487 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1488}; 1489 1490static void 1491setup_memwin(struct adapter *sc) 1492{ 1493 const struct memwin *mw; 1494 int i, n; 1495 uint32_t bar0; 1496 1497 if (is_t4(sc)) { 1498 /* 1499 * Read low 32b of bar0 indirectly via the hardware backdoor 1500 * mechanism. Works from within PCI passthrough environments 1501 * too, where rman_get_start() can return a different value. We 1502 * need to program the T4 memory window decoders with the actual 1503 * addresses that will be coming across the PCIe link. 1504 */ 1505 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1506 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1507 1508 mw = &t4_memwin[0]; 1509 n = nitems(t4_memwin); 1510 } else { 1511 /* T5 uses the relative offset inside the PCIe BAR */ 1512 bar0 = 0; 1513 1514 mw = &t5_memwin[0]; 1515 n = nitems(t5_memwin); 1516 } 1517 1518 for (i = 0; i < n; i++, mw++) { 1519 t4_write_reg(sc, 1520 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1521 (mw->base + bar0) | V_BIR(0) | 1522 V_WINDOW(ilog2(mw->aperture) - 10)); 1523 } 1524 1525 /* flush */ 1526 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1527} 1528 1529/* 1530 * Verify that the memory range specified by the addr/len pair is valid and lies 1531 * entirely within a single region (EDCx or MCx). 1532 */ 1533static int 1534validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1535{ 1536 uint32_t em, addr_len, maddr, mlen; 1537 1538 /* Memory can only be accessed in naturally aligned 4 byte units */ 1539 if (addr & 3 || len & 3 || len == 0) 1540 return (EINVAL); 1541 1542 /* Enabled memories */ 1543 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1544 if (em & F_EDRAM0_ENABLE) { 1545 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1546 maddr = G_EDRAM0_BASE(addr_len) << 20; 1547 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1548 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1549 addr + len <= maddr + mlen) 1550 return (0); 1551 } 1552 if (em & F_EDRAM1_ENABLE) { 1553 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1554 maddr = G_EDRAM1_BASE(addr_len) << 20; 1555 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1556 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1557 addr + len <= maddr + mlen) 1558 return (0); 1559 } 1560 if (em & F_EXT_MEM_ENABLE) { 1561 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1562 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1563 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1564 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1565 addr + len <= maddr + mlen) 1566 return (0); 1567 } 1568 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1569 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1570 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1571 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1572 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1573 addr + len <= maddr + mlen) 1574 return (0); 1575 } 1576 1577 return (EFAULT); 1578} 1579 1580static int 1581fwmtype_to_hwmtype(int mtype) 1582{ 1583 1584 switch (mtype) { 1585 case FW_MEMTYPE_EDC0: 1586 return (MEM_EDC0); 1587 case FW_MEMTYPE_EDC1: 1588 return (MEM_EDC1); 1589 case FW_MEMTYPE_EXTMEM: 1590 return (MEM_MC0); 1591 case FW_MEMTYPE_EXTMEM1: 1592 return (MEM_MC1); 1593 default: 1594 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1595 } 1596} 1597 1598/* 1599 * Verify that the memory range specified by the memtype/offset/len pair is 1600 * valid and lies entirely within the memtype specified. The global address of 1601 * the start of the range is returned in addr. 1602 */ 1603static int 1604validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1605 uint32_t *addr) 1606{ 1607 uint32_t em, addr_len, maddr, mlen; 1608 1609 /* Memory can only be accessed in naturally aligned 4 byte units */ 1610 if (off & 3 || len & 3 || len == 0) 1611 return (EINVAL); 1612 1613 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1614 switch (fwmtype_to_hwmtype(mtype)) { 1615 case MEM_EDC0: 1616 if (!(em & F_EDRAM0_ENABLE)) 1617 return (EINVAL); 1618 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1619 maddr = G_EDRAM0_BASE(addr_len) << 20; 1620 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1621 break; 1622 case MEM_EDC1: 1623 if (!(em & F_EDRAM1_ENABLE)) 1624 return (EINVAL); 1625 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1626 maddr = G_EDRAM1_BASE(addr_len) << 20; 1627 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1628 break; 1629 case MEM_MC: 1630 if (!(em & F_EXT_MEM_ENABLE)) 1631 return (EINVAL); 1632 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1633 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1634 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1635 break; 1636 case MEM_MC1: 1637 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1638 return (EINVAL); 1639 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1640 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1641 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1642 break; 1643 default: 1644 return (EINVAL); 1645 } 1646 1647 if (mlen > 0 && off < mlen && off + len <= mlen) { 1648 *addr = maddr + off; /* global address */ 1649 return (0); 1650 } 1651 1652 return (EFAULT); 1653} 1654 1655static void 1656memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1657{ 1658 const struct memwin *mw; 1659 1660 if (is_t4(sc)) { 1661 KASSERT(win >= 0 && win < nitems(t4_memwin), 1662 ("%s: incorrect memwin# (%d)", __func__, win)); 1663 mw = &t4_memwin[win]; 1664 } else { 1665 KASSERT(win >= 0 && win < nitems(t5_memwin), 1666 ("%s: incorrect memwin# (%d)", __func__, win)); 1667 mw = &t5_memwin[win]; 1668 } 1669 1670 if (base != NULL) 1671 *base = mw->base; 1672 if (aperture != NULL) 1673 *aperture = mw->aperture; 1674} 1675 1676/* 1677 * Positions the memory window such that it can be used to access the specified 1678 * address in the chip's address space. The return value is the offset of addr 1679 * from the start of the window. 1680 */ 1681static uint32_t 1682position_memwin(struct adapter *sc, int n, uint32_t addr) 1683{ 1684 uint32_t start, pf; 1685 uint32_t reg; 1686 1687 KASSERT(n >= 0 && n <= 3, 1688 ("%s: invalid window %d.", __func__, n)); 1689 KASSERT((addr & 3) == 0, 1690 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1691 1692 if (is_t4(sc)) { 1693 pf = 0; 1694 start = addr & ~0xf; /* start must be 16B aligned */ 1695 } else { 1696 pf = V_PFNUM(sc->pf); 1697 start = addr & ~0x7f; /* start must be 128B aligned */ 1698 } 1699 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1700 1701 t4_write_reg(sc, reg, start | pf); 1702 t4_read_reg(sc, reg); 1703 1704 return (addr - start); 1705} 1706 1707static int 1708cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1709 struct intrs_and_queues *iaq) 1710{ 1711 int rc, itype, navail, nrxq10g, nrxq1g, n; 1712 int nofldrxq10g = 0, nofldrxq1g = 0; 1713 1714 bzero(iaq, sizeof(*iaq)); 1715 1716 iaq->ntxq10g = t4_ntxq10g; 1717 iaq->ntxq1g = t4_ntxq1g; 1718 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1719 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1720 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1721#ifdef TCP_OFFLOAD 1722 if (is_offload(sc)) { 1723 iaq->nofldtxq10g = t4_nofldtxq10g; 1724 iaq->nofldtxq1g = t4_nofldtxq1g; 1725 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1726 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1727 } 1728#endif 1729 1730 for (itype = INTR_MSIX; itype; itype >>= 1) { 1731 1732 if ((itype & t4_intr_types) == 0) 1733 continue; /* not allowed */ 1734 1735 if (itype == INTR_MSIX) 1736 navail = pci_msix_count(sc->dev); 1737 else if (itype == INTR_MSI) 1738 navail = pci_msi_count(sc->dev); 1739 else 1740 navail = 1; 1741restart: 1742 if (navail == 0) 1743 continue; 1744 1745 iaq->intr_type = itype; 1746 iaq->intr_flags = 0; 1747 1748 /* 1749 * Best option: an interrupt vector for errors, one for the 1750 * firmware event queue, and one each for each rxq (NIC as well 1751 * as offload). 1752 */ 1753 iaq->nirq = T4_EXTRA_INTR; 1754 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1755 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1756 if (iaq->nirq <= navail && 1757 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1758 iaq->intr_flags |= INTR_DIRECT; 1759 goto allocate; 1760 } 1761 1762 /* 1763 * Second best option: an interrupt vector for errors, one for 1764 * the firmware event queue, and one each for either NIC or 1765 * offload rxq's. 1766 */ 1767 iaq->nirq = T4_EXTRA_INTR; 1768 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1769 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1770 if (iaq->nirq <= navail && 1771 (itype != INTR_MSI || powerof2(iaq->nirq))) 1772 goto allocate; 1773 1774 /* 1775 * Next best option: an interrupt vector for errors, one for the 1776 * firmware event queue, and at least one per port. At this 1777 * point we know we'll have to downsize nrxq or nofldrxq to fit 1778 * what's available to us. 1779 */ 1780 iaq->nirq = T4_EXTRA_INTR; 1781 iaq->nirq += n10g + n1g; 1782 if (iaq->nirq <= navail) { 1783 int leftover = navail - iaq->nirq; 1784 1785 if (n10g > 0) { 1786 int target = max(nrxq10g, nofldrxq10g); 1787 1788 n = 1; 1789 while (n < target && leftover >= n10g) { 1790 leftover -= n10g; 1791 iaq->nirq += n10g; 1792 n++; 1793 } 1794 iaq->nrxq10g = min(n, nrxq10g); 1795#ifdef TCP_OFFLOAD 1796 if (is_offload(sc)) 1797 iaq->nofldrxq10g = min(n, nofldrxq10g); 1798#endif 1799 } 1800 1801 if (n1g > 0) { 1802 int target = max(nrxq1g, nofldrxq1g); 1803 1804 n = 1; 1805 while (n < target && leftover >= n1g) { 1806 leftover -= n1g; 1807 iaq->nirq += n1g; 1808 n++; 1809 } 1810 iaq->nrxq1g = min(n, nrxq1g); 1811#ifdef TCP_OFFLOAD 1812 if (is_offload(sc)) 1813 iaq->nofldrxq1g = min(n, nofldrxq1g); 1814#endif 1815 } 1816 1817 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1818 goto allocate; 1819 } 1820 1821 /* 1822 * Least desirable option: one interrupt vector for everything. 1823 */ 1824 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1825#ifdef TCP_OFFLOAD 1826 if (is_offload(sc)) 1827 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1828#endif 1829 1830allocate: 1831 navail = iaq->nirq; 1832 rc = 0; 1833 if (itype == INTR_MSIX) 1834 rc = pci_alloc_msix(sc->dev, &navail); 1835 else if (itype == INTR_MSI) 1836 rc = pci_alloc_msi(sc->dev, &navail); 1837 1838 if (rc == 0) { 1839 if (navail == iaq->nirq) 1840 return (0); 1841 1842 /* 1843 * Didn't get the number requested. Use whatever number 1844 * the kernel is willing to allocate (it's in navail). 1845 */ 1846 device_printf(sc->dev, "fewer vectors than requested, " 1847 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1848 itype, iaq->nirq, navail); 1849 pci_release_msi(sc->dev); 1850 goto restart; 1851 } 1852 1853 device_printf(sc->dev, 1854 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1855 itype, rc, iaq->nirq, navail); 1856 } 1857 1858 device_printf(sc->dev, 1859 "failed to find a usable interrupt type. " 1860 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1861 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1862 1863 return (ENXIO); 1864} 1865 1866#define FW_VERSION(chip) ( \ 1867 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 1868 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 1869 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 1870 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 1871#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 1872 1873struct fw_info { 1874 uint8_t chip; 1875 char *kld_name; 1876 char *fw_mod_name; 1877 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 1878} fw_info[] = { 1879 { 1880 .chip = CHELSIO_T4, 1881 .kld_name = "t4fw_cfg", 1882 .fw_mod_name = "t4fw", 1883 .fw_hdr = { 1884 .chip = FW_HDR_CHIP_T4, 1885 .fw_ver = htobe32_const(FW_VERSION(T4)), 1886 .intfver_nic = FW_INTFVER(T4, NIC), 1887 .intfver_vnic = FW_INTFVER(T4, VNIC), 1888 .intfver_ofld = FW_INTFVER(T4, OFLD), 1889 .intfver_ri = FW_INTFVER(T4, RI), 1890 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 1891 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 1892 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 1893 .intfver_fcoe = FW_INTFVER(T4, FCOE), 1894 }, 1895 }, { 1896 .chip = CHELSIO_T5, 1897 .kld_name = "t5fw_cfg", 1898 .fw_mod_name = "t5fw", 1899 .fw_hdr = { 1900 .chip = FW_HDR_CHIP_T5, 1901 .fw_ver = htobe32_const(FW_VERSION(T5)), 1902 .intfver_nic = FW_INTFVER(T5, NIC), 1903 .intfver_vnic = FW_INTFVER(T5, VNIC), 1904 .intfver_ofld = FW_INTFVER(T5, OFLD), 1905 .intfver_ri = FW_INTFVER(T5, RI), 1906 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 1907 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 1908 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 1909 .intfver_fcoe = FW_INTFVER(T5, FCOE), 1910 }, 1911 } 1912}; 1913 1914static struct fw_info * 1915find_fw_info(int chip) 1916{ 1917 int i; 1918 1919 for (i = 0; i < nitems(fw_info); i++) { 1920 if (fw_info[i].chip == chip) 1921 return (&fw_info[i]); 1922 } 1923 return (NULL); 1924} 1925 1926/* 1927 * Is the given firmware API compatible with the one the driver was compiled 1928 * with? 1929 */ 1930static int 1931fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 1932{ 1933 1934 /* short circuit if it's the exact same firmware version */ 1935 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 1936 return (1); 1937 1938 /* 1939 * XXX: Is this too conservative? Perhaps I should limit this to the 1940 * features that are supported in the driver. 1941 */ 1942#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 1943 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 1944 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 1945 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 1946 return (1); 1947#undef SAME_INTF 1948 1949 return (0); 1950} 1951 1952/* 1953 * The firmware in the KLD is usable, but should it be installed? This routine 1954 * explains itself in detail if it indicates the KLD firmware should be 1955 * installed. 1956 */ 1957static int 1958should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 1959{ 1960 const char *reason; 1961 1962 if (!card_fw_usable) { 1963 reason = "incompatible or unusable"; 1964 goto install; 1965 } 1966 1967 if (k > c) { 1968 reason = "older than the version bundled with this driver"; 1969 goto install; 1970 } 1971 1972 if (t4_fw_install == 2 && k != c) { 1973 reason = "different than the version bundled with this driver"; 1974 goto install; 1975 } 1976 1977 return (0); 1978 1979install: 1980 if (t4_fw_install == 0) { 1981 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 1982 "but the driver is prohibited from installing a different " 1983 "firmware on the card.\n", 1984 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 1985 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 1986 1987 return (0); 1988 } 1989 1990 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 1991 "installing firmware %u.%u.%u.%u on card.\n", 1992 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 1993 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 1994 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 1995 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 1996 1997 return (1); 1998} 1999/* 2000 * Establish contact with the firmware and determine if we are the master driver 2001 * or not, and whether we are responsible for chip initialization. 2002 */ 2003static int 2004prep_firmware(struct adapter *sc) 2005{ 2006 const struct firmware *fw = NULL, *default_cfg; 2007 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2008 enum dev_state state; 2009 struct fw_info *fw_info; 2010 struct fw_hdr *card_fw; /* fw on the card */ 2011 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2012 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2013 against */ 2014 2015 /* Contact firmware. */ 2016 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2017 if (rc < 0 || state == DEV_STATE_ERR) { 2018 rc = -rc; 2019 device_printf(sc->dev, 2020 "failed to connect to the firmware: %d, %d.\n", rc, state); 2021 return (rc); 2022 } 2023 pf = rc; 2024 if (pf == sc->mbox) 2025 sc->flags |= MASTER_PF; 2026 else if (state == DEV_STATE_UNINIT) { 2027 /* 2028 * We didn't get to be the master so we definitely won't be 2029 * configuring the chip. It's a bug if someone else hasn't 2030 * configured it already. 2031 */ 2032 device_printf(sc->dev, "couldn't be master(%d), " 2033 "device not already initialized either(%d).\n", rc, state); 2034 return (EDOOFUS); 2035 } 2036 2037 /* This is the firmware whose headers the driver was compiled against */ 2038 fw_info = find_fw_info(chip_id(sc)); 2039 if (fw_info == NULL) { 2040 device_printf(sc->dev, 2041 "unable to look up firmware information for chip %d.\n", 2042 chip_id(sc)); 2043 return (EINVAL); 2044 } 2045 drv_fw = &fw_info->fw_hdr; 2046 2047 /* 2048 * The firmware KLD contains many modules. The KLD name is also the 2049 * name of the module that contains the default config file. 2050 */ 2051 default_cfg = firmware_get(fw_info->kld_name); 2052 2053 /* Read the header of the firmware on the card */ 2054 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2055 rc = -t4_read_flash(sc, FLASH_FW_START, 2056 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2057 if (rc == 0) 2058 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2059 else { 2060 device_printf(sc->dev, 2061 "Unable to read card's firmware header: %d\n", rc); 2062 card_fw_usable = 0; 2063 } 2064 2065 /* This is the firmware in the KLD */ 2066 fw = firmware_get(fw_info->fw_mod_name); 2067 if (fw != NULL) { 2068 kld_fw = (const void *)fw->data; 2069 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2070 } else { 2071 kld_fw = NULL; 2072 kld_fw_usable = 0; 2073 } 2074 2075 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2076 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2077 /* 2078 * Common case: the firmware on the card is an exact match and 2079 * the KLD is an exact match too, or the KLD is 2080 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2081 * here -- use cxgbetool loadfw if you want to reinstall the 2082 * same firmware as the one on the card. 2083 */ 2084 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2085 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2086 be32toh(card_fw->fw_ver))) { 2087 2088 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2089 if (rc != 0) { 2090 device_printf(sc->dev, 2091 "failed to install firmware: %d\n", rc); 2092 goto done; 2093 } 2094 2095 /* Installed successfully, update the cached header too. */ 2096 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2097 card_fw_usable = 1; 2098 need_fw_reset = 0; /* already reset as part of load_fw */ 2099 } 2100 2101 if (!card_fw_usable) { 2102 uint32_t d, c, k; 2103 2104 d = ntohl(drv_fw->fw_ver); 2105 c = ntohl(card_fw->fw_ver); 2106 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2107 2108 device_printf(sc->dev, "Cannot find a usable firmware: " 2109 "fw_install %d, chip state %d, " 2110 "driver compiled with %d.%d.%d.%d, " 2111 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2112 t4_fw_install, state, 2113 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2114 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2115 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2116 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2117 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2118 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2119 rc = EINVAL; 2120 goto done; 2121 } 2122 2123 /* We're using whatever's on the card and it's known to be good. */ 2124 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2125 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2126 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2127 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2128 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2129 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2130 t4_get_tp_version(sc, &sc->params.tp_vers); 2131 2132 /* Reset device */ 2133 if (need_fw_reset && 2134 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2135 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2136 if (rc != ETIMEDOUT && rc != EIO) 2137 t4_fw_bye(sc, sc->mbox); 2138 goto done; 2139 } 2140 sc->flags |= FW_OK; 2141 2142 rc = get_params__pre_init(sc); 2143 if (rc != 0) 2144 goto done; /* error message displayed already */ 2145 2146 /* Partition adapter resources as specified in the config file. */ 2147 if (state == DEV_STATE_UNINIT) { 2148 2149 KASSERT(sc->flags & MASTER_PF, 2150 ("%s: trying to change chip settings when not master.", 2151 __func__)); 2152 2153 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2154 if (rc != 0) 2155 goto done; /* error message displayed already */ 2156 2157 t4_tweak_chip_settings(sc); 2158 2159 /* get basic stuff going */ 2160 rc = -t4_fw_initialize(sc, sc->mbox); 2161 if (rc != 0) { 2162 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2163 goto done; 2164 } 2165 } else { 2166 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2167 sc->cfcsum = 0; 2168 } 2169 2170done: 2171 free(card_fw, M_CXGBE); 2172 if (fw != NULL) 2173 firmware_put(fw, FIRMWARE_UNLOAD); 2174 if (default_cfg != NULL) 2175 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2176 2177 return (rc); 2178} 2179 2180#define FW_PARAM_DEV(param) \ 2181 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2182 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2183#define FW_PARAM_PFVF(param) \ 2184 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2185 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2186 2187/* 2188 * Partition chip resources for use between various PFs, VFs, etc. 2189 */ 2190static int 2191partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2192 const char *name_prefix) 2193{ 2194 const struct firmware *cfg = NULL; 2195 int rc = 0; 2196 struct fw_caps_config_cmd caps; 2197 uint32_t mtype, moff, finicsum, cfcsum; 2198 2199 /* 2200 * Figure out what configuration file to use. Pick the default config 2201 * file for the card if the user hasn't specified one explicitly. 2202 */ 2203 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2204 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2205 /* Card specific overrides go here. */ 2206 if (pci_get_device(sc->dev) == 0x440a) 2207 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2208 if (is_fpga(sc)) 2209 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2210 } 2211 2212 /* 2213 * We need to load another module if the profile is anything except 2214 * "default" or "flash". 2215 */ 2216 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2217 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2218 char s[32]; 2219 2220 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2221 cfg = firmware_get(s); 2222 if (cfg == NULL) { 2223 if (default_cfg != NULL) { 2224 device_printf(sc->dev, 2225 "unable to load module \"%s\" for " 2226 "configuration profile \"%s\", will use " 2227 "the default config file instead.\n", 2228 s, sc->cfg_file); 2229 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2230 "%s", DEFAULT_CF); 2231 } else { 2232 device_printf(sc->dev, 2233 "unable to load module \"%s\" for " 2234 "configuration profile \"%s\", will use " 2235 "the config file on the card's flash " 2236 "instead.\n", s, sc->cfg_file); 2237 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2238 "%s", FLASH_CF); 2239 } 2240 } 2241 } 2242 2243 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2244 default_cfg == NULL) { 2245 device_printf(sc->dev, 2246 "default config file not available, will use the config " 2247 "file on the card's flash instead.\n"); 2248 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2249 } 2250 2251 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2252 u_int cflen, i, n; 2253 const uint32_t *cfdata; 2254 uint32_t param, val, addr, off, mw_base, mw_aperture; 2255 2256 KASSERT(cfg != NULL || default_cfg != NULL, 2257 ("%s: no config to upload", __func__)); 2258 2259 /* 2260 * Ask the firmware where it wants us to upload the config file. 2261 */ 2262 param = FW_PARAM_DEV(CF); 2263 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2264 if (rc != 0) { 2265 /* No support for config file? Shouldn't happen. */ 2266 device_printf(sc->dev, 2267 "failed to query config file location: %d.\n", rc); 2268 goto done; 2269 } 2270 mtype = G_FW_PARAMS_PARAM_Y(val); 2271 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2272 2273 /* 2274 * XXX: sheer laziness. We deliberately added 4 bytes of 2275 * useless stuffing/comments at the end of the config file so 2276 * it's ok to simply throw away the last remaining bytes when 2277 * the config file is not an exact multiple of 4. This also 2278 * helps with the validate_mt_off_len check. 2279 */ 2280 if (cfg != NULL) { 2281 cflen = cfg->datasize & ~3; 2282 cfdata = cfg->data; 2283 } else { 2284 cflen = default_cfg->datasize & ~3; 2285 cfdata = default_cfg->data; 2286 } 2287 2288 if (cflen > FLASH_CFG_MAX_SIZE) { 2289 device_printf(sc->dev, 2290 "config file too long (%d, max allowed is %d). " 2291 "Will try to use the config on the card, if any.\n", 2292 cflen, FLASH_CFG_MAX_SIZE); 2293 goto use_config_on_flash; 2294 } 2295 2296 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2297 if (rc != 0) { 2298 device_printf(sc->dev, 2299 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2300 "Will try to use the config on the card, if any.\n", 2301 __func__, mtype, moff, cflen, rc); 2302 goto use_config_on_flash; 2303 } 2304 2305 memwin_info(sc, 2, &mw_base, &mw_aperture); 2306 while (cflen) { 2307 off = position_memwin(sc, 2, addr); 2308 n = min(cflen, mw_aperture - off); 2309 for (i = 0; i < n; i += 4) 2310 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2311 cflen -= n; 2312 addr += n; 2313 } 2314 } else { 2315use_config_on_flash: 2316 mtype = FW_MEMTYPE_FLASH; 2317 moff = t4_flash_cfg_addr(sc); 2318 } 2319 2320 bzero(&caps, sizeof(caps)); 2321 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2322 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2323 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2324 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2325 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2326 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2327 if (rc != 0) { 2328 device_printf(sc->dev, 2329 "failed to pre-process config file: %d " 2330 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2331 goto done; 2332 } 2333 2334 finicsum = be32toh(caps.finicsum); 2335 cfcsum = be32toh(caps.cfcsum); 2336 if (finicsum != cfcsum) { 2337 device_printf(sc->dev, 2338 "WARNING: config file checksum mismatch: %08x %08x\n", 2339 finicsum, cfcsum); 2340 } 2341 sc->cfcsum = cfcsum; 2342 2343#define LIMIT_CAPS(x) do { \ 2344 caps.x &= htobe16(t4_##x##_allowed); \ 2345} while (0) 2346 2347 /* 2348 * Let the firmware know what features will (not) be used so it can tune 2349 * things accordingly. 2350 */ 2351 LIMIT_CAPS(linkcaps); 2352 LIMIT_CAPS(niccaps); 2353 LIMIT_CAPS(toecaps); 2354 LIMIT_CAPS(rdmacaps); 2355 LIMIT_CAPS(iscsicaps); 2356 LIMIT_CAPS(fcoecaps); 2357#undef LIMIT_CAPS 2358 2359 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2360 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2361 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2362 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2363 if (rc != 0) { 2364 device_printf(sc->dev, 2365 "failed to process config file: %d.\n", rc); 2366 } 2367done: 2368 if (cfg != NULL) 2369 firmware_put(cfg, FIRMWARE_UNLOAD); 2370 return (rc); 2371} 2372 2373/* 2374 * Retrieve parameters that are needed (or nice to have) very early. 2375 */ 2376static int 2377get_params__pre_init(struct adapter *sc) 2378{ 2379 int rc; 2380 uint32_t param[2], val[2]; 2381 struct fw_devlog_cmd cmd; 2382 struct devlog_params *dlog = &sc->params.devlog; 2383 2384 param[0] = FW_PARAM_DEV(PORTVEC); 2385 param[1] = FW_PARAM_DEV(CCLK); 2386 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2387 if (rc != 0) { 2388 device_printf(sc->dev, 2389 "failed to query parameters (pre_init): %d.\n", rc); 2390 return (rc); 2391 } 2392 2393 sc->params.portvec = val[0]; 2394 sc->params.nports = bitcount32(val[0]); 2395 sc->params.vpd.cclk = val[1]; 2396 2397 /* Read device log parameters. */ 2398 bzero(&cmd, sizeof(cmd)); 2399 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2400 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2401 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2402 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2403 if (rc != 0) { 2404 device_printf(sc->dev, 2405 "failed to get devlog parameters: %d.\n", rc); 2406 bzero(dlog, sizeof (*dlog)); 2407 rc = 0; /* devlog isn't critical for device operation */ 2408 } else { 2409 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2410 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2411 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2412 dlog->size = be32toh(cmd.memsize_devlog); 2413 } 2414 2415 return (rc); 2416} 2417 2418/* 2419 * Retrieve various parameters that are of interest to the driver. The device 2420 * has been initialized by the firmware at this point. 2421 */ 2422static int 2423get_params__post_init(struct adapter *sc) 2424{ 2425 int rc; 2426 uint32_t param[7], val[7]; 2427 struct fw_caps_config_cmd caps; 2428 2429 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2430 param[1] = FW_PARAM_PFVF(EQ_START); 2431 param[2] = FW_PARAM_PFVF(FILTER_START); 2432 param[3] = FW_PARAM_PFVF(FILTER_END); 2433 param[4] = FW_PARAM_PFVF(L2T_START); 2434 param[5] = FW_PARAM_PFVF(L2T_END); 2435 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2436 if (rc != 0) { 2437 device_printf(sc->dev, 2438 "failed to query parameters (post_init): %d.\n", rc); 2439 return (rc); 2440 } 2441 2442 sc->sge.iq_start = val[0]; 2443 sc->sge.eq_start = val[1]; 2444 sc->tids.ftid_base = val[2]; 2445 sc->tids.nftids = val[3] - val[2] + 1; 2446 sc->params.ftid_min = val[2]; 2447 sc->params.ftid_max = val[3]; 2448 sc->vres.l2t.start = val[4]; 2449 sc->vres.l2t.size = val[5] - val[4] + 1; 2450 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2451 ("%s: L2 table size (%u) larger than expected (%u)", 2452 __func__, sc->vres.l2t.size, L2T_SIZE)); 2453 2454 /* get capabilites */ 2455 bzero(&caps, sizeof(caps)); 2456 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2457 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2458 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2459 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2460 if (rc != 0) { 2461 device_printf(sc->dev, 2462 "failed to get card capabilities: %d.\n", rc); 2463 return (rc); 2464 } 2465 2466#define READ_CAPS(x) do { \ 2467 sc->x = htobe16(caps.x); \ 2468} while (0) 2469 READ_CAPS(linkcaps); 2470 READ_CAPS(niccaps); 2471 READ_CAPS(toecaps); 2472 READ_CAPS(rdmacaps); 2473 READ_CAPS(iscsicaps); 2474 READ_CAPS(fcoecaps); 2475 2476 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2477 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2478 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2479 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2480 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2481 if (rc != 0) { 2482 device_printf(sc->dev, 2483 "failed to query NIC parameters: %d.\n", rc); 2484 return (rc); 2485 } 2486 sc->tids.etid_base = val[0]; 2487 sc->params.etid_min = val[0]; 2488 sc->tids.netids = val[1] - val[0] + 1; 2489 sc->params.netids = sc->tids.netids; 2490 sc->params.eo_wr_cred = val[2]; 2491 sc->params.ethoffload = 1; 2492 } 2493 2494 if (sc->toecaps) { 2495 /* query offload-related parameters */ 2496 param[0] = FW_PARAM_DEV(NTID); 2497 param[1] = FW_PARAM_PFVF(SERVER_START); 2498 param[2] = FW_PARAM_PFVF(SERVER_END); 2499 param[3] = FW_PARAM_PFVF(TDDP_START); 2500 param[4] = FW_PARAM_PFVF(TDDP_END); 2501 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2502 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2503 if (rc != 0) { 2504 device_printf(sc->dev, 2505 "failed to query TOE parameters: %d.\n", rc); 2506 return (rc); 2507 } 2508 sc->tids.ntids = val[0]; 2509 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2510 sc->tids.stid_base = val[1]; 2511 sc->tids.nstids = val[2] - val[1] + 1; 2512 sc->vres.ddp.start = val[3]; 2513 sc->vres.ddp.size = val[4] - val[3] + 1; 2514 sc->params.ofldq_wr_cred = val[5]; 2515 sc->params.offload = 1; 2516 } 2517 if (sc->rdmacaps) { 2518 param[0] = FW_PARAM_PFVF(STAG_START); 2519 param[1] = FW_PARAM_PFVF(STAG_END); 2520 param[2] = FW_PARAM_PFVF(RQ_START); 2521 param[3] = FW_PARAM_PFVF(RQ_END); 2522 param[4] = FW_PARAM_PFVF(PBL_START); 2523 param[5] = FW_PARAM_PFVF(PBL_END); 2524 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2525 if (rc != 0) { 2526 device_printf(sc->dev, 2527 "failed to query RDMA parameters(1): %d.\n", rc); 2528 return (rc); 2529 } 2530 sc->vres.stag.start = val[0]; 2531 sc->vres.stag.size = val[1] - val[0] + 1; 2532 sc->vres.rq.start = val[2]; 2533 sc->vres.rq.size = val[3] - val[2] + 1; 2534 sc->vres.pbl.start = val[4]; 2535 sc->vres.pbl.size = val[5] - val[4] + 1; 2536 2537 param[0] = FW_PARAM_PFVF(SQRQ_START); 2538 param[1] = FW_PARAM_PFVF(SQRQ_END); 2539 param[2] = FW_PARAM_PFVF(CQ_START); 2540 param[3] = FW_PARAM_PFVF(CQ_END); 2541 param[4] = FW_PARAM_PFVF(OCQ_START); 2542 param[5] = FW_PARAM_PFVF(OCQ_END); 2543 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2544 if (rc != 0) { 2545 device_printf(sc->dev, 2546 "failed to query RDMA parameters(2): %d.\n", rc); 2547 return (rc); 2548 } 2549 sc->vres.qp.start = val[0]; 2550 sc->vres.qp.size = val[1] - val[0] + 1; 2551 sc->vres.cq.start = val[2]; 2552 sc->vres.cq.size = val[3] - val[2] + 1; 2553 sc->vres.ocq.start = val[4]; 2554 sc->vres.ocq.size = val[5] - val[4] + 1; 2555 } 2556 if (sc->iscsicaps) { 2557 param[0] = FW_PARAM_PFVF(ISCSI_START); 2558 param[1] = FW_PARAM_PFVF(ISCSI_END); 2559 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2560 if (rc != 0) { 2561 device_printf(sc->dev, 2562 "failed to query iSCSI parameters: %d.\n", rc); 2563 return (rc); 2564 } 2565 sc->vres.iscsi.start = val[0]; 2566 sc->vres.iscsi.size = val[1] - val[0] + 1; 2567 } 2568 2569 /* 2570 * We've got the params we wanted to query via the firmware. Now grab 2571 * some others directly from the chip. 2572 */ 2573 rc = t4_read_chip_settings(sc); 2574 2575 return (rc); 2576} 2577 2578static int 2579set_params__post_init(struct adapter *sc) 2580{ 2581 uint32_t param, val; 2582 2583 /* ask for encapsulated CPLs */ 2584 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2585 val = 1; 2586 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2587 2588 return (0); 2589} 2590 2591#undef FW_PARAM_PFVF 2592#undef FW_PARAM_DEV 2593 2594static void 2595t4_set_desc(struct adapter *sc) 2596{ 2597 char buf[128]; 2598 struct adapter_params *p = &sc->params; 2599 2600 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2601 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2602 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2603 2604 device_set_desc_copy(sc->dev, buf); 2605} 2606 2607static void 2608build_medialist(struct port_info *pi) 2609{ 2610 struct ifmedia *media = &pi->media; 2611 int data, m; 2612 2613 PORT_LOCK(pi); 2614 2615 ifmedia_removeall(media); 2616 2617 m = IFM_ETHER | IFM_FDX; 2618 data = (pi->port_type << 8) | pi->mod_type; 2619 2620 switch(pi->port_type) { 2621 case FW_PORT_TYPE_BT_XFI: 2622 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2623 break; 2624 2625 case FW_PORT_TYPE_BT_XAUI: 2626 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2627 /* fall through */ 2628 2629 case FW_PORT_TYPE_BT_SGMII: 2630 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2631 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2632 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2633 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2634 break; 2635 2636 case FW_PORT_TYPE_CX4: 2637 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2638 ifmedia_set(media, m | IFM_10G_CX4); 2639 break; 2640 2641 case FW_PORT_TYPE_QSFP_10G: 2642 case FW_PORT_TYPE_SFP: 2643 case FW_PORT_TYPE_FIBER_XFI: 2644 case FW_PORT_TYPE_FIBER_XAUI: 2645 switch (pi->mod_type) { 2646 2647 case FW_PORT_MOD_TYPE_LR: 2648 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2649 ifmedia_set(media, m | IFM_10G_LR); 2650 break; 2651 2652 case FW_PORT_MOD_TYPE_SR: 2653 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2654 ifmedia_set(media, m | IFM_10G_SR); 2655 break; 2656 2657 case FW_PORT_MOD_TYPE_LRM: 2658 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2659 ifmedia_set(media, m | IFM_10G_LRM); 2660 break; 2661 2662 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2663 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2664 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2665 ifmedia_set(media, m | IFM_10G_TWINAX); 2666 break; 2667 2668 case FW_PORT_MOD_TYPE_NONE: 2669 m &= ~IFM_FDX; 2670 ifmedia_add(media, m | IFM_NONE, data, NULL); 2671 ifmedia_set(media, m | IFM_NONE); 2672 break; 2673 2674 case FW_PORT_MOD_TYPE_NA: 2675 case FW_PORT_MOD_TYPE_ER: 2676 default: 2677 device_printf(pi->dev, 2678 "unknown port_type (%d), mod_type (%d)\n", 2679 pi->port_type, pi->mod_type); 2680 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2681 ifmedia_set(media, m | IFM_UNKNOWN); 2682 break; 2683 } 2684 break; 2685 2686 case FW_PORT_TYPE_QSFP: 2687 switch (pi->mod_type) { 2688 2689 case FW_PORT_MOD_TYPE_LR: 2690 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2691 ifmedia_set(media, m | IFM_40G_LR4); 2692 break; 2693 2694 case FW_PORT_MOD_TYPE_SR: 2695 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2696 ifmedia_set(media, m | IFM_40G_SR4); 2697 break; 2698 2699 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2700 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2701 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2702 ifmedia_set(media, m | IFM_40G_CR4); 2703 break; 2704 2705 case FW_PORT_MOD_TYPE_NONE: 2706 m &= ~IFM_FDX; 2707 ifmedia_add(media, m | IFM_NONE, data, NULL); 2708 ifmedia_set(media, m | IFM_NONE); 2709 break; 2710 2711 default: 2712 device_printf(pi->dev, 2713 "unknown port_type (%d), mod_type (%d)\n", 2714 pi->port_type, pi->mod_type); 2715 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2716 ifmedia_set(media, m | IFM_UNKNOWN); 2717 break; 2718 } 2719 break; 2720 2721 default: 2722 device_printf(pi->dev, 2723 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2724 pi->mod_type); 2725 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2726 ifmedia_set(media, m | IFM_UNKNOWN); 2727 break; 2728 } 2729 2730 PORT_UNLOCK(pi); 2731} 2732 2733#define FW_MAC_EXACT_CHUNK 7 2734 2735/* 2736 * Program the port's XGMAC based on parameters in ifnet. The caller also 2737 * indicates which parameters should be programmed (the rest are left alone). 2738 */ 2739static int 2740update_mac_settings(struct port_info *pi, int flags) 2741{ 2742 int rc; 2743 struct ifnet *ifp = pi->ifp; 2744 struct adapter *sc = pi->adapter; 2745 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2746 2747 ASSERT_SYNCHRONIZED_OP(sc); 2748 KASSERT(flags, ("%s: not told what to update.", __func__)); 2749 2750 if (flags & XGMAC_MTU) 2751 mtu = ifp->if_mtu; 2752 2753 if (flags & XGMAC_PROMISC) 2754 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2755 2756 if (flags & XGMAC_ALLMULTI) 2757 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2758 2759 if (flags & XGMAC_VLANEX) 2760 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2761 2762 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2763 vlanex, false); 2764 if (rc) { 2765 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2766 return (rc); 2767 } 2768 2769 if (flags & XGMAC_UCADDR) { 2770 uint8_t ucaddr[ETHER_ADDR_LEN]; 2771 2772 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2773 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2774 ucaddr, true, true); 2775 if (rc < 0) { 2776 rc = -rc; 2777 if_printf(ifp, "change_mac failed: %d\n", rc); 2778 return (rc); 2779 } else { 2780 pi->xact_addr_filt = rc; 2781 rc = 0; 2782 } 2783 } 2784 2785 if (flags & XGMAC_MCADDRS) { 2786 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2787 int del = 1; 2788 uint64_t hash = 0; 2789 struct ifmultiaddr *ifma; 2790 int i = 0, j; 2791 2792 if_maddr_rlock(ifp); 2793 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2794 if (ifma->ifma_addr->sa_family != AF_LINK) 2795 continue; 2796 mcaddr[i++] = 2797 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2798 2799 if (i == FW_MAC_EXACT_CHUNK) { 2800 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2801 del, i, mcaddr, NULL, &hash, 0); 2802 if (rc < 0) { 2803 rc = -rc; 2804 for (j = 0; j < i; j++) { 2805 if_printf(ifp, 2806 "failed to add mc address" 2807 " %02x:%02x:%02x:" 2808 "%02x:%02x:%02x rc=%d\n", 2809 mcaddr[j][0], mcaddr[j][1], 2810 mcaddr[j][2], mcaddr[j][3], 2811 mcaddr[j][4], mcaddr[j][5], 2812 rc); 2813 } 2814 goto mcfail; 2815 } 2816 del = 0; 2817 i = 0; 2818 } 2819 } 2820 if (i > 0) { 2821 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2822 del, i, mcaddr, NULL, &hash, 0); 2823 if (rc < 0) { 2824 rc = -rc; 2825 for (j = 0; j < i; j++) { 2826 if_printf(ifp, 2827 "failed to add mc address" 2828 " %02x:%02x:%02x:" 2829 "%02x:%02x:%02x rc=%d\n", 2830 mcaddr[j][0], mcaddr[j][1], 2831 mcaddr[j][2], mcaddr[j][3], 2832 mcaddr[j][4], mcaddr[j][5], 2833 rc); 2834 } 2835 goto mcfail; 2836 } 2837 } 2838 2839 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2840 if (rc != 0) 2841 if_printf(ifp, "failed to set mc address hash: %d", rc); 2842mcfail: 2843 if_maddr_runlock(ifp); 2844 } 2845 2846 return (rc); 2847} 2848 2849int 2850begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 2851 char *wmesg) 2852{ 2853 int rc, pri; 2854 2855#ifdef WITNESS 2856 /* the caller thinks it's ok to sleep, but is it really? */ 2857 if (flags & SLEEP_OK) 2858 pause("t4slptst", 1); 2859#endif 2860 2861 if (INTR_OK) 2862 pri = PCATCH; 2863 else 2864 pri = 0; 2865 2866 ADAPTER_LOCK(sc); 2867 for (;;) { 2868 2869 if (pi && IS_DOOMED(pi)) { 2870 rc = ENXIO; 2871 goto done; 2872 } 2873 2874 if (!IS_BUSY(sc)) { 2875 rc = 0; 2876 break; 2877 } 2878 2879 if (!(flags & SLEEP_OK)) { 2880 rc = EBUSY; 2881 goto done; 2882 } 2883 2884 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 2885 rc = EINTR; 2886 goto done; 2887 } 2888 } 2889 2890 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2891 SET_BUSY(sc); 2892#ifdef INVARIANTS 2893 sc->last_op = wmesg; 2894 sc->last_op_thr = curthread; 2895#endif 2896 2897done: 2898 if (!(flags & HOLD_LOCK) || rc) 2899 ADAPTER_UNLOCK(sc); 2900 2901 return (rc); 2902} 2903 2904void 2905end_synchronized_op(struct adapter *sc, int flags) 2906{ 2907 2908 if (flags & LOCK_HELD) 2909 ADAPTER_LOCK_ASSERT_OWNED(sc); 2910 else 2911 ADAPTER_LOCK(sc); 2912 2913 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2914 CLR_BUSY(sc); 2915 wakeup(&sc->flags); 2916 ADAPTER_UNLOCK(sc); 2917} 2918 2919static int 2920cxgbe_init_synchronized(struct port_info *pi) 2921{ 2922 struct adapter *sc = pi->adapter; 2923 struct ifnet *ifp = pi->ifp; 2924 int rc = 0; 2925 2926 ASSERT_SYNCHRONIZED_OP(sc); 2927 2928 if (isset(&sc->open_device_map, pi->port_id)) { 2929 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2930 ("mismatch between open_device_map and if_drv_flags")); 2931 return (0); /* already running */ 2932 } 2933 2934 if (!(sc->flags & FULL_INIT_DONE) && 2935 ((rc = adapter_full_init(sc)) != 0)) 2936 return (rc); /* error message displayed already */ 2937 2938 if (!(pi->flags & PORT_INIT_DONE) && 2939 ((rc = port_full_init(pi)) != 0)) 2940 return (rc); /* error message displayed already */ 2941 2942 rc = update_mac_settings(pi, XGMAC_ALL); 2943 if (rc) 2944 goto done; /* error message displayed already */ 2945 2946 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2947 if (rc != 0) { 2948 if_printf(ifp, "start_link failed: %d\n", rc); 2949 goto done; 2950 } 2951 2952 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2953 if (rc != 0) { 2954 if_printf(ifp, "enable_vi failed: %d\n", rc); 2955 goto done; 2956 } 2957 2958 /* all ok */ 2959 setbit(&sc->open_device_map, pi->port_id); 2960 PORT_LOCK(pi); 2961 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2962 PORT_UNLOCK(pi); 2963 2964 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2965done: 2966 if (rc != 0) 2967 cxgbe_uninit_synchronized(pi); 2968 2969 return (rc); 2970} 2971 2972/* 2973 * Idempotent. 2974 */ 2975static int 2976cxgbe_uninit_synchronized(struct port_info *pi) 2977{ 2978 struct adapter *sc = pi->adapter; 2979 struct ifnet *ifp = pi->ifp; 2980 int rc; 2981 2982 ASSERT_SYNCHRONIZED_OP(sc); 2983 2984 /* 2985 * Disable the VI so that all its data in either direction is discarded 2986 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2987 * tick) intact as the TP can deliver negative advice or data that it's 2988 * holding in its RAM (for an offloaded connection) even after the VI is 2989 * disabled. 2990 */ 2991 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2992 if (rc) { 2993 if_printf(ifp, "disable_vi failed: %d\n", rc); 2994 return (rc); 2995 } 2996 2997 clrbit(&sc->open_device_map, pi->port_id); 2998 PORT_LOCK(pi); 2999 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3000 PORT_UNLOCK(pi); 3001 3002 pi->link_cfg.link_ok = 0; 3003 pi->link_cfg.speed = 0; 3004 pi->linkdnrc = -1; 3005 t4_os_link_changed(sc, pi->port_id, 0, -1); 3006 3007 return (0); 3008} 3009 3010/* 3011 * It is ok for this function to fail midway and return right away. t4_detach 3012 * will walk the entire sc->irq list and clean up whatever is valid. 3013 */ 3014static int 3015setup_intr_handlers(struct adapter *sc) 3016{ 3017 int rc, rid, p, q; 3018 char s[8]; 3019 struct irq *irq; 3020 struct port_info *pi; 3021 struct sge_rxq *rxq; 3022#ifdef TCP_OFFLOAD 3023 struct sge_ofld_rxq *ofld_rxq; 3024#endif 3025 3026 /* 3027 * Setup interrupts. 3028 */ 3029 irq = &sc->irq[0]; 3030 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3031 if (sc->intr_count == 1) { 3032 KASSERT(!(sc->flags & INTR_DIRECT), 3033 ("%s: single interrupt && INTR_DIRECT?", __func__)); 3034 3035 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 3036 if (rc != 0) 3037 return (rc); 3038 } else { 3039 /* Multiple interrupts. */ 3040 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3041 ("%s: too few intr.", __func__)); 3042 3043 /* The first one is always error intr */ 3044 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3045 if (rc != 0) 3046 return (rc); 3047 irq++; 3048 rid++; 3049 3050 /* The second one is always the firmware event queue */ 3051 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 3052 "evt"); 3053 if (rc != 0) 3054 return (rc); 3055 irq++; 3056 rid++; 3057 3058 /* 3059 * Note that if INTR_DIRECT is not set then either the NIC rx 3060 * queues or (exclusive or) the TOE rx queueus will be taking 3061 * direct interrupts. 3062 * 3063 * There is no need to check for is_offload(sc) as nofldrxq 3064 * will be 0 if offload is disabled. 3065 */ 3066 for_each_port(sc, p) { 3067 pi = sc->port[p]; 3068 3069#ifdef TCP_OFFLOAD 3070 /* 3071 * Skip over the NIC queues if they aren't taking direct 3072 * interrupts. 3073 */ 3074 if (!(sc->flags & INTR_DIRECT) && 3075 pi->nofldrxq > pi->nrxq) 3076 goto ofld_queues; 3077#endif 3078 rxq = &sc->sge.rxq[pi->first_rxq]; 3079 for (q = 0; q < pi->nrxq; q++, rxq++) { 3080 snprintf(s, sizeof(s), "%d.%d", p, q); 3081 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3082 s); 3083 if (rc != 0) 3084 return (rc); 3085 irq++; 3086 rid++; 3087 } 3088 3089#ifdef TCP_OFFLOAD 3090 /* 3091 * Skip over the offload queues if they aren't taking 3092 * direct interrupts. 3093 */ 3094 if (!(sc->flags & INTR_DIRECT)) 3095 continue; 3096ofld_queues: 3097 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 3098 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 3099 snprintf(s, sizeof(s), "%d,%d", p, q); 3100 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3101 ofld_rxq, s); 3102 if (rc != 0) 3103 return (rc); 3104 irq++; 3105 rid++; 3106 } 3107#endif 3108 } 3109 } 3110 3111 return (0); 3112} 3113 3114static int 3115adapter_full_init(struct adapter *sc) 3116{ 3117 int rc, i; 3118 3119 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3120 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3121 ("%s: FULL_INIT_DONE already", __func__)); 3122 3123 /* 3124 * queues that belong to the adapter (not any particular port). 3125 */ 3126 rc = t4_setup_adapter_queues(sc); 3127 if (rc != 0) 3128 goto done; 3129 3130 for (i = 0; i < nitems(sc->tq); i++) { 3131 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3132 taskqueue_thread_enqueue, &sc->tq[i]); 3133 if (sc->tq[i] == NULL) { 3134 device_printf(sc->dev, 3135 "failed to allocate task queue %d\n", i); 3136 rc = ENOMEM; 3137 goto done; 3138 } 3139 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3140 device_get_nameunit(sc->dev), i); 3141 } 3142 3143 t4_intr_enable(sc); 3144 sc->flags |= FULL_INIT_DONE; 3145done: 3146 if (rc != 0) 3147 adapter_full_uninit(sc); 3148 3149 return (rc); 3150} 3151 3152static int 3153adapter_full_uninit(struct adapter *sc) 3154{ 3155 int i; 3156 3157 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3158 3159 t4_teardown_adapter_queues(sc); 3160 3161 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3162 taskqueue_free(sc->tq[i]); 3163 sc->tq[i] = NULL; 3164 } 3165 3166 sc->flags &= ~FULL_INIT_DONE; 3167 3168 return (0); 3169} 3170 3171static int 3172port_full_init(struct port_info *pi) 3173{ 3174 struct adapter *sc = pi->adapter; 3175 struct ifnet *ifp = pi->ifp; 3176 uint16_t *rss; 3177 struct sge_rxq *rxq; 3178 int rc, i, j; 3179 3180 ASSERT_SYNCHRONIZED_OP(sc); 3181 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3182 ("%s: PORT_INIT_DONE already", __func__)); 3183 3184 sysctl_ctx_init(&pi->ctx); 3185 pi->flags |= PORT_SYSCTL_CTX; 3186 3187 /* 3188 * Allocate tx/rx/fl queues for this port. 3189 */ 3190 rc = t4_setup_port_queues(pi); 3191 if (rc != 0) 3192 goto done; /* error message displayed already */ 3193 3194 /* 3195 * Setup RSS for this port. Save a copy of the RSS table for later use. 3196 */ 3197 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3198 for (i = 0; i < pi->rss_size;) { 3199 for_each_rxq(pi, j, rxq) { 3200 rss[i++] = rxq->iq.abs_id; 3201 if (i == pi->rss_size) 3202 break; 3203 } 3204 } 3205 3206 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3207 pi->rss_size); 3208 if (rc != 0) { 3209 if_printf(ifp, "rss_config failed: %d\n", rc); 3210 goto done; 3211 } 3212 3213 pi->rss = rss; 3214 pi->flags |= PORT_INIT_DONE; 3215done: 3216 if (rc != 0) 3217 port_full_uninit(pi); 3218 3219 return (rc); 3220} 3221 3222/* 3223 * Idempotent. 3224 */ 3225static int 3226port_full_uninit(struct port_info *pi) 3227{ 3228 struct adapter *sc = pi->adapter; 3229 int i; 3230 struct sge_rxq *rxq; 3231 struct sge_txq *txq; 3232#ifdef TCP_OFFLOAD 3233 struct sge_ofld_rxq *ofld_rxq; 3234 struct sge_wrq *ofld_txq; 3235#endif 3236 3237 if (pi->flags & PORT_INIT_DONE) { 3238 3239 /* Need to quiesce queues. XXX: ctrl queues? */ 3240 3241 for_each_txq(pi, i, txq) { 3242 quiesce_eq(sc, &txq->eq); 3243 } 3244 3245#ifdef TCP_OFFLOAD 3246 for_each_ofld_txq(pi, i, ofld_txq) { 3247 quiesce_eq(sc, &ofld_txq->eq); 3248 } 3249#endif 3250 3251 for_each_rxq(pi, i, rxq) { 3252 quiesce_iq(sc, &rxq->iq); 3253 quiesce_fl(sc, &rxq->fl); 3254 } 3255 3256#ifdef TCP_OFFLOAD 3257 for_each_ofld_rxq(pi, i, ofld_rxq) { 3258 quiesce_iq(sc, &ofld_rxq->iq); 3259 quiesce_fl(sc, &ofld_rxq->fl); 3260 } 3261#endif 3262 free(pi->rss, M_CXGBE); 3263 } 3264 3265 t4_teardown_port_queues(pi); 3266 pi->flags &= ~PORT_INIT_DONE; 3267 3268 return (0); 3269} 3270 3271static void 3272quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3273{ 3274 EQ_LOCK(eq); 3275 eq->flags |= EQ_DOOMED; 3276 3277 /* 3278 * Wait for the response to a credit flush if one's 3279 * pending. 3280 */ 3281 while (eq->flags & EQ_CRFLUSHED) 3282 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3283 EQ_UNLOCK(eq); 3284 3285 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3286 pause("callout", 10); /* Still iffy */ 3287 3288 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3289} 3290 3291static void 3292quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3293{ 3294 (void) sc; /* unused */ 3295 3296 /* Synchronize with the interrupt handler */ 3297 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3298 pause("iqfree", 1); 3299} 3300 3301static void 3302quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3303{ 3304 mtx_lock(&sc->sfl_lock); 3305 FL_LOCK(fl); 3306 fl->flags |= FL_DOOMED; 3307 FL_UNLOCK(fl); 3308 mtx_unlock(&sc->sfl_lock); 3309 3310 callout_drain(&sc->sfl_callout); 3311 KASSERT((fl->flags & FL_STARVING) == 0, 3312 ("%s: still starving", __func__)); 3313} 3314 3315static int 3316t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3317 driver_intr_t *handler, void *arg, char *name) 3318{ 3319 int rc; 3320 3321 irq->rid = rid; 3322 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3323 RF_SHAREABLE | RF_ACTIVE); 3324 if (irq->res == NULL) { 3325 device_printf(sc->dev, 3326 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3327 return (ENOMEM); 3328 } 3329 3330 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3331 NULL, handler, arg, &irq->tag); 3332 if (rc != 0) { 3333 device_printf(sc->dev, 3334 "failed to setup interrupt for rid %d, name %s: %d\n", 3335 rid, name, rc); 3336 } else if (name) 3337 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3338 3339 return (rc); 3340} 3341 3342static int 3343t4_free_irq(struct adapter *sc, struct irq *irq) 3344{ 3345 if (irq->tag) 3346 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3347 if (irq->res) 3348 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3349 3350 bzero(irq, sizeof(*irq)); 3351 3352 return (0); 3353} 3354 3355static void 3356reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3357 unsigned int end) 3358{ 3359 uint32_t *p = (uint32_t *)(buf + start); 3360 3361 for ( ; start <= end; start += sizeof(uint32_t)) 3362 *p++ = t4_read_reg(sc, start); 3363} 3364 3365static void 3366t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3367{ 3368 int i, n; 3369 const unsigned int *reg_ranges; 3370 static const unsigned int t4_reg_ranges[] = { 3371 0x1008, 0x1108, 3372 0x1180, 0x11b4, 3373 0x11fc, 0x123c, 3374 0x1300, 0x173c, 3375 0x1800, 0x18fc, 3376 0x3000, 0x30d8, 3377 0x30e0, 0x5924, 3378 0x5960, 0x59d4, 3379 0x5a00, 0x5af8, 3380 0x6000, 0x6098, 3381 0x6100, 0x6150, 3382 0x6200, 0x6208, 3383 0x6240, 0x6248, 3384 0x6280, 0x6338, 3385 0x6370, 0x638c, 3386 0x6400, 0x643c, 3387 0x6500, 0x6524, 3388 0x6a00, 0x6a38, 3389 0x6a60, 0x6a78, 3390 0x6b00, 0x6b84, 3391 0x6bf0, 0x6c84, 3392 0x6cf0, 0x6d84, 3393 0x6df0, 0x6e84, 3394 0x6ef0, 0x6f84, 3395 0x6ff0, 0x7084, 3396 0x70f0, 0x7184, 3397 0x71f0, 0x7284, 3398 0x72f0, 0x7384, 3399 0x73f0, 0x7450, 3400 0x7500, 0x7530, 3401 0x7600, 0x761c, 3402 0x7680, 0x76cc, 3403 0x7700, 0x7798, 3404 0x77c0, 0x77fc, 3405 0x7900, 0x79fc, 3406 0x7b00, 0x7c38, 3407 0x7d00, 0x7efc, 3408 0x8dc0, 0x8e1c, 3409 0x8e30, 0x8e78, 3410 0x8ea0, 0x8f6c, 3411 0x8fc0, 0x9074, 3412 0x90fc, 0x90fc, 3413 0x9400, 0x9458, 3414 0x9600, 0x96bc, 3415 0x9800, 0x9808, 3416 0x9820, 0x983c, 3417 0x9850, 0x9864, 3418 0x9c00, 0x9c6c, 3419 0x9c80, 0x9cec, 3420 0x9d00, 0x9d6c, 3421 0x9d80, 0x9dec, 3422 0x9e00, 0x9e6c, 3423 0x9e80, 0x9eec, 3424 0x9f00, 0x9f6c, 3425 0x9f80, 0x9fec, 3426 0xd004, 0xd03c, 3427 0xdfc0, 0xdfe0, 3428 0xe000, 0xea7c, 3429 0xf000, 0x11110, 3430 0x11118, 0x11190, 3431 0x19040, 0x1906c, 3432 0x19078, 0x19080, 3433 0x1908c, 0x19124, 3434 0x19150, 0x191b0, 3435 0x191d0, 0x191e8, 3436 0x19238, 0x1924c, 3437 0x193f8, 0x19474, 3438 0x19490, 0x194f8, 3439 0x19800, 0x19f30, 3440 0x1a000, 0x1a06c, 3441 0x1a0b0, 0x1a120, 3442 0x1a128, 0x1a138, 3443 0x1a190, 0x1a1c4, 3444 0x1a1fc, 0x1a1fc, 3445 0x1e040, 0x1e04c, 3446 0x1e284, 0x1e28c, 3447 0x1e2c0, 0x1e2c0, 3448 0x1e2e0, 0x1e2e0, 3449 0x1e300, 0x1e384, 3450 0x1e3c0, 0x1e3c8, 3451 0x1e440, 0x1e44c, 3452 0x1e684, 0x1e68c, 3453 0x1e6c0, 0x1e6c0, 3454 0x1e6e0, 0x1e6e0, 3455 0x1e700, 0x1e784, 3456 0x1e7c0, 0x1e7c8, 3457 0x1e840, 0x1e84c, 3458 0x1ea84, 0x1ea8c, 3459 0x1eac0, 0x1eac0, 3460 0x1eae0, 0x1eae0, 3461 0x1eb00, 0x1eb84, 3462 0x1ebc0, 0x1ebc8, 3463 0x1ec40, 0x1ec4c, 3464 0x1ee84, 0x1ee8c, 3465 0x1eec0, 0x1eec0, 3466 0x1eee0, 0x1eee0, 3467 0x1ef00, 0x1ef84, 3468 0x1efc0, 0x1efc8, 3469 0x1f040, 0x1f04c, 3470 0x1f284, 0x1f28c, 3471 0x1f2c0, 0x1f2c0, 3472 0x1f2e0, 0x1f2e0, 3473 0x1f300, 0x1f384, 3474 0x1f3c0, 0x1f3c8, 3475 0x1f440, 0x1f44c, 3476 0x1f684, 0x1f68c, 3477 0x1f6c0, 0x1f6c0, 3478 0x1f6e0, 0x1f6e0, 3479 0x1f700, 0x1f784, 3480 0x1f7c0, 0x1f7c8, 3481 0x1f840, 0x1f84c, 3482 0x1fa84, 0x1fa8c, 3483 0x1fac0, 0x1fac0, 3484 0x1fae0, 0x1fae0, 3485 0x1fb00, 0x1fb84, 3486 0x1fbc0, 0x1fbc8, 3487 0x1fc40, 0x1fc4c, 3488 0x1fe84, 0x1fe8c, 3489 0x1fec0, 0x1fec0, 3490 0x1fee0, 0x1fee0, 3491 0x1ff00, 0x1ff84, 3492 0x1ffc0, 0x1ffc8, 3493 0x20000, 0x2002c, 3494 0x20100, 0x2013c, 3495 0x20190, 0x201c8, 3496 0x20200, 0x20318, 3497 0x20400, 0x20528, 3498 0x20540, 0x20614, 3499 0x21000, 0x21040, 3500 0x2104c, 0x21060, 3501 0x210c0, 0x210ec, 3502 0x21200, 0x21268, 3503 0x21270, 0x21284, 3504 0x212fc, 0x21388, 3505 0x21400, 0x21404, 3506 0x21500, 0x21518, 3507 0x2152c, 0x2153c, 3508 0x21550, 0x21554, 3509 0x21600, 0x21600, 3510 0x21608, 0x21628, 3511 0x21630, 0x2163c, 3512 0x21700, 0x2171c, 3513 0x21780, 0x2178c, 3514 0x21800, 0x21c38, 3515 0x21c80, 0x21d7c, 3516 0x21e00, 0x21e04, 3517 0x22000, 0x2202c, 3518 0x22100, 0x2213c, 3519 0x22190, 0x221c8, 3520 0x22200, 0x22318, 3521 0x22400, 0x22528, 3522 0x22540, 0x22614, 3523 0x23000, 0x23040, 3524 0x2304c, 0x23060, 3525 0x230c0, 0x230ec, 3526 0x23200, 0x23268, 3527 0x23270, 0x23284, 3528 0x232fc, 0x23388, 3529 0x23400, 0x23404, 3530 0x23500, 0x23518, 3531 0x2352c, 0x2353c, 3532 0x23550, 0x23554, 3533 0x23600, 0x23600, 3534 0x23608, 0x23628, 3535 0x23630, 0x2363c, 3536 0x23700, 0x2371c, 3537 0x23780, 0x2378c, 3538 0x23800, 0x23c38, 3539 0x23c80, 0x23d7c, 3540 0x23e00, 0x23e04, 3541 0x24000, 0x2402c, 3542 0x24100, 0x2413c, 3543 0x24190, 0x241c8, 3544 0x24200, 0x24318, 3545 0x24400, 0x24528, 3546 0x24540, 0x24614, 3547 0x25000, 0x25040, 3548 0x2504c, 0x25060, 3549 0x250c0, 0x250ec, 3550 0x25200, 0x25268, 3551 0x25270, 0x25284, 3552 0x252fc, 0x25388, 3553 0x25400, 0x25404, 3554 0x25500, 0x25518, 3555 0x2552c, 0x2553c, 3556 0x25550, 0x25554, 3557 0x25600, 0x25600, 3558 0x25608, 0x25628, 3559 0x25630, 0x2563c, 3560 0x25700, 0x2571c, 3561 0x25780, 0x2578c, 3562 0x25800, 0x25c38, 3563 0x25c80, 0x25d7c, 3564 0x25e00, 0x25e04, 3565 0x26000, 0x2602c, 3566 0x26100, 0x2613c, 3567 0x26190, 0x261c8, 3568 0x26200, 0x26318, 3569 0x26400, 0x26528, 3570 0x26540, 0x26614, 3571 0x27000, 0x27040, 3572 0x2704c, 0x27060, 3573 0x270c0, 0x270ec, 3574 0x27200, 0x27268, 3575 0x27270, 0x27284, 3576 0x272fc, 0x27388, 3577 0x27400, 0x27404, 3578 0x27500, 0x27518, 3579 0x2752c, 0x2753c, 3580 0x27550, 0x27554, 3581 0x27600, 0x27600, 3582 0x27608, 0x27628, 3583 0x27630, 0x2763c, 3584 0x27700, 0x2771c, 3585 0x27780, 0x2778c, 3586 0x27800, 0x27c38, 3587 0x27c80, 0x27d7c, 3588 0x27e00, 0x27e04 3589 }; 3590 static const unsigned int t5_reg_ranges[] = { 3591 0x1008, 0x1148, 3592 0x1180, 0x11b4, 3593 0x11fc, 0x123c, 3594 0x1280, 0x173c, 3595 0x1800, 0x18fc, 3596 0x3000, 0x3028, 3597 0x3060, 0x30d8, 3598 0x30e0, 0x30fc, 3599 0x3140, 0x357c, 3600 0x35a8, 0x35cc, 3601 0x35ec, 0x35ec, 3602 0x3600, 0x5624, 3603 0x56cc, 0x575c, 3604 0x580c, 0x5814, 3605 0x5890, 0x58bc, 3606 0x5940, 0x59dc, 3607 0x59fc, 0x5a18, 3608 0x5a60, 0x5a9c, 3609 0x5b94, 0x5bfc, 3610 0x6000, 0x6040, 3611 0x6058, 0x614c, 3612 0x7700, 0x7798, 3613 0x77c0, 0x78fc, 3614 0x7b00, 0x7c54, 3615 0x7d00, 0x7efc, 3616 0x8dc0, 0x8de0, 3617 0x8df8, 0x8e84, 3618 0x8ea0, 0x8f84, 3619 0x8fc0, 0x90f8, 3620 0x9400, 0x9470, 3621 0x9600, 0x96f4, 3622 0x9800, 0x9808, 3623 0x9820, 0x983c, 3624 0x9850, 0x9864, 3625 0x9c00, 0x9c6c, 3626 0x9c80, 0x9cec, 3627 0x9d00, 0x9d6c, 3628 0x9d80, 0x9dec, 3629 0x9e00, 0x9e6c, 3630 0x9e80, 0x9eec, 3631 0x9f00, 0x9f6c, 3632 0x9f80, 0xa020, 3633 0xd004, 0xd03c, 3634 0xdfc0, 0xdfe0, 3635 0xe000, 0x11088, 3636 0x1109c, 0x11110, 3637 0x11118, 0x1117c, 3638 0x11190, 0x11204, 3639 0x19040, 0x1906c, 3640 0x19078, 0x19080, 3641 0x1908c, 0x19124, 3642 0x19150, 0x191b0, 3643 0x191d0, 0x191e8, 3644 0x19238, 0x19290, 3645 0x193f8, 0x19474, 3646 0x19490, 0x194cc, 3647 0x194f0, 0x194f8, 3648 0x19c00, 0x19c60, 3649 0x19c94, 0x19e10, 3650 0x19e50, 0x19f34, 3651 0x19f40, 0x19f50, 3652 0x19f90, 0x19fe4, 3653 0x1a000, 0x1a06c, 3654 0x1a0b0, 0x1a120, 3655 0x1a128, 0x1a138, 3656 0x1a190, 0x1a1c4, 3657 0x1a1fc, 0x1a1fc, 3658 0x1e008, 0x1e00c, 3659 0x1e040, 0x1e04c, 3660 0x1e284, 0x1e290, 3661 0x1e2c0, 0x1e2c0, 3662 0x1e2e0, 0x1e2e0, 3663 0x1e300, 0x1e384, 3664 0x1e3c0, 0x1e3c8, 3665 0x1e408, 0x1e40c, 3666 0x1e440, 0x1e44c, 3667 0x1e684, 0x1e690, 3668 0x1e6c0, 0x1e6c0, 3669 0x1e6e0, 0x1e6e0, 3670 0x1e700, 0x1e784, 3671 0x1e7c0, 0x1e7c8, 3672 0x1e808, 0x1e80c, 3673 0x1e840, 0x1e84c, 3674 0x1ea84, 0x1ea90, 3675 0x1eac0, 0x1eac0, 3676 0x1eae0, 0x1eae0, 3677 0x1eb00, 0x1eb84, 3678 0x1ebc0, 0x1ebc8, 3679 0x1ec08, 0x1ec0c, 3680 0x1ec40, 0x1ec4c, 3681 0x1ee84, 0x1ee90, 3682 0x1eec0, 0x1eec0, 3683 0x1eee0, 0x1eee0, 3684 0x1ef00, 0x1ef84, 3685 0x1efc0, 0x1efc8, 3686 0x1f008, 0x1f00c, 3687 0x1f040, 0x1f04c, 3688 0x1f284, 0x1f290, 3689 0x1f2c0, 0x1f2c0, 3690 0x1f2e0, 0x1f2e0, 3691 0x1f300, 0x1f384, 3692 0x1f3c0, 0x1f3c8, 3693 0x1f408, 0x1f40c, 3694 0x1f440, 0x1f44c, 3695 0x1f684, 0x1f690, 3696 0x1f6c0, 0x1f6c0, 3697 0x1f6e0, 0x1f6e0, 3698 0x1f700, 0x1f784, 3699 0x1f7c0, 0x1f7c8, 3700 0x1f808, 0x1f80c, 3701 0x1f840, 0x1f84c, 3702 0x1fa84, 0x1fa90, 3703 0x1fac0, 0x1fac0, 3704 0x1fae0, 0x1fae0, 3705 0x1fb00, 0x1fb84, 3706 0x1fbc0, 0x1fbc8, 3707 0x1fc08, 0x1fc0c, 3708 0x1fc40, 0x1fc4c, 3709 0x1fe84, 0x1fe90, 3710 0x1fec0, 0x1fec0, 3711 0x1fee0, 0x1fee0, 3712 0x1ff00, 0x1ff84, 3713 0x1ffc0, 0x1ffc8, 3714 0x30000, 0x30030, 3715 0x30100, 0x30144, 3716 0x30190, 0x301d0, 3717 0x30200, 0x30318, 3718 0x30400, 0x3052c, 3719 0x30540, 0x3061c, 3720 0x30800, 0x30834, 3721 0x308c0, 0x30908, 3722 0x30910, 0x309ac, 3723 0x30a00, 0x30a2c, 3724 0x30a44, 0x30a50, 3725 0x30a74, 0x30c24, 3726 0x30d00, 0x30d00, 3727 0x30d08, 0x30d14, 3728 0x30d1c, 0x30d20, 3729 0x30d3c, 0x30d50, 3730 0x31200, 0x3120c, 3731 0x31220, 0x31220, 3732 0x31240, 0x31240, 3733 0x31600, 0x3160c, 3734 0x31a00, 0x31a1c, 3735 0x31e00, 0x31e20, 3736 0x31e38, 0x31e3c, 3737 0x31e80, 0x31e80, 3738 0x31e88, 0x31ea8, 3739 0x31eb0, 0x31eb4, 3740 0x31ec8, 0x31ed4, 3741 0x31fb8, 0x32004, 3742 0x32200, 0x32200, 3743 0x32208, 0x32240, 3744 0x32248, 0x32280, 3745 0x32288, 0x322c0, 3746 0x322c8, 0x322fc, 3747 0x32600, 0x32630, 3748 0x32a00, 0x32abc, 3749 0x32b00, 0x32b70, 3750 0x33000, 0x33048, 3751 0x33060, 0x3309c, 3752 0x330f0, 0x33148, 3753 0x33160, 0x3319c, 3754 0x331f0, 0x332e4, 3755 0x332f8, 0x333e4, 3756 0x333f8, 0x33448, 3757 0x33460, 0x3349c, 3758 0x334f0, 0x33548, 3759 0x33560, 0x3359c, 3760 0x335f0, 0x336e4, 3761 0x336f8, 0x337e4, 3762 0x337f8, 0x337fc, 3763 0x33814, 0x33814, 3764 0x3382c, 0x3382c, 3765 0x33880, 0x3388c, 3766 0x338e8, 0x338ec, 3767 0x33900, 0x33948, 3768 0x33960, 0x3399c, 3769 0x339f0, 0x33ae4, 3770 0x33af8, 0x33b10, 3771 0x33b28, 0x33b28, 3772 0x33b3c, 0x33b50, 3773 0x33bf0, 0x33c10, 3774 0x33c28, 0x33c28, 3775 0x33c3c, 0x33c50, 3776 0x33cf0, 0x33cfc, 3777 0x34000, 0x34030, 3778 0x34100, 0x34144, 3779 0x34190, 0x341d0, 3780 0x34200, 0x34318, 3781 0x34400, 0x3452c, 3782 0x34540, 0x3461c, 3783 0x34800, 0x34834, 3784 0x348c0, 0x34908, 3785 0x34910, 0x349ac, 3786 0x34a00, 0x34a2c, 3787 0x34a44, 0x34a50, 3788 0x34a74, 0x34c24, 3789 0x34d00, 0x34d00, 3790 0x34d08, 0x34d14, 3791 0x34d1c, 0x34d20, 3792 0x34d3c, 0x34d50, 3793 0x35200, 0x3520c, 3794 0x35220, 0x35220, 3795 0x35240, 0x35240, 3796 0x35600, 0x3560c, 3797 0x35a00, 0x35a1c, 3798 0x35e00, 0x35e20, 3799 0x35e38, 0x35e3c, 3800 0x35e80, 0x35e80, 3801 0x35e88, 0x35ea8, 3802 0x35eb0, 0x35eb4, 3803 0x35ec8, 0x35ed4, 3804 0x35fb8, 0x36004, 3805 0x36200, 0x36200, 3806 0x36208, 0x36240, 3807 0x36248, 0x36280, 3808 0x36288, 0x362c0, 3809 0x362c8, 0x362fc, 3810 0x36600, 0x36630, 3811 0x36a00, 0x36abc, 3812 0x36b00, 0x36b70, 3813 0x37000, 0x37048, 3814 0x37060, 0x3709c, 3815 0x370f0, 0x37148, 3816 0x37160, 0x3719c, 3817 0x371f0, 0x372e4, 3818 0x372f8, 0x373e4, 3819 0x373f8, 0x37448, 3820 0x37460, 0x3749c, 3821 0x374f0, 0x37548, 3822 0x37560, 0x3759c, 3823 0x375f0, 0x376e4, 3824 0x376f8, 0x377e4, 3825 0x377f8, 0x377fc, 3826 0x37814, 0x37814, 3827 0x3782c, 0x3782c, 3828 0x37880, 0x3788c, 3829 0x378e8, 0x378ec, 3830 0x37900, 0x37948, 3831 0x37960, 0x3799c, 3832 0x379f0, 0x37ae4, 3833 0x37af8, 0x37b10, 3834 0x37b28, 0x37b28, 3835 0x37b3c, 0x37b50, 3836 0x37bf0, 0x37c10, 3837 0x37c28, 0x37c28, 3838 0x37c3c, 0x37c50, 3839 0x37cf0, 0x37cfc, 3840 0x38000, 0x38030, 3841 0x38100, 0x38144, 3842 0x38190, 0x381d0, 3843 0x38200, 0x38318, 3844 0x38400, 0x3852c, 3845 0x38540, 0x3861c, 3846 0x38800, 0x38834, 3847 0x388c0, 0x38908, 3848 0x38910, 0x389ac, 3849 0x38a00, 0x38a2c, 3850 0x38a44, 0x38a50, 3851 0x38a74, 0x38c24, 3852 0x38d00, 0x38d00, 3853 0x38d08, 0x38d14, 3854 0x38d1c, 0x38d20, 3855 0x38d3c, 0x38d50, 3856 0x39200, 0x3920c, 3857 0x39220, 0x39220, 3858 0x39240, 0x39240, 3859 0x39600, 0x3960c, 3860 0x39a00, 0x39a1c, 3861 0x39e00, 0x39e20, 3862 0x39e38, 0x39e3c, 3863 0x39e80, 0x39e80, 3864 0x39e88, 0x39ea8, 3865 0x39eb0, 0x39eb4, 3866 0x39ec8, 0x39ed4, 3867 0x39fb8, 0x3a004, 3868 0x3a200, 0x3a200, 3869 0x3a208, 0x3a240, 3870 0x3a248, 0x3a280, 3871 0x3a288, 0x3a2c0, 3872 0x3a2c8, 0x3a2fc, 3873 0x3a600, 0x3a630, 3874 0x3aa00, 0x3aabc, 3875 0x3ab00, 0x3ab70, 3876 0x3b000, 0x3b048, 3877 0x3b060, 0x3b09c, 3878 0x3b0f0, 0x3b148, 3879 0x3b160, 0x3b19c, 3880 0x3b1f0, 0x3b2e4, 3881 0x3b2f8, 0x3b3e4, 3882 0x3b3f8, 0x3b448, 3883 0x3b460, 0x3b49c, 3884 0x3b4f0, 0x3b548, 3885 0x3b560, 0x3b59c, 3886 0x3b5f0, 0x3b6e4, 3887 0x3b6f8, 0x3b7e4, 3888 0x3b7f8, 0x3b7fc, 3889 0x3b814, 0x3b814, 3890 0x3b82c, 0x3b82c, 3891 0x3b880, 0x3b88c, 3892 0x3b8e8, 0x3b8ec, 3893 0x3b900, 0x3b948, 3894 0x3b960, 0x3b99c, 3895 0x3b9f0, 0x3bae4, 3896 0x3baf8, 0x3bb10, 3897 0x3bb28, 0x3bb28, 3898 0x3bb3c, 0x3bb50, 3899 0x3bbf0, 0x3bc10, 3900 0x3bc28, 0x3bc28, 3901 0x3bc3c, 0x3bc50, 3902 0x3bcf0, 0x3bcfc, 3903 0x3c000, 0x3c030, 3904 0x3c100, 0x3c144, 3905 0x3c190, 0x3c1d0, 3906 0x3c200, 0x3c318, 3907 0x3c400, 0x3c52c, 3908 0x3c540, 0x3c61c, 3909 0x3c800, 0x3c834, 3910 0x3c8c0, 0x3c908, 3911 0x3c910, 0x3c9ac, 3912 0x3ca00, 0x3ca2c, 3913 0x3ca44, 0x3ca50, 3914 0x3ca74, 0x3cc24, 3915 0x3cd00, 0x3cd00, 3916 0x3cd08, 0x3cd14, 3917 0x3cd1c, 0x3cd20, 3918 0x3cd3c, 0x3cd50, 3919 0x3d200, 0x3d20c, 3920 0x3d220, 0x3d220, 3921 0x3d240, 0x3d240, 3922 0x3d600, 0x3d60c, 3923 0x3da00, 0x3da1c, 3924 0x3de00, 0x3de20, 3925 0x3de38, 0x3de3c, 3926 0x3de80, 0x3de80, 3927 0x3de88, 0x3dea8, 3928 0x3deb0, 0x3deb4, 3929 0x3dec8, 0x3ded4, 3930 0x3dfb8, 0x3e004, 3931 0x3e200, 0x3e200, 3932 0x3e208, 0x3e240, 3933 0x3e248, 0x3e280, 3934 0x3e288, 0x3e2c0, 3935 0x3e2c8, 0x3e2fc, 3936 0x3e600, 0x3e630, 3937 0x3ea00, 0x3eabc, 3938 0x3eb00, 0x3eb70, 3939 0x3f000, 0x3f048, 3940 0x3f060, 0x3f09c, 3941 0x3f0f0, 0x3f148, 3942 0x3f160, 0x3f19c, 3943 0x3f1f0, 0x3f2e4, 3944 0x3f2f8, 0x3f3e4, 3945 0x3f3f8, 0x3f448, 3946 0x3f460, 0x3f49c, 3947 0x3f4f0, 0x3f548, 3948 0x3f560, 0x3f59c, 3949 0x3f5f0, 0x3f6e4, 3950 0x3f6f8, 0x3f7e4, 3951 0x3f7f8, 0x3f7fc, 3952 0x3f814, 0x3f814, 3953 0x3f82c, 0x3f82c, 3954 0x3f880, 0x3f88c, 3955 0x3f8e8, 0x3f8ec, 3956 0x3f900, 0x3f948, 3957 0x3f960, 0x3f99c, 3958 0x3f9f0, 0x3fae4, 3959 0x3faf8, 0x3fb10, 3960 0x3fb28, 0x3fb28, 3961 0x3fb3c, 0x3fb50, 3962 0x3fbf0, 0x3fc10, 3963 0x3fc28, 0x3fc28, 3964 0x3fc3c, 0x3fc50, 3965 0x3fcf0, 0x3fcfc, 3966 0x40000, 0x4000c, 3967 0x40040, 0x40068, 3968 0x4007c, 0x40144, 3969 0x40180, 0x4018c, 3970 0x40200, 0x40298, 3971 0x402ac, 0x4033c, 3972 0x403f8, 0x403fc, 3973 0x41304, 0x413c4, 3974 0x41400, 0x4141c, 3975 0x41480, 0x414d0, 3976 0x44000, 0x44078, 3977 0x440c0, 0x44278, 3978 0x442c0, 0x44478, 3979 0x444c0, 0x44678, 3980 0x446c0, 0x44878, 3981 0x448c0, 0x449fc, 3982 0x45000, 0x45068, 3983 0x45080, 0x45084, 3984 0x450a0, 0x450b0, 3985 0x45200, 0x45268, 3986 0x45280, 0x45284, 3987 0x452a0, 0x452b0, 3988 0x460c0, 0x460e4, 3989 0x47000, 0x4708c, 3990 0x47200, 0x47250, 3991 0x47400, 0x47420, 3992 0x47600, 0x47618, 3993 0x47800, 0x47814, 3994 0x48000, 0x4800c, 3995 0x48040, 0x48068, 3996 0x4807c, 0x48144, 3997 0x48180, 0x4818c, 3998 0x48200, 0x48298, 3999 0x482ac, 0x4833c, 4000 0x483f8, 0x483fc, 4001 0x49304, 0x493c4, 4002 0x49400, 0x4941c, 4003 0x49480, 0x494d0, 4004 0x4c000, 0x4c078, 4005 0x4c0c0, 0x4c278, 4006 0x4c2c0, 0x4c478, 4007 0x4c4c0, 0x4c678, 4008 0x4c6c0, 0x4c878, 4009 0x4c8c0, 0x4c9fc, 4010 0x4d000, 0x4d068, 4011 0x4d080, 0x4d084, 4012 0x4d0a0, 0x4d0b0, 4013 0x4d200, 0x4d268, 4014 0x4d280, 0x4d284, 4015 0x4d2a0, 0x4d2b0, 4016 0x4e0c0, 0x4e0e4, 4017 0x4f000, 0x4f08c, 4018 0x4f200, 0x4f250, 4019 0x4f400, 0x4f420, 4020 0x4f600, 0x4f618, 4021 0x4f800, 0x4f814, 4022 0x50000, 0x500cc, 4023 0x50400, 0x50400, 4024 0x50800, 0x508cc, 4025 0x50c00, 0x50c00, 4026 0x51000, 0x5101c, 4027 0x51300, 0x51308, 4028 }; 4029 4030 if (is_t4(sc)) { 4031 reg_ranges = &t4_reg_ranges[0]; 4032 n = nitems(t4_reg_ranges); 4033 } else { 4034 reg_ranges = &t5_reg_ranges[0]; 4035 n = nitems(t5_reg_ranges); 4036 } 4037 4038 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4039 for (i = 0; i < n; i += 2) 4040 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4041} 4042 4043static void 4044cxgbe_tick(void *arg) 4045{ 4046 struct port_info *pi = arg; 4047 struct adapter *sc = pi->adapter; 4048 struct ifnet *ifp = pi->ifp; 4049 struct sge_txq *txq; 4050 int i, drops; 4051 struct port_stats *s = &pi->stats; 4052 4053 PORT_LOCK(pi); 4054 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4055 PORT_UNLOCK(pi); 4056 return; /* without scheduling another callout */ 4057 } 4058 4059 t4_get_port_stats(sc, pi->tx_chan, s); 4060 4061 ifp->if_opackets = s->tx_frames - s->tx_pause; 4062 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4063 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4064 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4065 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4066 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4067 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4068 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4069 s->rx_trunc3; 4070 for (i = 0; i < 4; i++) { 4071 if (pi->rx_chan_map & (1 << i)) { 4072 uint32_t v; 4073 4074 /* 4075 * XXX: indirect reads from the same ADDR/DATA pair can 4076 * race with each other. 4077 */ 4078 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4079 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4080 ifp->if_iqdrops += v; 4081 } 4082 } 4083 4084 drops = s->tx_drop; 4085 for_each_txq(pi, i, txq) 4086 drops += txq->br->br_drops; 4087 ifp->if_snd.ifq_drops = drops; 4088 4089 ifp->if_oerrors = s->tx_error_frames; 4090 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4091 s->rx_fcs_err + s->rx_len_err; 4092 4093 callout_schedule(&pi->tick, hz); 4094 PORT_UNLOCK(pi); 4095} 4096 4097static void 4098cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4099{ 4100 struct ifnet *vlan; 4101 4102 if (arg != ifp || ifp->if_type != IFT_ETHER) 4103 return; 4104 4105 vlan = VLAN_DEVAT(ifp, vid); 4106 VLAN_SETCOOKIE(vlan, ifp); 4107} 4108 4109static int 4110cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4111{ 4112 4113#ifdef INVARIANTS 4114 panic("%s: opcode 0x%02x on iq %p with payload %p", 4115 __func__, rss->opcode, iq, m); 4116#else 4117 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4118 __func__, rss->opcode, iq, m); 4119 m_freem(m); 4120#endif 4121 return (EDOOFUS); 4122} 4123 4124int 4125t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4126{ 4127 uintptr_t *loc, new; 4128 4129 if (opcode >= nitems(sc->cpl_handler)) 4130 return (EINVAL); 4131 4132 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4133 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4134 atomic_store_rel_ptr(loc, new); 4135 4136 return (0); 4137} 4138 4139static int 4140an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4141{ 4142 4143#ifdef INVARIANTS 4144 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4145#else 4146 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4147 __func__, iq, ctrl); 4148#endif 4149 return (EDOOFUS); 4150} 4151 4152int 4153t4_register_an_handler(struct adapter *sc, an_handler_t h) 4154{ 4155 uintptr_t *loc, new; 4156 4157 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4158 loc = (uintptr_t *) &sc->an_handler; 4159 atomic_store_rel_ptr(loc, new); 4160 4161 return (0); 4162} 4163 4164static int 4165fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4166{ 4167 const struct cpl_fw6_msg *cpl = 4168 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4169 4170#ifdef INVARIANTS 4171 panic("%s: fw_msg type %d", __func__, cpl->type); 4172#else 4173 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4174#endif 4175 return (EDOOFUS); 4176} 4177 4178int 4179t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4180{ 4181 uintptr_t *loc, new; 4182 4183 if (type >= nitems(sc->fw_msg_handler)) 4184 return (EINVAL); 4185 4186 /* 4187 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4188 * handler dispatch table. Reject any attempt to install a handler for 4189 * this subtype. 4190 */ 4191 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4192 return (EINVAL); 4193 4194 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4195 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4196 atomic_store_rel_ptr(loc, new); 4197 4198 return (0); 4199} 4200 4201static int 4202t4_sysctls(struct adapter *sc) 4203{ 4204 struct sysctl_ctx_list *ctx; 4205 struct sysctl_oid *oid; 4206 struct sysctl_oid_list *children, *c0; 4207 static char *caps[] = { 4208 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4209 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4210 "\6HASHFILTER\7ETHOFLD", 4211 "\20\1TOE", /* caps[2] toecaps */ 4212 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4213 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4214 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4215 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4216 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4217 "\4PO_INITIAOR\5PO_TARGET" 4218 }; 4219 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4220 4221 ctx = device_get_sysctl_ctx(sc->dev); 4222 4223 /* 4224 * dev.t4nex.X. 4225 */ 4226 oid = device_get_sysctl_tree(sc->dev); 4227 c0 = children = SYSCTL_CHILDREN(oid); 4228 4229 sc->sc_do_rxcopy = 1; 4230 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4231 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4232 4233 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4234 sc->params.nports, "# of ports"); 4235 4236 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4237 NULL, chip_rev(sc), "chip hardware revision"); 4238 4239 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4240 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4241 4242 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4243 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4244 4245 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4246 sc->cfcsum, "config file checksum"); 4247 4248 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4249 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4250 sysctl_bitfield, "A", "available doorbells"); 4251 4252 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4253 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4254 sysctl_bitfield, "A", "available link capabilities"); 4255 4256 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4257 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4258 sysctl_bitfield, "A", "available NIC capabilities"); 4259 4260 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4261 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4262 sysctl_bitfield, "A", "available TCP offload capabilities"); 4263 4264 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4265 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4266 sysctl_bitfield, "A", "available RDMA capabilities"); 4267 4268 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4269 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4270 sysctl_bitfield, "A", "available iSCSI capabilities"); 4271 4272 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4273 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4274 sysctl_bitfield, "A", "available FCoE capabilities"); 4275 4276 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4277 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4278 4279 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4280 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4281 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4282 "interrupt holdoff timer values (us)"); 4283 4284 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4285 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4286 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4287 "interrupt holdoff packet counter values"); 4288 4289 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4290 NULL, sc->tids.nftids, "number of filters"); 4291 4292 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4293 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4294 "chip temperature (in Celsius)"); 4295 4296 t4_sge_sysctls(sc, ctx, children); 4297 4298#ifdef SBUF_DRAIN 4299 /* 4300 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4301 */ 4302 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4303 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4304 "logs and miscellaneous information"); 4305 children = SYSCTL_CHILDREN(oid); 4306 4307 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4308 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4309 sysctl_cctrl, "A", "congestion control"); 4310 4311 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4312 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4313 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4314 4315 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4316 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4317 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4318 4319 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4320 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4321 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4322 4323 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4324 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4325 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4326 4327 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4328 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4329 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4330 4331 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4332 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4333 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4334 4335 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4336 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4337 sysctl_cim_la, "A", "CIM logic analyzer"); 4338 4339 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4340 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4341 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4342 4343 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4344 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4345 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4346 4347 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4348 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4349 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4350 4351 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4352 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4353 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4354 4355 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4356 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4357 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4358 4359 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4360 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4361 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4362 4363 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4364 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4365 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4366 4367 if (is_t5(sc)) { 4368 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4369 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4370 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4371 4372 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4373 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4374 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4375 } 4376 4377 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4378 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4379 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4380 4381 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4382 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4383 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4384 4385 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4386 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4387 sysctl_cpl_stats, "A", "CPL statistics"); 4388 4389 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4390 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4391 sysctl_ddp_stats, "A", "DDP statistics"); 4392 4393 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4394 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4395 sysctl_devlog, "A", "firmware's device log"); 4396 4397 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4398 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4399 sysctl_fcoe_stats, "A", "FCoE statistics"); 4400 4401 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4402 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4403 sysctl_hw_sched, "A", "hardware scheduler "); 4404 4405 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4406 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4407 sysctl_l2t, "A", "hardware L2 table"); 4408 4409 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4410 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4411 sysctl_lb_stats, "A", "loopback statistics"); 4412 4413 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4414 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4415 sysctl_meminfo, "A", "memory regions"); 4416 4417 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4418 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4419 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4420 4421 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4422 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4423 sysctl_path_mtus, "A", "path MTUs"); 4424 4425 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4426 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4427 sysctl_pm_stats, "A", "PM statistics"); 4428 4429 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4430 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4431 sysctl_rdma_stats, "A", "RDMA statistics"); 4432 4433 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4434 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4435 sysctl_tcp_stats, "A", "TCP statistics"); 4436 4437 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4438 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4439 sysctl_tids, "A", "TID information"); 4440 4441 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4442 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4443 sysctl_tp_err_stats, "A", "TP error statistics"); 4444 4445 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4446 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4447 sysctl_tp_la, "A", "TP logic analyzer"); 4448 4449 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4450 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4451 sysctl_tx_rate, "A", "Tx rate"); 4452 4453 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4454 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4455 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4456 4457 if (is_t5(sc)) { 4458 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4459 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4460 sysctl_wcwr_stats, "A", "write combined work requests"); 4461 } 4462#endif 4463 4464#ifdef TCP_OFFLOAD 4465 if (is_offload(sc)) { 4466 /* 4467 * dev.t4nex.X.toe. 4468 */ 4469 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4470 NULL, "TOE parameters"); 4471 children = SYSCTL_CHILDREN(oid); 4472 4473 sc->tt.sndbuf = 256 * 1024; 4474 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4475 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4476 4477 sc->tt.ddp = 0; 4478 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4479 &sc->tt.ddp, 0, "DDP allowed"); 4480 4481 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4482 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4483 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4484 4485 sc->tt.ddp_thres = 4486 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4487 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4488 &sc->tt.ddp_thres, 0, "DDP threshold"); 4489 4490 sc->tt.rx_coalesce = 1; 4491 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4492 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4493 } 4494#endif 4495 4496 4497 return (0); 4498} 4499 4500static int 4501cxgbe_sysctls(struct port_info *pi) 4502{ 4503 struct sysctl_ctx_list *ctx; 4504 struct sysctl_oid *oid; 4505 struct sysctl_oid_list *children; 4506 struct adapter *sc = pi->adapter; 4507 4508 ctx = device_get_sysctl_ctx(pi->dev); 4509 4510 /* 4511 * dev.cxgbe.X. 4512 */ 4513 oid = device_get_sysctl_tree(pi->dev); 4514 children = SYSCTL_CHILDREN(oid); 4515 4516 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4517 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4518 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4519 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4520 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4521 "PHY temperature (in Celsius)"); 4522 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4523 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4524 "PHY firmware version"); 4525 } 4526 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4527 &pi->nrxq, 0, "# of rx queues"); 4528 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4529 &pi->ntxq, 0, "# of tx queues"); 4530 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4531 &pi->first_rxq, 0, "index of first rx queue"); 4532 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4533 &pi->first_txq, 0, "index of first tx queue"); 4534 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4535 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4536 "Reserve queue 0 for non-flowid packets"); 4537 4538#ifdef TCP_OFFLOAD 4539 if (is_offload(sc)) { 4540 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4541 &pi->nofldrxq, 0, 4542 "# of rx queues for offloaded TCP connections"); 4543 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4544 &pi->nofldtxq, 0, 4545 "# of tx queues for offloaded TCP connections"); 4546 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4547 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4548 "index of first TOE rx queue"); 4549 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4550 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4551 "index of first TOE tx queue"); 4552 } 4553#endif 4554 4555 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4556 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4557 "holdoff timer index"); 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4559 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4560 "holdoff packet counter index"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4563 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4564 "rx queue size"); 4565 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4566 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4567 "tx queue size"); 4568 4569 /* 4570 * dev.cxgbe.X.stats. 4571 */ 4572 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4573 NULL, "port statistics"); 4574 children = SYSCTL_CHILDREN(oid); 4575 4576#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4577 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4578 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4579 sysctl_handle_t4_reg64, "QU", desc) 4580 4581 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4582 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4583 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4584 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4585 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4586 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4587 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4588 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4589 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4590 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4591 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4592 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4593 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4594 "# of tx frames in this range", 4595 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4596 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4597 "# of tx frames in this range", 4598 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4599 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4600 "# of tx frames in this range", 4601 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4602 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4603 "# of tx frames in this range", 4604 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4605 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4606 "# of tx frames in this range", 4607 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4608 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4609 "# of tx frames in this range", 4610 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4611 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4612 "# of tx frames in this range", 4613 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4614 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4615 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4616 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4617 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4618 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4619 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4620 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4621 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4622 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4623 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4624 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4625 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4626 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4627 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4628 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4629 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4630 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4631 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4632 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4633 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4634 4635 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4636 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4637 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4638 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4639 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4640 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4641 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4642 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4643 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4644 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4645 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4646 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4647 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4648 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4649 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4650 "# of frames received with bad FCS", 4651 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4652 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4653 "# of frames received with length error", 4654 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4655 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4656 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4657 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4658 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4659 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4660 "# of rx frames in this range", 4661 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4662 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4663 "# of rx frames in this range", 4664 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4665 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4666 "# of rx frames in this range", 4667 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4668 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4669 "# of rx frames in this range", 4670 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4671 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4672 "# of rx frames in this range", 4673 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4674 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4675 "# of rx frames in this range", 4676 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4677 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4678 "# of rx frames in this range", 4679 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4680 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4681 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4682 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4683 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4684 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4685 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4686 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4687 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4688 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4689 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4690 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4691 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4692 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4693 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4694 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4695 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4696 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4697 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4698 4699#undef SYSCTL_ADD_T4_REG64 4700 4701#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4702 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4703 &pi->stats.name, desc) 4704 4705 /* We get these from port_stats and they may be stale by upto 1s */ 4706 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4707 "# drops due to buffer-group 0 overflows"); 4708 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4709 "# drops due to buffer-group 1 overflows"); 4710 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4711 "# drops due to buffer-group 2 overflows"); 4712 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4713 "# drops due to buffer-group 3 overflows"); 4714 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4715 "# of buffer-group 0 truncated packets"); 4716 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4717 "# of buffer-group 1 truncated packets"); 4718 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4719 "# of buffer-group 2 truncated packets"); 4720 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4721 "# of buffer-group 3 truncated packets"); 4722 4723#undef SYSCTL_ADD_T4_PORTSTAT 4724 4725 return (0); 4726} 4727 4728static int 4729sysctl_int_array(SYSCTL_HANDLER_ARGS) 4730{ 4731 int rc, *i; 4732 struct sbuf sb; 4733 4734 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4735 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4736 sbuf_printf(&sb, "%d ", *i); 4737 sbuf_trim(&sb); 4738 sbuf_finish(&sb); 4739 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4740 sbuf_delete(&sb); 4741 return (rc); 4742} 4743 4744static int 4745sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4746{ 4747 int rc; 4748 struct sbuf *sb; 4749 4750 rc = sysctl_wire_old_buffer(req, 0); 4751 if (rc != 0) 4752 return(rc); 4753 4754 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4755 if (sb == NULL) 4756 return (ENOMEM); 4757 4758 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4759 rc = sbuf_finish(sb); 4760 sbuf_delete(sb); 4761 4762 return (rc); 4763} 4764 4765static int 4766sysctl_btphy(SYSCTL_HANDLER_ARGS) 4767{ 4768 struct port_info *pi = arg1; 4769 int op = arg2; 4770 struct adapter *sc = pi->adapter; 4771 u_int v; 4772 int rc; 4773 4774 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 4775 if (rc) 4776 return (rc); 4777 /* XXX: magic numbers */ 4778 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 4779 &v); 4780 end_synchronized_op(sc, 0); 4781 if (rc) 4782 return (rc); 4783 if (op == 0) 4784 v /= 256; 4785 4786 rc = sysctl_handle_int(oidp, &v, 0, req); 4787 return (rc); 4788} 4789 4790static int 4791sysctl_noflowq(SYSCTL_HANDLER_ARGS) 4792{ 4793 struct port_info *pi = arg1; 4794 int rc, val; 4795 4796 val = pi->rsrv_noflowq; 4797 rc = sysctl_handle_int(oidp, &val, 0, req); 4798 if (rc != 0 || req->newptr == NULL) 4799 return (rc); 4800 4801 if ((val >= 1) && (pi->ntxq > 1)) 4802 pi->rsrv_noflowq = 1; 4803 else 4804 pi->rsrv_noflowq = 0; 4805 4806 return (rc); 4807} 4808 4809static int 4810sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4811{ 4812 struct port_info *pi = arg1; 4813 struct adapter *sc = pi->adapter; 4814 int idx, rc, i; 4815 struct sge_rxq *rxq; 4816#ifdef TCP_OFFLOAD 4817 struct sge_ofld_rxq *ofld_rxq; 4818#endif 4819 uint8_t v; 4820 4821 idx = pi->tmr_idx; 4822 4823 rc = sysctl_handle_int(oidp, &idx, 0, req); 4824 if (rc != 0 || req->newptr == NULL) 4825 return (rc); 4826 4827 if (idx < 0 || idx >= SGE_NTIMERS) 4828 return (EINVAL); 4829 4830 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4831 "t4tmr"); 4832 if (rc) 4833 return (rc); 4834 4835 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 4836 for_each_rxq(pi, i, rxq) { 4837#ifdef atomic_store_rel_8 4838 atomic_store_rel_8(&rxq->iq.intr_params, v); 4839#else 4840 rxq->iq.intr_params = v; 4841#endif 4842 } 4843#ifdef TCP_OFFLOAD 4844 for_each_ofld_rxq(pi, i, ofld_rxq) { 4845#ifdef atomic_store_rel_8 4846 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 4847#else 4848 ofld_rxq->iq.intr_params = v; 4849#endif 4850 } 4851#endif 4852 pi->tmr_idx = idx; 4853 4854 end_synchronized_op(sc, LOCK_HELD); 4855 return (0); 4856} 4857 4858static int 4859sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 4860{ 4861 struct port_info *pi = arg1; 4862 struct adapter *sc = pi->adapter; 4863 int idx, rc; 4864 4865 idx = pi->pktc_idx; 4866 4867 rc = sysctl_handle_int(oidp, &idx, 0, req); 4868 if (rc != 0 || req->newptr == NULL) 4869 return (rc); 4870 4871 if (idx < -1 || idx >= SGE_NCOUNTERS) 4872 return (EINVAL); 4873 4874 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4875 "t4pktc"); 4876 if (rc) 4877 return (rc); 4878 4879 if (pi->flags & PORT_INIT_DONE) 4880 rc = EBUSY; /* cannot be changed once the queues are created */ 4881 else 4882 pi->pktc_idx = idx; 4883 4884 end_synchronized_op(sc, LOCK_HELD); 4885 return (rc); 4886} 4887 4888static int 4889sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 4890{ 4891 struct port_info *pi = arg1; 4892 struct adapter *sc = pi->adapter; 4893 int qsize, rc; 4894 4895 qsize = pi->qsize_rxq; 4896 4897 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4898 if (rc != 0 || req->newptr == NULL) 4899 return (rc); 4900 4901 if (qsize < 128 || (qsize & 7)) 4902 return (EINVAL); 4903 4904 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4905 "t4rxqs"); 4906 if (rc) 4907 return (rc); 4908 4909 if (pi->flags & PORT_INIT_DONE) 4910 rc = EBUSY; /* cannot be changed once the queues are created */ 4911 else 4912 pi->qsize_rxq = qsize; 4913 4914 end_synchronized_op(sc, LOCK_HELD); 4915 return (rc); 4916} 4917 4918static int 4919sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 4920{ 4921 struct port_info *pi = arg1; 4922 struct adapter *sc = pi->adapter; 4923 int qsize, rc; 4924 4925 qsize = pi->qsize_txq; 4926 4927 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4928 if (rc != 0 || req->newptr == NULL) 4929 return (rc); 4930 4931 /* bufring size must be powerof2 */ 4932 if (qsize < 128 || !powerof2(qsize)) 4933 return (EINVAL); 4934 4935 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4936 "t4txqs"); 4937 if (rc) 4938 return (rc); 4939 4940 if (pi->flags & PORT_INIT_DONE) 4941 rc = EBUSY; /* cannot be changed once the queues are created */ 4942 else 4943 pi->qsize_txq = qsize; 4944 4945 end_synchronized_op(sc, LOCK_HELD); 4946 return (rc); 4947} 4948 4949static int 4950sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 4951{ 4952 struct adapter *sc = arg1; 4953 int reg = arg2; 4954 uint64_t val; 4955 4956 val = t4_read_reg64(sc, reg); 4957 4958 return (sysctl_handle_64(oidp, &val, 0, req)); 4959} 4960 4961static int 4962sysctl_temperature(SYSCTL_HANDLER_ARGS) 4963{ 4964 struct adapter *sc = arg1; 4965 int rc, t; 4966 uint32_t param, val; 4967 4968 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 4969 if (rc) 4970 return (rc); 4971 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 4972 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 4973 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 4974 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 4975 end_synchronized_op(sc, 0); 4976 if (rc) 4977 return (rc); 4978 4979 /* unknown is returned as 0 but we display -1 in that case */ 4980 t = val == 0 ? -1 : val; 4981 4982 rc = sysctl_handle_int(oidp, &t, 0, req); 4983 return (rc); 4984} 4985 4986#ifdef SBUF_DRAIN 4987static int 4988sysctl_cctrl(SYSCTL_HANDLER_ARGS) 4989{ 4990 struct adapter *sc = arg1; 4991 struct sbuf *sb; 4992 int rc, i; 4993 uint16_t incr[NMTUS][NCCTRL_WIN]; 4994 static const char *dec_fac[] = { 4995 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 4996 "0.9375" 4997 }; 4998 4999 rc = sysctl_wire_old_buffer(req, 0); 5000 if (rc != 0) 5001 return (rc); 5002 5003 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5004 if (sb == NULL) 5005 return (ENOMEM); 5006 5007 t4_read_cong_tbl(sc, incr); 5008 5009 for (i = 0; i < NCCTRL_WIN; ++i) { 5010 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5011 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5012 incr[5][i], incr[6][i], incr[7][i]); 5013 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5014 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5015 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5016 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5017 } 5018 5019 rc = sbuf_finish(sb); 5020 sbuf_delete(sb); 5021 5022 return (rc); 5023} 5024 5025static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5026 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5027 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5028 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5029}; 5030 5031static int 5032sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5033{ 5034 struct adapter *sc = arg1; 5035 struct sbuf *sb; 5036 int rc, i, n, qid = arg2; 5037 uint32_t *buf, *p; 5038 char *qtype; 5039 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5040 5041 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5042 ("%s: bad qid %d\n", __func__, qid)); 5043 5044 if (qid < CIM_NUM_IBQ) { 5045 /* inbound queue */ 5046 qtype = "IBQ"; 5047 n = 4 * CIM_IBQ_SIZE; 5048 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5049 rc = t4_read_cim_ibq(sc, qid, buf, n); 5050 } else { 5051 /* outbound queue */ 5052 qtype = "OBQ"; 5053 qid -= CIM_NUM_IBQ; 5054 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5055 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5056 rc = t4_read_cim_obq(sc, qid, buf, n); 5057 } 5058 5059 if (rc < 0) { 5060 rc = -rc; 5061 goto done; 5062 } 5063 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5064 5065 rc = sysctl_wire_old_buffer(req, 0); 5066 if (rc != 0) 5067 goto done; 5068 5069 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5070 if (sb == NULL) { 5071 rc = ENOMEM; 5072 goto done; 5073 } 5074 5075 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5076 for (i = 0, p = buf; i < n; i += 16, p += 4) 5077 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5078 p[2], p[3]); 5079 5080 rc = sbuf_finish(sb); 5081 sbuf_delete(sb); 5082done: 5083 free(buf, M_CXGBE); 5084 return (rc); 5085} 5086 5087static int 5088sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5089{ 5090 struct adapter *sc = arg1; 5091 u_int cfg; 5092 struct sbuf *sb; 5093 uint32_t *buf, *p; 5094 int rc; 5095 5096 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5097 if (rc != 0) 5098 return (rc); 5099 5100 rc = sysctl_wire_old_buffer(req, 0); 5101 if (rc != 0) 5102 return (rc); 5103 5104 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5105 if (sb == NULL) 5106 return (ENOMEM); 5107 5108 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5109 M_ZERO | M_WAITOK); 5110 5111 rc = -t4_cim_read_la(sc, buf, NULL); 5112 if (rc != 0) 5113 goto done; 5114 5115 sbuf_printf(sb, "Status Data PC%s", 5116 cfg & F_UPDBGLACAPTPCONLY ? "" : 5117 " LS0Stat LS0Addr LS0Data"); 5118 5119 KASSERT((sc->params.cim_la_size & 7) == 0, 5120 ("%s: p will walk off the end of buf", __func__)); 5121 5122 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5123 if (cfg & F_UPDBGLACAPTPCONLY) { 5124 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5125 p[6], p[7]); 5126 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5127 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5128 p[4] & 0xff, p[5] >> 8); 5129 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5130 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5131 p[1] & 0xf, p[2] >> 4); 5132 } else { 5133 sbuf_printf(sb, 5134 "\n %02x %x%07x %x%07x %08x %08x " 5135 "%08x%08x%08x%08x", 5136 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5137 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5138 p[6], p[7]); 5139 } 5140 } 5141 5142 rc = sbuf_finish(sb); 5143 sbuf_delete(sb); 5144done: 5145 free(buf, M_CXGBE); 5146 return (rc); 5147} 5148 5149static int 5150sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5151{ 5152 struct adapter *sc = arg1; 5153 u_int i; 5154 struct sbuf *sb; 5155 uint32_t *buf, *p; 5156 int rc; 5157 5158 rc = sysctl_wire_old_buffer(req, 0); 5159 if (rc != 0) 5160 return (rc); 5161 5162 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5163 if (sb == NULL) 5164 return (ENOMEM); 5165 5166 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5167 M_ZERO | M_WAITOK); 5168 5169 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5170 p = buf; 5171 5172 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5173 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5174 p[1], p[0]); 5175 } 5176 5177 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5178 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5179 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5180 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5181 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5182 (p[1] >> 2) | ((p[2] & 3) << 30), 5183 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5184 p[0] & 1); 5185 } 5186 5187 rc = sbuf_finish(sb); 5188 sbuf_delete(sb); 5189 free(buf, M_CXGBE); 5190 return (rc); 5191} 5192 5193static int 5194sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5195{ 5196 struct adapter *sc = arg1; 5197 u_int i; 5198 struct sbuf *sb; 5199 uint32_t *buf, *p; 5200 int rc; 5201 5202 rc = sysctl_wire_old_buffer(req, 0); 5203 if (rc != 0) 5204 return (rc); 5205 5206 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5207 if (sb == NULL) 5208 return (ENOMEM); 5209 5210 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5211 M_ZERO | M_WAITOK); 5212 5213 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5214 p = buf; 5215 5216 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5217 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5218 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5219 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5220 p[4], p[3], p[2], p[1], p[0]); 5221 } 5222 5223 sbuf_printf(sb, "\n\nCntl ID Data"); 5224 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5225 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5226 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5227 } 5228 5229 rc = sbuf_finish(sb); 5230 sbuf_delete(sb); 5231 free(buf, M_CXGBE); 5232 return (rc); 5233} 5234 5235static int 5236sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5237{ 5238 struct adapter *sc = arg1; 5239 struct sbuf *sb; 5240 int rc, i; 5241 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5242 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5243 uint16_t thres[CIM_NUM_IBQ]; 5244 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5245 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5246 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5247 5248 if (is_t4(sc)) { 5249 cim_num_obq = CIM_NUM_OBQ; 5250 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5251 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5252 } else { 5253 cim_num_obq = CIM_NUM_OBQ_T5; 5254 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5255 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5256 } 5257 nq = CIM_NUM_IBQ + cim_num_obq; 5258 5259 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5260 if (rc == 0) 5261 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5262 if (rc != 0) 5263 return (rc); 5264 5265 t4_read_cimq_cfg(sc, base, size, thres); 5266 5267 rc = sysctl_wire_old_buffer(req, 0); 5268 if (rc != 0) 5269 return (rc); 5270 5271 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5272 if (sb == NULL) 5273 return (ENOMEM); 5274 5275 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5276 5277 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5278 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5279 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5280 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5281 G_QUEREMFLITS(p[2]) * 16); 5282 for ( ; i < nq; i++, p += 4, wr += 2) 5283 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5284 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5285 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5286 G_QUEREMFLITS(p[2]) * 16); 5287 5288 rc = sbuf_finish(sb); 5289 sbuf_delete(sb); 5290 5291 return (rc); 5292} 5293 5294static int 5295sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5296{ 5297 struct adapter *sc = arg1; 5298 struct sbuf *sb; 5299 int rc; 5300 struct tp_cpl_stats stats; 5301 5302 rc = sysctl_wire_old_buffer(req, 0); 5303 if (rc != 0) 5304 return (rc); 5305 5306 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5307 if (sb == NULL) 5308 return (ENOMEM); 5309 5310 t4_tp_get_cpl_stats(sc, &stats); 5311 5312 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5313 "channel 3\n"); 5314 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5315 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5316 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5317 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5318 5319 rc = sbuf_finish(sb); 5320 sbuf_delete(sb); 5321 5322 return (rc); 5323} 5324 5325static int 5326sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5327{ 5328 struct adapter *sc = arg1; 5329 struct sbuf *sb; 5330 int rc; 5331 struct tp_usm_stats stats; 5332 5333 rc = sysctl_wire_old_buffer(req, 0); 5334 if (rc != 0) 5335 return(rc); 5336 5337 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5338 if (sb == NULL) 5339 return (ENOMEM); 5340 5341 t4_get_usm_stats(sc, &stats); 5342 5343 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5344 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5345 sbuf_printf(sb, "Drops: %u", stats.drops); 5346 5347 rc = sbuf_finish(sb); 5348 sbuf_delete(sb); 5349 5350 return (rc); 5351} 5352 5353const char *devlog_level_strings[] = { 5354 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5355 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5356 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5357 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5358 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5359 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5360}; 5361 5362const char *devlog_facility_strings[] = { 5363 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5364 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5365 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5366 [FW_DEVLOG_FACILITY_RES] = "RES", 5367 [FW_DEVLOG_FACILITY_HW] = "HW", 5368 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5369 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5370 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5371 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5372 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5373 [FW_DEVLOG_FACILITY_VI] = "VI", 5374 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5375 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5376 [FW_DEVLOG_FACILITY_TM] = "TM", 5377 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5378 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5379 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5380 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5381 [FW_DEVLOG_FACILITY_RI] = "RI", 5382 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5383 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5384 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5385 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5386}; 5387 5388static int 5389sysctl_devlog(SYSCTL_HANDLER_ARGS) 5390{ 5391 struct adapter *sc = arg1; 5392 struct devlog_params *dparams = &sc->params.devlog; 5393 struct fw_devlog_e *buf, *e; 5394 int i, j, rc, nentries, first = 0, m; 5395 struct sbuf *sb; 5396 uint64_t ftstamp = UINT64_MAX; 5397 5398 if (dparams->start == 0) { 5399 dparams->memtype = FW_MEMTYPE_EDC0; 5400 dparams->start = 0x84000; 5401 dparams->size = 32768; 5402 } 5403 5404 nentries = dparams->size / sizeof(struct fw_devlog_e); 5405 5406 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5407 if (buf == NULL) 5408 return (ENOMEM); 5409 5410 m = fwmtype_to_hwmtype(dparams->memtype); 5411 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5412 if (rc != 0) 5413 goto done; 5414 5415 for (i = 0; i < nentries; i++) { 5416 e = &buf[i]; 5417 5418 if (e->timestamp == 0) 5419 break; /* end */ 5420 5421 e->timestamp = be64toh(e->timestamp); 5422 e->seqno = be32toh(e->seqno); 5423 for (j = 0; j < 8; j++) 5424 e->params[j] = be32toh(e->params[j]); 5425 5426 if (e->timestamp < ftstamp) { 5427 ftstamp = e->timestamp; 5428 first = i; 5429 } 5430 } 5431 5432 if (buf[first].timestamp == 0) 5433 goto done; /* nothing in the log */ 5434 5435 rc = sysctl_wire_old_buffer(req, 0); 5436 if (rc != 0) 5437 goto done; 5438 5439 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5440 if (sb == NULL) { 5441 rc = ENOMEM; 5442 goto done; 5443 } 5444 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5445 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5446 5447 i = first; 5448 do { 5449 e = &buf[i]; 5450 if (e->timestamp == 0) 5451 break; /* end */ 5452 5453 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5454 e->seqno, e->timestamp, 5455 (e->level < nitems(devlog_level_strings) ? 5456 devlog_level_strings[e->level] : "UNKNOWN"), 5457 (e->facility < nitems(devlog_facility_strings) ? 5458 devlog_facility_strings[e->facility] : "UNKNOWN")); 5459 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5460 e->params[2], e->params[3], e->params[4], 5461 e->params[5], e->params[6], e->params[7]); 5462 5463 if (++i == nentries) 5464 i = 0; 5465 } while (i != first); 5466 5467 rc = sbuf_finish(sb); 5468 sbuf_delete(sb); 5469done: 5470 free(buf, M_CXGBE); 5471 return (rc); 5472} 5473 5474static int 5475sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5476{ 5477 struct adapter *sc = arg1; 5478 struct sbuf *sb; 5479 int rc; 5480 struct tp_fcoe_stats stats[4]; 5481 5482 rc = sysctl_wire_old_buffer(req, 0); 5483 if (rc != 0) 5484 return (rc); 5485 5486 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5487 if (sb == NULL) 5488 return (ENOMEM); 5489 5490 t4_get_fcoe_stats(sc, 0, &stats[0]); 5491 t4_get_fcoe_stats(sc, 1, &stats[1]); 5492 t4_get_fcoe_stats(sc, 2, &stats[2]); 5493 t4_get_fcoe_stats(sc, 3, &stats[3]); 5494 5495 sbuf_printf(sb, " channel 0 channel 1 " 5496 "channel 2 channel 3\n"); 5497 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5498 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5499 stats[3].octetsDDP); 5500 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5501 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5502 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5503 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5504 stats[3].framesDrop); 5505 5506 rc = sbuf_finish(sb); 5507 sbuf_delete(sb); 5508 5509 return (rc); 5510} 5511 5512static int 5513sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5514{ 5515 struct adapter *sc = arg1; 5516 struct sbuf *sb; 5517 int rc, i; 5518 unsigned int map, kbps, ipg, mode; 5519 unsigned int pace_tab[NTX_SCHED]; 5520 5521 rc = sysctl_wire_old_buffer(req, 0); 5522 if (rc != 0) 5523 return (rc); 5524 5525 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5526 if (sb == NULL) 5527 return (ENOMEM); 5528 5529 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5530 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5531 t4_read_pace_tbl(sc, pace_tab); 5532 5533 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5534 "Class IPG (0.1 ns) Flow IPG (us)"); 5535 5536 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5537 t4_get_tx_sched(sc, i, &kbps, &ipg); 5538 sbuf_printf(sb, "\n %u %-5s %u ", i, 5539 (mode & (1 << i)) ? "flow" : "class", map & 3); 5540 if (kbps) 5541 sbuf_printf(sb, "%9u ", kbps); 5542 else 5543 sbuf_printf(sb, " disabled "); 5544 5545 if (ipg) 5546 sbuf_printf(sb, "%13u ", ipg); 5547 else 5548 sbuf_printf(sb, " disabled "); 5549 5550 if (pace_tab[i]) 5551 sbuf_printf(sb, "%10u", pace_tab[i]); 5552 else 5553 sbuf_printf(sb, " disabled"); 5554 } 5555 5556 rc = sbuf_finish(sb); 5557 sbuf_delete(sb); 5558 5559 return (rc); 5560} 5561 5562static int 5563sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5564{ 5565 struct adapter *sc = arg1; 5566 struct sbuf *sb; 5567 int rc, i, j; 5568 uint64_t *p0, *p1; 5569 struct lb_port_stats s[2]; 5570 static const char *stat_name[] = { 5571 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5572 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5573 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5574 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5575 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5576 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5577 "BG2FramesTrunc:", "BG3FramesTrunc:" 5578 }; 5579 5580 rc = sysctl_wire_old_buffer(req, 0); 5581 if (rc != 0) 5582 return (rc); 5583 5584 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5585 if (sb == NULL) 5586 return (ENOMEM); 5587 5588 memset(s, 0, sizeof(s)); 5589 5590 for (i = 0; i < 4; i += 2) { 5591 t4_get_lb_stats(sc, i, &s[0]); 5592 t4_get_lb_stats(sc, i + 1, &s[1]); 5593 5594 p0 = &s[0].octets; 5595 p1 = &s[1].octets; 5596 sbuf_printf(sb, "%s Loopback %u" 5597 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5598 5599 for (j = 0; j < nitems(stat_name); j++) 5600 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5601 *p0++, *p1++); 5602 } 5603 5604 rc = sbuf_finish(sb); 5605 sbuf_delete(sb); 5606 5607 return (rc); 5608} 5609 5610static int 5611sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5612{ 5613 int rc = 0; 5614 struct port_info *pi = arg1; 5615 struct sbuf *sb; 5616 static const char *linkdnreasons[] = { 5617 "non-specific", "remote fault", "autoneg failed", "reserved3", 5618 "PHY overheated", "unknown", "rx los", "reserved7" 5619 }; 5620 5621 rc = sysctl_wire_old_buffer(req, 0); 5622 if (rc != 0) 5623 return(rc); 5624 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5625 if (sb == NULL) 5626 return (ENOMEM); 5627 5628 if (pi->linkdnrc < 0) 5629 sbuf_printf(sb, "n/a"); 5630 else if (pi->linkdnrc < nitems(linkdnreasons)) 5631 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5632 else 5633 sbuf_printf(sb, "%d", pi->linkdnrc); 5634 5635 rc = sbuf_finish(sb); 5636 sbuf_delete(sb); 5637 5638 return (rc); 5639} 5640 5641struct mem_desc { 5642 unsigned int base; 5643 unsigned int limit; 5644 unsigned int idx; 5645}; 5646 5647static int 5648mem_desc_cmp(const void *a, const void *b) 5649{ 5650 return ((const struct mem_desc *)a)->base - 5651 ((const struct mem_desc *)b)->base; 5652} 5653 5654static void 5655mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5656 unsigned int to) 5657{ 5658 unsigned int size; 5659 5660 size = to - from + 1; 5661 if (size == 0) 5662 return; 5663 5664 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5665 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5666} 5667 5668static int 5669sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5670{ 5671 struct adapter *sc = arg1; 5672 struct sbuf *sb; 5673 int rc, i, n; 5674 uint32_t lo, hi, used, alloc; 5675 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5676 static const char *region[] = { 5677 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5678 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5679 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5680 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5681 "RQUDP region:", "PBL region:", "TXPBL region:", 5682 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5683 "On-chip queues:" 5684 }; 5685 struct mem_desc avail[4]; 5686 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5687 struct mem_desc *md = mem; 5688 5689 rc = sysctl_wire_old_buffer(req, 0); 5690 if (rc != 0) 5691 return (rc); 5692 5693 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5694 if (sb == NULL) 5695 return (ENOMEM); 5696 5697 for (i = 0; i < nitems(mem); i++) { 5698 mem[i].limit = 0; 5699 mem[i].idx = i; 5700 } 5701 5702 /* Find and sort the populated memory ranges */ 5703 i = 0; 5704 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5705 if (lo & F_EDRAM0_ENABLE) { 5706 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5707 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5708 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5709 avail[i].idx = 0; 5710 i++; 5711 } 5712 if (lo & F_EDRAM1_ENABLE) { 5713 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5714 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5715 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5716 avail[i].idx = 1; 5717 i++; 5718 } 5719 if (lo & F_EXT_MEM_ENABLE) { 5720 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5721 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5722 avail[i].limit = avail[i].base + 5723 (G_EXT_MEM_SIZE(hi) << 20); 5724 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5725 i++; 5726 } 5727 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5728 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5729 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5730 avail[i].limit = avail[i].base + 5731 (G_EXT_MEM1_SIZE(hi) << 20); 5732 avail[i].idx = 4; 5733 i++; 5734 } 5735 if (!i) /* no memory available */ 5736 return 0; 5737 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5738 5739 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5740 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5741 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5742 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5743 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5744 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5745 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5746 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5747 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5748 5749 /* the next few have explicit upper bounds */ 5750 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5751 md->limit = md->base - 1 + 5752 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5753 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5754 md++; 5755 5756 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5757 md->limit = md->base - 1 + 5758 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5759 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5760 md++; 5761 5762 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5763 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5764 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5765 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5766 } else { 5767 md->base = 0; 5768 md->idx = nitems(region); /* hide it */ 5769 } 5770 md++; 5771 5772#define ulp_region(reg) \ 5773 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5774 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5775 5776 ulp_region(RX_ISCSI); 5777 ulp_region(RX_TDDP); 5778 ulp_region(TX_TPT); 5779 ulp_region(RX_STAG); 5780 ulp_region(RX_RQ); 5781 ulp_region(RX_RQUDP); 5782 ulp_region(RX_PBL); 5783 ulp_region(TX_PBL); 5784#undef ulp_region 5785 5786 md->base = 0; 5787 md->idx = nitems(region); 5788 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5789 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5790 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5791 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5792 } 5793 md++; 5794 5795 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5796 md->limit = md->base + sc->tids.ntids - 1; 5797 md++; 5798 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5799 md->limit = md->base + sc->tids.ntids - 1; 5800 md++; 5801 5802 md->base = sc->vres.ocq.start; 5803 if (sc->vres.ocq.size) 5804 md->limit = md->base + sc->vres.ocq.size - 1; 5805 else 5806 md->idx = nitems(region); /* hide it */ 5807 md++; 5808 5809 /* add any address-space holes, there can be up to 3 */ 5810 for (n = 0; n < i - 1; n++) 5811 if (avail[n].limit < avail[n + 1].base) 5812 (md++)->base = avail[n].limit; 5813 if (avail[n].limit) 5814 (md++)->base = avail[n].limit; 5815 5816 n = md - mem; 5817 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 5818 5819 for (lo = 0; lo < i; lo++) 5820 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 5821 avail[lo].limit - 1); 5822 5823 sbuf_printf(sb, "\n"); 5824 for (i = 0; i < n; i++) { 5825 if (mem[i].idx >= nitems(region)) 5826 continue; /* skip holes */ 5827 if (!mem[i].limit) 5828 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 5829 mem_region_show(sb, region[mem[i].idx], mem[i].base, 5830 mem[i].limit); 5831 } 5832 5833 sbuf_printf(sb, "\n"); 5834 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 5835 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 5836 mem_region_show(sb, "uP RAM:", lo, hi); 5837 5838 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 5839 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 5840 mem_region_show(sb, "uP Extmem2:", lo, hi); 5841 5842 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 5843 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 5844 G_PMRXMAXPAGE(lo), 5845 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 5846 (lo & F_PMRXNUMCHN) ? 2 : 1); 5847 5848 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 5849 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 5850 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 5851 G_PMTXMAXPAGE(lo), 5852 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 5853 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 5854 sbuf_printf(sb, "%u p-structs\n", 5855 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 5856 5857 for (i = 0; i < 4; i++) { 5858 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 5859 if (is_t4(sc)) { 5860 used = G_USED(lo); 5861 alloc = G_ALLOC(lo); 5862 } else { 5863 used = G_T5_USED(lo); 5864 alloc = G_T5_ALLOC(lo); 5865 } 5866 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 5867 i, used, alloc); 5868 } 5869 for (i = 0; i < 4; i++) { 5870 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 5871 if (is_t4(sc)) { 5872 used = G_USED(lo); 5873 alloc = G_ALLOC(lo); 5874 } else { 5875 used = G_T5_USED(lo); 5876 alloc = G_T5_ALLOC(lo); 5877 } 5878 sbuf_printf(sb, 5879 "\nLoopback %d using %u pages out of %u allocated", 5880 i, used, alloc); 5881 } 5882 5883 rc = sbuf_finish(sb); 5884 sbuf_delete(sb); 5885 5886 return (rc); 5887} 5888 5889static inline void 5890tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 5891{ 5892 *mask = x | y; 5893 y = htobe64(y); 5894 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 5895} 5896 5897static int 5898sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 5899{ 5900 struct adapter *sc = arg1; 5901 struct sbuf *sb; 5902 int rc, i, n; 5903 5904 rc = sysctl_wire_old_buffer(req, 0); 5905 if (rc != 0) 5906 return (rc); 5907 5908 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5909 if (sb == NULL) 5910 return (ENOMEM); 5911 5912 sbuf_printf(sb, 5913 "Idx Ethernet address Mask Vld Ports PF" 5914 " VF Replication P0 P1 P2 P3 ML"); 5915 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 5916 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 5917 for (i = 0; i < n; i++) { 5918 uint64_t tcamx, tcamy, mask; 5919 uint32_t cls_lo, cls_hi; 5920 uint8_t addr[ETHER_ADDR_LEN]; 5921 5922 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 5923 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 5924 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 5925 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 5926 5927 if (tcamx & tcamy) 5928 continue; 5929 5930 tcamxy2valmask(tcamx, tcamy, addr, &mask); 5931 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 5932 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 5933 addr[3], addr[4], addr[5], (uintmax_t)mask, 5934 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 5935 G_PORTMAP(cls_hi), G_PF(cls_lo), 5936 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 5937 5938 if (cls_lo & F_REPLICATE) { 5939 struct fw_ldst_cmd ldst_cmd; 5940 5941 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 5942 ldst_cmd.op_to_addrspace = 5943 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 5944 F_FW_CMD_REQUEST | F_FW_CMD_READ | 5945 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 5946 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 5947 ldst_cmd.u.mps.fid_ctl = 5948 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 5949 V_FW_LDST_CMD_CTL(i)); 5950 5951 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 5952 "t4mps"); 5953 if (rc) 5954 break; 5955 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 5956 sizeof(ldst_cmd), &ldst_cmd); 5957 end_synchronized_op(sc, 0); 5958 5959 if (rc != 0) { 5960 sbuf_printf(sb, 5961 " ------------ error %3u ------------", rc); 5962 rc = 0; 5963 } else { 5964 sbuf_printf(sb, " %08x %08x %08x %08x", 5965 be32toh(ldst_cmd.u.mps.rplc127_96), 5966 be32toh(ldst_cmd.u.mps.rplc95_64), 5967 be32toh(ldst_cmd.u.mps.rplc63_32), 5968 be32toh(ldst_cmd.u.mps.rplc31_0)); 5969 } 5970 } else 5971 sbuf_printf(sb, "%36s", ""); 5972 5973 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 5974 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 5975 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 5976 } 5977 5978 if (rc) 5979 (void) sbuf_finish(sb); 5980 else 5981 rc = sbuf_finish(sb); 5982 sbuf_delete(sb); 5983 5984 return (rc); 5985} 5986 5987static int 5988sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 5989{ 5990 struct adapter *sc = arg1; 5991 struct sbuf *sb; 5992 int rc; 5993 uint16_t mtus[NMTUS]; 5994 5995 rc = sysctl_wire_old_buffer(req, 0); 5996 if (rc != 0) 5997 return (rc); 5998 5999 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6000 if (sb == NULL) 6001 return (ENOMEM); 6002 6003 t4_read_mtu_tbl(sc, mtus, NULL); 6004 6005 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6006 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6007 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6008 mtus[14], mtus[15]); 6009 6010 rc = sbuf_finish(sb); 6011 sbuf_delete(sb); 6012 6013 return (rc); 6014} 6015 6016static int 6017sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6018{ 6019 struct adapter *sc = arg1; 6020 struct sbuf *sb; 6021 int rc, i; 6022 uint32_t cnt[PM_NSTATS]; 6023 uint64_t cyc[PM_NSTATS]; 6024 static const char *rx_stats[] = { 6025 "Read:", "Write bypass:", "Write mem:", "Flush:" 6026 }; 6027 static const char *tx_stats[] = { 6028 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6029 }; 6030 6031 rc = sysctl_wire_old_buffer(req, 0); 6032 if (rc != 0) 6033 return (rc); 6034 6035 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6036 if (sb == NULL) 6037 return (ENOMEM); 6038 6039 t4_pmtx_get_stats(sc, cnt, cyc); 6040 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6041 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6042 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6043 cyc[i]); 6044 6045 t4_pmrx_get_stats(sc, cnt, cyc); 6046 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6047 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6048 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6049 cyc[i]); 6050 6051 rc = sbuf_finish(sb); 6052 sbuf_delete(sb); 6053 6054 return (rc); 6055} 6056 6057static int 6058sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6059{ 6060 struct adapter *sc = arg1; 6061 struct sbuf *sb; 6062 int rc; 6063 struct tp_rdma_stats stats; 6064 6065 rc = sysctl_wire_old_buffer(req, 0); 6066 if (rc != 0) 6067 return (rc); 6068 6069 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6070 if (sb == NULL) 6071 return (ENOMEM); 6072 6073 t4_tp_get_rdma_stats(sc, &stats); 6074 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6075 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6076 6077 rc = sbuf_finish(sb); 6078 sbuf_delete(sb); 6079 6080 return (rc); 6081} 6082 6083static int 6084sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6085{ 6086 struct adapter *sc = arg1; 6087 struct sbuf *sb; 6088 int rc; 6089 struct tp_tcp_stats v4, v6; 6090 6091 rc = sysctl_wire_old_buffer(req, 0); 6092 if (rc != 0) 6093 return (rc); 6094 6095 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6096 if (sb == NULL) 6097 return (ENOMEM); 6098 6099 t4_tp_get_tcp_stats(sc, &v4, &v6); 6100 sbuf_printf(sb, 6101 " IP IPv6\n"); 6102 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6103 v4.tcpOutRsts, v6.tcpOutRsts); 6104 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6105 v4.tcpInSegs, v6.tcpInSegs); 6106 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6107 v4.tcpOutSegs, v6.tcpOutSegs); 6108 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6109 v4.tcpRetransSegs, v6.tcpRetransSegs); 6110 6111 rc = sbuf_finish(sb); 6112 sbuf_delete(sb); 6113 6114 return (rc); 6115} 6116 6117static int 6118sysctl_tids(SYSCTL_HANDLER_ARGS) 6119{ 6120 struct adapter *sc = arg1; 6121 struct sbuf *sb; 6122 int rc; 6123 struct tid_info *t = &sc->tids; 6124 6125 rc = sysctl_wire_old_buffer(req, 0); 6126 if (rc != 0) 6127 return (rc); 6128 6129 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6130 if (sb == NULL) 6131 return (ENOMEM); 6132 6133 if (t->natids) { 6134 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6135 t->atids_in_use); 6136 } 6137 6138 if (t->ntids) { 6139 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6140 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6141 6142 if (b) { 6143 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6144 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6145 t->ntids - 1); 6146 } else { 6147 sbuf_printf(sb, "TID range: %u-%u", 6148 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6149 t->ntids - 1); 6150 } 6151 } else 6152 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6153 sbuf_printf(sb, ", in use: %u\n", 6154 atomic_load_acq_int(&t->tids_in_use)); 6155 } 6156 6157 if (t->nstids) { 6158 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6159 t->stid_base + t->nstids - 1, t->stids_in_use); 6160 } 6161 6162 if (t->nftids) { 6163 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6164 t->ftid_base + t->nftids - 1); 6165 } 6166 6167 if (t->netids) { 6168 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6169 t->etid_base + t->netids - 1); 6170 } 6171 6172 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6173 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6174 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6175 6176 rc = sbuf_finish(sb); 6177 sbuf_delete(sb); 6178 6179 return (rc); 6180} 6181 6182static int 6183sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6184{ 6185 struct adapter *sc = arg1; 6186 struct sbuf *sb; 6187 int rc; 6188 struct tp_err_stats stats; 6189 6190 rc = sysctl_wire_old_buffer(req, 0); 6191 if (rc != 0) 6192 return (rc); 6193 6194 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6195 if (sb == NULL) 6196 return (ENOMEM); 6197 6198 t4_tp_get_err_stats(sc, &stats); 6199 6200 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6201 "channel 3\n"); 6202 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6203 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6204 stats.macInErrs[3]); 6205 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6206 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6207 stats.hdrInErrs[3]); 6208 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6209 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6210 stats.tcpInErrs[3]); 6211 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6212 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6213 stats.tcp6InErrs[3]); 6214 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6215 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6216 stats.tnlCongDrops[3]); 6217 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6218 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6219 stats.tnlTxDrops[3]); 6220 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6221 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6222 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6223 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6224 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6225 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6226 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6227 stats.ofldNoNeigh, stats.ofldCongDefer); 6228 6229 rc = sbuf_finish(sb); 6230 sbuf_delete(sb); 6231 6232 return (rc); 6233} 6234 6235struct field_desc { 6236 const char *name; 6237 u_int start; 6238 u_int width; 6239}; 6240 6241static void 6242field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6243{ 6244 char buf[32]; 6245 int line_size = 0; 6246 6247 while (f->name) { 6248 uint64_t mask = (1ULL << f->width) - 1; 6249 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6250 ((uintmax_t)v >> f->start) & mask); 6251 6252 if (line_size + len >= 79) { 6253 line_size = 8; 6254 sbuf_printf(sb, "\n "); 6255 } 6256 sbuf_printf(sb, "%s ", buf); 6257 line_size += len + 1; 6258 f++; 6259 } 6260 sbuf_printf(sb, "\n"); 6261} 6262 6263static struct field_desc tp_la0[] = { 6264 { "RcfOpCodeOut", 60, 4 }, 6265 { "State", 56, 4 }, 6266 { "WcfState", 52, 4 }, 6267 { "RcfOpcSrcOut", 50, 2 }, 6268 { "CRxError", 49, 1 }, 6269 { "ERxError", 48, 1 }, 6270 { "SanityFailed", 47, 1 }, 6271 { "SpuriousMsg", 46, 1 }, 6272 { "FlushInputMsg", 45, 1 }, 6273 { "FlushInputCpl", 44, 1 }, 6274 { "RssUpBit", 43, 1 }, 6275 { "RssFilterHit", 42, 1 }, 6276 { "Tid", 32, 10 }, 6277 { "InitTcb", 31, 1 }, 6278 { "LineNumber", 24, 7 }, 6279 { "Emsg", 23, 1 }, 6280 { "EdataOut", 22, 1 }, 6281 { "Cmsg", 21, 1 }, 6282 { "CdataOut", 20, 1 }, 6283 { "EreadPdu", 19, 1 }, 6284 { "CreadPdu", 18, 1 }, 6285 { "TunnelPkt", 17, 1 }, 6286 { "RcfPeerFin", 16, 1 }, 6287 { "RcfReasonOut", 12, 4 }, 6288 { "TxCchannel", 10, 2 }, 6289 { "RcfTxChannel", 8, 2 }, 6290 { "RxEchannel", 6, 2 }, 6291 { "RcfRxChannel", 5, 1 }, 6292 { "RcfDataOutSrdy", 4, 1 }, 6293 { "RxDvld", 3, 1 }, 6294 { "RxOoDvld", 2, 1 }, 6295 { "RxCongestion", 1, 1 }, 6296 { "TxCongestion", 0, 1 }, 6297 { NULL } 6298}; 6299 6300static struct field_desc tp_la1[] = { 6301 { "CplCmdIn", 56, 8 }, 6302 { "CplCmdOut", 48, 8 }, 6303 { "ESynOut", 47, 1 }, 6304 { "EAckOut", 46, 1 }, 6305 { "EFinOut", 45, 1 }, 6306 { "ERstOut", 44, 1 }, 6307 { "SynIn", 43, 1 }, 6308 { "AckIn", 42, 1 }, 6309 { "FinIn", 41, 1 }, 6310 { "RstIn", 40, 1 }, 6311 { "DataIn", 39, 1 }, 6312 { "DataInVld", 38, 1 }, 6313 { "PadIn", 37, 1 }, 6314 { "RxBufEmpty", 36, 1 }, 6315 { "RxDdp", 35, 1 }, 6316 { "RxFbCongestion", 34, 1 }, 6317 { "TxFbCongestion", 33, 1 }, 6318 { "TxPktSumSrdy", 32, 1 }, 6319 { "RcfUlpType", 28, 4 }, 6320 { "Eread", 27, 1 }, 6321 { "Ebypass", 26, 1 }, 6322 { "Esave", 25, 1 }, 6323 { "Static0", 24, 1 }, 6324 { "Cread", 23, 1 }, 6325 { "Cbypass", 22, 1 }, 6326 { "Csave", 21, 1 }, 6327 { "CPktOut", 20, 1 }, 6328 { "RxPagePoolFull", 18, 2 }, 6329 { "RxLpbkPkt", 17, 1 }, 6330 { "TxLpbkPkt", 16, 1 }, 6331 { "RxVfValid", 15, 1 }, 6332 { "SynLearned", 14, 1 }, 6333 { "SetDelEntry", 13, 1 }, 6334 { "SetInvEntry", 12, 1 }, 6335 { "CpcmdDvld", 11, 1 }, 6336 { "CpcmdSave", 10, 1 }, 6337 { "RxPstructsFull", 8, 2 }, 6338 { "EpcmdDvld", 7, 1 }, 6339 { "EpcmdFlush", 6, 1 }, 6340 { "EpcmdTrimPrefix", 5, 1 }, 6341 { "EpcmdTrimPostfix", 4, 1 }, 6342 { "ERssIp4Pkt", 3, 1 }, 6343 { "ERssIp6Pkt", 2, 1 }, 6344 { "ERssTcpUdpPkt", 1, 1 }, 6345 { "ERssFceFipPkt", 0, 1 }, 6346 { NULL } 6347}; 6348 6349static struct field_desc tp_la2[] = { 6350 { "CplCmdIn", 56, 8 }, 6351 { "MpsVfVld", 55, 1 }, 6352 { "MpsPf", 52, 3 }, 6353 { "MpsVf", 44, 8 }, 6354 { "SynIn", 43, 1 }, 6355 { "AckIn", 42, 1 }, 6356 { "FinIn", 41, 1 }, 6357 { "RstIn", 40, 1 }, 6358 { "DataIn", 39, 1 }, 6359 { "DataInVld", 38, 1 }, 6360 { "PadIn", 37, 1 }, 6361 { "RxBufEmpty", 36, 1 }, 6362 { "RxDdp", 35, 1 }, 6363 { "RxFbCongestion", 34, 1 }, 6364 { "TxFbCongestion", 33, 1 }, 6365 { "TxPktSumSrdy", 32, 1 }, 6366 { "RcfUlpType", 28, 4 }, 6367 { "Eread", 27, 1 }, 6368 { "Ebypass", 26, 1 }, 6369 { "Esave", 25, 1 }, 6370 { "Static0", 24, 1 }, 6371 { "Cread", 23, 1 }, 6372 { "Cbypass", 22, 1 }, 6373 { "Csave", 21, 1 }, 6374 { "CPktOut", 20, 1 }, 6375 { "RxPagePoolFull", 18, 2 }, 6376 { "RxLpbkPkt", 17, 1 }, 6377 { "TxLpbkPkt", 16, 1 }, 6378 { "RxVfValid", 15, 1 }, 6379 { "SynLearned", 14, 1 }, 6380 { "SetDelEntry", 13, 1 }, 6381 { "SetInvEntry", 12, 1 }, 6382 { "CpcmdDvld", 11, 1 }, 6383 { "CpcmdSave", 10, 1 }, 6384 { "RxPstructsFull", 8, 2 }, 6385 { "EpcmdDvld", 7, 1 }, 6386 { "EpcmdFlush", 6, 1 }, 6387 { "EpcmdTrimPrefix", 5, 1 }, 6388 { "EpcmdTrimPostfix", 4, 1 }, 6389 { "ERssIp4Pkt", 3, 1 }, 6390 { "ERssIp6Pkt", 2, 1 }, 6391 { "ERssTcpUdpPkt", 1, 1 }, 6392 { "ERssFceFipPkt", 0, 1 }, 6393 { NULL } 6394}; 6395 6396static void 6397tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6398{ 6399 6400 field_desc_show(sb, *p, tp_la0); 6401} 6402 6403static void 6404tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6405{ 6406 6407 if (idx) 6408 sbuf_printf(sb, "\n"); 6409 field_desc_show(sb, p[0], tp_la0); 6410 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6411 field_desc_show(sb, p[1], tp_la0); 6412} 6413 6414static void 6415tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6416{ 6417 6418 if (idx) 6419 sbuf_printf(sb, "\n"); 6420 field_desc_show(sb, p[0], tp_la0); 6421 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6422 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6423} 6424 6425static int 6426sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6427{ 6428 struct adapter *sc = arg1; 6429 struct sbuf *sb; 6430 uint64_t *buf, *p; 6431 int rc; 6432 u_int i, inc; 6433 void (*show_func)(struct sbuf *, uint64_t *, int); 6434 6435 rc = sysctl_wire_old_buffer(req, 0); 6436 if (rc != 0) 6437 return (rc); 6438 6439 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6440 if (sb == NULL) 6441 return (ENOMEM); 6442 6443 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6444 6445 t4_tp_read_la(sc, buf, NULL); 6446 p = buf; 6447 6448 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6449 case 2: 6450 inc = 2; 6451 show_func = tp_la_show2; 6452 break; 6453 case 3: 6454 inc = 2; 6455 show_func = tp_la_show3; 6456 break; 6457 default: 6458 inc = 1; 6459 show_func = tp_la_show; 6460 } 6461 6462 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6463 (*show_func)(sb, p, i); 6464 6465 rc = sbuf_finish(sb); 6466 sbuf_delete(sb); 6467 free(buf, M_CXGBE); 6468 return (rc); 6469} 6470 6471static int 6472sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6473{ 6474 struct adapter *sc = arg1; 6475 struct sbuf *sb; 6476 int rc; 6477 u64 nrate[NCHAN], orate[NCHAN]; 6478 6479 rc = sysctl_wire_old_buffer(req, 0); 6480 if (rc != 0) 6481 return (rc); 6482 6483 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6484 if (sb == NULL) 6485 return (ENOMEM); 6486 6487 t4_get_chan_txrate(sc, nrate, orate); 6488 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6489 "channel 3\n"); 6490 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6491 nrate[0], nrate[1], nrate[2], nrate[3]); 6492 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6493 orate[0], orate[1], orate[2], orate[3]); 6494 6495 rc = sbuf_finish(sb); 6496 sbuf_delete(sb); 6497 6498 return (rc); 6499} 6500 6501static int 6502sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6503{ 6504 struct adapter *sc = arg1; 6505 struct sbuf *sb; 6506 uint32_t *buf, *p; 6507 int rc, i; 6508 6509 rc = sysctl_wire_old_buffer(req, 0); 6510 if (rc != 0) 6511 return (rc); 6512 6513 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6514 if (sb == NULL) 6515 return (ENOMEM); 6516 6517 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6518 M_ZERO | M_WAITOK); 6519 6520 t4_ulprx_read_la(sc, buf); 6521 p = buf; 6522 6523 sbuf_printf(sb, " Pcmd Type Message" 6524 " Data"); 6525 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6526 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6527 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6528 } 6529 6530 rc = sbuf_finish(sb); 6531 sbuf_delete(sb); 6532 free(buf, M_CXGBE); 6533 return (rc); 6534} 6535 6536static int 6537sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6538{ 6539 struct adapter *sc = arg1; 6540 struct sbuf *sb; 6541 int rc, v; 6542 6543 rc = sysctl_wire_old_buffer(req, 0); 6544 if (rc != 0) 6545 return (rc); 6546 6547 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6548 if (sb == NULL) 6549 return (ENOMEM); 6550 6551 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6552 if (G_STATSOURCE_T5(v) == 7) { 6553 if (G_STATMODE(v) == 0) { 6554 sbuf_printf(sb, "total %d, incomplete %d", 6555 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6556 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6557 } else if (G_STATMODE(v) == 1) { 6558 sbuf_printf(sb, "total %d, data overflow %d", 6559 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6560 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6561 } 6562 } 6563 rc = sbuf_finish(sb); 6564 sbuf_delete(sb); 6565 6566 return (rc); 6567} 6568#endif 6569 6570static inline void 6571txq_start(struct ifnet *ifp, struct sge_txq *txq) 6572{ 6573 struct buf_ring *br; 6574 struct mbuf *m; 6575 6576 TXQ_LOCK_ASSERT_OWNED(txq); 6577 6578 br = txq->br; 6579 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6580 if (m) 6581 t4_eth_tx(ifp, txq, m); 6582} 6583 6584void 6585t4_tx_callout(void *arg) 6586{ 6587 struct sge_eq *eq = arg; 6588 struct adapter *sc; 6589 6590 if (EQ_TRYLOCK(eq) == 0) 6591 goto reschedule; 6592 6593 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6594 EQ_UNLOCK(eq); 6595reschedule: 6596 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6597 callout_schedule(&eq->tx_callout, 1); 6598 return; 6599 } 6600 6601 EQ_LOCK_ASSERT_OWNED(eq); 6602 6603 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6604 6605 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6606 struct sge_txq *txq = arg; 6607 struct port_info *pi = txq->ifp->if_softc; 6608 6609 sc = pi->adapter; 6610 } else { 6611 struct sge_wrq *wrq = arg; 6612 6613 sc = wrq->adapter; 6614 } 6615 6616 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6617 } 6618 6619 EQ_UNLOCK(eq); 6620} 6621 6622void 6623t4_tx_task(void *arg, int count) 6624{ 6625 struct sge_eq *eq = arg; 6626 6627 EQ_LOCK(eq); 6628 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6629 struct sge_txq *txq = arg; 6630 txq_start(txq->ifp, txq); 6631 } else { 6632 struct sge_wrq *wrq = arg; 6633 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6634 } 6635 EQ_UNLOCK(eq); 6636} 6637 6638static uint32_t 6639fconf_to_mode(uint32_t fconf) 6640{ 6641 uint32_t mode; 6642 6643 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6644 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6645 6646 if (fconf & F_FRAGMENTATION) 6647 mode |= T4_FILTER_IP_FRAGMENT; 6648 6649 if (fconf & F_MPSHITTYPE) 6650 mode |= T4_FILTER_MPS_HIT_TYPE; 6651 6652 if (fconf & F_MACMATCH) 6653 mode |= T4_FILTER_MAC_IDX; 6654 6655 if (fconf & F_ETHERTYPE) 6656 mode |= T4_FILTER_ETH_TYPE; 6657 6658 if (fconf & F_PROTOCOL) 6659 mode |= T4_FILTER_IP_PROTO; 6660 6661 if (fconf & F_TOS) 6662 mode |= T4_FILTER_IP_TOS; 6663 6664 if (fconf & F_VLAN) 6665 mode |= T4_FILTER_VLAN; 6666 6667 if (fconf & F_VNIC_ID) 6668 mode |= T4_FILTER_VNIC; 6669 6670 if (fconf & F_PORT) 6671 mode |= T4_FILTER_PORT; 6672 6673 if (fconf & F_FCOE) 6674 mode |= T4_FILTER_FCoE; 6675 6676 return (mode); 6677} 6678 6679static uint32_t 6680mode_to_fconf(uint32_t mode) 6681{ 6682 uint32_t fconf = 0; 6683 6684 if (mode & T4_FILTER_IP_FRAGMENT) 6685 fconf |= F_FRAGMENTATION; 6686 6687 if (mode & T4_FILTER_MPS_HIT_TYPE) 6688 fconf |= F_MPSHITTYPE; 6689 6690 if (mode & T4_FILTER_MAC_IDX) 6691 fconf |= F_MACMATCH; 6692 6693 if (mode & T4_FILTER_ETH_TYPE) 6694 fconf |= F_ETHERTYPE; 6695 6696 if (mode & T4_FILTER_IP_PROTO) 6697 fconf |= F_PROTOCOL; 6698 6699 if (mode & T4_FILTER_IP_TOS) 6700 fconf |= F_TOS; 6701 6702 if (mode & T4_FILTER_VLAN) 6703 fconf |= F_VLAN; 6704 6705 if (mode & T4_FILTER_VNIC) 6706 fconf |= F_VNIC_ID; 6707 6708 if (mode & T4_FILTER_PORT) 6709 fconf |= F_PORT; 6710 6711 if (mode & T4_FILTER_FCoE) 6712 fconf |= F_FCOE; 6713 6714 return (fconf); 6715} 6716 6717static uint32_t 6718fspec_to_fconf(struct t4_filter_specification *fs) 6719{ 6720 uint32_t fconf = 0; 6721 6722 if (fs->val.frag || fs->mask.frag) 6723 fconf |= F_FRAGMENTATION; 6724 6725 if (fs->val.matchtype || fs->mask.matchtype) 6726 fconf |= F_MPSHITTYPE; 6727 6728 if (fs->val.macidx || fs->mask.macidx) 6729 fconf |= F_MACMATCH; 6730 6731 if (fs->val.ethtype || fs->mask.ethtype) 6732 fconf |= F_ETHERTYPE; 6733 6734 if (fs->val.proto || fs->mask.proto) 6735 fconf |= F_PROTOCOL; 6736 6737 if (fs->val.tos || fs->mask.tos) 6738 fconf |= F_TOS; 6739 6740 if (fs->val.vlan_vld || fs->mask.vlan_vld) 6741 fconf |= F_VLAN; 6742 6743 if (fs->val.vnic_vld || fs->mask.vnic_vld) 6744 fconf |= F_VNIC_ID; 6745 6746 if (fs->val.iport || fs->mask.iport) 6747 fconf |= F_PORT; 6748 6749 if (fs->val.fcoe || fs->mask.fcoe) 6750 fconf |= F_FCOE; 6751 6752 return (fconf); 6753} 6754 6755static int 6756get_filter_mode(struct adapter *sc, uint32_t *mode) 6757{ 6758 int rc; 6759 uint32_t fconf; 6760 6761 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6762 "t4getfm"); 6763 if (rc) 6764 return (rc); 6765 6766 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 6767 A_TP_VLAN_PRI_MAP); 6768 6769 if (sc->params.tp.vlan_pri_map != fconf) { 6770 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 6771 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 6772 fconf); 6773 sc->params.tp.vlan_pri_map = fconf; 6774 } 6775 6776 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 6777 6778 end_synchronized_op(sc, LOCK_HELD); 6779 return (0); 6780} 6781 6782static int 6783set_filter_mode(struct adapter *sc, uint32_t mode) 6784{ 6785 uint32_t fconf; 6786 int rc; 6787 6788 fconf = mode_to_fconf(mode); 6789 6790 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6791 "t4setfm"); 6792 if (rc) 6793 return (rc); 6794 6795 if (sc->tids.ftids_in_use > 0) { 6796 rc = EBUSY; 6797 goto done; 6798 } 6799 6800#ifdef TCP_OFFLOAD 6801 if (sc->offload_map) { 6802 rc = EBUSY; 6803 goto done; 6804 } 6805#endif 6806 6807#ifdef notyet 6808 rc = -t4_set_filter_mode(sc, fconf); 6809 if (rc == 0) 6810 sc->filter_mode = fconf; 6811#else 6812 rc = ENOTSUP; 6813#endif 6814 6815done: 6816 end_synchronized_op(sc, LOCK_HELD); 6817 return (rc); 6818} 6819 6820static inline uint64_t 6821get_filter_hits(struct adapter *sc, uint32_t fid) 6822{ 6823 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6824 uint64_t hits; 6825 6826 memwin_info(sc, 0, &mw_base, NULL); 6827 off = position_memwin(sc, 0, 6828 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 6829 if (is_t4(sc)) { 6830 hits = t4_read_reg64(sc, mw_base + off + 16); 6831 hits = be64toh(hits); 6832 } else { 6833 hits = t4_read_reg(sc, mw_base + off + 24); 6834 hits = be32toh(hits); 6835 } 6836 6837 return (hits); 6838} 6839 6840static int 6841get_filter(struct adapter *sc, struct t4_filter *t) 6842{ 6843 int i, rc, nfilters = sc->tids.nftids; 6844 struct filter_entry *f; 6845 6846 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6847 "t4getf"); 6848 if (rc) 6849 return (rc); 6850 6851 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 6852 t->idx >= nfilters) { 6853 t->idx = 0xffffffff; 6854 goto done; 6855 } 6856 6857 f = &sc->tids.ftid_tab[t->idx]; 6858 for (i = t->idx; i < nfilters; i++, f++) { 6859 if (f->valid) { 6860 t->idx = i; 6861 t->l2tidx = f->l2t ? f->l2t->idx : 0; 6862 t->smtidx = f->smtidx; 6863 if (f->fs.hitcnts) 6864 t->hits = get_filter_hits(sc, t->idx); 6865 else 6866 t->hits = UINT64_MAX; 6867 t->fs = f->fs; 6868 6869 goto done; 6870 } 6871 } 6872 6873 t->idx = 0xffffffff; 6874done: 6875 end_synchronized_op(sc, LOCK_HELD); 6876 return (0); 6877} 6878 6879static int 6880set_filter(struct adapter *sc, struct t4_filter *t) 6881{ 6882 unsigned int nfilters, nports; 6883 struct filter_entry *f; 6884 int i, rc; 6885 6886 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 6887 if (rc) 6888 return (rc); 6889 6890 nfilters = sc->tids.nftids; 6891 nports = sc->params.nports; 6892 6893 if (nfilters == 0) { 6894 rc = ENOTSUP; 6895 goto done; 6896 } 6897 6898 if (!(sc->flags & FULL_INIT_DONE)) { 6899 rc = EAGAIN; 6900 goto done; 6901 } 6902 6903 if (t->idx >= nfilters) { 6904 rc = EINVAL; 6905 goto done; 6906 } 6907 6908 /* Validate against the global filter mode */ 6909 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 6910 sc->params.tp.vlan_pri_map) { 6911 rc = E2BIG; 6912 goto done; 6913 } 6914 6915 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 6916 rc = EINVAL; 6917 goto done; 6918 } 6919 6920 if (t->fs.val.iport >= nports) { 6921 rc = EINVAL; 6922 goto done; 6923 } 6924 6925 /* Can't specify an iq if not steering to it */ 6926 if (!t->fs.dirsteer && t->fs.iq) { 6927 rc = EINVAL; 6928 goto done; 6929 } 6930 6931 /* IPv6 filter idx must be 4 aligned */ 6932 if (t->fs.type == 1 && 6933 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 6934 rc = EINVAL; 6935 goto done; 6936 } 6937 6938 if (sc->tids.ftid_tab == NULL) { 6939 KASSERT(sc->tids.ftids_in_use == 0, 6940 ("%s: no memory allocated but filters_in_use > 0", 6941 __func__)); 6942 6943 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 6944 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 6945 if (sc->tids.ftid_tab == NULL) { 6946 rc = ENOMEM; 6947 goto done; 6948 } 6949 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 6950 } 6951 6952 for (i = 0; i < 4; i++) { 6953 f = &sc->tids.ftid_tab[t->idx + i]; 6954 6955 if (f->pending || f->valid) { 6956 rc = EBUSY; 6957 goto done; 6958 } 6959 if (f->locked) { 6960 rc = EPERM; 6961 goto done; 6962 } 6963 6964 if (t->fs.type == 0) 6965 break; 6966 } 6967 6968 f = &sc->tids.ftid_tab[t->idx]; 6969 f->fs = t->fs; 6970 6971 rc = set_filter_wr(sc, t->idx); 6972done: 6973 end_synchronized_op(sc, 0); 6974 6975 if (rc == 0) { 6976 mtx_lock(&sc->tids.ftid_lock); 6977 for (;;) { 6978 if (f->pending == 0) { 6979 rc = f->valid ? 0 : EIO; 6980 break; 6981 } 6982 6983 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6984 PCATCH, "t4setfw", 0)) { 6985 rc = EINPROGRESS; 6986 break; 6987 } 6988 } 6989 mtx_unlock(&sc->tids.ftid_lock); 6990 } 6991 return (rc); 6992} 6993 6994static int 6995del_filter(struct adapter *sc, struct t4_filter *t) 6996{ 6997 unsigned int nfilters; 6998 struct filter_entry *f; 6999 int rc; 7000 7001 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7002 if (rc) 7003 return (rc); 7004 7005 nfilters = sc->tids.nftids; 7006 7007 if (nfilters == 0) { 7008 rc = ENOTSUP; 7009 goto done; 7010 } 7011 7012 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7013 t->idx >= nfilters) { 7014 rc = EINVAL; 7015 goto done; 7016 } 7017 7018 if (!(sc->flags & FULL_INIT_DONE)) { 7019 rc = EAGAIN; 7020 goto done; 7021 } 7022 7023 f = &sc->tids.ftid_tab[t->idx]; 7024 7025 if (f->pending) { 7026 rc = EBUSY; 7027 goto done; 7028 } 7029 if (f->locked) { 7030 rc = EPERM; 7031 goto done; 7032 } 7033 7034 if (f->valid) { 7035 t->fs = f->fs; /* extra info for the caller */ 7036 rc = del_filter_wr(sc, t->idx); 7037 } 7038 7039done: 7040 end_synchronized_op(sc, 0); 7041 7042 if (rc == 0) { 7043 mtx_lock(&sc->tids.ftid_lock); 7044 for (;;) { 7045 if (f->pending == 0) { 7046 rc = f->valid ? EIO : 0; 7047 break; 7048 } 7049 7050 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7051 PCATCH, "t4delfw", 0)) { 7052 rc = EINPROGRESS; 7053 break; 7054 } 7055 } 7056 mtx_unlock(&sc->tids.ftid_lock); 7057 } 7058 7059 return (rc); 7060} 7061 7062static void 7063clear_filter(struct filter_entry *f) 7064{ 7065 if (f->l2t) 7066 t4_l2t_release(f->l2t); 7067 7068 bzero(f, sizeof (*f)); 7069} 7070 7071static int 7072set_filter_wr(struct adapter *sc, int fidx) 7073{ 7074 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7075 struct wrqe *wr; 7076 struct fw_filter_wr *fwr; 7077 unsigned int ftid; 7078 7079 ASSERT_SYNCHRONIZED_OP(sc); 7080 7081 if (f->fs.newdmac || f->fs.newvlan) { 7082 /* This filter needs an L2T entry; allocate one. */ 7083 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7084 if (f->l2t == NULL) 7085 return (EAGAIN); 7086 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7087 f->fs.dmac)) { 7088 t4_l2t_release(f->l2t); 7089 f->l2t = NULL; 7090 return (ENOMEM); 7091 } 7092 } 7093 7094 ftid = sc->tids.ftid_base + fidx; 7095 7096 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7097 if (wr == NULL) 7098 return (ENOMEM); 7099 7100 fwr = wrtod(wr); 7101 bzero(fwr, sizeof (*fwr)); 7102 7103 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7104 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7105 fwr->tid_to_iq = 7106 htobe32(V_FW_FILTER_WR_TID(ftid) | 7107 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7108 V_FW_FILTER_WR_NOREPLY(0) | 7109 V_FW_FILTER_WR_IQ(f->fs.iq)); 7110 fwr->del_filter_to_l2tix = 7111 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7112 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7113 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7114 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7115 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7116 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7117 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7118 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7119 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7120 f->fs.newvlan == VLAN_REWRITE) | 7121 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7122 f->fs.newvlan == VLAN_REWRITE) | 7123 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7124 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7125 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7126 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7127 fwr->ethtype = htobe16(f->fs.val.ethtype); 7128 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7129 fwr->frag_to_ovlan_vldm = 7130 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7131 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7132 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7133 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7134 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7135 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7136 fwr->smac_sel = 0; 7137 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7138 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7139 fwr->maci_to_matchtypem = 7140 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7141 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7142 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7143 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7144 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7145 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7146 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7147 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7148 fwr->ptcl = f->fs.val.proto; 7149 fwr->ptclm = f->fs.mask.proto; 7150 fwr->ttyp = f->fs.val.tos; 7151 fwr->ttypm = f->fs.mask.tos; 7152 fwr->ivlan = htobe16(f->fs.val.vlan); 7153 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7154 fwr->ovlan = htobe16(f->fs.val.vnic); 7155 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7156 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7157 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7158 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7159 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7160 fwr->lp = htobe16(f->fs.val.dport); 7161 fwr->lpm = htobe16(f->fs.mask.dport); 7162 fwr->fp = htobe16(f->fs.val.sport); 7163 fwr->fpm = htobe16(f->fs.mask.sport); 7164 if (f->fs.newsmac) 7165 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7166 7167 f->pending = 1; 7168 sc->tids.ftids_in_use++; 7169 7170 t4_wrq_tx(sc, wr); 7171 return (0); 7172} 7173 7174static int 7175del_filter_wr(struct adapter *sc, int fidx) 7176{ 7177 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7178 struct wrqe *wr; 7179 struct fw_filter_wr *fwr; 7180 unsigned int ftid; 7181 7182 ftid = sc->tids.ftid_base + fidx; 7183 7184 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7185 if (wr == NULL) 7186 return (ENOMEM); 7187 fwr = wrtod(wr); 7188 bzero(fwr, sizeof (*fwr)); 7189 7190 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7191 7192 f->pending = 1; 7193 t4_wrq_tx(sc, wr); 7194 return (0); 7195} 7196 7197int 7198t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7199{ 7200 struct adapter *sc = iq->adapter; 7201 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7202 unsigned int idx = GET_TID(rpl); 7203 unsigned int rc; 7204 struct filter_entry *f; 7205 7206 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7207 rss->opcode)); 7208 7209 if (is_ftid(sc, idx)) { 7210 7211 idx -= sc->tids.ftid_base; 7212 f = &sc->tids.ftid_tab[idx]; 7213 rc = G_COOKIE(rpl->cookie); 7214 7215 mtx_lock(&sc->tids.ftid_lock); 7216 if (rc == FW_FILTER_WR_FLT_ADDED) { 7217 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7218 __func__, idx)); 7219 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7220 f->pending = 0; /* asynchronous setup completed */ 7221 f->valid = 1; 7222 } else { 7223 if (rc != FW_FILTER_WR_FLT_DELETED) { 7224 /* Add or delete failed, display an error */ 7225 log(LOG_ERR, 7226 "filter %u setup failed with error %u\n", 7227 idx, rc); 7228 } 7229 7230 clear_filter(f); 7231 sc->tids.ftids_in_use--; 7232 } 7233 wakeup(&sc->tids.ftid_tab); 7234 mtx_unlock(&sc->tids.ftid_lock); 7235 } 7236 7237 return (0); 7238} 7239 7240static int 7241get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7242{ 7243 int rc; 7244 7245 if (cntxt->cid > M_CTXTQID) 7246 return (EINVAL); 7247 7248 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7249 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7250 return (EINVAL); 7251 7252 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7253 if (rc) 7254 return (rc); 7255 7256 if (sc->flags & FW_OK) { 7257 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7258 &cntxt->data[0]); 7259 if (rc == 0) 7260 goto done; 7261 } 7262 7263 /* 7264 * Read via firmware failed or wasn't even attempted. Read directly via 7265 * the backdoor. 7266 */ 7267 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7268done: 7269 end_synchronized_op(sc, 0); 7270 return (rc); 7271} 7272 7273static int 7274load_fw(struct adapter *sc, struct t4_data *fw) 7275{ 7276 int rc; 7277 uint8_t *fw_data; 7278 7279 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7280 if (rc) 7281 return (rc); 7282 7283 if (sc->flags & FULL_INIT_DONE) { 7284 rc = EBUSY; 7285 goto done; 7286 } 7287 7288 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7289 if (fw_data == NULL) { 7290 rc = ENOMEM; 7291 goto done; 7292 } 7293 7294 rc = copyin(fw->data, fw_data, fw->len); 7295 if (rc == 0) 7296 rc = -t4_load_fw(sc, fw_data, fw->len); 7297 7298 free(fw_data, M_CXGBE); 7299done: 7300 end_synchronized_op(sc, 0); 7301 return (rc); 7302} 7303 7304static int 7305read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7306{ 7307 uint32_t addr, off, remaining, i, n; 7308 uint32_t *buf, *b; 7309 uint32_t mw_base, mw_aperture; 7310 int rc; 7311 uint8_t *dst; 7312 7313 rc = validate_mem_range(sc, mr->addr, mr->len); 7314 if (rc != 0) 7315 return (rc); 7316 7317 memwin_info(sc, win, &mw_base, &mw_aperture); 7318 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7319 addr = mr->addr; 7320 remaining = mr->len; 7321 dst = (void *)mr->data; 7322 7323 while (remaining) { 7324 off = position_memwin(sc, win, addr); 7325 7326 /* number of bytes that we'll copy in the inner loop */ 7327 n = min(remaining, mw_aperture - off); 7328 for (i = 0; i < n; i += 4) 7329 *b++ = t4_read_reg(sc, mw_base + off + i); 7330 7331 rc = copyout(buf, dst, n); 7332 if (rc != 0) 7333 break; 7334 7335 b = buf; 7336 dst += n; 7337 remaining -= n; 7338 addr += n; 7339 } 7340 7341 free(buf, M_CXGBE); 7342 return (rc); 7343} 7344 7345static int 7346read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7347{ 7348 int rc; 7349 7350 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7351 return (EINVAL); 7352 7353 if (i2cd->len > 1) { 7354 /* XXX: need fw support for longer reads in one go */ 7355 return (ENOTSUP); 7356 } 7357 7358 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7359 if (rc) 7360 return (rc); 7361 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7362 i2cd->offset, &i2cd->data[0]); 7363 end_synchronized_op(sc, 0); 7364 7365 return (rc); 7366} 7367 7368static int 7369in_range(int val, int lo, int hi) 7370{ 7371 7372 return (val < 0 || (val <= hi && val >= lo)); 7373} 7374 7375static int 7376set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7377{ 7378 int fw_subcmd, fw_type, rc; 7379 7380 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7381 if (rc) 7382 return (rc); 7383 7384 if (!(sc->flags & FULL_INIT_DONE)) { 7385 rc = EAGAIN; 7386 goto done; 7387 } 7388 7389 /* 7390 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7391 * sub-command and type are in common locations.) 7392 */ 7393 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7394 fw_subcmd = FW_SCHED_SC_CONFIG; 7395 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7396 fw_subcmd = FW_SCHED_SC_PARAMS; 7397 else { 7398 rc = EINVAL; 7399 goto done; 7400 } 7401 if (p->type == SCHED_CLASS_TYPE_PACKET) 7402 fw_type = FW_SCHED_TYPE_PKTSCHED; 7403 else { 7404 rc = EINVAL; 7405 goto done; 7406 } 7407 7408 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7409 /* Vet our parameters ..*/ 7410 if (p->u.config.minmax < 0) { 7411 rc = EINVAL; 7412 goto done; 7413 } 7414 7415 /* And pass the request to the firmware ...*/ 7416 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax); 7417 goto done; 7418 } 7419 7420 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7421 int fw_level; 7422 int fw_mode; 7423 int fw_rateunit; 7424 int fw_ratemode; 7425 7426 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7427 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7428 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7429 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7430 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7431 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7432 else { 7433 rc = EINVAL; 7434 goto done; 7435 } 7436 7437 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7438 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7439 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7440 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7441 else { 7442 rc = EINVAL; 7443 goto done; 7444 } 7445 7446 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7447 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7448 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7449 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7450 else { 7451 rc = EINVAL; 7452 goto done; 7453 } 7454 7455 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7456 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7457 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7458 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7459 else { 7460 rc = EINVAL; 7461 goto done; 7462 } 7463 7464 /* Vet our parameters ... */ 7465 if (!in_range(p->u.params.channel, 0, 3) || 7466 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7467 !in_range(p->u.params.minrate, 0, 10000000) || 7468 !in_range(p->u.params.maxrate, 0, 10000000) || 7469 !in_range(p->u.params.weight, 0, 100)) { 7470 rc = ERANGE; 7471 goto done; 7472 } 7473 7474 /* 7475 * Translate any unset parameters into the firmware's 7476 * nomenclature and/or fail the call if the parameters 7477 * are required ... 7478 */ 7479 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7480 p->u.params.channel < 0 || p->u.params.cl < 0) { 7481 rc = EINVAL; 7482 goto done; 7483 } 7484 if (p->u.params.minrate < 0) 7485 p->u.params.minrate = 0; 7486 if (p->u.params.maxrate < 0) { 7487 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7488 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7489 rc = EINVAL; 7490 goto done; 7491 } else 7492 p->u.params.maxrate = 0; 7493 } 7494 if (p->u.params.weight < 0) { 7495 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7496 rc = EINVAL; 7497 goto done; 7498 } else 7499 p->u.params.weight = 0; 7500 } 7501 if (p->u.params.pktsize < 0) { 7502 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7503 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7504 rc = EINVAL; 7505 goto done; 7506 } else 7507 p->u.params.pktsize = 0; 7508 } 7509 7510 /* See what the firmware thinks of the request ... */ 7511 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7512 fw_rateunit, fw_ratemode, p->u.params.channel, 7513 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7514 p->u.params.weight, p->u.params.pktsize); 7515 goto done; 7516 } 7517 7518 rc = EINVAL; 7519done: 7520 end_synchronized_op(sc, 0); 7521 return (rc); 7522} 7523 7524static int 7525set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7526{ 7527 struct port_info *pi = NULL; 7528 struct sge_txq *txq; 7529 uint32_t fw_mnem, fw_queue, fw_class; 7530 int i, rc; 7531 7532 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7533 if (rc) 7534 return (rc); 7535 7536 if (!(sc->flags & FULL_INIT_DONE)) { 7537 rc = EAGAIN; 7538 goto done; 7539 } 7540 7541 if (p->port >= sc->params.nports) { 7542 rc = EINVAL; 7543 goto done; 7544 } 7545 7546 pi = sc->port[p->port]; 7547 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7548 rc = EINVAL; 7549 goto done; 7550 } 7551 7552 /* 7553 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7554 * Scheduling Class in this case). 7555 */ 7556 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7557 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7558 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7559 7560 /* 7561 * If op.queue is non-negative, then we're only changing the scheduling 7562 * on a single specified TX queue. 7563 */ 7564 if (p->queue >= 0) { 7565 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7566 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7567 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7568 &fw_class); 7569 goto done; 7570 } 7571 7572 /* 7573 * Change the scheduling on all the TX queues for the 7574 * interface. 7575 */ 7576 for_each_txq(pi, i, txq) { 7577 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7578 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7579 &fw_class); 7580 if (rc) 7581 goto done; 7582 } 7583 7584 rc = 0; 7585done: 7586 end_synchronized_op(sc, 0); 7587 return (rc); 7588} 7589 7590int 7591t4_os_find_pci_capability(struct adapter *sc, int cap) 7592{ 7593 int i; 7594 7595 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7596} 7597 7598int 7599t4_os_pci_save_state(struct adapter *sc) 7600{ 7601 device_t dev; 7602 struct pci_devinfo *dinfo; 7603 7604 dev = sc->dev; 7605 dinfo = device_get_ivars(dev); 7606 7607 pci_cfg_save(dev, dinfo, 0); 7608 return (0); 7609} 7610 7611int 7612t4_os_pci_restore_state(struct adapter *sc) 7613{ 7614 device_t dev; 7615 struct pci_devinfo *dinfo; 7616 7617 dev = sc->dev; 7618 dinfo = device_get_ivars(dev); 7619 7620 pci_cfg_restore(dev, dinfo); 7621 return (0); 7622} 7623 7624void 7625t4_os_portmod_changed(const struct adapter *sc, int idx) 7626{ 7627 struct port_info *pi = sc->port[idx]; 7628 static const char *mod_str[] = { 7629 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7630 }; 7631 7632 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7633 if_printf(pi->ifp, "transceiver unplugged.\n"); 7634 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7635 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7636 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7637 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7638 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7639 if_printf(pi->ifp, "%s transceiver inserted.\n", 7640 mod_str[pi->mod_type]); 7641 } else { 7642 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7643 pi->mod_type); 7644 } 7645} 7646 7647void 7648t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7649{ 7650 struct port_info *pi = sc->port[idx]; 7651 struct ifnet *ifp = pi->ifp; 7652 7653 if (link_stat) { 7654 pi->linkdnrc = -1; 7655 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7656 if_link_state_change(ifp, LINK_STATE_UP); 7657 } else { 7658 if (reason >= 0) 7659 pi->linkdnrc = reason; 7660 if_link_state_change(ifp, LINK_STATE_DOWN); 7661 } 7662} 7663 7664void 7665t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7666{ 7667 struct adapter *sc; 7668 7669 mtx_lock(&t4_list_lock); 7670 SLIST_FOREACH(sc, &t4_list, link) { 7671 /* 7672 * func should not make any assumptions about what state sc is 7673 * in - the only guarantee is that sc->sc_lock is a valid lock. 7674 */ 7675 func(sc, arg); 7676 } 7677 mtx_unlock(&t4_list_lock); 7678} 7679 7680static int 7681t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7682{ 7683 return (0); 7684} 7685 7686static int 7687t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7688{ 7689 return (0); 7690} 7691 7692static int 7693t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7694 struct thread *td) 7695{ 7696 int rc; 7697 struct adapter *sc = dev->si_drv1; 7698 7699 rc = priv_check(td, PRIV_DRIVER); 7700 if (rc != 0) 7701 return (rc); 7702 7703 switch (cmd) { 7704 case CHELSIO_T4_GETREG: { 7705 struct t4_reg *edata = (struct t4_reg *)data; 7706 7707 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7708 return (EFAULT); 7709 7710 if (edata->size == 4) 7711 edata->val = t4_read_reg(sc, edata->addr); 7712 else if (edata->size == 8) 7713 edata->val = t4_read_reg64(sc, edata->addr); 7714 else 7715 return (EINVAL); 7716 7717 break; 7718 } 7719 case CHELSIO_T4_SETREG: { 7720 struct t4_reg *edata = (struct t4_reg *)data; 7721 7722 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7723 return (EFAULT); 7724 7725 if (edata->size == 4) { 7726 if (edata->val & 0xffffffff00000000) 7727 return (EINVAL); 7728 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 7729 } else if (edata->size == 8) 7730 t4_write_reg64(sc, edata->addr, edata->val); 7731 else 7732 return (EINVAL); 7733 break; 7734 } 7735 case CHELSIO_T4_REGDUMP: { 7736 struct t4_regdump *regs = (struct t4_regdump *)data; 7737 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 7738 uint8_t *buf; 7739 7740 if (regs->len < reglen) { 7741 regs->len = reglen; /* hint to the caller */ 7742 return (ENOBUFS); 7743 } 7744 7745 regs->len = reglen; 7746 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 7747 t4_get_regs(sc, regs, buf); 7748 rc = copyout(buf, regs->data, reglen); 7749 free(buf, M_CXGBE); 7750 break; 7751 } 7752 case CHELSIO_T4_GET_FILTER_MODE: 7753 rc = get_filter_mode(sc, (uint32_t *)data); 7754 break; 7755 case CHELSIO_T4_SET_FILTER_MODE: 7756 rc = set_filter_mode(sc, *(uint32_t *)data); 7757 break; 7758 case CHELSIO_T4_GET_FILTER: 7759 rc = get_filter(sc, (struct t4_filter *)data); 7760 break; 7761 case CHELSIO_T4_SET_FILTER: 7762 rc = set_filter(sc, (struct t4_filter *)data); 7763 break; 7764 case CHELSIO_T4_DEL_FILTER: 7765 rc = del_filter(sc, (struct t4_filter *)data); 7766 break; 7767 case CHELSIO_T4_GET_SGE_CONTEXT: 7768 rc = get_sge_context(sc, (struct t4_sge_context *)data); 7769 break; 7770 case CHELSIO_T4_LOAD_FW: 7771 rc = load_fw(sc, (struct t4_data *)data); 7772 break; 7773 case CHELSIO_T4_GET_MEM: 7774 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 7775 break; 7776 case CHELSIO_T4_GET_I2C: 7777 rc = read_i2c(sc, (struct t4_i2c_data *)data); 7778 break; 7779 case CHELSIO_T4_CLEAR_STATS: { 7780 int i; 7781 u_int port_id = *(uint32_t *)data; 7782 struct port_info *pi; 7783 7784 if (port_id >= sc->params.nports) 7785 return (EINVAL); 7786 pi = sc->port[port_id]; 7787 7788 /* MAC stats */ 7789 t4_clr_port_stats(sc, pi->tx_chan); 7790 7791 if (pi->flags & PORT_INIT_DONE) { 7792 struct sge_rxq *rxq; 7793 struct sge_txq *txq; 7794 struct sge_wrq *wrq; 7795 7796 for_each_rxq(pi, i, rxq) { 7797#if defined(INET) || defined(INET6) 7798 rxq->lro.lro_queued = 0; 7799 rxq->lro.lro_flushed = 0; 7800#endif 7801 rxq->rxcsum = 0; 7802 rxq->vlan_extraction = 0; 7803 } 7804 7805 for_each_txq(pi, i, txq) { 7806 txq->txcsum = 0; 7807 txq->tso_wrs = 0; 7808 txq->vlan_insertion = 0; 7809 txq->imm_wrs = 0; 7810 txq->sgl_wrs = 0; 7811 txq->txpkt_wrs = 0; 7812 txq->txpkts_wrs = 0; 7813 txq->txpkts_pkts = 0; 7814 txq->br->br_drops = 0; 7815 txq->no_dmamap = 0; 7816 txq->no_desc = 0; 7817 } 7818 7819#ifdef TCP_OFFLOAD 7820 /* nothing to clear for each ofld_rxq */ 7821 7822 for_each_ofld_txq(pi, i, wrq) { 7823 wrq->tx_wrs = 0; 7824 wrq->no_desc = 0; 7825 } 7826#endif 7827 wrq = &sc->sge.ctrlq[pi->port_id]; 7828 wrq->tx_wrs = 0; 7829 wrq->no_desc = 0; 7830 } 7831 break; 7832 } 7833 case CHELSIO_T4_SCHED_CLASS: 7834 rc = set_sched_class(sc, (struct t4_sched_params *)data); 7835 break; 7836 case CHELSIO_T4_SCHED_QUEUE: 7837 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 7838 break; 7839 default: 7840 rc = EINVAL; 7841 } 7842 7843 return (rc); 7844} 7845 7846#ifdef TCP_OFFLOAD 7847static int 7848toe_capability(struct port_info *pi, int enable) 7849{ 7850 int rc; 7851 struct adapter *sc = pi->adapter; 7852 7853 ASSERT_SYNCHRONIZED_OP(sc); 7854 7855 if (!is_offload(sc)) 7856 return (ENODEV); 7857 7858 if (enable) { 7859 if (!(sc->flags & FULL_INIT_DONE)) { 7860 rc = cxgbe_init_synchronized(pi); 7861 if (rc) 7862 return (rc); 7863 } 7864 7865 if (isset(&sc->offload_map, pi->port_id)) 7866 return (0); 7867 7868 if (!(sc->flags & TOM_INIT_DONE)) { 7869 rc = t4_activate_uld(sc, ULD_TOM); 7870 if (rc == EAGAIN) { 7871 log(LOG_WARNING, 7872 "You must kldload t4_tom.ko before trying " 7873 "to enable TOE on a cxgbe interface.\n"); 7874 } 7875 if (rc != 0) 7876 return (rc); 7877 KASSERT(sc->tom_softc != NULL, 7878 ("%s: TOM activated but softc NULL", __func__)); 7879 KASSERT(sc->flags & TOM_INIT_DONE, 7880 ("%s: TOM activated but flag not set", __func__)); 7881 } 7882 7883 setbit(&sc->offload_map, pi->port_id); 7884 } else { 7885 if (!isset(&sc->offload_map, pi->port_id)) 7886 return (0); 7887 7888 KASSERT(sc->flags & TOM_INIT_DONE, 7889 ("%s: TOM never initialized?", __func__)); 7890 clrbit(&sc->offload_map, pi->port_id); 7891 } 7892 7893 return (0); 7894} 7895 7896/* 7897 * Add an upper layer driver to the global list. 7898 */ 7899int 7900t4_register_uld(struct uld_info *ui) 7901{ 7902 int rc = 0; 7903 struct uld_info *u; 7904 7905 mtx_lock(&t4_uld_list_lock); 7906 SLIST_FOREACH(u, &t4_uld_list, link) { 7907 if (u->uld_id == ui->uld_id) { 7908 rc = EEXIST; 7909 goto done; 7910 } 7911 } 7912 7913 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 7914 ui->refcount = 0; 7915done: 7916 mtx_unlock(&t4_uld_list_lock); 7917 return (rc); 7918} 7919 7920int 7921t4_unregister_uld(struct uld_info *ui) 7922{ 7923 int rc = EINVAL; 7924 struct uld_info *u; 7925 7926 mtx_lock(&t4_uld_list_lock); 7927 7928 SLIST_FOREACH(u, &t4_uld_list, link) { 7929 if (u == ui) { 7930 if (ui->refcount > 0) { 7931 rc = EBUSY; 7932 goto done; 7933 } 7934 7935 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 7936 rc = 0; 7937 goto done; 7938 } 7939 } 7940done: 7941 mtx_unlock(&t4_uld_list_lock); 7942 return (rc); 7943} 7944 7945int 7946t4_activate_uld(struct adapter *sc, int id) 7947{ 7948 int rc = EAGAIN; 7949 struct uld_info *ui; 7950 7951 ASSERT_SYNCHRONIZED_OP(sc); 7952 7953 mtx_lock(&t4_uld_list_lock); 7954 7955 SLIST_FOREACH(ui, &t4_uld_list, link) { 7956 if (ui->uld_id == id) { 7957 rc = ui->activate(sc); 7958 if (rc == 0) 7959 ui->refcount++; 7960 goto done; 7961 } 7962 } 7963done: 7964 mtx_unlock(&t4_uld_list_lock); 7965 7966 return (rc); 7967} 7968 7969int 7970t4_deactivate_uld(struct adapter *sc, int id) 7971{ 7972 int rc = EINVAL; 7973 struct uld_info *ui; 7974 7975 ASSERT_SYNCHRONIZED_OP(sc); 7976 7977 mtx_lock(&t4_uld_list_lock); 7978 7979 SLIST_FOREACH(ui, &t4_uld_list, link) { 7980 if (ui->uld_id == id) { 7981 rc = ui->deactivate(sc); 7982 if (rc == 0) 7983 ui->refcount--; 7984 goto done; 7985 } 7986 } 7987done: 7988 mtx_unlock(&t4_uld_list_lock); 7989 7990 return (rc); 7991} 7992#endif 7993 7994/* 7995 * Come up with reasonable defaults for some of the tunables, provided they're 7996 * not set by the user (in which case we'll use the values as is). 7997 */ 7998static void 7999tweak_tunables(void) 8000{ 8001 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8002 8003 if (t4_ntxq10g < 1) 8004 t4_ntxq10g = min(nc, NTXQ_10G); 8005 8006 if (t4_ntxq1g < 1) 8007 t4_ntxq1g = min(nc, NTXQ_1G); 8008 8009 if (t4_nrxq10g < 1) 8010 t4_nrxq10g = min(nc, NRXQ_10G); 8011 8012 if (t4_nrxq1g < 1) 8013 t4_nrxq1g = min(nc, NRXQ_1G); 8014 8015#ifdef TCP_OFFLOAD 8016 if (t4_nofldtxq10g < 1) 8017 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8018 8019 if (t4_nofldtxq1g < 1) 8020 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8021 8022 if (t4_nofldrxq10g < 1) 8023 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8024 8025 if (t4_nofldrxq1g < 1) 8026 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8027 8028 if (t4_toecaps_allowed == -1) 8029 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8030#else 8031 if (t4_toecaps_allowed == -1) 8032 t4_toecaps_allowed = 0; 8033#endif 8034 8035 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8036 t4_tmr_idx_10g = TMR_IDX_10G; 8037 8038 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8039 t4_pktc_idx_10g = PKTC_IDX_10G; 8040 8041 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8042 t4_tmr_idx_1g = TMR_IDX_1G; 8043 8044 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8045 t4_pktc_idx_1g = PKTC_IDX_1G; 8046 8047 if (t4_qsize_txq < 128) 8048 t4_qsize_txq = 128; 8049 8050 if (t4_qsize_rxq < 128) 8051 t4_qsize_rxq = 128; 8052 while (t4_qsize_rxq & 7) 8053 t4_qsize_rxq++; 8054 8055 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8056} 8057 8058static int 8059mod_event(module_t mod, int cmd, void *arg) 8060{ 8061 int rc = 0; 8062 static int loaded = 0; 8063 8064 switch (cmd) { 8065 case MOD_LOAD: 8066 if (atomic_fetchadd_int(&loaded, 1)) 8067 break; 8068 t4_sge_modload(); 8069 mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 8070 SLIST_INIT(&t4_list); 8071#ifdef TCP_OFFLOAD 8072 mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 8073 SLIST_INIT(&t4_uld_list); 8074#endif 8075 tweak_tunables(); 8076 break; 8077 8078 case MOD_UNLOAD: 8079 if (atomic_fetchadd_int(&loaded, -1) > 1) 8080 break; 8081#ifdef TCP_OFFLOAD 8082 mtx_lock(&t4_uld_list_lock); 8083 if (!SLIST_EMPTY(&t4_uld_list)) { 8084 rc = EBUSY; 8085 mtx_unlock(&t4_uld_list_lock); 8086 break; 8087 } 8088 mtx_unlock(&t4_uld_list_lock); 8089 mtx_destroy(&t4_uld_list_lock); 8090#endif 8091 mtx_lock(&t4_list_lock); 8092 if (!SLIST_EMPTY(&t4_list)) { 8093 rc = EBUSY; 8094 mtx_unlock(&t4_list_lock); 8095 break; 8096 } 8097 mtx_unlock(&t4_list_lock); 8098 mtx_destroy(&t4_list_lock); 8099 break; 8100 } 8101 8102 return (rc); 8103} 8104 8105static devclass_t t4_devclass, t5_devclass; 8106static devclass_t cxgbe_devclass, cxl_devclass; 8107 8108DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8109MODULE_VERSION(t4nex, 1); 8110MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8111 8112DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8113MODULE_VERSION(t5nex, 1); 8114MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8115 8116DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8117MODULE_VERSION(cxgbe, 1); 8118 8119DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8120MODULE_VERSION(cxl, 1); 8121