t4_main.c revision 265425
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 265425 2014-05-06 06:49:39Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58#if defined(__i386__) || defined(__amd64__) 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" 69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80}; 81static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85}; 86 87 88/* T4 port (cxgbe) interface */ 89static int cxgbe_probe(device_t); 90static int cxgbe_attach(device_t); 91static int cxgbe_detach(device_t); 92static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97}; 98static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102}; 103 104static d_ioctl_t t4_ioctl; 105static d_open_t t4_open; 106static d_close_t t4_close; 107 108static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115}; 116 117/* T5 bus driver interface */ 118static int t5_probe(device_t); 119static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125}; 126static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130}; 131 132 133/* T5 port (cxl) interface */ 134static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138}; 139 140static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147}; 148 149/* ifnet + media interface */ 150static void cxgbe_init(void *); 151static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153static void cxgbe_qflush(struct ifnet *); 154static int cxgbe_media_change(struct ifnet *); 155static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159/* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163static struct sx t4_list_lock; 164SLIST_HEAD(, adapter) t4_list; 165#ifdef TCP_OFFLOAD 166static struct sx t4_uld_list_lock; 167SLIST_HEAD(, uld_info) t4_uld_list; 168#endif 169 170/* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181/* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184#define NTXQ_10G 16 185static int t4_ntxq10g = -1; 186TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188#define NRXQ_10G 8 189static int t4_nrxq10g = -1; 190TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192#define NTXQ_1G 4 193static int t4_ntxq1g = -1; 194TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196#define NRXQ_1G 2 197static int t4_nrxq1g = -1; 198TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200static int t4_rsrv_noflowq = 0; 201TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203#ifdef TCP_OFFLOAD 204#define NOFLDTXQ_10G 8 205static int t4_nofldtxq10g = -1; 206TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208#define NOFLDRXQ_10G 2 209static int t4_nofldrxq10g = -1; 210TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212#define NOFLDTXQ_1G 2 213static int t4_nofldtxq1g = -1; 214TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216#define NOFLDRXQ_1G 1 217static int t4_nofldrxq1g = -1; 218TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219#endif 220 221/* 222 * Holdoff parameters for 10G and 1G ports. 223 */ 224#define TMR_IDX_10G 1 225static int t4_tmr_idx_10g = TMR_IDX_10G; 226TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 227 228#define PKTC_IDX_10G (-1) 229static int t4_pktc_idx_10g = PKTC_IDX_10G; 230TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 231 232#define TMR_IDX_1G 1 233static int t4_tmr_idx_1g = TMR_IDX_1G; 234TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 235 236#define PKTC_IDX_1G (-1) 237static int t4_pktc_idx_1g = PKTC_IDX_1G; 238TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 239 240/* 241 * Size (# of entries) of each tx and rx queue. 242 */ 243static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 244TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 245 246static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 247TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 248 249/* 250 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 251 */ 252static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 253TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 254 255/* 256 * Configuration file. 257 */ 258#define DEFAULT_CF "default" 259#define FLASH_CF "flash" 260#define UWIRE_CF "uwire" 261#define FPGA_CF "fpga" 262static char t4_cfg_file[32] = DEFAULT_CF; 263TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 264 265/* 266 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 267 * encouraged respectively). 268 */ 269static unsigned int t4_fw_install = 1; 270TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 271 272/* 273 * ASIC features that will be used. Disable the ones you don't want so that the 274 * chip resources aren't wasted on features that will not be used. 275 */ 276static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 277TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 278 279static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 280TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 281 282static int t4_toecaps_allowed = -1; 283TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 284 285static int t4_rdmacaps_allowed = 0; 286TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 287 288static int t4_iscsicaps_allowed = 0; 289TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 290 291static int t4_fcoecaps_allowed = 0; 292TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 293 294static int t5_write_combine = 0; 295TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 296 297struct intrs_and_queues { 298 int intr_type; /* INTx, MSI, or MSI-X */ 299 int nirq; /* Number of vectors */ 300 int intr_flags; 301 int ntxq10g; /* # of NIC txq's for each 10G port */ 302 int nrxq10g; /* # of NIC rxq's for each 10G port */ 303 int ntxq1g; /* # of NIC txq's for each 1G port */ 304 int nrxq1g; /* # of NIC rxq's for each 1G port */ 305 int rsrv_noflowq; /* Flag whether to reserve queue 0 */ 306#ifdef TCP_OFFLOAD 307 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 308 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 309 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 310 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 311#endif 312}; 313 314struct filter_entry { 315 uint32_t valid:1; /* filter allocated and valid */ 316 uint32_t locked:1; /* filter is administratively locked */ 317 uint32_t pending:1; /* filter action is pending firmware reply */ 318 uint32_t smtidx:8; /* Source MAC Table index for smac */ 319 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 320 321 struct t4_filter_specification fs; 322}; 323 324enum { 325 XGMAC_MTU = (1 << 0), 326 XGMAC_PROMISC = (1 << 1), 327 XGMAC_ALLMULTI = (1 << 2), 328 XGMAC_VLANEX = (1 << 3), 329 XGMAC_UCADDR = (1 << 4), 330 XGMAC_MCADDRS = (1 << 5), 331 332 XGMAC_ALL = 0xffff 333}; 334 335static int map_bars_0_and_4(struct adapter *); 336static int map_bar_2(struct adapter *); 337static void setup_memwin(struct adapter *); 338static int validate_mem_range(struct adapter *, uint32_t, int); 339static int fwmtype_to_hwmtype(int); 340static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 341 uint32_t *); 342static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 343static uint32_t position_memwin(struct adapter *, int, uint32_t); 344static int cfg_itype_and_nqueues(struct adapter *, int, int, 345 struct intrs_and_queues *); 346static int prep_firmware(struct adapter *); 347static int partition_resources(struct adapter *, const struct firmware *, 348 const char *); 349static int get_params__pre_init(struct adapter *); 350static int get_params__post_init(struct adapter *); 351static int set_params__post_init(struct adapter *); 352static void t4_set_desc(struct adapter *); 353static void build_medialist(struct port_info *); 354static int update_mac_settings(struct port_info *, int); 355static int cxgbe_init_synchronized(struct port_info *); 356static int cxgbe_uninit_synchronized(struct port_info *); 357static int setup_intr_handlers(struct adapter *); 358static int adapter_full_init(struct adapter *); 359static int adapter_full_uninit(struct adapter *); 360static int port_full_init(struct port_info *); 361static int port_full_uninit(struct port_info *); 362static void quiesce_eq(struct adapter *, struct sge_eq *); 363static void quiesce_iq(struct adapter *, struct sge_iq *); 364static void quiesce_fl(struct adapter *, struct sge_fl *); 365static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 366 driver_intr_t *, void *, char *); 367static int t4_free_irq(struct adapter *, struct irq *); 368static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 369 unsigned int); 370static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 371static void cxgbe_tick(void *); 372static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 373static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 374 struct mbuf *); 375static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 376static int fw_msg_not_handled(struct adapter *, const __be64 *); 377static int t4_sysctls(struct adapter *); 378static int cxgbe_sysctls(struct port_info *); 379static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 380static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 381static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 382static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 383static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 384static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 385static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 386static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 387static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 388static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 389#ifdef SBUF_DRAIN 390static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 391static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 392static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 393static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 394static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 395static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 396static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 397static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 398static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 399static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 400static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 401static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 402static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 403static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 404static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 405static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 406static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 407static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 408static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 409static int sysctl_tids(SYSCTL_HANDLER_ARGS); 410static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 411static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 412static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 413static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 414static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 415#endif 416static inline void txq_start(struct ifnet *, struct sge_txq *); 417static uint32_t fconf_to_mode(uint32_t); 418static uint32_t mode_to_fconf(uint32_t); 419static uint32_t fspec_to_fconf(struct t4_filter_specification *); 420static int get_filter_mode(struct adapter *, uint32_t *); 421static int set_filter_mode(struct adapter *, uint32_t); 422static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 423static int get_filter(struct adapter *, struct t4_filter *); 424static int set_filter(struct adapter *, struct t4_filter *); 425static int del_filter(struct adapter *, struct t4_filter *); 426static void clear_filter(struct filter_entry *); 427static int set_filter_wr(struct adapter *, int); 428static int del_filter_wr(struct adapter *, int); 429static int get_sge_context(struct adapter *, struct t4_sge_context *); 430static int load_fw(struct adapter *, struct t4_data *); 431static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 432static int read_i2c(struct adapter *, struct t4_i2c_data *); 433static int set_sched_class(struct adapter *, struct t4_sched_params *); 434static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 435#ifdef TCP_OFFLOAD 436static int toe_capability(struct port_info *, int); 437#endif 438static int mod_event(module_t, int, void *); 439 440struct { 441 uint16_t device; 442 char *desc; 443} t4_pciids[] = { 444 {0xa000, "Chelsio Terminator 4 FPGA"}, 445 {0x4400, "Chelsio T440-dbg"}, 446 {0x4401, "Chelsio T420-CR"}, 447 {0x4402, "Chelsio T422-CR"}, 448 {0x4403, "Chelsio T440-CR"}, 449 {0x4404, "Chelsio T420-BCH"}, 450 {0x4405, "Chelsio T440-BCH"}, 451 {0x4406, "Chelsio T440-CH"}, 452 {0x4407, "Chelsio T420-SO"}, 453 {0x4408, "Chelsio T420-CX"}, 454 {0x4409, "Chelsio T420-BT"}, 455 {0x440a, "Chelsio T404-BT"}, 456 {0x440e, "Chelsio T440-LP-CR"}, 457}, t5_pciids[] = { 458 {0xb000, "Chelsio Terminator 5 FPGA"}, 459 {0x5400, "Chelsio T580-dbg"}, 460 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 461 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 462 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 463 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 464 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 465 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 466 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 467 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 468 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 469 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 470 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 471 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 472#ifdef notyet 473 {0x5404, "Chelsio T520-BCH"}, 474 {0x5405, "Chelsio T540-BCH"}, 475 {0x5406, "Chelsio T540-CH"}, 476 {0x5408, "Chelsio T520-CX"}, 477 {0x540b, "Chelsio B520-SR"}, 478 {0x540c, "Chelsio B504-BT"}, 479 {0x540f, "Chelsio Amsterdam"}, 480 {0x5413, "Chelsio T580-CHR"}, 481#endif 482}; 483 484#ifdef TCP_OFFLOAD 485/* 486 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 487 * exactly the same for both rxq and ofld_rxq. 488 */ 489CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 490CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 491#endif 492 493/* No easy way to include t4_msg.h before adapter.h so we check this way */ 494CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 495CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 496 497CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 498 499static int 500t4_probe(device_t dev) 501{ 502 int i; 503 uint16_t v = pci_get_vendor(dev); 504 uint16_t d = pci_get_device(dev); 505 uint8_t f = pci_get_function(dev); 506 507 if (v != PCI_VENDOR_ID_CHELSIO) 508 return (ENXIO); 509 510 /* Attach only to PF0 of the FPGA */ 511 if (d == 0xa000 && f != 0) 512 return (ENXIO); 513 514 for (i = 0; i < nitems(t4_pciids); i++) { 515 if (d == t4_pciids[i].device) { 516 device_set_desc(dev, t4_pciids[i].desc); 517 return (BUS_PROBE_DEFAULT); 518 } 519 } 520 521 return (ENXIO); 522} 523 524static int 525t5_probe(device_t dev) 526{ 527 int i; 528 uint16_t v = pci_get_vendor(dev); 529 uint16_t d = pci_get_device(dev); 530 uint8_t f = pci_get_function(dev); 531 532 if (v != PCI_VENDOR_ID_CHELSIO) 533 return (ENXIO); 534 535 /* Attach only to PF0 of the FPGA */ 536 if (d == 0xb000 && f != 0) 537 return (ENXIO); 538 539 for (i = 0; i < nitems(t5_pciids); i++) { 540 if (d == t5_pciids[i].device) { 541 device_set_desc(dev, t5_pciids[i].desc); 542 return (BUS_PROBE_DEFAULT); 543 } 544 } 545 546 return (ENXIO); 547} 548 549static int 550t4_attach(device_t dev) 551{ 552 struct adapter *sc; 553 int rc = 0, i, n10g, n1g, rqidx, tqidx; 554 struct intrs_and_queues iaq; 555 struct sge *s; 556#ifdef TCP_OFFLOAD 557 int ofld_rqidx, ofld_tqidx; 558#endif 559 560 sc = device_get_softc(dev); 561 sc->dev = dev; 562 563 pci_enable_busmaster(dev); 564 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 565 uint32_t v; 566 567 pci_set_max_read_req(dev, 4096); 568 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 569 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 570 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 571 } 572 573 sc->traceq = -1; 574 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 575 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 576 device_get_nameunit(dev)); 577 578 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 579 device_get_nameunit(dev)); 580 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 581 sx_xlock(&t4_list_lock); 582 SLIST_INSERT_HEAD(&t4_list, sc, link); 583 sx_xunlock(&t4_list_lock); 584 585 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 586 TAILQ_INIT(&sc->sfl); 587 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 588 589 rc = map_bars_0_and_4(sc); 590 if (rc != 0) 591 goto done; /* error message displayed already */ 592 593 /* 594 * This is the real PF# to which we're attaching. Works from within PCI 595 * passthrough environments too, where pci_get_function() could return a 596 * different PF# depending on the passthrough configuration. We need to 597 * use the real PF# in all our communication with the firmware. 598 */ 599 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 600 sc->mbox = sc->pf; 601 602 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 603 sc->an_handler = an_not_handled; 604 for (i = 0; i < nitems(sc->cpl_handler); i++) 605 sc->cpl_handler[i] = cpl_not_handled; 606 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 607 sc->fw_msg_handler[i] = fw_msg_not_handled; 608 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 609 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 610 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 611 t4_init_sge_cpl_handlers(sc); 612 613 /* Prepare the adapter for operation */ 614 rc = -t4_prep_adapter(sc); 615 if (rc != 0) { 616 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 617 goto done; 618 } 619 620 /* 621 * Do this really early, with the memory windows set up even before the 622 * character device. The userland tool's register i/o and mem read 623 * will work even in "recovery mode". 624 */ 625 setup_memwin(sc); 626 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 627 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 628 device_get_nameunit(dev)); 629 if (sc->cdev == NULL) 630 device_printf(dev, "failed to create nexus char device.\n"); 631 else 632 sc->cdev->si_drv1 = sc; 633 634 /* Go no further if recovery mode has been requested. */ 635 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 636 device_printf(dev, "recovery mode.\n"); 637 goto done; 638 } 639 640 /* Prepare the firmware for operation */ 641 rc = prep_firmware(sc); 642 if (rc != 0) 643 goto done; /* error message displayed already */ 644 645 rc = get_params__post_init(sc); 646 if (rc != 0) 647 goto done; /* error message displayed already */ 648 649 rc = set_params__post_init(sc); 650 if (rc != 0) 651 goto done; /* error message displayed already */ 652 653 rc = map_bar_2(sc); 654 if (rc != 0) 655 goto done; /* error message displayed already */ 656 657 rc = t4_create_dma_tag(sc); 658 if (rc != 0) 659 goto done; /* error message displayed already */ 660 661 /* 662 * First pass over all the ports - allocate VIs and initialize some 663 * basic parameters like mac address, port type, etc. We also figure 664 * out whether a port is 10G or 1G and use that information when 665 * calculating how many interrupts to attempt to allocate. 666 */ 667 n10g = n1g = 0; 668 for_each_port(sc, i) { 669 struct port_info *pi; 670 671 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 672 sc->port[i] = pi; 673 674 /* These must be set before t4_port_init */ 675 pi->adapter = sc; 676 pi->port_id = i; 677 678 /* Allocate the vi and initialize parameters like mac addr */ 679 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 680 if (rc != 0) { 681 device_printf(dev, "unable to initialize port %d: %d\n", 682 i, rc); 683 free(pi, M_CXGBE); 684 sc->port[i] = NULL; 685 goto done; 686 } 687 688 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 689 device_get_nameunit(dev), i); 690 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 691 sc->chan_map[pi->tx_chan] = i; 692 693 if (is_10G_port(pi) || is_40G_port(pi)) { 694 n10g++; 695 pi->tmr_idx = t4_tmr_idx_10g; 696 pi->pktc_idx = t4_pktc_idx_10g; 697 } else { 698 n1g++; 699 pi->tmr_idx = t4_tmr_idx_1g; 700 pi->pktc_idx = t4_pktc_idx_1g; 701 } 702 703 pi->xact_addr_filt = -1; 704 pi->linkdnrc = -1; 705 706 pi->qsize_rxq = t4_qsize_rxq; 707 pi->qsize_txq = t4_qsize_txq; 708 709 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 710 if (pi->dev == NULL) { 711 device_printf(dev, 712 "failed to add device for port %d.\n", i); 713 rc = ENXIO; 714 goto done; 715 } 716 device_set_softc(pi->dev, pi); 717 } 718 719 /* 720 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 721 */ 722 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 723 if (rc != 0) 724 goto done; /* error message displayed already */ 725 726 sc->intr_type = iaq.intr_type; 727 sc->intr_count = iaq.nirq; 728 sc->flags |= iaq.intr_flags; 729 730 s = &sc->sge; 731 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 732 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 733 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 734 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 735 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 736 737#ifdef TCP_OFFLOAD 738 if (is_offload(sc)) { 739 740 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 741 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 742 s->neq += s->nofldtxq + s->nofldrxq; 743 s->niq += s->nofldrxq; 744 745 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 746 M_CXGBE, M_ZERO | M_WAITOK); 747 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 748 M_CXGBE, M_ZERO | M_WAITOK); 749 } 750#endif 751 752 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 753 M_ZERO | M_WAITOK); 754 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 755 M_ZERO | M_WAITOK); 756 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 757 M_ZERO | M_WAITOK); 758 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 759 M_ZERO | M_WAITOK); 760 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 761 M_ZERO | M_WAITOK); 762 763 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 764 M_ZERO | M_WAITOK); 765 766 t4_init_l2t(sc, M_WAITOK); 767 768 /* 769 * Second pass over the ports. This time we know the number of rx and 770 * tx queues that each port should get. 771 */ 772 rqidx = tqidx = 0; 773#ifdef TCP_OFFLOAD 774 ofld_rqidx = ofld_tqidx = 0; 775#endif 776 for_each_port(sc, i) { 777 struct port_info *pi = sc->port[i]; 778 779 if (pi == NULL) 780 continue; 781 782 pi->first_rxq = rqidx; 783 pi->first_txq = tqidx; 784 if (is_10G_port(pi) || is_40G_port(pi)) { 785 pi->nrxq = iaq.nrxq10g; 786 pi->ntxq = iaq.ntxq10g; 787 } else { 788 pi->nrxq = iaq.nrxq1g; 789 pi->ntxq = iaq.ntxq1g; 790 } 791 792 if (pi->ntxq > 1) 793 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 794 else 795 pi->rsrv_noflowq = 0; 796 797 rqidx += pi->nrxq; 798 tqidx += pi->ntxq; 799 800#ifdef TCP_OFFLOAD 801 if (is_offload(sc)) { 802 pi->first_ofld_rxq = ofld_rqidx; 803 pi->first_ofld_txq = ofld_tqidx; 804 if (is_10G_port(pi) || is_40G_port(pi)) { 805 pi->nofldrxq = iaq.nofldrxq10g; 806 pi->nofldtxq = iaq.nofldtxq10g; 807 } else { 808 pi->nofldrxq = iaq.nofldrxq1g; 809 pi->nofldtxq = iaq.nofldtxq1g; 810 } 811 ofld_rqidx += pi->nofldrxq; 812 ofld_tqidx += pi->nofldtxq; 813 } 814#endif 815 } 816 817 rc = setup_intr_handlers(sc); 818 if (rc != 0) { 819 device_printf(dev, 820 "failed to setup interrupt handlers: %d\n", rc); 821 goto done; 822 } 823 824 rc = bus_generic_attach(dev); 825 if (rc != 0) { 826 device_printf(dev, 827 "failed to attach all child ports: %d\n", rc); 828 goto done; 829 } 830 831 device_printf(dev, 832 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 833 sc->params.pci.width, sc->params.nports, sc->intr_count, 834 sc->intr_type == INTR_MSIX ? "MSI-X" : 835 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 836 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 837 838 t4_set_desc(sc); 839 840done: 841 if (rc != 0 && sc->cdev) { 842 /* cdev was created and so cxgbetool works; recover that way. */ 843 device_printf(dev, 844 "error during attach, adapter is now in recovery mode.\n"); 845 rc = 0; 846 } 847 848 if (rc != 0) 849 t4_detach(dev); 850 else 851 t4_sysctls(sc); 852 853 return (rc); 854} 855 856/* 857 * Idempotent 858 */ 859static int 860t4_detach(device_t dev) 861{ 862 struct adapter *sc; 863 struct port_info *pi; 864 int i, rc; 865 866 sc = device_get_softc(dev); 867 868 if (sc->flags & FULL_INIT_DONE) 869 t4_intr_disable(sc); 870 871 if (sc->cdev) { 872 destroy_dev(sc->cdev); 873 sc->cdev = NULL; 874 } 875 876 rc = bus_generic_detach(dev); 877 if (rc) { 878 device_printf(dev, 879 "failed to detach child devices: %d\n", rc); 880 return (rc); 881 } 882 883 for (i = 0; i < sc->intr_count; i++) 884 t4_free_irq(sc, &sc->irq[i]); 885 886 for (i = 0; i < MAX_NPORTS; i++) { 887 pi = sc->port[i]; 888 if (pi) { 889 t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 890 if (pi->dev) 891 device_delete_child(dev, pi->dev); 892 893 mtx_destroy(&pi->pi_lock); 894 free(pi, M_CXGBE); 895 } 896 } 897 898 if (sc->flags & FULL_INIT_DONE) 899 adapter_full_uninit(sc); 900 901 if (sc->flags & FW_OK) 902 t4_fw_bye(sc, sc->mbox); 903 904 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 905 pci_release_msi(dev); 906 907 if (sc->regs_res) 908 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 909 sc->regs_res); 910 911 if (sc->udbs_res) 912 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 913 sc->udbs_res); 914 915 if (sc->msix_res) 916 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 917 sc->msix_res); 918 919 if (sc->l2t) 920 t4_free_l2t(sc->l2t); 921 922#ifdef TCP_OFFLOAD 923 free(sc->sge.ofld_rxq, M_CXGBE); 924 free(sc->sge.ofld_txq, M_CXGBE); 925#endif 926 free(sc->irq, M_CXGBE); 927 free(sc->sge.rxq, M_CXGBE); 928 free(sc->sge.txq, M_CXGBE); 929 free(sc->sge.ctrlq, M_CXGBE); 930 free(sc->sge.iqmap, M_CXGBE); 931 free(sc->sge.eqmap, M_CXGBE); 932 free(sc->tids.ftid_tab, M_CXGBE); 933 t4_destroy_dma_tag(sc); 934 if (mtx_initialized(&sc->sc_lock)) { 935 sx_xlock(&t4_list_lock); 936 SLIST_REMOVE(&t4_list, sc, adapter, link); 937 sx_xunlock(&t4_list_lock); 938 mtx_destroy(&sc->sc_lock); 939 } 940 941 if (mtx_initialized(&sc->tids.ftid_lock)) 942 mtx_destroy(&sc->tids.ftid_lock); 943 if (mtx_initialized(&sc->sfl_lock)) 944 mtx_destroy(&sc->sfl_lock); 945 if (mtx_initialized(&sc->ifp_lock)) 946 mtx_destroy(&sc->ifp_lock); 947 948 bzero(sc, sizeof(*sc)); 949 950 return (0); 951} 952 953 954static int 955cxgbe_probe(device_t dev) 956{ 957 char buf[128]; 958 struct port_info *pi = device_get_softc(dev); 959 960 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 961 device_set_desc_copy(dev, buf); 962 963 return (BUS_PROBE_DEFAULT); 964} 965 966#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 967 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 968 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 969#define T4_CAP_ENABLE (T4_CAP) 970 971static int 972cxgbe_attach(device_t dev) 973{ 974 struct port_info *pi = device_get_softc(dev); 975 struct ifnet *ifp; 976 977 /* Allocate an ifnet and set it up */ 978 ifp = if_alloc(IFT_ETHER); 979 if (ifp == NULL) { 980 device_printf(dev, "Cannot allocate ifnet\n"); 981 return (ENOMEM); 982 } 983 pi->ifp = ifp; 984 ifp->if_softc = pi; 985 986 callout_init(&pi->tick, CALLOUT_MPSAFE); 987 988 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 989 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 990 991 ifp->if_init = cxgbe_init; 992 ifp->if_ioctl = cxgbe_ioctl; 993 ifp->if_transmit = cxgbe_transmit; 994 ifp->if_qflush = cxgbe_qflush; 995 996 ifp->if_capabilities = T4_CAP; 997#ifdef TCP_OFFLOAD 998 if (is_offload(pi->adapter)) 999 ifp->if_capabilities |= IFCAP_TOE; 1000#endif 1001 ifp->if_capenable = T4_CAP_ENABLE; 1002 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1003 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1004 1005 /* Initialize ifmedia for this port */ 1006 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1007 cxgbe_media_status); 1008 build_medialist(pi); 1009 1010 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1011 EVENTHANDLER_PRI_ANY); 1012 1013 ether_ifattach(ifp, pi->hw_addr); 1014 1015#ifdef TCP_OFFLOAD 1016 if (is_offload(pi->adapter)) { 1017 device_printf(dev, 1018 "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 1019 pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 1020 } else 1021#endif 1022 device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 1023 1024 cxgbe_sysctls(pi); 1025 1026 return (0); 1027} 1028 1029static int 1030cxgbe_detach(device_t dev) 1031{ 1032 struct port_info *pi = device_get_softc(dev); 1033 struct adapter *sc = pi->adapter; 1034 struct ifnet *ifp = pi->ifp; 1035 1036 /* Tell if_ioctl and if_init that the port is going away */ 1037 ADAPTER_LOCK(sc); 1038 SET_DOOMED(pi); 1039 wakeup(&sc->flags); 1040 while (IS_BUSY(sc)) 1041 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1042 SET_BUSY(sc); 1043#ifdef INVARIANTS 1044 sc->last_op = "t4detach"; 1045 sc->last_op_thr = curthread; 1046#endif 1047 ADAPTER_UNLOCK(sc); 1048 1049 if (pi->flags & HAS_TRACEQ) { 1050 sc->traceq = -1; /* cloner should not create ifnet */ 1051 t4_tracer_port_detach(sc); 1052 } 1053 1054 if (pi->vlan_c) 1055 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1056 1057 PORT_LOCK(pi); 1058 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1059 callout_stop(&pi->tick); 1060 PORT_UNLOCK(pi); 1061 callout_drain(&pi->tick); 1062 1063 /* Let detach proceed even if these fail. */ 1064 cxgbe_uninit_synchronized(pi); 1065 port_full_uninit(pi); 1066 1067 ifmedia_removeall(&pi->media); 1068 ether_ifdetach(pi->ifp); 1069 if_free(pi->ifp); 1070 1071 ADAPTER_LOCK(sc); 1072 CLR_BUSY(sc); 1073 wakeup(&sc->flags); 1074 ADAPTER_UNLOCK(sc); 1075 1076 return (0); 1077} 1078 1079static void 1080cxgbe_init(void *arg) 1081{ 1082 struct port_info *pi = arg; 1083 struct adapter *sc = pi->adapter; 1084 1085 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1086 return; 1087 cxgbe_init_synchronized(pi); 1088 end_synchronized_op(sc, 0); 1089} 1090 1091static int 1092cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1093{ 1094 int rc = 0, mtu, flags; 1095 struct port_info *pi = ifp->if_softc; 1096 struct adapter *sc = pi->adapter; 1097 struct ifreq *ifr = (struct ifreq *)data; 1098 uint32_t mask; 1099 1100 switch (cmd) { 1101 case SIOCSIFMTU: 1102 mtu = ifr->ifr_mtu; 1103 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1104 return (EINVAL); 1105 1106 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1107 if (rc) 1108 return (rc); 1109 ifp->if_mtu = mtu; 1110 if (pi->flags & PORT_INIT_DONE) { 1111 t4_update_fl_bufsize(ifp); 1112 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1113 rc = update_mac_settings(pi, XGMAC_MTU); 1114 } 1115 end_synchronized_op(sc, 0); 1116 break; 1117 1118 case SIOCSIFFLAGS: 1119 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1120 if (rc) 1121 return (rc); 1122 1123 if (ifp->if_flags & IFF_UP) { 1124 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1125 flags = pi->if_flags; 1126 if ((ifp->if_flags ^ flags) & 1127 (IFF_PROMISC | IFF_ALLMULTI)) { 1128 rc = update_mac_settings(pi, 1129 XGMAC_PROMISC | XGMAC_ALLMULTI); 1130 } 1131 } else 1132 rc = cxgbe_init_synchronized(pi); 1133 pi->if_flags = ifp->if_flags; 1134 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1135 rc = cxgbe_uninit_synchronized(pi); 1136 end_synchronized_op(sc, 0); 1137 break; 1138 1139 case SIOCADDMULTI: 1140 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1141 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1142 if (rc) 1143 return (rc); 1144 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1145 rc = update_mac_settings(pi, XGMAC_MCADDRS); 1146 end_synchronized_op(sc, LOCK_HELD); 1147 break; 1148 1149 case SIOCSIFCAP: 1150 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1151 if (rc) 1152 return (rc); 1153 1154 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1155 if (mask & IFCAP_TXCSUM) { 1156 ifp->if_capenable ^= IFCAP_TXCSUM; 1157 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1158 1159 if (IFCAP_TSO4 & ifp->if_capenable && 1160 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1161 ifp->if_capenable &= ~IFCAP_TSO4; 1162 if_printf(ifp, 1163 "tso4 disabled due to -txcsum.\n"); 1164 } 1165 } 1166 if (mask & IFCAP_TXCSUM_IPV6) { 1167 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1168 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1169 1170 if (IFCAP_TSO6 & ifp->if_capenable && 1171 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1172 ifp->if_capenable &= ~IFCAP_TSO6; 1173 if_printf(ifp, 1174 "tso6 disabled due to -txcsum6.\n"); 1175 } 1176 } 1177 if (mask & IFCAP_RXCSUM) 1178 ifp->if_capenable ^= IFCAP_RXCSUM; 1179 if (mask & IFCAP_RXCSUM_IPV6) 1180 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1181 1182 /* 1183 * Note that we leave CSUM_TSO alone (it is always set). The 1184 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1185 * sending a TSO request our way, so it's sufficient to toggle 1186 * IFCAP_TSOx only. 1187 */ 1188 if (mask & IFCAP_TSO4) { 1189 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1190 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1191 if_printf(ifp, "enable txcsum first.\n"); 1192 rc = EAGAIN; 1193 goto fail; 1194 } 1195 ifp->if_capenable ^= IFCAP_TSO4; 1196 } 1197 if (mask & IFCAP_TSO6) { 1198 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1199 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1200 if_printf(ifp, "enable txcsum6 first.\n"); 1201 rc = EAGAIN; 1202 goto fail; 1203 } 1204 ifp->if_capenable ^= IFCAP_TSO6; 1205 } 1206 if (mask & IFCAP_LRO) { 1207#if defined(INET) || defined(INET6) 1208 int i; 1209 struct sge_rxq *rxq; 1210 1211 ifp->if_capenable ^= IFCAP_LRO; 1212 for_each_rxq(pi, i, rxq) { 1213 if (ifp->if_capenable & IFCAP_LRO) 1214 rxq->iq.flags |= IQ_LRO_ENABLED; 1215 else 1216 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1217 } 1218#endif 1219 } 1220#ifdef TCP_OFFLOAD 1221 if (mask & IFCAP_TOE) { 1222 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1223 1224 rc = toe_capability(pi, enable); 1225 if (rc != 0) 1226 goto fail; 1227 1228 ifp->if_capenable ^= mask; 1229 } 1230#endif 1231 if (mask & IFCAP_VLAN_HWTAGGING) { 1232 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1233 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1234 rc = update_mac_settings(pi, XGMAC_VLANEX); 1235 } 1236 if (mask & IFCAP_VLAN_MTU) { 1237 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1238 1239 /* Need to find out how to disable auto-mtu-inflation */ 1240 } 1241 if (mask & IFCAP_VLAN_HWTSO) 1242 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1243 if (mask & IFCAP_VLAN_HWCSUM) 1244 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1245 1246#ifdef VLAN_CAPABILITIES 1247 VLAN_CAPABILITIES(ifp); 1248#endif 1249fail: 1250 end_synchronized_op(sc, 0); 1251 break; 1252 1253 case SIOCSIFMEDIA: 1254 case SIOCGIFMEDIA: 1255 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1256 break; 1257 1258 default: 1259 rc = ether_ioctl(ifp, cmd, data); 1260 } 1261 1262 return (rc); 1263} 1264 1265static int 1266cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1267{ 1268 struct port_info *pi = ifp->if_softc; 1269 struct adapter *sc = pi->adapter; 1270 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1271 struct buf_ring *br; 1272 int rc; 1273 1274 M_ASSERTPKTHDR(m); 1275 1276 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1277 m_freem(m); 1278 return (ENETDOWN); 1279 } 1280 1281 if (m->m_flags & M_FLOWID) 1282 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1283 + pi->rsrv_noflowq); 1284 br = txq->br; 1285 1286 if (TXQ_TRYLOCK(txq) == 0) { 1287 struct sge_eq *eq = &txq->eq; 1288 1289 /* 1290 * It is possible that t4_eth_tx finishes up and releases the 1291 * lock between the TRYLOCK above and the drbr_enqueue here. We 1292 * need to make sure that this mbuf doesn't just sit there in 1293 * the drbr. 1294 */ 1295 1296 rc = drbr_enqueue(ifp, br, m); 1297 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1298 !(eq->flags & EQ_DOOMED)) 1299 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1300 return (rc); 1301 } 1302 1303 /* 1304 * txq->m is the mbuf that is held up due to a temporary shortage of 1305 * resources and it should be put on the wire first. Then what's in 1306 * drbr and finally the mbuf that was just passed in to us. 1307 * 1308 * Return code should indicate the fate of the mbuf that was passed in 1309 * this time. 1310 */ 1311 1312 TXQ_LOCK_ASSERT_OWNED(txq); 1313 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1314 1315 /* Queued for transmission. */ 1316 1317 rc = drbr_enqueue(ifp, br, m); 1318 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1319 (void) t4_eth_tx(ifp, txq, m); 1320 TXQ_UNLOCK(txq); 1321 return (rc); 1322 } 1323 1324 /* Direct transmission. */ 1325 rc = t4_eth_tx(ifp, txq, m); 1326 if (rc != 0 && txq->m) 1327 rc = 0; /* held, will be transmitted soon (hopefully) */ 1328 1329 TXQ_UNLOCK(txq); 1330 return (rc); 1331} 1332 1333static void 1334cxgbe_qflush(struct ifnet *ifp) 1335{ 1336 struct port_info *pi = ifp->if_softc; 1337 struct sge_txq *txq; 1338 int i; 1339 struct mbuf *m; 1340 1341 /* queues do not exist if !PORT_INIT_DONE. */ 1342 if (pi->flags & PORT_INIT_DONE) { 1343 for_each_txq(pi, i, txq) { 1344 TXQ_LOCK(txq); 1345 m_freem(txq->m); 1346 txq->m = NULL; 1347 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1348 m_freem(m); 1349 TXQ_UNLOCK(txq); 1350 } 1351 } 1352 if_qflush(ifp); 1353} 1354 1355static int 1356cxgbe_media_change(struct ifnet *ifp) 1357{ 1358 struct port_info *pi = ifp->if_softc; 1359 1360 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1361 1362 return (EOPNOTSUPP); 1363} 1364 1365static void 1366cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1367{ 1368 struct port_info *pi = ifp->if_softc; 1369 struct ifmedia_entry *cur = pi->media.ifm_cur; 1370 int speed = pi->link_cfg.speed; 1371 int data = (pi->port_type << 8) | pi->mod_type; 1372 1373 if (cur->ifm_data != data) { 1374 build_medialist(pi); 1375 cur = pi->media.ifm_cur; 1376 } 1377 1378 ifmr->ifm_status = IFM_AVALID; 1379 if (!pi->link_cfg.link_ok) 1380 return; 1381 1382 ifmr->ifm_status |= IFM_ACTIVE; 1383 1384 /* active and current will differ iff current media is autoselect. */ 1385 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1386 return; 1387 1388 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1389 if (speed == SPEED_10000) 1390 ifmr->ifm_active |= IFM_10G_T; 1391 else if (speed == SPEED_1000) 1392 ifmr->ifm_active |= IFM_1000_T; 1393 else if (speed == SPEED_100) 1394 ifmr->ifm_active |= IFM_100_TX; 1395 else if (speed == SPEED_10) 1396 ifmr->ifm_active |= IFM_10_T; 1397 else 1398 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1399 speed)); 1400} 1401 1402void 1403t4_fatal_err(struct adapter *sc) 1404{ 1405 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1406 t4_intr_disable(sc); 1407 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1408 device_get_nameunit(sc->dev)); 1409} 1410 1411static int 1412map_bars_0_and_4(struct adapter *sc) 1413{ 1414 sc->regs_rid = PCIR_BAR(0); 1415 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1416 &sc->regs_rid, RF_ACTIVE); 1417 if (sc->regs_res == NULL) { 1418 device_printf(sc->dev, "cannot map registers.\n"); 1419 return (ENXIO); 1420 } 1421 sc->bt = rman_get_bustag(sc->regs_res); 1422 sc->bh = rman_get_bushandle(sc->regs_res); 1423 sc->mmio_len = rman_get_size(sc->regs_res); 1424 setbit(&sc->doorbells, DOORBELL_KDB); 1425 1426 sc->msix_rid = PCIR_BAR(4); 1427 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1428 &sc->msix_rid, RF_ACTIVE); 1429 if (sc->msix_res == NULL) { 1430 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1431 return (ENXIO); 1432 } 1433 1434 return (0); 1435} 1436 1437static int 1438map_bar_2(struct adapter *sc) 1439{ 1440 1441 /* 1442 * T4: only iWARP driver uses the userspace doorbells. There is no need 1443 * to map it if RDMA is disabled. 1444 */ 1445 if (is_t4(sc) && sc->rdmacaps == 0) 1446 return (0); 1447 1448 sc->udbs_rid = PCIR_BAR(2); 1449 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1450 &sc->udbs_rid, RF_ACTIVE); 1451 if (sc->udbs_res == NULL) { 1452 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1453 return (ENXIO); 1454 } 1455 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1456 1457 if (is_t5(sc)) { 1458 setbit(&sc->doorbells, DOORBELL_UDB); 1459#if defined(__i386__) || defined(__amd64__) 1460 if (t5_write_combine) { 1461 int rc; 1462 1463 /* 1464 * Enable write combining on BAR2. This is the 1465 * userspace doorbell BAR and is split into 128B 1466 * (UDBS_SEG_SIZE) doorbell regions, each associated 1467 * with an egress queue. The first 64B has the doorbell 1468 * and the second 64B can be used to submit a tx work 1469 * request with an implicit doorbell. 1470 */ 1471 1472 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1473 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1474 if (rc == 0) { 1475 clrbit(&sc->doorbells, DOORBELL_UDB); 1476 setbit(&sc->doorbells, DOORBELL_WCWR); 1477 setbit(&sc->doorbells, DOORBELL_UDBWC); 1478 } else { 1479 device_printf(sc->dev, 1480 "couldn't enable write combining: %d\n", 1481 rc); 1482 } 1483 1484 t4_write_reg(sc, A_SGE_STAT_CFG, 1485 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1486 } 1487#endif 1488 } 1489 1490 return (0); 1491} 1492 1493static const struct memwin t4_memwin[] = { 1494 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1495 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1496 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1497}; 1498 1499static const struct memwin t5_memwin[] = { 1500 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1501 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1502 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1503}; 1504 1505static void 1506setup_memwin(struct adapter *sc) 1507{ 1508 const struct memwin *mw; 1509 int i, n; 1510 uint32_t bar0; 1511 1512 if (is_t4(sc)) { 1513 /* 1514 * Read low 32b of bar0 indirectly via the hardware backdoor 1515 * mechanism. Works from within PCI passthrough environments 1516 * too, where rman_get_start() can return a different value. We 1517 * need to program the T4 memory window decoders with the actual 1518 * addresses that will be coming across the PCIe link. 1519 */ 1520 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1521 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1522 1523 mw = &t4_memwin[0]; 1524 n = nitems(t4_memwin); 1525 } else { 1526 /* T5 uses the relative offset inside the PCIe BAR */ 1527 bar0 = 0; 1528 1529 mw = &t5_memwin[0]; 1530 n = nitems(t5_memwin); 1531 } 1532 1533 for (i = 0; i < n; i++, mw++) { 1534 t4_write_reg(sc, 1535 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1536 (mw->base + bar0) | V_BIR(0) | 1537 V_WINDOW(ilog2(mw->aperture) - 10)); 1538 } 1539 1540 /* flush */ 1541 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1542} 1543 1544/* 1545 * Verify that the memory range specified by the addr/len pair is valid and lies 1546 * entirely within a single region (EDCx or MCx). 1547 */ 1548static int 1549validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1550{ 1551 uint32_t em, addr_len, maddr, mlen; 1552 1553 /* Memory can only be accessed in naturally aligned 4 byte units */ 1554 if (addr & 3 || len & 3 || len == 0) 1555 return (EINVAL); 1556 1557 /* Enabled memories */ 1558 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1559 if (em & F_EDRAM0_ENABLE) { 1560 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1561 maddr = G_EDRAM0_BASE(addr_len) << 20; 1562 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1563 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1564 addr + len <= maddr + mlen) 1565 return (0); 1566 } 1567 if (em & F_EDRAM1_ENABLE) { 1568 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1569 maddr = G_EDRAM1_BASE(addr_len) << 20; 1570 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1571 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1572 addr + len <= maddr + mlen) 1573 return (0); 1574 } 1575 if (em & F_EXT_MEM_ENABLE) { 1576 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1577 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1578 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1579 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1580 addr + len <= maddr + mlen) 1581 return (0); 1582 } 1583 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1584 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1585 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1586 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1587 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1588 addr + len <= maddr + mlen) 1589 return (0); 1590 } 1591 1592 return (EFAULT); 1593} 1594 1595static int 1596fwmtype_to_hwmtype(int mtype) 1597{ 1598 1599 switch (mtype) { 1600 case FW_MEMTYPE_EDC0: 1601 return (MEM_EDC0); 1602 case FW_MEMTYPE_EDC1: 1603 return (MEM_EDC1); 1604 case FW_MEMTYPE_EXTMEM: 1605 return (MEM_MC0); 1606 case FW_MEMTYPE_EXTMEM1: 1607 return (MEM_MC1); 1608 default: 1609 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1610 } 1611} 1612 1613/* 1614 * Verify that the memory range specified by the memtype/offset/len pair is 1615 * valid and lies entirely within the memtype specified. The global address of 1616 * the start of the range is returned in addr. 1617 */ 1618static int 1619validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1620 uint32_t *addr) 1621{ 1622 uint32_t em, addr_len, maddr, mlen; 1623 1624 /* Memory can only be accessed in naturally aligned 4 byte units */ 1625 if (off & 3 || len & 3 || len == 0) 1626 return (EINVAL); 1627 1628 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1629 switch (fwmtype_to_hwmtype(mtype)) { 1630 case MEM_EDC0: 1631 if (!(em & F_EDRAM0_ENABLE)) 1632 return (EINVAL); 1633 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1634 maddr = G_EDRAM0_BASE(addr_len) << 20; 1635 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1636 break; 1637 case MEM_EDC1: 1638 if (!(em & F_EDRAM1_ENABLE)) 1639 return (EINVAL); 1640 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1641 maddr = G_EDRAM1_BASE(addr_len) << 20; 1642 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1643 break; 1644 case MEM_MC: 1645 if (!(em & F_EXT_MEM_ENABLE)) 1646 return (EINVAL); 1647 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1648 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1649 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1650 break; 1651 case MEM_MC1: 1652 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1653 return (EINVAL); 1654 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1655 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1656 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1657 break; 1658 default: 1659 return (EINVAL); 1660 } 1661 1662 if (mlen > 0 && off < mlen && off + len <= mlen) { 1663 *addr = maddr + off; /* global address */ 1664 return (0); 1665 } 1666 1667 return (EFAULT); 1668} 1669 1670static void 1671memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1672{ 1673 const struct memwin *mw; 1674 1675 if (is_t4(sc)) { 1676 KASSERT(win >= 0 && win < nitems(t4_memwin), 1677 ("%s: incorrect memwin# (%d)", __func__, win)); 1678 mw = &t4_memwin[win]; 1679 } else { 1680 KASSERT(win >= 0 && win < nitems(t5_memwin), 1681 ("%s: incorrect memwin# (%d)", __func__, win)); 1682 mw = &t5_memwin[win]; 1683 } 1684 1685 if (base != NULL) 1686 *base = mw->base; 1687 if (aperture != NULL) 1688 *aperture = mw->aperture; 1689} 1690 1691/* 1692 * Positions the memory window such that it can be used to access the specified 1693 * address in the chip's address space. The return value is the offset of addr 1694 * from the start of the window. 1695 */ 1696static uint32_t 1697position_memwin(struct adapter *sc, int n, uint32_t addr) 1698{ 1699 uint32_t start, pf; 1700 uint32_t reg; 1701 1702 KASSERT(n >= 0 && n <= 3, 1703 ("%s: invalid window %d.", __func__, n)); 1704 KASSERT((addr & 3) == 0, 1705 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1706 1707 if (is_t4(sc)) { 1708 pf = 0; 1709 start = addr & ~0xf; /* start must be 16B aligned */ 1710 } else { 1711 pf = V_PFNUM(sc->pf); 1712 start = addr & ~0x7f; /* start must be 128B aligned */ 1713 } 1714 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1715 1716 t4_write_reg(sc, reg, start | pf); 1717 t4_read_reg(sc, reg); 1718 1719 return (addr - start); 1720} 1721 1722static int 1723cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1724 struct intrs_and_queues *iaq) 1725{ 1726 int rc, itype, navail, nrxq10g, nrxq1g, n; 1727 int nofldrxq10g = 0, nofldrxq1g = 0; 1728 1729 bzero(iaq, sizeof(*iaq)); 1730 1731 iaq->ntxq10g = t4_ntxq10g; 1732 iaq->ntxq1g = t4_ntxq1g; 1733 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1734 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1735 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1736#ifdef TCP_OFFLOAD 1737 if (is_offload(sc)) { 1738 iaq->nofldtxq10g = t4_nofldtxq10g; 1739 iaq->nofldtxq1g = t4_nofldtxq1g; 1740 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1741 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1742 } 1743#endif 1744 1745 for (itype = INTR_MSIX; itype; itype >>= 1) { 1746 1747 if ((itype & t4_intr_types) == 0) 1748 continue; /* not allowed */ 1749 1750 if (itype == INTR_MSIX) 1751 navail = pci_msix_count(sc->dev); 1752 else if (itype == INTR_MSI) 1753 navail = pci_msi_count(sc->dev); 1754 else 1755 navail = 1; 1756restart: 1757 if (navail == 0) 1758 continue; 1759 1760 iaq->intr_type = itype; 1761 iaq->intr_flags = 0; 1762 1763 /* 1764 * Best option: an interrupt vector for errors, one for the 1765 * firmware event queue, and one each for each rxq (NIC as well 1766 * as offload). 1767 */ 1768 iaq->nirq = T4_EXTRA_INTR; 1769 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1770 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1771 if (iaq->nirq <= navail && 1772 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1773 iaq->intr_flags |= INTR_DIRECT; 1774 goto allocate; 1775 } 1776 1777 /* 1778 * Second best option: an interrupt vector for errors, one for 1779 * the firmware event queue, and one each for either NIC or 1780 * offload rxq's. 1781 */ 1782 iaq->nirq = T4_EXTRA_INTR; 1783 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1784 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1785 if (iaq->nirq <= navail && 1786 (itype != INTR_MSI || powerof2(iaq->nirq))) 1787 goto allocate; 1788 1789 /* 1790 * Next best option: an interrupt vector for errors, one for the 1791 * firmware event queue, and at least one per port. At this 1792 * point we know we'll have to downsize nrxq or nofldrxq to fit 1793 * what's available to us. 1794 */ 1795 iaq->nirq = T4_EXTRA_INTR; 1796 iaq->nirq += n10g + n1g; 1797 if (iaq->nirq <= navail) { 1798 int leftover = navail - iaq->nirq; 1799 1800 if (n10g > 0) { 1801 int target = max(nrxq10g, nofldrxq10g); 1802 1803 n = 1; 1804 while (n < target && leftover >= n10g) { 1805 leftover -= n10g; 1806 iaq->nirq += n10g; 1807 n++; 1808 } 1809 iaq->nrxq10g = min(n, nrxq10g); 1810#ifdef TCP_OFFLOAD 1811 if (is_offload(sc)) 1812 iaq->nofldrxq10g = min(n, nofldrxq10g); 1813#endif 1814 } 1815 1816 if (n1g > 0) { 1817 int target = max(nrxq1g, nofldrxq1g); 1818 1819 n = 1; 1820 while (n < target && leftover >= n1g) { 1821 leftover -= n1g; 1822 iaq->nirq += n1g; 1823 n++; 1824 } 1825 iaq->nrxq1g = min(n, nrxq1g); 1826#ifdef TCP_OFFLOAD 1827 if (is_offload(sc)) 1828 iaq->nofldrxq1g = min(n, nofldrxq1g); 1829#endif 1830 } 1831 1832 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1833 goto allocate; 1834 } 1835 1836 /* 1837 * Least desirable option: one interrupt vector for everything. 1838 */ 1839 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1840#ifdef TCP_OFFLOAD 1841 if (is_offload(sc)) 1842 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1843#endif 1844 1845allocate: 1846 navail = iaq->nirq; 1847 rc = 0; 1848 if (itype == INTR_MSIX) 1849 rc = pci_alloc_msix(sc->dev, &navail); 1850 else if (itype == INTR_MSI) 1851 rc = pci_alloc_msi(sc->dev, &navail); 1852 1853 if (rc == 0) { 1854 if (navail == iaq->nirq) 1855 return (0); 1856 1857 /* 1858 * Didn't get the number requested. Use whatever number 1859 * the kernel is willing to allocate (it's in navail). 1860 */ 1861 device_printf(sc->dev, "fewer vectors than requested, " 1862 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1863 itype, iaq->nirq, navail); 1864 pci_release_msi(sc->dev); 1865 goto restart; 1866 } 1867 1868 device_printf(sc->dev, 1869 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1870 itype, rc, iaq->nirq, navail); 1871 } 1872 1873 device_printf(sc->dev, 1874 "failed to find a usable interrupt type. " 1875 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1876 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1877 1878 return (ENXIO); 1879} 1880 1881#define FW_VERSION(chip) ( \ 1882 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 1883 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 1884 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 1885 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 1886#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 1887 1888struct fw_info { 1889 uint8_t chip; 1890 char *kld_name; 1891 char *fw_mod_name; 1892 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 1893} fw_info[] = { 1894 { 1895 .chip = CHELSIO_T4, 1896 .kld_name = "t4fw_cfg", 1897 .fw_mod_name = "t4fw", 1898 .fw_hdr = { 1899 .chip = FW_HDR_CHIP_T4, 1900 .fw_ver = htobe32_const(FW_VERSION(T4)), 1901 .intfver_nic = FW_INTFVER(T4, NIC), 1902 .intfver_vnic = FW_INTFVER(T4, VNIC), 1903 .intfver_ofld = FW_INTFVER(T4, OFLD), 1904 .intfver_ri = FW_INTFVER(T4, RI), 1905 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 1906 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 1907 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 1908 .intfver_fcoe = FW_INTFVER(T4, FCOE), 1909 }, 1910 }, { 1911 .chip = CHELSIO_T5, 1912 .kld_name = "t5fw_cfg", 1913 .fw_mod_name = "t5fw", 1914 .fw_hdr = { 1915 .chip = FW_HDR_CHIP_T5, 1916 .fw_ver = htobe32_const(FW_VERSION(T5)), 1917 .intfver_nic = FW_INTFVER(T5, NIC), 1918 .intfver_vnic = FW_INTFVER(T5, VNIC), 1919 .intfver_ofld = FW_INTFVER(T5, OFLD), 1920 .intfver_ri = FW_INTFVER(T5, RI), 1921 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 1922 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 1923 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 1924 .intfver_fcoe = FW_INTFVER(T5, FCOE), 1925 }, 1926 } 1927}; 1928 1929static struct fw_info * 1930find_fw_info(int chip) 1931{ 1932 int i; 1933 1934 for (i = 0; i < nitems(fw_info); i++) { 1935 if (fw_info[i].chip == chip) 1936 return (&fw_info[i]); 1937 } 1938 return (NULL); 1939} 1940 1941/* 1942 * Is the given firmware API compatible with the one the driver was compiled 1943 * with? 1944 */ 1945static int 1946fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 1947{ 1948 1949 /* short circuit if it's the exact same firmware version */ 1950 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 1951 return (1); 1952 1953 /* 1954 * XXX: Is this too conservative? Perhaps I should limit this to the 1955 * features that are supported in the driver. 1956 */ 1957#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 1958 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 1959 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 1960 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 1961 return (1); 1962#undef SAME_INTF 1963 1964 return (0); 1965} 1966 1967/* 1968 * The firmware in the KLD is usable, but should it be installed? This routine 1969 * explains itself in detail if it indicates the KLD firmware should be 1970 * installed. 1971 */ 1972static int 1973should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 1974{ 1975 const char *reason; 1976 1977 if (!card_fw_usable) { 1978 reason = "incompatible or unusable"; 1979 goto install; 1980 } 1981 1982 if (k > c) { 1983 reason = "older than the version bundled with this driver"; 1984 goto install; 1985 } 1986 1987 if (t4_fw_install == 2 && k != c) { 1988 reason = "different than the version bundled with this driver"; 1989 goto install; 1990 } 1991 1992 return (0); 1993 1994install: 1995 if (t4_fw_install == 0) { 1996 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 1997 "but the driver is prohibited from installing a different " 1998 "firmware on the card.\n", 1999 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2000 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2001 2002 return (0); 2003 } 2004 2005 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2006 "installing firmware %u.%u.%u.%u on card.\n", 2007 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2008 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2009 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2010 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2011 2012 return (1); 2013} 2014/* 2015 * Establish contact with the firmware and determine if we are the master driver 2016 * or not, and whether we are responsible for chip initialization. 2017 */ 2018static int 2019prep_firmware(struct adapter *sc) 2020{ 2021 const struct firmware *fw = NULL, *default_cfg; 2022 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2023 enum dev_state state; 2024 struct fw_info *fw_info; 2025 struct fw_hdr *card_fw; /* fw on the card */ 2026 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2027 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2028 against */ 2029 2030 /* Contact firmware. */ 2031 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2032 if (rc < 0 || state == DEV_STATE_ERR) { 2033 rc = -rc; 2034 device_printf(sc->dev, 2035 "failed to connect to the firmware: %d, %d.\n", rc, state); 2036 return (rc); 2037 } 2038 pf = rc; 2039 if (pf == sc->mbox) 2040 sc->flags |= MASTER_PF; 2041 else if (state == DEV_STATE_UNINIT) { 2042 /* 2043 * We didn't get to be the master so we definitely won't be 2044 * configuring the chip. It's a bug if someone else hasn't 2045 * configured it already. 2046 */ 2047 device_printf(sc->dev, "couldn't be master(%d), " 2048 "device not already initialized either(%d).\n", rc, state); 2049 return (EDOOFUS); 2050 } 2051 2052 /* This is the firmware whose headers the driver was compiled against */ 2053 fw_info = find_fw_info(chip_id(sc)); 2054 if (fw_info == NULL) { 2055 device_printf(sc->dev, 2056 "unable to look up firmware information for chip %d.\n", 2057 chip_id(sc)); 2058 return (EINVAL); 2059 } 2060 drv_fw = &fw_info->fw_hdr; 2061 2062 /* 2063 * The firmware KLD contains many modules. The KLD name is also the 2064 * name of the module that contains the default config file. 2065 */ 2066 default_cfg = firmware_get(fw_info->kld_name); 2067 2068 /* Read the header of the firmware on the card */ 2069 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2070 rc = -t4_read_flash(sc, FLASH_FW_START, 2071 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2072 if (rc == 0) 2073 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2074 else { 2075 device_printf(sc->dev, 2076 "Unable to read card's firmware header: %d\n", rc); 2077 card_fw_usable = 0; 2078 } 2079 2080 /* This is the firmware in the KLD */ 2081 fw = firmware_get(fw_info->fw_mod_name); 2082 if (fw != NULL) { 2083 kld_fw = (const void *)fw->data; 2084 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2085 } else { 2086 kld_fw = NULL; 2087 kld_fw_usable = 0; 2088 } 2089 2090 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2091 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2092 /* 2093 * Common case: the firmware on the card is an exact match and 2094 * the KLD is an exact match too, or the KLD is 2095 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2096 * here -- use cxgbetool loadfw if you want to reinstall the 2097 * same firmware as the one on the card. 2098 */ 2099 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2100 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2101 be32toh(card_fw->fw_ver))) { 2102 2103 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2104 if (rc != 0) { 2105 device_printf(sc->dev, 2106 "failed to install firmware: %d\n", rc); 2107 goto done; 2108 } 2109 2110 /* Installed successfully, update the cached header too. */ 2111 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2112 card_fw_usable = 1; 2113 need_fw_reset = 0; /* already reset as part of load_fw */ 2114 } 2115 2116 if (!card_fw_usable) { 2117 uint32_t d, c, k; 2118 2119 d = ntohl(drv_fw->fw_ver); 2120 c = ntohl(card_fw->fw_ver); 2121 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2122 2123 device_printf(sc->dev, "Cannot find a usable firmware: " 2124 "fw_install %d, chip state %d, " 2125 "driver compiled with %d.%d.%d.%d, " 2126 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2127 t4_fw_install, state, 2128 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2129 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2130 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2131 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2132 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2133 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2134 rc = EINVAL; 2135 goto done; 2136 } 2137 2138 /* We're using whatever's on the card and it's known to be good. */ 2139 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2140 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2141 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2142 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2143 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2144 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2145 t4_get_tp_version(sc, &sc->params.tp_vers); 2146 2147 /* Reset device */ 2148 if (need_fw_reset && 2149 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2150 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2151 if (rc != ETIMEDOUT && rc != EIO) 2152 t4_fw_bye(sc, sc->mbox); 2153 goto done; 2154 } 2155 sc->flags |= FW_OK; 2156 2157 rc = get_params__pre_init(sc); 2158 if (rc != 0) 2159 goto done; /* error message displayed already */ 2160 2161 /* Partition adapter resources as specified in the config file. */ 2162 if (state == DEV_STATE_UNINIT) { 2163 2164 KASSERT(sc->flags & MASTER_PF, 2165 ("%s: trying to change chip settings when not master.", 2166 __func__)); 2167 2168 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2169 if (rc != 0) 2170 goto done; /* error message displayed already */ 2171 2172 t4_tweak_chip_settings(sc); 2173 2174 /* get basic stuff going */ 2175 rc = -t4_fw_initialize(sc, sc->mbox); 2176 if (rc != 0) { 2177 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2178 goto done; 2179 } 2180 } else { 2181 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2182 sc->cfcsum = 0; 2183 } 2184 2185done: 2186 free(card_fw, M_CXGBE); 2187 if (fw != NULL) 2188 firmware_put(fw, FIRMWARE_UNLOAD); 2189 if (default_cfg != NULL) 2190 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2191 2192 return (rc); 2193} 2194 2195#define FW_PARAM_DEV(param) \ 2196 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2197 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2198#define FW_PARAM_PFVF(param) \ 2199 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2200 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2201 2202/* 2203 * Partition chip resources for use between various PFs, VFs, etc. 2204 */ 2205static int 2206partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2207 const char *name_prefix) 2208{ 2209 const struct firmware *cfg = NULL; 2210 int rc = 0; 2211 struct fw_caps_config_cmd caps; 2212 uint32_t mtype, moff, finicsum, cfcsum; 2213 2214 /* 2215 * Figure out what configuration file to use. Pick the default config 2216 * file for the card if the user hasn't specified one explicitly. 2217 */ 2218 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2219 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2220 /* Card specific overrides go here. */ 2221 if (pci_get_device(sc->dev) == 0x440a) 2222 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2223 if (is_fpga(sc)) 2224 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2225 } 2226 2227 /* 2228 * We need to load another module if the profile is anything except 2229 * "default" or "flash". 2230 */ 2231 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2232 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2233 char s[32]; 2234 2235 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2236 cfg = firmware_get(s); 2237 if (cfg == NULL) { 2238 if (default_cfg != NULL) { 2239 device_printf(sc->dev, 2240 "unable to load module \"%s\" for " 2241 "configuration profile \"%s\", will use " 2242 "the default config file instead.\n", 2243 s, sc->cfg_file); 2244 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2245 "%s", DEFAULT_CF); 2246 } else { 2247 device_printf(sc->dev, 2248 "unable to load module \"%s\" for " 2249 "configuration profile \"%s\", will use " 2250 "the config file on the card's flash " 2251 "instead.\n", s, sc->cfg_file); 2252 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2253 "%s", FLASH_CF); 2254 } 2255 } 2256 } 2257 2258 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2259 default_cfg == NULL) { 2260 device_printf(sc->dev, 2261 "default config file not available, will use the config " 2262 "file on the card's flash instead.\n"); 2263 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2264 } 2265 2266 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2267 u_int cflen, i, n; 2268 const uint32_t *cfdata; 2269 uint32_t param, val, addr, off, mw_base, mw_aperture; 2270 2271 KASSERT(cfg != NULL || default_cfg != NULL, 2272 ("%s: no config to upload", __func__)); 2273 2274 /* 2275 * Ask the firmware where it wants us to upload the config file. 2276 */ 2277 param = FW_PARAM_DEV(CF); 2278 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2279 if (rc != 0) { 2280 /* No support for config file? Shouldn't happen. */ 2281 device_printf(sc->dev, 2282 "failed to query config file location: %d.\n", rc); 2283 goto done; 2284 } 2285 mtype = G_FW_PARAMS_PARAM_Y(val); 2286 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2287 2288 /* 2289 * XXX: sheer laziness. We deliberately added 4 bytes of 2290 * useless stuffing/comments at the end of the config file so 2291 * it's ok to simply throw away the last remaining bytes when 2292 * the config file is not an exact multiple of 4. This also 2293 * helps with the validate_mt_off_len check. 2294 */ 2295 if (cfg != NULL) { 2296 cflen = cfg->datasize & ~3; 2297 cfdata = cfg->data; 2298 } else { 2299 cflen = default_cfg->datasize & ~3; 2300 cfdata = default_cfg->data; 2301 } 2302 2303 if (cflen > FLASH_CFG_MAX_SIZE) { 2304 device_printf(sc->dev, 2305 "config file too long (%d, max allowed is %d). " 2306 "Will try to use the config on the card, if any.\n", 2307 cflen, FLASH_CFG_MAX_SIZE); 2308 goto use_config_on_flash; 2309 } 2310 2311 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2312 if (rc != 0) { 2313 device_printf(sc->dev, 2314 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2315 "Will try to use the config on the card, if any.\n", 2316 __func__, mtype, moff, cflen, rc); 2317 goto use_config_on_flash; 2318 } 2319 2320 memwin_info(sc, 2, &mw_base, &mw_aperture); 2321 while (cflen) { 2322 off = position_memwin(sc, 2, addr); 2323 n = min(cflen, mw_aperture - off); 2324 for (i = 0; i < n; i += 4) 2325 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2326 cflen -= n; 2327 addr += n; 2328 } 2329 } else { 2330use_config_on_flash: 2331 mtype = FW_MEMTYPE_FLASH; 2332 moff = t4_flash_cfg_addr(sc); 2333 } 2334 2335 bzero(&caps, sizeof(caps)); 2336 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2337 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2338 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2339 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2340 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2341 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2342 if (rc != 0) { 2343 device_printf(sc->dev, 2344 "failed to pre-process config file: %d " 2345 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2346 goto done; 2347 } 2348 2349 finicsum = be32toh(caps.finicsum); 2350 cfcsum = be32toh(caps.cfcsum); 2351 if (finicsum != cfcsum) { 2352 device_printf(sc->dev, 2353 "WARNING: config file checksum mismatch: %08x %08x\n", 2354 finicsum, cfcsum); 2355 } 2356 sc->cfcsum = cfcsum; 2357 2358#define LIMIT_CAPS(x) do { \ 2359 caps.x &= htobe16(t4_##x##_allowed); \ 2360 sc->x = htobe16(caps.x); \ 2361} while (0) 2362 2363 /* 2364 * Let the firmware know what features will (not) be used so it can tune 2365 * things accordingly. 2366 */ 2367 LIMIT_CAPS(linkcaps); 2368 LIMIT_CAPS(niccaps); 2369 LIMIT_CAPS(toecaps); 2370 LIMIT_CAPS(rdmacaps); 2371 LIMIT_CAPS(iscsicaps); 2372 LIMIT_CAPS(fcoecaps); 2373#undef LIMIT_CAPS 2374 2375 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2376 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2377 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2378 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2379 if (rc != 0) { 2380 device_printf(sc->dev, 2381 "failed to process config file: %d.\n", rc); 2382 } 2383done: 2384 if (cfg != NULL) 2385 firmware_put(cfg, FIRMWARE_UNLOAD); 2386 return (rc); 2387} 2388 2389/* 2390 * Retrieve parameters that are needed (or nice to have) very early. 2391 */ 2392static int 2393get_params__pre_init(struct adapter *sc) 2394{ 2395 int rc; 2396 uint32_t param[2], val[2]; 2397 struct fw_devlog_cmd cmd; 2398 struct devlog_params *dlog = &sc->params.devlog; 2399 2400 param[0] = FW_PARAM_DEV(PORTVEC); 2401 param[1] = FW_PARAM_DEV(CCLK); 2402 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2403 if (rc != 0) { 2404 device_printf(sc->dev, 2405 "failed to query parameters (pre_init): %d.\n", rc); 2406 return (rc); 2407 } 2408 2409 sc->params.portvec = val[0]; 2410 sc->params.nports = bitcount32(val[0]); 2411 sc->params.vpd.cclk = val[1]; 2412 2413 /* Read device log parameters. */ 2414 bzero(&cmd, sizeof(cmd)); 2415 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2416 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2417 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2418 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2419 if (rc != 0) { 2420 device_printf(sc->dev, 2421 "failed to get devlog parameters: %d.\n", rc); 2422 bzero(dlog, sizeof (*dlog)); 2423 rc = 0; /* devlog isn't critical for device operation */ 2424 } else { 2425 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2426 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2427 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2428 dlog->size = be32toh(cmd.memsize_devlog); 2429 } 2430 2431 return (rc); 2432} 2433 2434/* 2435 * Retrieve various parameters that are of interest to the driver. The device 2436 * has been initialized by the firmware at this point. 2437 */ 2438static int 2439get_params__post_init(struct adapter *sc) 2440{ 2441 int rc; 2442 uint32_t param[7], val[7]; 2443 struct fw_caps_config_cmd caps; 2444 2445 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2446 param[1] = FW_PARAM_PFVF(EQ_START); 2447 param[2] = FW_PARAM_PFVF(FILTER_START); 2448 param[3] = FW_PARAM_PFVF(FILTER_END); 2449 param[4] = FW_PARAM_PFVF(L2T_START); 2450 param[5] = FW_PARAM_PFVF(L2T_END); 2451 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2452 if (rc != 0) { 2453 device_printf(sc->dev, 2454 "failed to query parameters (post_init): %d.\n", rc); 2455 return (rc); 2456 } 2457 2458 sc->sge.iq_start = val[0]; 2459 sc->sge.eq_start = val[1]; 2460 sc->tids.ftid_base = val[2]; 2461 sc->tids.nftids = val[3] - val[2] + 1; 2462 sc->vres.l2t.start = val[4]; 2463 sc->vres.l2t.size = val[5] - val[4] + 1; 2464 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2465 ("%s: L2 table size (%u) larger than expected (%u)", 2466 __func__, sc->vres.l2t.size, L2T_SIZE)); 2467 2468 /* get capabilites */ 2469 bzero(&caps, sizeof(caps)); 2470 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2471 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2472 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2473 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2474 if (rc != 0) { 2475 device_printf(sc->dev, 2476 "failed to get card capabilities: %d.\n", rc); 2477 return (rc); 2478 } 2479 2480 if (caps.toecaps) { 2481 /* query offload-related parameters */ 2482 param[0] = FW_PARAM_DEV(NTID); 2483 param[1] = FW_PARAM_PFVF(SERVER_START); 2484 param[2] = FW_PARAM_PFVF(SERVER_END); 2485 param[3] = FW_PARAM_PFVF(TDDP_START); 2486 param[4] = FW_PARAM_PFVF(TDDP_END); 2487 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2488 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2489 if (rc != 0) { 2490 device_printf(sc->dev, 2491 "failed to query TOE parameters: %d.\n", rc); 2492 return (rc); 2493 } 2494 sc->tids.ntids = val[0]; 2495 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2496 sc->tids.stid_base = val[1]; 2497 sc->tids.nstids = val[2] - val[1] + 1; 2498 sc->vres.ddp.start = val[3]; 2499 sc->vres.ddp.size = val[4] - val[3] + 1; 2500 sc->params.ofldq_wr_cred = val[5]; 2501 sc->params.offload = 1; 2502 } 2503 if (caps.rdmacaps) { 2504 param[0] = FW_PARAM_PFVF(STAG_START); 2505 param[1] = FW_PARAM_PFVF(STAG_END); 2506 param[2] = FW_PARAM_PFVF(RQ_START); 2507 param[3] = FW_PARAM_PFVF(RQ_END); 2508 param[4] = FW_PARAM_PFVF(PBL_START); 2509 param[5] = FW_PARAM_PFVF(PBL_END); 2510 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2511 if (rc != 0) { 2512 device_printf(sc->dev, 2513 "failed to query RDMA parameters(1): %d.\n", rc); 2514 return (rc); 2515 } 2516 sc->vres.stag.start = val[0]; 2517 sc->vres.stag.size = val[1] - val[0] + 1; 2518 sc->vres.rq.start = val[2]; 2519 sc->vres.rq.size = val[3] - val[2] + 1; 2520 sc->vres.pbl.start = val[4]; 2521 sc->vres.pbl.size = val[5] - val[4] + 1; 2522 2523 param[0] = FW_PARAM_PFVF(SQRQ_START); 2524 param[1] = FW_PARAM_PFVF(SQRQ_END); 2525 param[2] = FW_PARAM_PFVF(CQ_START); 2526 param[3] = FW_PARAM_PFVF(CQ_END); 2527 param[4] = FW_PARAM_PFVF(OCQ_START); 2528 param[5] = FW_PARAM_PFVF(OCQ_END); 2529 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2530 if (rc != 0) { 2531 device_printf(sc->dev, 2532 "failed to query RDMA parameters(2): %d.\n", rc); 2533 return (rc); 2534 } 2535 sc->vres.qp.start = val[0]; 2536 sc->vres.qp.size = val[1] - val[0] + 1; 2537 sc->vres.cq.start = val[2]; 2538 sc->vres.cq.size = val[3] - val[2] + 1; 2539 sc->vres.ocq.start = val[4]; 2540 sc->vres.ocq.size = val[5] - val[4] + 1; 2541 } 2542 if (caps.iscsicaps) { 2543 param[0] = FW_PARAM_PFVF(ISCSI_START); 2544 param[1] = FW_PARAM_PFVF(ISCSI_END); 2545 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2546 if (rc != 0) { 2547 device_printf(sc->dev, 2548 "failed to query iSCSI parameters: %d.\n", rc); 2549 return (rc); 2550 } 2551 sc->vres.iscsi.start = val[0]; 2552 sc->vres.iscsi.size = val[1] - val[0] + 1; 2553 } 2554 2555 /* 2556 * We've got the params we wanted to query via the firmware. Now grab 2557 * some others directly from the chip. 2558 */ 2559 rc = t4_read_chip_settings(sc); 2560 2561 return (rc); 2562} 2563 2564static int 2565set_params__post_init(struct adapter *sc) 2566{ 2567 uint32_t param, val; 2568 2569 /* ask for encapsulated CPLs */ 2570 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2571 val = 1; 2572 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2573 2574 return (0); 2575} 2576 2577#undef FW_PARAM_PFVF 2578#undef FW_PARAM_DEV 2579 2580static void 2581t4_set_desc(struct adapter *sc) 2582{ 2583 char buf[128]; 2584 struct adapter_params *p = &sc->params; 2585 2586 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2587 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2588 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2589 2590 device_set_desc_copy(sc->dev, buf); 2591} 2592 2593static void 2594build_medialist(struct port_info *pi) 2595{ 2596 struct ifmedia *media = &pi->media; 2597 int data, m; 2598 2599 PORT_LOCK(pi); 2600 2601 ifmedia_removeall(media); 2602 2603 m = IFM_ETHER | IFM_FDX; 2604 data = (pi->port_type << 8) | pi->mod_type; 2605 2606 switch(pi->port_type) { 2607 case FW_PORT_TYPE_BT_XFI: 2608 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2609 break; 2610 2611 case FW_PORT_TYPE_BT_XAUI: 2612 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2613 /* fall through */ 2614 2615 case FW_PORT_TYPE_BT_SGMII: 2616 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2617 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2618 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2619 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2620 break; 2621 2622 case FW_PORT_TYPE_CX4: 2623 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2624 ifmedia_set(media, m | IFM_10G_CX4); 2625 break; 2626 2627 case FW_PORT_TYPE_QSFP_10G: 2628 case FW_PORT_TYPE_SFP: 2629 case FW_PORT_TYPE_FIBER_XFI: 2630 case FW_PORT_TYPE_FIBER_XAUI: 2631 switch (pi->mod_type) { 2632 2633 case FW_PORT_MOD_TYPE_LR: 2634 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2635 ifmedia_set(media, m | IFM_10G_LR); 2636 break; 2637 2638 case FW_PORT_MOD_TYPE_SR: 2639 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2640 ifmedia_set(media, m | IFM_10G_SR); 2641 break; 2642 2643 case FW_PORT_MOD_TYPE_LRM: 2644 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2645 ifmedia_set(media, m | IFM_10G_LRM); 2646 break; 2647 2648 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2649 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2650 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2651 ifmedia_set(media, m | IFM_10G_TWINAX); 2652 break; 2653 2654 case FW_PORT_MOD_TYPE_NONE: 2655 m &= ~IFM_FDX; 2656 ifmedia_add(media, m | IFM_NONE, data, NULL); 2657 ifmedia_set(media, m | IFM_NONE); 2658 break; 2659 2660 case FW_PORT_MOD_TYPE_NA: 2661 case FW_PORT_MOD_TYPE_ER: 2662 default: 2663 device_printf(pi->dev, 2664 "unknown port_type (%d), mod_type (%d)\n", 2665 pi->port_type, pi->mod_type); 2666 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2667 ifmedia_set(media, m | IFM_UNKNOWN); 2668 break; 2669 } 2670 break; 2671 2672 case FW_PORT_TYPE_QSFP: 2673 switch (pi->mod_type) { 2674 2675 case FW_PORT_MOD_TYPE_LR: 2676 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2677 ifmedia_set(media, m | IFM_40G_LR4); 2678 break; 2679 2680 case FW_PORT_MOD_TYPE_SR: 2681 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2682 ifmedia_set(media, m | IFM_40G_SR4); 2683 break; 2684 2685 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2686 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2687 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2688 ifmedia_set(media, m | IFM_40G_CR4); 2689 break; 2690 2691 case FW_PORT_MOD_TYPE_NONE: 2692 m &= ~IFM_FDX; 2693 ifmedia_add(media, m | IFM_NONE, data, NULL); 2694 ifmedia_set(media, m | IFM_NONE); 2695 break; 2696 2697 default: 2698 device_printf(pi->dev, 2699 "unknown port_type (%d), mod_type (%d)\n", 2700 pi->port_type, pi->mod_type); 2701 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2702 ifmedia_set(media, m | IFM_UNKNOWN); 2703 break; 2704 } 2705 break; 2706 2707 default: 2708 device_printf(pi->dev, 2709 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2710 pi->mod_type); 2711 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2712 ifmedia_set(media, m | IFM_UNKNOWN); 2713 break; 2714 } 2715 2716 PORT_UNLOCK(pi); 2717} 2718 2719#define FW_MAC_EXACT_CHUNK 7 2720 2721/* 2722 * Program the port's XGMAC based on parameters in ifnet. The caller also 2723 * indicates which parameters should be programmed (the rest are left alone). 2724 */ 2725static int 2726update_mac_settings(struct port_info *pi, int flags) 2727{ 2728 int rc; 2729 struct ifnet *ifp = pi->ifp; 2730 struct adapter *sc = pi->adapter; 2731 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2732 2733 ASSERT_SYNCHRONIZED_OP(sc); 2734 KASSERT(flags, ("%s: not told what to update.", __func__)); 2735 2736 if (flags & XGMAC_MTU) 2737 mtu = ifp->if_mtu; 2738 2739 if (flags & XGMAC_PROMISC) 2740 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2741 2742 if (flags & XGMAC_ALLMULTI) 2743 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2744 2745 if (flags & XGMAC_VLANEX) 2746 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2747 2748 rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2749 vlanex, false); 2750 if (rc) { 2751 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2752 return (rc); 2753 } 2754 2755 if (flags & XGMAC_UCADDR) { 2756 uint8_t ucaddr[ETHER_ADDR_LEN]; 2757 2758 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2759 rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2760 ucaddr, true, true); 2761 if (rc < 0) { 2762 rc = -rc; 2763 if_printf(ifp, "change_mac failed: %d\n", rc); 2764 return (rc); 2765 } else { 2766 pi->xact_addr_filt = rc; 2767 rc = 0; 2768 } 2769 } 2770 2771 if (flags & XGMAC_MCADDRS) { 2772 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2773 int del = 1; 2774 uint64_t hash = 0; 2775 struct ifmultiaddr *ifma; 2776 int i = 0, j; 2777 2778 if_maddr_rlock(ifp); 2779 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2780 if (ifma->ifma_addr->sa_family != AF_LINK) 2781 continue; 2782 mcaddr[i++] = 2783 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2784 2785 if (i == FW_MAC_EXACT_CHUNK) { 2786 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2787 del, i, mcaddr, NULL, &hash, 0); 2788 if (rc < 0) { 2789 rc = -rc; 2790 for (j = 0; j < i; j++) { 2791 if_printf(ifp, 2792 "failed to add mc address" 2793 " %02x:%02x:%02x:" 2794 "%02x:%02x:%02x rc=%d\n", 2795 mcaddr[j][0], mcaddr[j][1], 2796 mcaddr[j][2], mcaddr[j][3], 2797 mcaddr[j][4], mcaddr[j][5], 2798 rc); 2799 } 2800 goto mcfail; 2801 } 2802 del = 0; 2803 i = 0; 2804 } 2805 } 2806 if (i > 0) { 2807 rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2808 del, i, mcaddr, NULL, &hash, 0); 2809 if (rc < 0) { 2810 rc = -rc; 2811 for (j = 0; j < i; j++) { 2812 if_printf(ifp, 2813 "failed to add mc address" 2814 " %02x:%02x:%02x:" 2815 "%02x:%02x:%02x rc=%d\n", 2816 mcaddr[j][0], mcaddr[j][1], 2817 mcaddr[j][2], mcaddr[j][3], 2818 mcaddr[j][4], mcaddr[j][5], 2819 rc); 2820 } 2821 goto mcfail; 2822 } 2823 } 2824 2825 rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2826 if (rc != 0) 2827 if_printf(ifp, "failed to set mc address hash: %d", rc); 2828mcfail: 2829 if_maddr_runlock(ifp); 2830 } 2831 2832 return (rc); 2833} 2834 2835int 2836begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 2837 char *wmesg) 2838{ 2839 int rc, pri; 2840 2841#ifdef WITNESS 2842 /* the caller thinks it's ok to sleep, but is it really? */ 2843 if (flags & SLEEP_OK) 2844 pause("t4slptst", 1); 2845#endif 2846 2847 if (INTR_OK) 2848 pri = PCATCH; 2849 else 2850 pri = 0; 2851 2852 ADAPTER_LOCK(sc); 2853 for (;;) { 2854 2855 if (pi && IS_DOOMED(pi)) { 2856 rc = ENXIO; 2857 goto done; 2858 } 2859 2860 if (!IS_BUSY(sc)) { 2861 rc = 0; 2862 break; 2863 } 2864 2865 if (!(flags & SLEEP_OK)) { 2866 rc = EBUSY; 2867 goto done; 2868 } 2869 2870 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 2871 rc = EINTR; 2872 goto done; 2873 } 2874 } 2875 2876 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2877 SET_BUSY(sc); 2878#ifdef INVARIANTS 2879 sc->last_op = wmesg; 2880 sc->last_op_thr = curthread; 2881#endif 2882 2883done: 2884 if (!(flags & HOLD_LOCK) || rc) 2885 ADAPTER_UNLOCK(sc); 2886 2887 return (rc); 2888} 2889 2890void 2891end_synchronized_op(struct adapter *sc, int flags) 2892{ 2893 2894 if (flags & LOCK_HELD) 2895 ADAPTER_LOCK_ASSERT_OWNED(sc); 2896 else 2897 ADAPTER_LOCK(sc); 2898 2899 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2900 CLR_BUSY(sc); 2901 wakeup(&sc->flags); 2902 ADAPTER_UNLOCK(sc); 2903} 2904 2905static int 2906cxgbe_init_synchronized(struct port_info *pi) 2907{ 2908 struct adapter *sc = pi->adapter; 2909 struct ifnet *ifp = pi->ifp; 2910 int rc = 0; 2911 2912 ASSERT_SYNCHRONIZED_OP(sc); 2913 2914 if (isset(&sc->open_device_map, pi->port_id)) { 2915 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2916 ("mismatch between open_device_map and if_drv_flags")); 2917 return (0); /* already running */ 2918 } 2919 2920 if (!(sc->flags & FULL_INIT_DONE) && 2921 ((rc = adapter_full_init(sc)) != 0)) 2922 return (rc); /* error message displayed already */ 2923 2924 if (!(pi->flags & PORT_INIT_DONE) && 2925 ((rc = port_full_init(pi)) != 0)) 2926 return (rc); /* error message displayed already */ 2927 2928 rc = update_mac_settings(pi, XGMAC_ALL); 2929 if (rc) 2930 goto done; /* error message displayed already */ 2931 2932 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2933 if (rc != 0) { 2934 if_printf(ifp, "start_link failed: %d\n", rc); 2935 goto done; 2936 } 2937 2938 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2939 if (rc != 0) { 2940 if_printf(ifp, "enable_vi failed: %d\n", rc); 2941 goto done; 2942 } 2943 2944 /* 2945 * The first iq of the first port to come up is used for tracing. 2946 */ 2947 if (sc->traceq < 0) { 2948 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 2949 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 2950 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 2951 V_QUEUENUMBER(sc->traceq)); 2952 pi->flags |= HAS_TRACEQ; 2953 } 2954 2955 /* all ok */ 2956 setbit(&sc->open_device_map, pi->port_id); 2957 PORT_LOCK(pi); 2958 ifp->if_drv_flags |= IFF_DRV_RUNNING; 2959 PORT_UNLOCK(pi); 2960 2961 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2962done: 2963 if (rc != 0) 2964 cxgbe_uninit_synchronized(pi); 2965 2966 return (rc); 2967} 2968 2969/* 2970 * Idempotent. 2971 */ 2972static int 2973cxgbe_uninit_synchronized(struct port_info *pi) 2974{ 2975 struct adapter *sc = pi->adapter; 2976 struct ifnet *ifp = pi->ifp; 2977 int rc; 2978 2979 ASSERT_SYNCHRONIZED_OP(sc); 2980 2981 /* 2982 * Disable the VI so that all its data in either direction is discarded 2983 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2984 * tick) intact as the TP can deliver negative advice or data that it's 2985 * holding in its RAM (for an offloaded connection) even after the VI is 2986 * disabled. 2987 */ 2988 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2989 if (rc) { 2990 if_printf(ifp, "disable_vi failed: %d\n", rc); 2991 return (rc); 2992 } 2993 2994 clrbit(&sc->open_device_map, pi->port_id); 2995 PORT_LOCK(pi); 2996 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2997 PORT_UNLOCK(pi); 2998 2999 pi->link_cfg.link_ok = 0; 3000 pi->link_cfg.speed = 0; 3001 pi->linkdnrc = -1; 3002 t4_os_link_changed(sc, pi->port_id, 0, -1); 3003 3004 return (0); 3005} 3006 3007/* 3008 * It is ok for this function to fail midway and return right away. t4_detach 3009 * will walk the entire sc->irq list and clean up whatever is valid. 3010 */ 3011static int 3012setup_intr_handlers(struct adapter *sc) 3013{ 3014 int rc, rid, p, q; 3015 char s[8]; 3016 struct irq *irq; 3017 struct port_info *pi; 3018 struct sge_rxq *rxq; 3019#ifdef TCP_OFFLOAD 3020 struct sge_ofld_rxq *ofld_rxq; 3021#endif 3022 3023 /* 3024 * Setup interrupts. 3025 */ 3026 irq = &sc->irq[0]; 3027 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3028 if (sc->intr_count == 1) { 3029 KASSERT(!(sc->flags & INTR_DIRECT), 3030 ("%s: single interrupt && INTR_DIRECT?", __func__)); 3031 3032 rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 3033 if (rc != 0) 3034 return (rc); 3035 } else { 3036 /* Multiple interrupts. */ 3037 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3038 ("%s: too few intr.", __func__)); 3039 3040 /* The first one is always error intr */ 3041 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3042 if (rc != 0) 3043 return (rc); 3044 irq++; 3045 rid++; 3046 3047 /* The second one is always the firmware event queue */ 3048 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 3049 "evt"); 3050 if (rc != 0) 3051 return (rc); 3052 irq++; 3053 rid++; 3054 3055 /* 3056 * Note that if INTR_DIRECT is not set then either the NIC rx 3057 * queues or (exclusive or) the TOE rx queueus will be taking 3058 * direct interrupts. 3059 * 3060 * There is no need to check for is_offload(sc) as nofldrxq 3061 * will be 0 if offload is disabled. 3062 */ 3063 for_each_port(sc, p) { 3064 pi = sc->port[p]; 3065 3066#ifdef TCP_OFFLOAD 3067 /* 3068 * Skip over the NIC queues if they aren't taking direct 3069 * interrupts. 3070 */ 3071 if (!(sc->flags & INTR_DIRECT) && 3072 pi->nofldrxq > pi->nrxq) 3073 goto ofld_queues; 3074#endif 3075 rxq = &sc->sge.rxq[pi->first_rxq]; 3076 for (q = 0; q < pi->nrxq; q++, rxq++) { 3077 snprintf(s, sizeof(s), "%d.%d", p, q); 3078 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3079 s); 3080 if (rc != 0) 3081 return (rc); 3082 irq++; 3083 rid++; 3084 } 3085 3086#ifdef TCP_OFFLOAD 3087 /* 3088 * Skip over the offload queues if they aren't taking 3089 * direct interrupts. 3090 */ 3091 if (!(sc->flags & INTR_DIRECT)) 3092 continue; 3093ofld_queues: 3094 ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 3095 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 3096 snprintf(s, sizeof(s), "%d,%d", p, q); 3097 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3098 ofld_rxq, s); 3099 if (rc != 0) 3100 return (rc); 3101 irq++; 3102 rid++; 3103 } 3104#endif 3105 } 3106 } 3107 3108 return (0); 3109} 3110 3111static int 3112adapter_full_init(struct adapter *sc) 3113{ 3114 int rc, i; 3115 3116 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3117 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3118 ("%s: FULL_INIT_DONE already", __func__)); 3119 3120 /* 3121 * queues that belong to the adapter (not any particular port). 3122 */ 3123 rc = t4_setup_adapter_queues(sc); 3124 if (rc != 0) 3125 goto done; 3126 3127 for (i = 0; i < nitems(sc->tq); i++) { 3128 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3129 taskqueue_thread_enqueue, &sc->tq[i]); 3130 if (sc->tq[i] == NULL) { 3131 device_printf(sc->dev, 3132 "failed to allocate task queue %d\n", i); 3133 rc = ENOMEM; 3134 goto done; 3135 } 3136 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3137 device_get_nameunit(sc->dev), i); 3138 } 3139 3140 t4_intr_enable(sc); 3141 sc->flags |= FULL_INIT_DONE; 3142done: 3143 if (rc != 0) 3144 adapter_full_uninit(sc); 3145 3146 return (rc); 3147} 3148 3149static int 3150adapter_full_uninit(struct adapter *sc) 3151{ 3152 int i; 3153 3154 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3155 3156 t4_teardown_adapter_queues(sc); 3157 3158 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3159 taskqueue_free(sc->tq[i]); 3160 sc->tq[i] = NULL; 3161 } 3162 3163 sc->flags &= ~FULL_INIT_DONE; 3164 3165 return (0); 3166} 3167 3168static int 3169port_full_init(struct port_info *pi) 3170{ 3171 struct adapter *sc = pi->adapter; 3172 struct ifnet *ifp = pi->ifp; 3173 uint16_t *rss; 3174 struct sge_rxq *rxq; 3175 int rc, i, j; 3176 3177 ASSERT_SYNCHRONIZED_OP(sc); 3178 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3179 ("%s: PORT_INIT_DONE already", __func__)); 3180 3181 sysctl_ctx_init(&pi->ctx); 3182 pi->flags |= PORT_SYSCTL_CTX; 3183 3184 /* 3185 * Allocate tx/rx/fl queues for this port. 3186 */ 3187 rc = t4_setup_port_queues(pi); 3188 if (rc != 0) 3189 goto done; /* error message displayed already */ 3190 3191 /* 3192 * Setup RSS for this port. Save a copy of the RSS table for later use. 3193 */ 3194 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3195 for (i = 0; i < pi->rss_size;) { 3196 for_each_rxq(pi, j, rxq) { 3197 rss[i++] = rxq->iq.abs_id; 3198 if (i == pi->rss_size) 3199 break; 3200 } 3201 } 3202 3203 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3204 pi->rss_size); 3205 if (rc != 0) { 3206 if_printf(ifp, "rss_config failed: %d\n", rc); 3207 goto done; 3208 } 3209 3210 pi->rss = rss; 3211 pi->flags |= PORT_INIT_DONE; 3212done: 3213 if (rc != 0) 3214 port_full_uninit(pi); 3215 3216 return (rc); 3217} 3218 3219/* 3220 * Idempotent. 3221 */ 3222static int 3223port_full_uninit(struct port_info *pi) 3224{ 3225 struct adapter *sc = pi->adapter; 3226 int i; 3227 struct sge_rxq *rxq; 3228 struct sge_txq *txq; 3229#ifdef TCP_OFFLOAD 3230 struct sge_ofld_rxq *ofld_rxq; 3231 struct sge_wrq *ofld_txq; 3232#endif 3233 3234 if (pi->flags & PORT_INIT_DONE) { 3235 3236 /* Need to quiesce queues. XXX: ctrl queues? */ 3237 3238 for_each_txq(pi, i, txq) { 3239 quiesce_eq(sc, &txq->eq); 3240 } 3241 3242#ifdef TCP_OFFLOAD 3243 for_each_ofld_txq(pi, i, ofld_txq) { 3244 quiesce_eq(sc, &ofld_txq->eq); 3245 } 3246#endif 3247 3248 for_each_rxq(pi, i, rxq) { 3249 quiesce_iq(sc, &rxq->iq); 3250 quiesce_fl(sc, &rxq->fl); 3251 } 3252 3253#ifdef TCP_OFFLOAD 3254 for_each_ofld_rxq(pi, i, ofld_rxq) { 3255 quiesce_iq(sc, &ofld_rxq->iq); 3256 quiesce_fl(sc, &ofld_rxq->fl); 3257 } 3258#endif 3259 free(pi->rss, M_CXGBE); 3260 } 3261 3262 t4_teardown_port_queues(pi); 3263 pi->flags &= ~PORT_INIT_DONE; 3264 3265 return (0); 3266} 3267 3268static void 3269quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3270{ 3271 EQ_LOCK(eq); 3272 eq->flags |= EQ_DOOMED; 3273 3274 /* 3275 * Wait for the response to a credit flush if one's 3276 * pending. 3277 */ 3278 while (eq->flags & EQ_CRFLUSHED) 3279 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3280 EQ_UNLOCK(eq); 3281 3282 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3283 pause("callout", 10); /* Still iffy */ 3284 3285 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3286} 3287 3288static void 3289quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3290{ 3291 (void) sc; /* unused */ 3292 3293 /* Synchronize with the interrupt handler */ 3294 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3295 pause("iqfree", 1); 3296} 3297 3298static void 3299quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3300{ 3301 mtx_lock(&sc->sfl_lock); 3302 FL_LOCK(fl); 3303 fl->flags |= FL_DOOMED; 3304 FL_UNLOCK(fl); 3305 mtx_unlock(&sc->sfl_lock); 3306 3307 callout_drain(&sc->sfl_callout); 3308 KASSERT((fl->flags & FL_STARVING) == 0, 3309 ("%s: still starving", __func__)); 3310} 3311 3312static int 3313t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3314 driver_intr_t *handler, void *arg, char *name) 3315{ 3316 int rc; 3317 3318 irq->rid = rid; 3319 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3320 RF_SHAREABLE | RF_ACTIVE); 3321 if (irq->res == NULL) { 3322 device_printf(sc->dev, 3323 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3324 return (ENOMEM); 3325 } 3326 3327 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3328 NULL, handler, arg, &irq->tag); 3329 if (rc != 0) { 3330 device_printf(sc->dev, 3331 "failed to setup interrupt for rid %d, name %s: %d\n", 3332 rid, name, rc); 3333 } else if (name) 3334 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3335 3336 return (rc); 3337} 3338 3339static int 3340t4_free_irq(struct adapter *sc, struct irq *irq) 3341{ 3342 if (irq->tag) 3343 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3344 if (irq->res) 3345 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3346 3347 bzero(irq, sizeof(*irq)); 3348 3349 return (0); 3350} 3351 3352static void 3353reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3354 unsigned int end) 3355{ 3356 uint32_t *p = (uint32_t *)(buf + start); 3357 3358 for ( ; start <= end; start += sizeof(uint32_t)) 3359 *p++ = t4_read_reg(sc, start); 3360} 3361 3362static void 3363t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3364{ 3365 int i, n; 3366 const unsigned int *reg_ranges; 3367 static const unsigned int t4_reg_ranges[] = { 3368 0x1008, 0x1108, 3369 0x1180, 0x11b4, 3370 0x11fc, 0x123c, 3371 0x1300, 0x173c, 3372 0x1800, 0x18fc, 3373 0x3000, 0x30d8, 3374 0x30e0, 0x5924, 3375 0x5960, 0x59d4, 3376 0x5a00, 0x5af8, 3377 0x6000, 0x6098, 3378 0x6100, 0x6150, 3379 0x6200, 0x6208, 3380 0x6240, 0x6248, 3381 0x6280, 0x6338, 3382 0x6370, 0x638c, 3383 0x6400, 0x643c, 3384 0x6500, 0x6524, 3385 0x6a00, 0x6a38, 3386 0x6a60, 0x6a78, 3387 0x6b00, 0x6b84, 3388 0x6bf0, 0x6c84, 3389 0x6cf0, 0x6d84, 3390 0x6df0, 0x6e84, 3391 0x6ef0, 0x6f84, 3392 0x6ff0, 0x7084, 3393 0x70f0, 0x7184, 3394 0x71f0, 0x7284, 3395 0x72f0, 0x7384, 3396 0x73f0, 0x7450, 3397 0x7500, 0x7530, 3398 0x7600, 0x761c, 3399 0x7680, 0x76cc, 3400 0x7700, 0x7798, 3401 0x77c0, 0x77fc, 3402 0x7900, 0x79fc, 3403 0x7b00, 0x7c38, 3404 0x7d00, 0x7efc, 3405 0x8dc0, 0x8e1c, 3406 0x8e30, 0x8e78, 3407 0x8ea0, 0x8f6c, 3408 0x8fc0, 0x9074, 3409 0x90fc, 0x90fc, 3410 0x9400, 0x9458, 3411 0x9600, 0x96bc, 3412 0x9800, 0x9808, 3413 0x9820, 0x983c, 3414 0x9850, 0x9864, 3415 0x9c00, 0x9c6c, 3416 0x9c80, 0x9cec, 3417 0x9d00, 0x9d6c, 3418 0x9d80, 0x9dec, 3419 0x9e00, 0x9e6c, 3420 0x9e80, 0x9eec, 3421 0x9f00, 0x9f6c, 3422 0x9f80, 0x9fec, 3423 0xd004, 0xd03c, 3424 0xdfc0, 0xdfe0, 3425 0xe000, 0xea7c, 3426 0xf000, 0x11110, 3427 0x11118, 0x11190, 3428 0x19040, 0x1906c, 3429 0x19078, 0x19080, 3430 0x1908c, 0x19124, 3431 0x19150, 0x191b0, 3432 0x191d0, 0x191e8, 3433 0x19238, 0x1924c, 3434 0x193f8, 0x19474, 3435 0x19490, 0x194f8, 3436 0x19800, 0x19f30, 3437 0x1a000, 0x1a06c, 3438 0x1a0b0, 0x1a120, 3439 0x1a128, 0x1a138, 3440 0x1a190, 0x1a1c4, 3441 0x1a1fc, 0x1a1fc, 3442 0x1e040, 0x1e04c, 3443 0x1e284, 0x1e28c, 3444 0x1e2c0, 0x1e2c0, 3445 0x1e2e0, 0x1e2e0, 3446 0x1e300, 0x1e384, 3447 0x1e3c0, 0x1e3c8, 3448 0x1e440, 0x1e44c, 3449 0x1e684, 0x1e68c, 3450 0x1e6c0, 0x1e6c0, 3451 0x1e6e0, 0x1e6e0, 3452 0x1e700, 0x1e784, 3453 0x1e7c0, 0x1e7c8, 3454 0x1e840, 0x1e84c, 3455 0x1ea84, 0x1ea8c, 3456 0x1eac0, 0x1eac0, 3457 0x1eae0, 0x1eae0, 3458 0x1eb00, 0x1eb84, 3459 0x1ebc0, 0x1ebc8, 3460 0x1ec40, 0x1ec4c, 3461 0x1ee84, 0x1ee8c, 3462 0x1eec0, 0x1eec0, 3463 0x1eee0, 0x1eee0, 3464 0x1ef00, 0x1ef84, 3465 0x1efc0, 0x1efc8, 3466 0x1f040, 0x1f04c, 3467 0x1f284, 0x1f28c, 3468 0x1f2c0, 0x1f2c0, 3469 0x1f2e0, 0x1f2e0, 3470 0x1f300, 0x1f384, 3471 0x1f3c0, 0x1f3c8, 3472 0x1f440, 0x1f44c, 3473 0x1f684, 0x1f68c, 3474 0x1f6c0, 0x1f6c0, 3475 0x1f6e0, 0x1f6e0, 3476 0x1f700, 0x1f784, 3477 0x1f7c0, 0x1f7c8, 3478 0x1f840, 0x1f84c, 3479 0x1fa84, 0x1fa8c, 3480 0x1fac0, 0x1fac0, 3481 0x1fae0, 0x1fae0, 3482 0x1fb00, 0x1fb84, 3483 0x1fbc0, 0x1fbc8, 3484 0x1fc40, 0x1fc4c, 3485 0x1fe84, 0x1fe8c, 3486 0x1fec0, 0x1fec0, 3487 0x1fee0, 0x1fee0, 3488 0x1ff00, 0x1ff84, 3489 0x1ffc0, 0x1ffc8, 3490 0x20000, 0x2002c, 3491 0x20100, 0x2013c, 3492 0x20190, 0x201c8, 3493 0x20200, 0x20318, 3494 0x20400, 0x20528, 3495 0x20540, 0x20614, 3496 0x21000, 0x21040, 3497 0x2104c, 0x21060, 3498 0x210c0, 0x210ec, 3499 0x21200, 0x21268, 3500 0x21270, 0x21284, 3501 0x212fc, 0x21388, 3502 0x21400, 0x21404, 3503 0x21500, 0x21518, 3504 0x2152c, 0x2153c, 3505 0x21550, 0x21554, 3506 0x21600, 0x21600, 3507 0x21608, 0x21628, 3508 0x21630, 0x2163c, 3509 0x21700, 0x2171c, 3510 0x21780, 0x2178c, 3511 0x21800, 0x21c38, 3512 0x21c80, 0x21d7c, 3513 0x21e00, 0x21e04, 3514 0x22000, 0x2202c, 3515 0x22100, 0x2213c, 3516 0x22190, 0x221c8, 3517 0x22200, 0x22318, 3518 0x22400, 0x22528, 3519 0x22540, 0x22614, 3520 0x23000, 0x23040, 3521 0x2304c, 0x23060, 3522 0x230c0, 0x230ec, 3523 0x23200, 0x23268, 3524 0x23270, 0x23284, 3525 0x232fc, 0x23388, 3526 0x23400, 0x23404, 3527 0x23500, 0x23518, 3528 0x2352c, 0x2353c, 3529 0x23550, 0x23554, 3530 0x23600, 0x23600, 3531 0x23608, 0x23628, 3532 0x23630, 0x2363c, 3533 0x23700, 0x2371c, 3534 0x23780, 0x2378c, 3535 0x23800, 0x23c38, 3536 0x23c80, 0x23d7c, 3537 0x23e00, 0x23e04, 3538 0x24000, 0x2402c, 3539 0x24100, 0x2413c, 3540 0x24190, 0x241c8, 3541 0x24200, 0x24318, 3542 0x24400, 0x24528, 3543 0x24540, 0x24614, 3544 0x25000, 0x25040, 3545 0x2504c, 0x25060, 3546 0x250c0, 0x250ec, 3547 0x25200, 0x25268, 3548 0x25270, 0x25284, 3549 0x252fc, 0x25388, 3550 0x25400, 0x25404, 3551 0x25500, 0x25518, 3552 0x2552c, 0x2553c, 3553 0x25550, 0x25554, 3554 0x25600, 0x25600, 3555 0x25608, 0x25628, 3556 0x25630, 0x2563c, 3557 0x25700, 0x2571c, 3558 0x25780, 0x2578c, 3559 0x25800, 0x25c38, 3560 0x25c80, 0x25d7c, 3561 0x25e00, 0x25e04, 3562 0x26000, 0x2602c, 3563 0x26100, 0x2613c, 3564 0x26190, 0x261c8, 3565 0x26200, 0x26318, 3566 0x26400, 0x26528, 3567 0x26540, 0x26614, 3568 0x27000, 0x27040, 3569 0x2704c, 0x27060, 3570 0x270c0, 0x270ec, 3571 0x27200, 0x27268, 3572 0x27270, 0x27284, 3573 0x272fc, 0x27388, 3574 0x27400, 0x27404, 3575 0x27500, 0x27518, 3576 0x2752c, 0x2753c, 3577 0x27550, 0x27554, 3578 0x27600, 0x27600, 3579 0x27608, 0x27628, 3580 0x27630, 0x2763c, 3581 0x27700, 0x2771c, 3582 0x27780, 0x2778c, 3583 0x27800, 0x27c38, 3584 0x27c80, 0x27d7c, 3585 0x27e00, 0x27e04 3586 }; 3587 static const unsigned int t5_reg_ranges[] = { 3588 0x1008, 0x1148, 3589 0x1180, 0x11b4, 3590 0x11fc, 0x123c, 3591 0x1280, 0x173c, 3592 0x1800, 0x18fc, 3593 0x3000, 0x3028, 3594 0x3060, 0x30d8, 3595 0x30e0, 0x30fc, 3596 0x3140, 0x357c, 3597 0x35a8, 0x35cc, 3598 0x35ec, 0x35ec, 3599 0x3600, 0x5624, 3600 0x56cc, 0x575c, 3601 0x580c, 0x5814, 3602 0x5890, 0x58bc, 3603 0x5940, 0x59dc, 3604 0x59fc, 0x5a18, 3605 0x5a60, 0x5a9c, 3606 0x5b94, 0x5bfc, 3607 0x6000, 0x6040, 3608 0x6058, 0x614c, 3609 0x7700, 0x7798, 3610 0x77c0, 0x78fc, 3611 0x7b00, 0x7c54, 3612 0x7d00, 0x7efc, 3613 0x8dc0, 0x8de0, 3614 0x8df8, 0x8e84, 3615 0x8ea0, 0x8f84, 3616 0x8fc0, 0x90f8, 3617 0x9400, 0x9470, 3618 0x9600, 0x96f4, 3619 0x9800, 0x9808, 3620 0x9820, 0x983c, 3621 0x9850, 0x9864, 3622 0x9c00, 0x9c6c, 3623 0x9c80, 0x9cec, 3624 0x9d00, 0x9d6c, 3625 0x9d80, 0x9dec, 3626 0x9e00, 0x9e6c, 3627 0x9e80, 0x9eec, 3628 0x9f00, 0x9f6c, 3629 0x9f80, 0xa020, 3630 0xd004, 0xd03c, 3631 0xdfc0, 0xdfe0, 3632 0xe000, 0x11088, 3633 0x1109c, 0x11110, 3634 0x11118, 0x1117c, 3635 0x11190, 0x11204, 3636 0x19040, 0x1906c, 3637 0x19078, 0x19080, 3638 0x1908c, 0x19124, 3639 0x19150, 0x191b0, 3640 0x191d0, 0x191e8, 3641 0x19238, 0x19290, 3642 0x193f8, 0x19474, 3643 0x19490, 0x194cc, 3644 0x194f0, 0x194f8, 3645 0x19c00, 0x19c60, 3646 0x19c94, 0x19e10, 3647 0x19e50, 0x19f34, 3648 0x19f40, 0x19f50, 3649 0x19f90, 0x19fe4, 3650 0x1a000, 0x1a06c, 3651 0x1a0b0, 0x1a120, 3652 0x1a128, 0x1a138, 3653 0x1a190, 0x1a1c4, 3654 0x1a1fc, 0x1a1fc, 3655 0x1e008, 0x1e00c, 3656 0x1e040, 0x1e04c, 3657 0x1e284, 0x1e290, 3658 0x1e2c0, 0x1e2c0, 3659 0x1e2e0, 0x1e2e0, 3660 0x1e300, 0x1e384, 3661 0x1e3c0, 0x1e3c8, 3662 0x1e408, 0x1e40c, 3663 0x1e440, 0x1e44c, 3664 0x1e684, 0x1e690, 3665 0x1e6c0, 0x1e6c0, 3666 0x1e6e0, 0x1e6e0, 3667 0x1e700, 0x1e784, 3668 0x1e7c0, 0x1e7c8, 3669 0x1e808, 0x1e80c, 3670 0x1e840, 0x1e84c, 3671 0x1ea84, 0x1ea90, 3672 0x1eac0, 0x1eac0, 3673 0x1eae0, 0x1eae0, 3674 0x1eb00, 0x1eb84, 3675 0x1ebc0, 0x1ebc8, 3676 0x1ec08, 0x1ec0c, 3677 0x1ec40, 0x1ec4c, 3678 0x1ee84, 0x1ee90, 3679 0x1eec0, 0x1eec0, 3680 0x1eee0, 0x1eee0, 3681 0x1ef00, 0x1ef84, 3682 0x1efc0, 0x1efc8, 3683 0x1f008, 0x1f00c, 3684 0x1f040, 0x1f04c, 3685 0x1f284, 0x1f290, 3686 0x1f2c0, 0x1f2c0, 3687 0x1f2e0, 0x1f2e0, 3688 0x1f300, 0x1f384, 3689 0x1f3c0, 0x1f3c8, 3690 0x1f408, 0x1f40c, 3691 0x1f440, 0x1f44c, 3692 0x1f684, 0x1f690, 3693 0x1f6c0, 0x1f6c0, 3694 0x1f6e0, 0x1f6e0, 3695 0x1f700, 0x1f784, 3696 0x1f7c0, 0x1f7c8, 3697 0x1f808, 0x1f80c, 3698 0x1f840, 0x1f84c, 3699 0x1fa84, 0x1fa90, 3700 0x1fac0, 0x1fac0, 3701 0x1fae0, 0x1fae0, 3702 0x1fb00, 0x1fb84, 3703 0x1fbc0, 0x1fbc8, 3704 0x1fc08, 0x1fc0c, 3705 0x1fc40, 0x1fc4c, 3706 0x1fe84, 0x1fe90, 3707 0x1fec0, 0x1fec0, 3708 0x1fee0, 0x1fee0, 3709 0x1ff00, 0x1ff84, 3710 0x1ffc0, 0x1ffc8, 3711 0x30000, 0x30030, 3712 0x30100, 0x30144, 3713 0x30190, 0x301d0, 3714 0x30200, 0x30318, 3715 0x30400, 0x3052c, 3716 0x30540, 0x3061c, 3717 0x30800, 0x30834, 3718 0x308c0, 0x30908, 3719 0x30910, 0x309ac, 3720 0x30a00, 0x30a2c, 3721 0x30a44, 0x30a50, 3722 0x30a74, 0x30c24, 3723 0x30d00, 0x30d00, 3724 0x30d08, 0x30d14, 3725 0x30d1c, 0x30d20, 3726 0x30d3c, 0x30d50, 3727 0x31200, 0x3120c, 3728 0x31220, 0x31220, 3729 0x31240, 0x31240, 3730 0x31600, 0x3160c, 3731 0x31a00, 0x31a1c, 3732 0x31e00, 0x31e20, 3733 0x31e38, 0x31e3c, 3734 0x31e80, 0x31e80, 3735 0x31e88, 0x31ea8, 3736 0x31eb0, 0x31eb4, 3737 0x31ec8, 0x31ed4, 3738 0x31fb8, 0x32004, 3739 0x32200, 0x32200, 3740 0x32208, 0x32240, 3741 0x32248, 0x32280, 3742 0x32288, 0x322c0, 3743 0x322c8, 0x322fc, 3744 0x32600, 0x32630, 3745 0x32a00, 0x32abc, 3746 0x32b00, 0x32b70, 3747 0x33000, 0x33048, 3748 0x33060, 0x3309c, 3749 0x330f0, 0x33148, 3750 0x33160, 0x3319c, 3751 0x331f0, 0x332e4, 3752 0x332f8, 0x333e4, 3753 0x333f8, 0x33448, 3754 0x33460, 0x3349c, 3755 0x334f0, 0x33548, 3756 0x33560, 0x3359c, 3757 0x335f0, 0x336e4, 3758 0x336f8, 0x337e4, 3759 0x337f8, 0x337fc, 3760 0x33814, 0x33814, 3761 0x3382c, 0x3382c, 3762 0x33880, 0x3388c, 3763 0x338e8, 0x338ec, 3764 0x33900, 0x33948, 3765 0x33960, 0x3399c, 3766 0x339f0, 0x33ae4, 3767 0x33af8, 0x33b10, 3768 0x33b28, 0x33b28, 3769 0x33b3c, 0x33b50, 3770 0x33bf0, 0x33c10, 3771 0x33c28, 0x33c28, 3772 0x33c3c, 0x33c50, 3773 0x33cf0, 0x33cfc, 3774 0x34000, 0x34030, 3775 0x34100, 0x34144, 3776 0x34190, 0x341d0, 3777 0x34200, 0x34318, 3778 0x34400, 0x3452c, 3779 0x34540, 0x3461c, 3780 0x34800, 0x34834, 3781 0x348c0, 0x34908, 3782 0x34910, 0x349ac, 3783 0x34a00, 0x34a2c, 3784 0x34a44, 0x34a50, 3785 0x34a74, 0x34c24, 3786 0x34d00, 0x34d00, 3787 0x34d08, 0x34d14, 3788 0x34d1c, 0x34d20, 3789 0x34d3c, 0x34d50, 3790 0x35200, 0x3520c, 3791 0x35220, 0x35220, 3792 0x35240, 0x35240, 3793 0x35600, 0x3560c, 3794 0x35a00, 0x35a1c, 3795 0x35e00, 0x35e20, 3796 0x35e38, 0x35e3c, 3797 0x35e80, 0x35e80, 3798 0x35e88, 0x35ea8, 3799 0x35eb0, 0x35eb4, 3800 0x35ec8, 0x35ed4, 3801 0x35fb8, 0x36004, 3802 0x36200, 0x36200, 3803 0x36208, 0x36240, 3804 0x36248, 0x36280, 3805 0x36288, 0x362c0, 3806 0x362c8, 0x362fc, 3807 0x36600, 0x36630, 3808 0x36a00, 0x36abc, 3809 0x36b00, 0x36b70, 3810 0x37000, 0x37048, 3811 0x37060, 0x3709c, 3812 0x370f0, 0x37148, 3813 0x37160, 0x3719c, 3814 0x371f0, 0x372e4, 3815 0x372f8, 0x373e4, 3816 0x373f8, 0x37448, 3817 0x37460, 0x3749c, 3818 0x374f0, 0x37548, 3819 0x37560, 0x3759c, 3820 0x375f0, 0x376e4, 3821 0x376f8, 0x377e4, 3822 0x377f8, 0x377fc, 3823 0x37814, 0x37814, 3824 0x3782c, 0x3782c, 3825 0x37880, 0x3788c, 3826 0x378e8, 0x378ec, 3827 0x37900, 0x37948, 3828 0x37960, 0x3799c, 3829 0x379f0, 0x37ae4, 3830 0x37af8, 0x37b10, 3831 0x37b28, 0x37b28, 3832 0x37b3c, 0x37b50, 3833 0x37bf0, 0x37c10, 3834 0x37c28, 0x37c28, 3835 0x37c3c, 0x37c50, 3836 0x37cf0, 0x37cfc, 3837 0x38000, 0x38030, 3838 0x38100, 0x38144, 3839 0x38190, 0x381d0, 3840 0x38200, 0x38318, 3841 0x38400, 0x3852c, 3842 0x38540, 0x3861c, 3843 0x38800, 0x38834, 3844 0x388c0, 0x38908, 3845 0x38910, 0x389ac, 3846 0x38a00, 0x38a2c, 3847 0x38a44, 0x38a50, 3848 0x38a74, 0x38c24, 3849 0x38d00, 0x38d00, 3850 0x38d08, 0x38d14, 3851 0x38d1c, 0x38d20, 3852 0x38d3c, 0x38d50, 3853 0x39200, 0x3920c, 3854 0x39220, 0x39220, 3855 0x39240, 0x39240, 3856 0x39600, 0x3960c, 3857 0x39a00, 0x39a1c, 3858 0x39e00, 0x39e20, 3859 0x39e38, 0x39e3c, 3860 0x39e80, 0x39e80, 3861 0x39e88, 0x39ea8, 3862 0x39eb0, 0x39eb4, 3863 0x39ec8, 0x39ed4, 3864 0x39fb8, 0x3a004, 3865 0x3a200, 0x3a200, 3866 0x3a208, 0x3a240, 3867 0x3a248, 0x3a280, 3868 0x3a288, 0x3a2c0, 3869 0x3a2c8, 0x3a2fc, 3870 0x3a600, 0x3a630, 3871 0x3aa00, 0x3aabc, 3872 0x3ab00, 0x3ab70, 3873 0x3b000, 0x3b048, 3874 0x3b060, 0x3b09c, 3875 0x3b0f0, 0x3b148, 3876 0x3b160, 0x3b19c, 3877 0x3b1f0, 0x3b2e4, 3878 0x3b2f8, 0x3b3e4, 3879 0x3b3f8, 0x3b448, 3880 0x3b460, 0x3b49c, 3881 0x3b4f0, 0x3b548, 3882 0x3b560, 0x3b59c, 3883 0x3b5f0, 0x3b6e4, 3884 0x3b6f8, 0x3b7e4, 3885 0x3b7f8, 0x3b7fc, 3886 0x3b814, 0x3b814, 3887 0x3b82c, 0x3b82c, 3888 0x3b880, 0x3b88c, 3889 0x3b8e8, 0x3b8ec, 3890 0x3b900, 0x3b948, 3891 0x3b960, 0x3b99c, 3892 0x3b9f0, 0x3bae4, 3893 0x3baf8, 0x3bb10, 3894 0x3bb28, 0x3bb28, 3895 0x3bb3c, 0x3bb50, 3896 0x3bbf0, 0x3bc10, 3897 0x3bc28, 0x3bc28, 3898 0x3bc3c, 0x3bc50, 3899 0x3bcf0, 0x3bcfc, 3900 0x3c000, 0x3c030, 3901 0x3c100, 0x3c144, 3902 0x3c190, 0x3c1d0, 3903 0x3c200, 0x3c318, 3904 0x3c400, 0x3c52c, 3905 0x3c540, 0x3c61c, 3906 0x3c800, 0x3c834, 3907 0x3c8c0, 0x3c908, 3908 0x3c910, 0x3c9ac, 3909 0x3ca00, 0x3ca2c, 3910 0x3ca44, 0x3ca50, 3911 0x3ca74, 0x3cc24, 3912 0x3cd00, 0x3cd00, 3913 0x3cd08, 0x3cd14, 3914 0x3cd1c, 0x3cd20, 3915 0x3cd3c, 0x3cd50, 3916 0x3d200, 0x3d20c, 3917 0x3d220, 0x3d220, 3918 0x3d240, 0x3d240, 3919 0x3d600, 0x3d60c, 3920 0x3da00, 0x3da1c, 3921 0x3de00, 0x3de20, 3922 0x3de38, 0x3de3c, 3923 0x3de80, 0x3de80, 3924 0x3de88, 0x3dea8, 3925 0x3deb0, 0x3deb4, 3926 0x3dec8, 0x3ded4, 3927 0x3dfb8, 0x3e004, 3928 0x3e200, 0x3e200, 3929 0x3e208, 0x3e240, 3930 0x3e248, 0x3e280, 3931 0x3e288, 0x3e2c0, 3932 0x3e2c8, 0x3e2fc, 3933 0x3e600, 0x3e630, 3934 0x3ea00, 0x3eabc, 3935 0x3eb00, 0x3eb70, 3936 0x3f000, 0x3f048, 3937 0x3f060, 0x3f09c, 3938 0x3f0f0, 0x3f148, 3939 0x3f160, 0x3f19c, 3940 0x3f1f0, 0x3f2e4, 3941 0x3f2f8, 0x3f3e4, 3942 0x3f3f8, 0x3f448, 3943 0x3f460, 0x3f49c, 3944 0x3f4f0, 0x3f548, 3945 0x3f560, 0x3f59c, 3946 0x3f5f0, 0x3f6e4, 3947 0x3f6f8, 0x3f7e4, 3948 0x3f7f8, 0x3f7fc, 3949 0x3f814, 0x3f814, 3950 0x3f82c, 0x3f82c, 3951 0x3f880, 0x3f88c, 3952 0x3f8e8, 0x3f8ec, 3953 0x3f900, 0x3f948, 3954 0x3f960, 0x3f99c, 3955 0x3f9f0, 0x3fae4, 3956 0x3faf8, 0x3fb10, 3957 0x3fb28, 0x3fb28, 3958 0x3fb3c, 0x3fb50, 3959 0x3fbf0, 0x3fc10, 3960 0x3fc28, 0x3fc28, 3961 0x3fc3c, 0x3fc50, 3962 0x3fcf0, 0x3fcfc, 3963 0x40000, 0x4000c, 3964 0x40040, 0x40068, 3965 0x4007c, 0x40144, 3966 0x40180, 0x4018c, 3967 0x40200, 0x40298, 3968 0x402ac, 0x4033c, 3969 0x403f8, 0x403fc, 3970 0x41304, 0x413c4, 3971 0x41400, 0x4141c, 3972 0x41480, 0x414d0, 3973 0x44000, 0x44078, 3974 0x440c0, 0x44278, 3975 0x442c0, 0x44478, 3976 0x444c0, 0x44678, 3977 0x446c0, 0x44878, 3978 0x448c0, 0x449fc, 3979 0x45000, 0x45068, 3980 0x45080, 0x45084, 3981 0x450a0, 0x450b0, 3982 0x45200, 0x45268, 3983 0x45280, 0x45284, 3984 0x452a0, 0x452b0, 3985 0x460c0, 0x460e4, 3986 0x47000, 0x4708c, 3987 0x47200, 0x47250, 3988 0x47400, 0x47420, 3989 0x47600, 0x47618, 3990 0x47800, 0x47814, 3991 0x48000, 0x4800c, 3992 0x48040, 0x48068, 3993 0x4807c, 0x48144, 3994 0x48180, 0x4818c, 3995 0x48200, 0x48298, 3996 0x482ac, 0x4833c, 3997 0x483f8, 0x483fc, 3998 0x49304, 0x493c4, 3999 0x49400, 0x4941c, 4000 0x49480, 0x494d0, 4001 0x4c000, 0x4c078, 4002 0x4c0c0, 0x4c278, 4003 0x4c2c0, 0x4c478, 4004 0x4c4c0, 0x4c678, 4005 0x4c6c0, 0x4c878, 4006 0x4c8c0, 0x4c9fc, 4007 0x4d000, 0x4d068, 4008 0x4d080, 0x4d084, 4009 0x4d0a0, 0x4d0b0, 4010 0x4d200, 0x4d268, 4011 0x4d280, 0x4d284, 4012 0x4d2a0, 0x4d2b0, 4013 0x4e0c0, 0x4e0e4, 4014 0x4f000, 0x4f08c, 4015 0x4f200, 0x4f250, 4016 0x4f400, 0x4f420, 4017 0x4f600, 0x4f618, 4018 0x4f800, 0x4f814, 4019 0x50000, 0x500cc, 4020 0x50400, 0x50400, 4021 0x50800, 0x508cc, 4022 0x50c00, 0x50c00, 4023 0x51000, 0x5101c, 4024 0x51300, 0x51308, 4025 }; 4026 4027 if (is_t4(sc)) { 4028 reg_ranges = &t4_reg_ranges[0]; 4029 n = nitems(t4_reg_ranges); 4030 } else { 4031 reg_ranges = &t5_reg_ranges[0]; 4032 n = nitems(t5_reg_ranges); 4033 } 4034 4035 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4036 for (i = 0; i < n; i += 2) 4037 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4038} 4039 4040static void 4041cxgbe_tick(void *arg) 4042{ 4043 struct port_info *pi = arg; 4044 struct adapter *sc = pi->adapter; 4045 struct ifnet *ifp = pi->ifp; 4046 struct sge_txq *txq; 4047 int i, drops; 4048 struct port_stats *s = &pi->stats; 4049 4050 PORT_LOCK(pi); 4051 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4052 PORT_UNLOCK(pi); 4053 return; /* without scheduling another callout */ 4054 } 4055 4056 t4_get_port_stats(sc, pi->tx_chan, s); 4057 4058 ifp->if_opackets = s->tx_frames - s->tx_pause; 4059 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4060 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4061 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4062 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4063 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4064 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4065 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4066 s->rx_trunc3; 4067 for (i = 0; i < 4; i++) { 4068 if (pi->rx_chan_map & (1 << i)) { 4069 uint32_t v; 4070 4071 /* 4072 * XXX: indirect reads from the same ADDR/DATA pair can 4073 * race with each other. 4074 */ 4075 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4076 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4077 ifp->if_iqdrops += v; 4078 } 4079 } 4080 4081 drops = s->tx_drop; 4082 for_each_txq(pi, i, txq) 4083 drops += txq->br->br_drops; 4084 ifp->if_snd.ifq_drops = drops; 4085 4086 ifp->if_oerrors = s->tx_error_frames; 4087 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4088 s->rx_fcs_err + s->rx_len_err; 4089 4090 callout_schedule(&pi->tick, hz); 4091 PORT_UNLOCK(pi); 4092} 4093 4094static void 4095cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4096{ 4097 struct ifnet *vlan; 4098 4099 if (arg != ifp || ifp->if_type != IFT_ETHER) 4100 return; 4101 4102 vlan = VLAN_DEVAT(ifp, vid); 4103 VLAN_SETCOOKIE(vlan, ifp); 4104} 4105 4106static int 4107cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4108{ 4109 4110#ifdef INVARIANTS 4111 panic("%s: opcode 0x%02x on iq %p with payload %p", 4112 __func__, rss->opcode, iq, m); 4113#else 4114 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4115 __func__, rss->opcode, iq, m); 4116 m_freem(m); 4117#endif 4118 return (EDOOFUS); 4119} 4120 4121int 4122t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4123{ 4124 uintptr_t *loc, new; 4125 4126 if (opcode >= nitems(sc->cpl_handler)) 4127 return (EINVAL); 4128 4129 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4130 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4131 atomic_store_rel_ptr(loc, new); 4132 4133 return (0); 4134} 4135 4136static int 4137an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4138{ 4139 4140#ifdef INVARIANTS 4141 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4142#else 4143 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4144 __func__, iq, ctrl); 4145#endif 4146 return (EDOOFUS); 4147} 4148 4149int 4150t4_register_an_handler(struct adapter *sc, an_handler_t h) 4151{ 4152 uintptr_t *loc, new; 4153 4154 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4155 loc = (uintptr_t *) &sc->an_handler; 4156 atomic_store_rel_ptr(loc, new); 4157 4158 return (0); 4159} 4160 4161static int 4162fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4163{ 4164 const struct cpl_fw6_msg *cpl = 4165 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4166 4167#ifdef INVARIANTS 4168 panic("%s: fw_msg type %d", __func__, cpl->type); 4169#else 4170 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4171#endif 4172 return (EDOOFUS); 4173} 4174 4175int 4176t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4177{ 4178 uintptr_t *loc, new; 4179 4180 if (type >= nitems(sc->fw_msg_handler)) 4181 return (EINVAL); 4182 4183 /* 4184 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4185 * handler dispatch table. Reject any attempt to install a handler for 4186 * this subtype. 4187 */ 4188 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4189 return (EINVAL); 4190 4191 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4192 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4193 atomic_store_rel_ptr(loc, new); 4194 4195 return (0); 4196} 4197 4198static int 4199t4_sysctls(struct adapter *sc) 4200{ 4201 struct sysctl_ctx_list *ctx; 4202 struct sysctl_oid *oid; 4203 struct sysctl_oid_list *children, *c0; 4204 static char *caps[] = { 4205 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4206 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4207 "\6HASHFILTER\7ETHOFLD", 4208 "\20\1TOE", /* caps[2] toecaps */ 4209 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4210 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4211 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4212 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4213 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4214 "\4PO_INITIAOR\5PO_TARGET" 4215 }; 4216 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4217 4218 ctx = device_get_sysctl_ctx(sc->dev); 4219 4220 /* 4221 * dev.t4nex.X. 4222 */ 4223 oid = device_get_sysctl_tree(sc->dev); 4224 c0 = children = SYSCTL_CHILDREN(oid); 4225 4226 sc->sc_do_rxcopy = 1; 4227 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4228 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4229 4230 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4231 sc->params.nports, "# of ports"); 4232 4233 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4234 NULL, chip_rev(sc), "chip hardware revision"); 4235 4236 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4237 CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4238 4239 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4240 CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4241 4242 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4243 sc->cfcsum, "config file checksum"); 4244 4245 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4246 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4247 sysctl_bitfield, "A", "available doorbells"); 4248 4249 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4250 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4251 sysctl_bitfield, "A", "available link capabilities"); 4252 4253 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4254 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4255 sysctl_bitfield, "A", "available NIC capabilities"); 4256 4257 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4258 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4259 sysctl_bitfield, "A", "available TCP offload capabilities"); 4260 4261 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4262 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4263 sysctl_bitfield, "A", "available RDMA capabilities"); 4264 4265 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4266 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4267 sysctl_bitfield, "A", "available iSCSI capabilities"); 4268 4269 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4270 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4271 sysctl_bitfield, "A", "available FCoE capabilities"); 4272 4273 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4274 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4275 4276 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4277 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4278 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4279 "interrupt holdoff timer values (us)"); 4280 4281 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4282 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4283 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4284 "interrupt holdoff packet counter values"); 4285 4286 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4287 NULL, sc->tids.nftids, "number of filters"); 4288 4289 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4290 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4291 "chip temperature (in Celsius)"); 4292 4293 t4_sge_sysctls(sc, ctx, children); 4294 4295 sc->lro_timeout = 100; 4296 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4297 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4298 4299#ifdef SBUF_DRAIN 4300 /* 4301 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4302 */ 4303 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4304 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4305 "logs and miscellaneous information"); 4306 children = SYSCTL_CHILDREN(oid); 4307 4308 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4309 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4310 sysctl_cctrl, "A", "congestion control"); 4311 4312 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4313 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4314 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4315 4316 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4317 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4318 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4319 4320 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4321 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4322 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4323 4324 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4325 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4326 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4327 4328 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4329 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4330 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4331 4332 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4333 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4334 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4335 4336 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4337 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4338 sysctl_cim_la, "A", "CIM logic analyzer"); 4339 4340 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4341 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4342 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4343 4344 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4345 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4346 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4347 4348 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4349 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4350 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4351 4352 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4353 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4354 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4355 4356 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4357 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4358 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4359 4360 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4361 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4362 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4363 4364 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4365 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4366 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4367 4368 if (is_t5(sc)) { 4369 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4370 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4371 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4372 4373 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4374 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4375 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4376 } 4377 4378 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4379 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4380 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4381 4382 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4383 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4384 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4385 4386 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4387 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4388 sysctl_cpl_stats, "A", "CPL statistics"); 4389 4390 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4391 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4392 sysctl_ddp_stats, "A", "DDP statistics"); 4393 4394 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4395 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4396 sysctl_devlog, "A", "firmware's device log"); 4397 4398 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4399 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4400 sysctl_fcoe_stats, "A", "FCoE statistics"); 4401 4402 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4403 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4404 sysctl_hw_sched, "A", "hardware scheduler "); 4405 4406 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4407 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4408 sysctl_l2t, "A", "hardware L2 table"); 4409 4410 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4411 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4412 sysctl_lb_stats, "A", "loopback statistics"); 4413 4414 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4415 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4416 sysctl_meminfo, "A", "memory regions"); 4417 4418 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4419 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4420 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4421 4422 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4423 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4424 sysctl_path_mtus, "A", "path MTUs"); 4425 4426 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4427 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4428 sysctl_pm_stats, "A", "PM statistics"); 4429 4430 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4431 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4432 sysctl_rdma_stats, "A", "RDMA statistics"); 4433 4434 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4435 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4436 sysctl_tcp_stats, "A", "TCP statistics"); 4437 4438 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4439 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4440 sysctl_tids, "A", "TID information"); 4441 4442 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4443 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4444 sysctl_tp_err_stats, "A", "TP error statistics"); 4445 4446 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4447 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4448 sysctl_tp_la, "A", "TP logic analyzer"); 4449 4450 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4451 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4452 sysctl_tx_rate, "A", "Tx rate"); 4453 4454 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4455 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4456 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4457 4458 if (is_t5(sc)) { 4459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4460 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4461 sysctl_wcwr_stats, "A", "write combined work requests"); 4462 } 4463#endif 4464 4465#ifdef TCP_OFFLOAD 4466 if (is_offload(sc)) { 4467 /* 4468 * dev.t4nex.X.toe. 4469 */ 4470 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4471 NULL, "TOE parameters"); 4472 children = SYSCTL_CHILDREN(oid); 4473 4474 sc->tt.sndbuf = 256 * 1024; 4475 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4476 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4477 4478 sc->tt.ddp = 0; 4479 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4480 &sc->tt.ddp, 0, "DDP allowed"); 4481 4482 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4483 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4484 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4485 4486 sc->tt.ddp_thres = 4487 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4488 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4489 &sc->tt.ddp_thres, 0, "DDP threshold"); 4490 4491 sc->tt.rx_coalesce = 1; 4492 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4493 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4494 } 4495#endif 4496 4497 4498 return (0); 4499} 4500 4501static int 4502cxgbe_sysctls(struct port_info *pi) 4503{ 4504 struct sysctl_ctx_list *ctx; 4505 struct sysctl_oid *oid; 4506 struct sysctl_oid_list *children; 4507 4508 ctx = device_get_sysctl_ctx(pi->dev); 4509 4510 /* 4511 * dev.cxgbe.X. 4512 */ 4513 oid = device_get_sysctl_tree(pi->dev); 4514 children = SYSCTL_CHILDREN(oid); 4515 4516 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4517 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4518 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4519 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4520 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4521 "PHY temperature (in Celsius)"); 4522 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4523 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4524 "PHY firmware version"); 4525 } 4526 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4527 &pi->nrxq, 0, "# of rx queues"); 4528 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4529 &pi->ntxq, 0, "# of tx queues"); 4530 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4531 &pi->first_rxq, 0, "index of first rx queue"); 4532 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4533 &pi->first_txq, 0, "index of first tx queue"); 4534 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4535 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4536 "Reserve queue 0 for non-flowid packets"); 4537 4538#ifdef TCP_OFFLOAD 4539 if (is_offload(pi->adapter)) { 4540 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4541 &pi->nofldrxq, 0, 4542 "# of rx queues for offloaded TCP connections"); 4543 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4544 &pi->nofldtxq, 0, 4545 "# of tx queues for offloaded TCP connections"); 4546 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4547 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4548 "index of first TOE rx queue"); 4549 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4550 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4551 "index of first TOE tx queue"); 4552 } 4553#endif 4554 4555 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4556 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4557 "holdoff timer index"); 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4559 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4560 "holdoff packet counter index"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4563 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4564 "rx queue size"); 4565 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4566 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4567 "tx queue size"); 4568 4569 /* 4570 * dev.cxgbe.X.stats. 4571 */ 4572 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4573 NULL, "port statistics"); 4574 children = SYSCTL_CHILDREN(oid); 4575 4576#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4577 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4578 CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 4579 sysctl_handle_t4_reg64, "QU", desc) 4580 4581 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4582 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4583 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4584 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4585 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4586 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4587 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4588 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4589 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4590 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4591 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4592 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4593 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4594 "# of tx frames in this range", 4595 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4596 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4597 "# of tx frames in this range", 4598 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4599 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4600 "# of tx frames in this range", 4601 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4602 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4603 "# of tx frames in this range", 4604 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4605 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4606 "# of tx frames in this range", 4607 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4608 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4609 "# of tx frames in this range", 4610 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4611 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4612 "# of tx frames in this range", 4613 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4614 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4615 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4616 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4617 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4618 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4619 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4620 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4621 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4622 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4623 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4624 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4625 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4626 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4627 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4628 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4629 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4630 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4631 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4632 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4633 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4634 4635 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4636 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4637 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4638 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4639 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4640 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4641 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4642 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4643 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4644 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4645 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4646 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4647 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4648 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4649 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4650 "# of frames received with bad FCS", 4651 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4652 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4653 "# of frames received with length error", 4654 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4655 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4656 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4657 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4658 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4659 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4660 "# of rx frames in this range", 4661 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4662 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4663 "# of rx frames in this range", 4664 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4665 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4666 "# of rx frames in this range", 4667 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4668 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4669 "# of rx frames in this range", 4670 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4671 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4672 "# of rx frames in this range", 4673 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4674 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4675 "# of rx frames in this range", 4676 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4677 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4678 "# of rx frames in this range", 4679 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4680 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4681 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4682 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4683 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4684 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4685 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4686 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4687 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4688 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4689 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4690 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4691 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4692 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4693 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4694 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4695 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4696 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4697 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4698 4699#undef SYSCTL_ADD_T4_REG64 4700 4701#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4702 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4703 &pi->stats.name, desc) 4704 4705 /* We get these from port_stats and they may be stale by upto 1s */ 4706 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4707 "# drops due to buffer-group 0 overflows"); 4708 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4709 "# drops due to buffer-group 1 overflows"); 4710 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4711 "# drops due to buffer-group 2 overflows"); 4712 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4713 "# drops due to buffer-group 3 overflows"); 4714 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4715 "# of buffer-group 0 truncated packets"); 4716 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4717 "# of buffer-group 1 truncated packets"); 4718 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4719 "# of buffer-group 2 truncated packets"); 4720 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4721 "# of buffer-group 3 truncated packets"); 4722 4723#undef SYSCTL_ADD_T4_PORTSTAT 4724 4725 return (0); 4726} 4727 4728static int 4729sysctl_int_array(SYSCTL_HANDLER_ARGS) 4730{ 4731 int rc, *i; 4732 struct sbuf sb; 4733 4734 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4735 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4736 sbuf_printf(&sb, "%d ", *i); 4737 sbuf_trim(&sb); 4738 sbuf_finish(&sb); 4739 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4740 sbuf_delete(&sb); 4741 return (rc); 4742} 4743 4744static int 4745sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4746{ 4747 int rc; 4748 struct sbuf *sb; 4749 4750 rc = sysctl_wire_old_buffer(req, 0); 4751 if (rc != 0) 4752 return(rc); 4753 4754 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4755 if (sb == NULL) 4756 return (ENOMEM); 4757 4758 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4759 rc = sbuf_finish(sb); 4760 sbuf_delete(sb); 4761 4762 return (rc); 4763} 4764 4765static int 4766sysctl_btphy(SYSCTL_HANDLER_ARGS) 4767{ 4768 struct port_info *pi = arg1; 4769 int op = arg2; 4770 struct adapter *sc = pi->adapter; 4771 u_int v; 4772 int rc; 4773 4774 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 4775 if (rc) 4776 return (rc); 4777 /* XXX: magic numbers */ 4778 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 4779 &v); 4780 end_synchronized_op(sc, 0); 4781 if (rc) 4782 return (rc); 4783 if (op == 0) 4784 v /= 256; 4785 4786 rc = sysctl_handle_int(oidp, &v, 0, req); 4787 return (rc); 4788} 4789 4790static int 4791sysctl_noflowq(SYSCTL_HANDLER_ARGS) 4792{ 4793 struct port_info *pi = arg1; 4794 int rc, val; 4795 4796 val = pi->rsrv_noflowq; 4797 rc = sysctl_handle_int(oidp, &val, 0, req); 4798 if (rc != 0 || req->newptr == NULL) 4799 return (rc); 4800 4801 if ((val >= 1) && (pi->ntxq > 1)) 4802 pi->rsrv_noflowq = 1; 4803 else 4804 pi->rsrv_noflowq = 0; 4805 4806 return (rc); 4807} 4808 4809static int 4810sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4811{ 4812 struct port_info *pi = arg1; 4813 struct adapter *sc = pi->adapter; 4814 int idx, rc, i; 4815 struct sge_rxq *rxq; 4816#ifdef TCP_OFFLOAD 4817 struct sge_ofld_rxq *ofld_rxq; 4818#endif 4819 uint8_t v; 4820 4821 idx = pi->tmr_idx; 4822 4823 rc = sysctl_handle_int(oidp, &idx, 0, req); 4824 if (rc != 0 || req->newptr == NULL) 4825 return (rc); 4826 4827 if (idx < 0 || idx >= SGE_NTIMERS) 4828 return (EINVAL); 4829 4830 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4831 "t4tmr"); 4832 if (rc) 4833 return (rc); 4834 4835 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 4836 for_each_rxq(pi, i, rxq) { 4837#ifdef atomic_store_rel_8 4838 atomic_store_rel_8(&rxq->iq.intr_params, v); 4839#else 4840 rxq->iq.intr_params = v; 4841#endif 4842 } 4843#ifdef TCP_OFFLOAD 4844 for_each_ofld_rxq(pi, i, ofld_rxq) { 4845#ifdef atomic_store_rel_8 4846 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 4847#else 4848 ofld_rxq->iq.intr_params = v; 4849#endif 4850 } 4851#endif 4852 pi->tmr_idx = idx; 4853 4854 end_synchronized_op(sc, LOCK_HELD); 4855 return (0); 4856} 4857 4858static int 4859sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 4860{ 4861 struct port_info *pi = arg1; 4862 struct adapter *sc = pi->adapter; 4863 int idx, rc; 4864 4865 idx = pi->pktc_idx; 4866 4867 rc = sysctl_handle_int(oidp, &idx, 0, req); 4868 if (rc != 0 || req->newptr == NULL) 4869 return (rc); 4870 4871 if (idx < -1 || idx >= SGE_NCOUNTERS) 4872 return (EINVAL); 4873 4874 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4875 "t4pktc"); 4876 if (rc) 4877 return (rc); 4878 4879 if (pi->flags & PORT_INIT_DONE) 4880 rc = EBUSY; /* cannot be changed once the queues are created */ 4881 else 4882 pi->pktc_idx = idx; 4883 4884 end_synchronized_op(sc, LOCK_HELD); 4885 return (rc); 4886} 4887 4888static int 4889sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 4890{ 4891 struct port_info *pi = arg1; 4892 struct adapter *sc = pi->adapter; 4893 int qsize, rc; 4894 4895 qsize = pi->qsize_rxq; 4896 4897 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4898 if (rc != 0 || req->newptr == NULL) 4899 return (rc); 4900 4901 if (qsize < 128 || (qsize & 7)) 4902 return (EINVAL); 4903 4904 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4905 "t4rxqs"); 4906 if (rc) 4907 return (rc); 4908 4909 if (pi->flags & PORT_INIT_DONE) 4910 rc = EBUSY; /* cannot be changed once the queues are created */ 4911 else 4912 pi->qsize_rxq = qsize; 4913 4914 end_synchronized_op(sc, LOCK_HELD); 4915 return (rc); 4916} 4917 4918static int 4919sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 4920{ 4921 struct port_info *pi = arg1; 4922 struct adapter *sc = pi->adapter; 4923 int qsize, rc; 4924 4925 qsize = pi->qsize_txq; 4926 4927 rc = sysctl_handle_int(oidp, &qsize, 0, req); 4928 if (rc != 0 || req->newptr == NULL) 4929 return (rc); 4930 4931 /* bufring size must be powerof2 */ 4932 if (qsize < 128 || !powerof2(qsize)) 4933 return (EINVAL); 4934 4935 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4936 "t4txqs"); 4937 if (rc) 4938 return (rc); 4939 4940 if (pi->flags & PORT_INIT_DONE) 4941 rc = EBUSY; /* cannot be changed once the queues are created */ 4942 else 4943 pi->qsize_txq = qsize; 4944 4945 end_synchronized_op(sc, LOCK_HELD); 4946 return (rc); 4947} 4948 4949static int 4950sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 4951{ 4952 struct adapter *sc = arg1; 4953 int reg = arg2; 4954 uint64_t val; 4955 4956 val = t4_read_reg64(sc, reg); 4957 4958 return (sysctl_handle_64(oidp, &val, 0, req)); 4959} 4960 4961static int 4962sysctl_temperature(SYSCTL_HANDLER_ARGS) 4963{ 4964 struct adapter *sc = arg1; 4965 int rc, t; 4966 uint32_t param, val; 4967 4968 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 4969 if (rc) 4970 return (rc); 4971 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 4972 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 4973 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 4974 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 4975 end_synchronized_op(sc, 0); 4976 if (rc) 4977 return (rc); 4978 4979 /* unknown is returned as 0 but we display -1 in that case */ 4980 t = val == 0 ? -1 : val; 4981 4982 rc = sysctl_handle_int(oidp, &t, 0, req); 4983 return (rc); 4984} 4985 4986#ifdef SBUF_DRAIN 4987static int 4988sysctl_cctrl(SYSCTL_HANDLER_ARGS) 4989{ 4990 struct adapter *sc = arg1; 4991 struct sbuf *sb; 4992 int rc, i; 4993 uint16_t incr[NMTUS][NCCTRL_WIN]; 4994 static const char *dec_fac[] = { 4995 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 4996 "0.9375" 4997 }; 4998 4999 rc = sysctl_wire_old_buffer(req, 0); 5000 if (rc != 0) 5001 return (rc); 5002 5003 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5004 if (sb == NULL) 5005 return (ENOMEM); 5006 5007 t4_read_cong_tbl(sc, incr); 5008 5009 for (i = 0; i < NCCTRL_WIN; ++i) { 5010 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5011 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5012 incr[5][i], incr[6][i], incr[7][i]); 5013 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5014 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5015 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5016 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5017 } 5018 5019 rc = sbuf_finish(sb); 5020 sbuf_delete(sb); 5021 5022 return (rc); 5023} 5024 5025static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5026 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5027 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5028 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5029}; 5030 5031static int 5032sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5033{ 5034 struct adapter *sc = arg1; 5035 struct sbuf *sb; 5036 int rc, i, n, qid = arg2; 5037 uint32_t *buf, *p; 5038 char *qtype; 5039 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5040 5041 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5042 ("%s: bad qid %d\n", __func__, qid)); 5043 5044 if (qid < CIM_NUM_IBQ) { 5045 /* inbound queue */ 5046 qtype = "IBQ"; 5047 n = 4 * CIM_IBQ_SIZE; 5048 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5049 rc = t4_read_cim_ibq(sc, qid, buf, n); 5050 } else { 5051 /* outbound queue */ 5052 qtype = "OBQ"; 5053 qid -= CIM_NUM_IBQ; 5054 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5055 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5056 rc = t4_read_cim_obq(sc, qid, buf, n); 5057 } 5058 5059 if (rc < 0) { 5060 rc = -rc; 5061 goto done; 5062 } 5063 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5064 5065 rc = sysctl_wire_old_buffer(req, 0); 5066 if (rc != 0) 5067 goto done; 5068 5069 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5070 if (sb == NULL) { 5071 rc = ENOMEM; 5072 goto done; 5073 } 5074 5075 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5076 for (i = 0, p = buf; i < n; i += 16, p += 4) 5077 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5078 p[2], p[3]); 5079 5080 rc = sbuf_finish(sb); 5081 sbuf_delete(sb); 5082done: 5083 free(buf, M_CXGBE); 5084 return (rc); 5085} 5086 5087static int 5088sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5089{ 5090 struct adapter *sc = arg1; 5091 u_int cfg; 5092 struct sbuf *sb; 5093 uint32_t *buf, *p; 5094 int rc; 5095 5096 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5097 if (rc != 0) 5098 return (rc); 5099 5100 rc = sysctl_wire_old_buffer(req, 0); 5101 if (rc != 0) 5102 return (rc); 5103 5104 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5105 if (sb == NULL) 5106 return (ENOMEM); 5107 5108 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5109 M_ZERO | M_WAITOK); 5110 5111 rc = -t4_cim_read_la(sc, buf, NULL); 5112 if (rc != 0) 5113 goto done; 5114 5115 sbuf_printf(sb, "Status Data PC%s", 5116 cfg & F_UPDBGLACAPTPCONLY ? "" : 5117 " LS0Stat LS0Addr LS0Data"); 5118 5119 KASSERT((sc->params.cim_la_size & 7) == 0, 5120 ("%s: p will walk off the end of buf", __func__)); 5121 5122 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5123 if (cfg & F_UPDBGLACAPTPCONLY) { 5124 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5125 p[6], p[7]); 5126 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5127 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5128 p[4] & 0xff, p[5] >> 8); 5129 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5130 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5131 p[1] & 0xf, p[2] >> 4); 5132 } else { 5133 sbuf_printf(sb, 5134 "\n %02x %x%07x %x%07x %08x %08x " 5135 "%08x%08x%08x%08x", 5136 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5137 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5138 p[6], p[7]); 5139 } 5140 } 5141 5142 rc = sbuf_finish(sb); 5143 sbuf_delete(sb); 5144done: 5145 free(buf, M_CXGBE); 5146 return (rc); 5147} 5148 5149static int 5150sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5151{ 5152 struct adapter *sc = arg1; 5153 u_int i; 5154 struct sbuf *sb; 5155 uint32_t *buf, *p; 5156 int rc; 5157 5158 rc = sysctl_wire_old_buffer(req, 0); 5159 if (rc != 0) 5160 return (rc); 5161 5162 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5163 if (sb == NULL) 5164 return (ENOMEM); 5165 5166 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5167 M_ZERO | M_WAITOK); 5168 5169 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5170 p = buf; 5171 5172 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5173 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5174 p[1], p[0]); 5175 } 5176 5177 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5178 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5179 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5180 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5181 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5182 (p[1] >> 2) | ((p[2] & 3) << 30), 5183 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5184 p[0] & 1); 5185 } 5186 5187 rc = sbuf_finish(sb); 5188 sbuf_delete(sb); 5189 free(buf, M_CXGBE); 5190 return (rc); 5191} 5192 5193static int 5194sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5195{ 5196 struct adapter *sc = arg1; 5197 u_int i; 5198 struct sbuf *sb; 5199 uint32_t *buf, *p; 5200 int rc; 5201 5202 rc = sysctl_wire_old_buffer(req, 0); 5203 if (rc != 0) 5204 return (rc); 5205 5206 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5207 if (sb == NULL) 5208 return (ENOMEM); 5209 5210 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5211 M_ZERO | M_WAITOK); 5212 5213 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5214 p = buf; 5215 5216 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5217 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5218 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5219 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5220 p[4], p[3], p[2], p[1], p[0]); 5221 } 5222 5223 sbuf_printf(sb, "\n\nCntl ID Data"); 5224 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5225 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5226 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5227 } 5228 5229 rc = sbuf_finish(sb); 5230 sbuf_delete(sb); 5231 free(buf, M_CXGBE); 5232 return (rc); 5233} 5234 5235static int 5236sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5237{ 5238 struct adapter *sc = arg1; 5239 struct sbuf *sb; 5240 int rc, i; 5241 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5242 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5243 uint16_t thres[CIM_NUM_IBQ]; 5244 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5245 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5246 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5247 5248 if (is_t4(sc)) { 5249 cim_num_obq = CIM_NUM_OBQ; 5250 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5251 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5252 } else { 5253 cim_num_obq = CIM_NUM_OBQ_T5; 5254 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5255 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5256 } 5257 nq = CIM_NUM_IBQ + cim_num_obq; 5258 5259 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5260 if (rc == 0) 5261 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5262 if (rc != 0) 5263 return (rc); 5264 5265 t4_read_cimq_cfg(sc, base, size, thres); 5266 5267 rc = sysctl_wire_old_buffer(req, 0); 5268 if (rc != 0) 5269 return (rc); 5270 5271 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5272 if (sb == NULL) 5273 return (ENOMEM); 5274 5275 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5276 5277 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5278 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5279 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5280 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5281 G_QUEREMFLITS(p[2]) * 16); 5282 for ( ; i < nq; i++, p += 4, wr += 2) 5283 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5284 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5285 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5286 G_QUEREMFLITS(p[2]) * 16); 5287 5288 rc = sbuf_finish(sb); 5289 sbuf_delete(sb); 5290 5291 return (rc); 5292} 5293 5294static int 5295sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5296{ 5297 struct adapter *sc = arg1; 5298 struct sbuf *sb; 5299 int rc; 5300 struct tp_cpl_stats stats; 5301 5302 rc = sysctl_wire_old_buffer(req, 0); 5303 if (rc != 0) 5304 return (rc); 5305 5306 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5307 if (sb == NULL) 5308 return (ENOMEM); 5309 5310 t4_tp_get_cpl_stats(sc, &stats); 5311 5312 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5313 "channel 3\n"); 5314 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5315 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5316 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5317 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5318 5319 rc = sbuf_finish(sb); 5320 sbuf_delete(sb); 5321 5322 return (rc); 5323} 5324 5325static int 5326sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5327{ 5328 struct adapter *sc = arg1; 5329 struct sbuf *sb; 5330 int rc; 5331 struct tp_usm_stats stats; 5332 5333 rc = sysctl_wire_old_buffer(req, 0); 5334 if (rc != 0) 5335 return(rc); 5336 5337 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5338 if (sb == NULL) 5339 return (ENOMEM); 5340 5341 t4_get_usm_stats(sc, &stats); 5342 5343 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5344 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5345 sbuf_printf(sb, "Drops: %u", stats.drops); 5346 5347 rc = sbuf_finish(sb); 5348 sbuf_delete(sb); 5349 5350 return (rc); 5351} 5352 5353const char *devlog_level_strings[] = { 5354 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5355 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5356 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5357 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5358 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5359 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5360}; 5361 5362const char *devlog_facility_strings[] = { 5363 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5364 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5365 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5366 [FW_DEVLOG_FACILITY_RES] = "RES", 5367 [FW_DEVLOG_FACILITY_HW] = "HW", 5368 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5369 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5370 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5371 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5372 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5373 [FW_DEVLOG_FACILITY_VI] = "VI", 5374 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5375 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5376 [FW_DEVLOG_FACILITY_TM] = "TM", 5377 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5378 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5379 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5380 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5381 [FW_DEVLOG_FACILITY_RI] = "RI", 5382 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5383 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5384 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5385 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5386}; 5387 5388static int 5389sysctl_devlog(SYSCTL_HANDLER_ARGS) 5390{ 5391 struct adapter *sc = arg1; 5392 struct devlog_params *dparams = &sc->params.devlog; 5393 struct fw_devlog_e *buf, *e; 5394 int i, j, rc, nentries, first = 0, m; 5395 struct sbuf *sb; 5396 uint64_t ftstamp = UINT64_MAX; 5397 5398 if (dparams->start == 0) { 5399 dparams->memtype = FW_MEMTYPE_EDC0; 5400 dparams->start = 0x84000; 5401 dparams->size = 32768; 5402 } 5403 5404 nentries = dparams->size / sizeof(struct fw_devlog_e); 5405 5406 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5407 if (buf == NULL) 5408 return (ENOMEM); 5409 5410 m = fwmtype_to_hwmtype(dparams->memtype); 5411 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5412 if (rc != 0) 5413 goto done; 5414 5415 for (i = 0; i < nentries; i++) { 5416 e = &buf[i]; 5417 5418 if (e->timestamp == 0) 5419 break; /* end */ 5420 5421 e->timestamp = be64toh(e->timestamp); 5422 e->seqno = be32toh(e->seqno); 5423 for (j = 0; j < 8; j++) 5424 e->params[j] = be32toh(e->params[j]); 5425 5426 if (e->timestamp < ftstamp) { 5427 ftstamp = e->timestamp; 5428 first = i; 5429 } 5430 } 5431 5432 if (buf[first].timestamp == 0) 5433 goto done; /* nothing in the log */ 5434 5435 rc = sysctl_wire_old_buffer(req, 0); 5436 if (rc != 0) 5437 goto done; 5438 5439 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5440 if (sb == NULL) { 5441 rc = ENOMEM; 5442 goto done; 5443 } 5444 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5445 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5446 5447 i = first; 5448 do { 5449 e = &buf[i]; 5450 if (e->timestamp == 0) 5451 break; /* end */ 5452 5453 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5454 e->seqno, e->timestamp, 5455 (e->level < nitems(devlog_level_strings) ? 5456 devlog_level_strings[e->level] : "UNKNOWN"), 5457 (e->facility < nitems(devlog_facility_strings) ? 5458 devlog_facility_strings[e->facility] : "UNKNOWN")); 5459 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5460 e->params[2], e->params[3], e->params[4], 5461 e->params[5], e->params[6], e->params[7]); 5462 5463 if (++i == nentries) 5464 i = 0; 5465 } while (i != first); 5466 5467 rc = sbuf_finish(sb); 5468 sbuf_delete(sb); 5469done: 5470 free(buf, M_CXGBE); 5471 return (rc); 5472} 5473 5474static int 5475sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5476{ 5477 struct adapter *sc = arg1; 5478 struct sbuf *sb; 5479 int rc; 5480 struct tp_fcoe_stats stats[4]; 5481 5482 rc = sysctl_wire_old_buffer(req, 0); 5483 if (rc != 0) 5484 return (rc); 5485 5486 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5487 if (sb == NULL) 5488 return (ENOMEM); 5489 5490 t4_get_fcoe_stats(sc, 0, &stats[0]); 5491 t4_get_fcoe_stats(sc, 1, &stats[1]); 5492 t4_get_fcoe_stats(sc, 2, &stats[2]); 5493 t4_get_fcoe_stats(sc, 3, &stats[3]); 5494 5495 sbuf_printf(sb, " channel 0 channel 1 " 5496 "channel 2 channel 3\n"); 5497 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5498 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5499 stats[3].octetsDDP); 5500 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5501 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5502 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5503 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5504 stats[3].framesDrop); 5505 5506 rc = sbuf_finish(sb); 5507 sbuf_delete(sb); 5508 5509 return (rc); 5510} 5511 5512static int 5513sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5514{ 5515 struct adapter *sc = arg1; 5516 struct sbuf *sb; 5517 int rc, i; 5518 unsigned int map, kbps, ipg, mode; 5519 unsigned int pace_tab[NTX_SCHED]; 5520 5521 rc = sysctl_wire_old_buffer(req, 0); 5522 if (rc != 0) 5523 return (rc); 5524 5525 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5526 if (sb == NULL) 5527 return (ENOMEM); 5528 5529 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5530 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5531 t4_read_pace_tbl(sc, pace_tab); 5532 5533 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5534 "Class IPG (0.1 ns) Flow IPG (us)"); 5535 5536 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5537 t4_get_tx_sched(sc, i, &kbps, &ipg); 5538 sbuf_printf(sb, "\n %u %-5s %u ", i, 5539 (mode & (1 << i)) ? "flow" : "class", map & 3); 5540 if (kbps) 5541 sbuf_printf(sb, "%9u ", kbps); 5542 else 5543 sbuf_printf(sb, " disabled "); 5544 5545 if (ipg) 5546 sbuf_printf(sb, "%13u ", ipg); 5547 else 5548 sbuf_printf(sb, " disabled "); 5549 5550 if (pace_tab[i]) 5551 sbuf_printf(sb, "%10u", pace_tab[i]); 5552 else 5553 sbuf_printf(sb, " disabled"); 5554 } 5555 5556 rc = sbuf_finish(sb); 5557 sbuf_delete(sb); 5558 5559 return (rc); 5560} 5561 5562static int 5563sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5564{ 5565 struct adapter *sc = arg1; 5566 struct sbuf *sb; 5567 int rc, i, j; 5568 uint64_t *p0, *p1; 5569 struct lb_port_stats s[2]; 5570 static const char *stat_name[] = { 5571 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5572 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5573 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5574 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5575 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5576 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5577 "BG2FramesTrunc:", "BG3FramesTrunc:" 5578 }; 5579 5580 rc = sysctl_wire_old_buffer(req, 0); 5581 if (rc != 0) 5582 return (rc); 5583 5584 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5585 if (sb == NULL) 5586 return (ENOMEM); 5587 5588 memset(s, 0, sizeof(s)); 5589 5590 for (i = 0; i < 4; i += 2) { 5591 t4_get_lb_stats(sc, i, &s[0]); 5592 t4_get_lb_stats(sc, i + 1, &s[1]); 5593 5594 p0 = &s[0].octets; 5595 p1 = &s[1].octets; 5596 sbuf_printf(sb, "%s Loopback %u" 5597 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5598 5599 for (j = 0; j < nitems(stat_name); j++) 5600 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5601 *p0++, *p1++); 5602 } 5603 5604 rc = sbuf_finish(sb); 5605 sbuf_delete(sb); 5606 5607 return (rc); 5608} 5609 5610static int 5611sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5612{ 5613 int rc = 0; 5614 struct port_info *pi = arg1; 5615 struct sbuf *sb; 5616 static const char *linkdnreasons[] = { 5617 "non-specific", "remote fault", "autoneg failed", "reserved3", 5618 "PHY overheated", "unknown", "rx los", "reserved7" 5619 }; 5620 5621 rc = sysctl_wire_old_buffer(req, 0); 5622 if (rc != 0) 5623 return(rc); 5624 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5625 if (sb == NULL) 5626 return (ENOMEM); 5627 5628 if (pi->linkdnrc < 0) 5629 sbuf_printf(sb, "n/a"); 5630 else if (pi->linkdnrc < nitems(linkdnreasons)) 5631 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5632 else 5633 sbuf_printf(sb, "%d", pi->linkdnrc); 5634 5635 rc = sbuf_finish(sb); 5636 sbuf_delete(sb); 5637 5638 return (rc); 5639} 5640 5641struct mem_desc { 5642 unsigned int base; 5643 unsigned int limit; 5644 unsigned int idx; 5645}; 5646 5647static int 5648mem_desc_cmp(const void *a, const void *b) 5649{ 5650 return ((const struct mem_desc *)a)->base - 5651 ((const struct mem_desc *)b)->base; 5652} 5653 5654static void 5655mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5656 unsigned int to) 5657{ 5658 unsigned int size; 5659 5660 size = to - from + 1; 5661 if (size == 0) 5662 return; 5663 5664 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5665 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5666} 5667 5668static int 5669sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5670{ 5671 struct adapter *sc = arg1; 5672 struct sbuf *sb; 5673 int rc, i, n; 5674 uint32_t lo, hi, used, alloc; 5675 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5676 static const char *region[] = { 5677 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5678 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5679 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5680 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5681 "RQUDP region:", "PBL region:", "TXPBL region:", 5682 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5683 "On-chip queues:" 5684 }; 5685 struct mem_desc avail[4]; 5686 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5687 struct mem_desc *md = mem; 5688 5689 rc = sysctl_wire_old_buffer(req, 0); 5690 if (rc != 0) 5691 return (rc); 5692 5693 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5694 if (sb == NULL) 5695 return (ENOMEM); 5696 5697 for (i = 0; i < nitems(mem); i++) { 5698 mem[i].limit = 0; 5699 mem[i].idx = i; 5700 } 5701 5702 /* Find and sort the populated memory ranges */ 5703 i = 0; 5704 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5705 if (lo & F_EDRAM0_ENABLE) { 5706 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5707 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5708 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5709 avail[i].idx = 0; 5710 i++; 5711 } 5712 if (lo & F_EDRAM1_ENABLE) { 5713 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5714 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5715 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5716 avail[i].idx = 1; 5717 i++; 5718 } 5719 if (lo & F_EXT_MEM_ENABLE) { 5720 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5721 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5722 avail[i].limit = avail[i].base + 5723 (G_EXT_MEM_SIZE(hi) << 20); 5724 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5725 i++; 5726 } 5727 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5728 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5729 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5730 avail[i].limit = avail[i].base + 5731 (G_EXT_MEM1_SIZE(hi) << 20); 5732 avail[i].idx = 4; 5733 i++; 5734 } 5735 if (!i) /* no memory available */ 5736 return 0; 5737 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5738 5739 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5740 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5741 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5742 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5743 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5744 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5745 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5746 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5747 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5748 5749 /* the next few have explicit upper bounds */ 5750 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5751 md->limit = md->base - 1 + 5752 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5753 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5754 md++; 5755 5756 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5757 md->limit = md->base - 1 + 5758 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5759 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5760 md++; 5761 5762 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5763 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5764 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5765 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5766 } else { 5767 md->base = 0; 5768 md->idx = nitems(region); /* hide it */ 5769 } 5770 md++; 5771 5772#define ulp_region(reg) \ 5773 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5774 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5775 5776 ulp_region(RX_ISCSI); 5777 ulp_region(RX_TDDP); 5778 ulp_region(TX_TPT); 5779 ulp_region(RX_STAG); 5780 ulp_region(RX_RQ); 5781 ulp_region(RX_RQUDP); 5782 ulp_region(RX_PBL); 5783 ulp_region(TX_PBL); 5784#undef ulp_region 5785 5786 md->base = 0; 5787 md->idx = nitems(region); 5788 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5789 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5790 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5791 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5792 } 5793 md++; 5794 5795 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5796 md->limit = md->base + sc->tids.ntids - 1; 5797 md++; 5798 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5799 md->limit = md->base + sc->tids.ntids - 1; 5800 md++; 5801 5802 md->base = sc->vres.ocq.start; 5803 if (sc->vres.ocq.size) 5804 md->limit = md->base + sc->vres.ocq.size - 1; 5805 else 5806 md->idx = nitems(region); /* hide it */ 5807 md++; 5808 5809 /* add any address-space holes, there can be up to 3 */ 5810 for (n = 0; n < i - 1; n++) 5811 if (avail[n].limit < avail[n + 1].base) 5812 (md++)->base = avail[n].limit; 5813 if (avail[n].limit) 5814 (md++)->base = avail[n].limit; 5815 5816 n = md - mem; 5817 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 5818 5819 for (lo = 0; lo < i; lo++) 5820 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 5821 avail[lo].limit - 1); 5822 5823 sbuf_printf(sb, "\n"); 5824 for (i = 0; i < n; i++) { 5825 if (mem[i].idx >= nitems(region)) 5826 continue; /* skip holes */ 5827 if (!mem[i].limit) 5828 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 5829 mem_region_show(sb, region[mem[i].idx], mem[i].base, 5830 mem[i].limit); 5831 } 5832 5833 sbuf_printf(sb, "\n"); 5834 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 5835 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 5836 mem_region_show(sb, "uP RAM:", lo, hi); 5837 5838 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 5839 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 5840 mem_region_show(sb, "uP Extmem2:", lo, hi); 5841 5842 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 5843 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 5844 G_PMRXMAXPAGE(lo), 5845 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 5846 (lo & F_PMRXNUMCHN) ? 2 : 1); 5847 5848 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 5849 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 5850 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 5851 G_PMTXMAXPAGE(lo), 5852 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 5853 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 5854 sbuf_printf(sb, "%u p-structs\n", 5855 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 5856 5857 for (i = 0; i < 4; i++) { 5858 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 5859 if (is_t4(sc)) { 5860 used = G_USED(lo); 5861 alloc = G_ALLOC(lo); 5862 } else { 5863 used = G_T5_USED(lo); 5864 alloc = G_T5_ALLOC(lo); 5865 } 5866 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 5867 i, used, alloc); 5868 } 5869 for (i = 0; i < 4; i++) { 5870 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 5871 if (is_t4(sc)) { 5872 used = G_USED(lo); 5873 alloc = G_ALLOC(lo); 5874 } else { 5875 used = G_T5_USED(lo); 5876 alloc = G_T5_ALLOC(lo); 5877 } 5878 sbuf_printf(sb, 5879 "\nLoopback %d using %u pages out of %u allocated", 5880 i, used, alloc); 5881 } 5882 5883 rc = sbuf_finish(sb); 5884 sbuf_delete(sb); 5885 5886 return (rc); 5887} 5888 5889static inline void 5890tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 5891{ 5892 *mask = x | y; 5893 y = htobe64(y); 5894 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 5895} 5896 5897static int 5898sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 5899{ 5900 struct adapter *sc = arg1; 5901 struct sbuf *sb; 5902 int rc, i, n; 5903 5904 rc = sysctl_wire_old_buffer(req, 0); 5905 if (rc != 0) 5906 return (rc); 5907 5908 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5909 if (sb == NULL) 5910 return (ENOMEM); 5911 5912 sbuf_printf(sb, 5913 "Idx Ethernet address Mask Vld Ports PF" 5914 " VF Replication P0 P1 P2 P3 ML"); 5915 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 5916 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 5917 for (i = 0; i < n; i++) { 5918 uint64_t tcamx, tcamy, mask; 5919 uint32_t cls_lo, cls_hi; 5920 uint8_t addr[ETHER_ADDR_LEN]; 5921 5922 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 5923 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 5924 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 5925 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 5926 5927 if (tcamx & tcamy) 5928 continue; 5929 5930 tcamxy2valmask(tcamx, tcamy, addr, &mask); 5931 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 5932 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 5933 addr[3], addr[4], addr[5], (uintmax_t)mask, 5934 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 5935 G_PORTMAP(cls_hi), G_PF(cls_lo), 5936 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 5937 5938 if (cls_lo & F_REPLICATE) { 5939 struct fw_ldst_cmd ldst_cmd; 5940 5941 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 5942 ldst_cmd.op_to_addrspace = 5943 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 5944 F_FW_CMD_REQUEST | F_FW_CMD_READ | 5945 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 5946 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 5947 ldst_cmd.u.mps.fid_ctl = 5948 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 5949 V_FW_LDST_CMD_CTL(i)); 5950 5951 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 5952 "t4mps"); 5953 if (rc) 5954 break; 5955 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 5956 sizeof(ldst_cmd), &ldst_cmd); 5957 end_synchronized_op(sc, 0); 5958 5959 if (rc != 0) { 5960 sbuf_printf(sb, 5961 " ------------ error %3u ------------", rc); 5962 rc = 0; 5963 } else { 5964 sbuf_printf(sb, " %08x %08x %08x %08x", 5965 be32toh(ldst_cmd.u.mps.rplc127_96), 5966 be32toh(ldst_cmd.u.mps.rplc95_64), 5967 be32toh(ldst_cmd.u.mps.rplc63_32), 5968 be32toh(ldst_cmd.u.mps.rplc31_0)); 5969 } 5970 } else 5971 sbuf_printf(sb, "%36s", ""); 5972 5973 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 5974 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 5975 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 5976 } 5977 5978 if (rc) 5979 (void) sbuf_finish(sb); 5980 else 5981 rc = sbuf_finish(sb); 5982 sbuf_delete(sb); 5983 5984 return (rc); 5985} 5986 5987static int 5988sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 5989{ 5990 struct adapter *sc = arg1; 5991 struct sbuf *sb; 5992 int rc; 5993 uint16_t mtus[NMTUS]; 5994 5995 rc = sysctl_wire_old_buffer(req, 0); 5996 if (rc != 0) 5997 return (rc); 5998 5999 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6000 if (sb == NULL) 6001 return (ENOMEM); 6002 6003 t4_read_mtu_tbl(sc, mtus, NULL); 6004 6005 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6006 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6007 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6008 mtus[14], mtus[15]); 6009 6010 rc = sbuf_finish(sb); 6011 sbuf_delete(sb); 6012 6013 return (rc); 6014} 6015 6016static int 6017sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6018{ 6019 struct adapter *sc = arg1; 6020 struct sbuf *sb; 6021 int rc, i; 6022 uint32_t cnt[PM_NSTATS]; 6023 uint64_t cyc[PM_NSTATS]; 6024 static const char *rx_stats[] = { 6025 "Read:", "Write bypass:", "Write mem:", "Flush:" 6026 }; 6027 static const char *tx_stats[] = { 6028 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6029 }; 6030 6031 rc = sysctl_wire_old_buffer(req, 0); 6032 if (rc != 0) 6033 return (rc); 6034 6035 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6036 if (sb == NULL) 6037 return (ENOMEM); 6038 6039 t4_pmtx_get_stats(sc, cnt, cyc); 6040 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6041 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6042 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6043 cyc[i]); 6044 6045 t4_pmrx_get_stats(sc, cnt, cyc); 6046 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6047 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6048 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6049 cyc[i]); 6050 6051 rc = sbuf_finish(sb); 6052 sbuf_delete(sb); 6053 6054 return (rc); 6055} 6056 6057static int 6058sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6059{ 6060 struct adapter *sc = arg1; 6061 struct sbuf *sb; 6062 int rc; 6063 struct tp_rdma_stats stats; 6064 6065 rc = sysctl_wire_old_buffer(req, 0); 6066 if (rc != 0) 6067 return (rc); 6068 6069 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6070 if (sb == NULL) 6071 return (ENOMEM); 6072 6073 t4_tp_get_rdma_stats(sc, &stats); 6074 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6075 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6076 6077 rc = sbuf_finish(sb); 6078 sbuf_delete(sb); 6079 6080 return (rc); 6081} 6082 6083static int 6084sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6085{ 6086 struct adapter *sc = arg1; 6087 struct sbuf *sb; 6088 int rc; 6089 struct tp_tcp_stats v4, v6; 6090 6091 rc = sysctl_wire_old_buffer(req, 0); 6092 if (rc != 0) 6093 return (rc); 6094 6095 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6096 if (sb == NULL) 6097 return (ENOMEM); 6098 6099 t4_tp_get_tcp_stats(sc, &v4, &v6); 6100 sbuf_printf(sb, 6101 " IP IPv6\n"); 6102 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6103 v4.tcpOutRsts, v6.tcpOutRsts); 6104 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6105 v4.tcpInSegs, v6.tcpInSegs); 6106 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6107 v4.tcpOutSegs, v6.tcpOutSegs); 6108 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6109 v4.tcpRetransSegs, v6.tcpRetransSegs); 6110 6111 rc = sbuf_finish(sb); 6112 sbuf_delete(sb); 6113 6114 return (rc); 6115} 6116 6117static int 6118sysctl_tids(SYSCTL_HANDLER_ARGS) 6119{ 6120 struct adapter *sc = arg1; 6121 struct sbuf *sb; 6122 int rc; 6123 struct tid_info *t = &sc->tids; 6124 6125 rc = sysctl_wire_old_buffer(req, 0); 6126 if (rc != 0) 6127 return (rc); 6128 6129 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6130 if (sb == NULL) 6131 return (ENOMEM); 6132 6133 if (t->natids) { 6134 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6135 t->atids_in_use); 6136 } 6137 6138 if (t->ntids) { 6139 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6140 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6141 6142 if (b) { 6143 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6144 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6145 t->ntids - 1); 6146 } else { 6147 sbuf_printf(sb, "TID range: %u-%u", 6148 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6149 t->ntids - 1); 6150 } 6151 } else 6152 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6153 sbuf_printf(sb, ", in use: %u\n", 6154 atomic_load_acq_int(&t->tids_in_use)); 6155 } 6156 6157 if (t->nstids) { 6158 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6159 t->stid_base + t->nstids - 1, t->stids_in_use); 6160 } 6161 6162 if (t->nftids) { 6163 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6164 t->ftid_base + t->nftids - 1); 6165 } 6166 6167 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6168 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6169 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6170 6171 rc = sbuf_finish(sb); 6172 sbuf_delete(sb); 6173 6174 return (rc); 6175} 6176 6177static int 6178sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6179{ 6180 struct adapter *sc = arg1; 6181 struct sbuf *sb; 6182 int rc; 6183 struct tp_err_stats stats; 6184 6185 rc = sysctl_wire_old_buffer(req, 0); 6186 if (rc != 0) 6187 return (rc); 6188 6189 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6190 if (sb == NULL) 6191 return (ENOMEM); 6192 6193 t4_tp_get_err_stats(sc, &stats); 6194 6195 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6196 "channel 3\n"); 6197 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6198 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6199 stats.macInErrs[3]); 6200 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6201 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6202 stats.hdrInErrs[3]); 6203 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6204 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6205 stats.tcpInErrs[3]); 6206 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6207 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6208 stats.tcp6InErrs[3]); 6209 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6210 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6211 stats.tnlCongDrops[3]); 6212 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6213 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6214 stats.tnlTxDrops[3]); 6215 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6216 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6217 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6218 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6219 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6220 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6221 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6222 stats.ofldNoNeigh, stats.ofldCongDefer); 6223 6224 rc = sbuf_finish(sb); 6225 sbuf_delete(sb); 6226 6227 return (rc); 6228} 6229 6230struct field_desc { 6231 const char *name; 6232 u_int start; 6233 u_int width; 6234}; 6235 6236static void 6237field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6238{ 6239 char buf[32]; 6240 int line_size = 0; 6241 6242 while (f->name) { 6243 uint64_t mask = (1ULL << f->width) - 1; 6244 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6245 ((uintmax_t)v >> f->start) & mask); 6246 6247 if (line_size + len >= 79) { 6248 line_size = 8; 6249 sbuf_printf(sb, "\n "); 6250 } 6251 sbuf_printf(sb, "%s ", buf); 6252 line_size += len + 1; 6253 f++; 6254 } 6255 sbuf_printf(sb, "\n"); 6256} 6257 6258static struct field_desc tp_la0[] = { 6259 { "RcfOpCodeOut", 60, 4 }, 6260 { "State", 56, 4 }, 6261 { "WcfState", 52, 4 }, 6262 { "RcfOpcSrcOut", 50, 2 }, 6263 { "CRxError", 49, 1 }, 6264 { "ERxError", 48, 1 }, 6265 { "SanityFailed", 47, 1 }, 6266 { "SpuriousMsg", 46, 1 }, 6267 { "FlushInputMsg", 45, 1 }, 6268 { "FlushInputCpl", 44, 1 }, 6269 { "RssUpBit", 43, 1 }, 6270 { "RssFilterHit", 42, 1 }, 6271 { "Tid", 32, 10 }, 6272 { "InitTcb", 31, 1 }, 6273 { "LineNumber", 24, 7 }, 6274 { "Emsg", 23, 1 }, 6275 { "EdataOut", 22, 1 }, 6276 { "Cmsg", 21, 1 }, 6277 { "CdataOut", 20, 1 }, 6278 { "EreadPdu", 19, 1 }, 6279 { "CreadPdu", 18, 1 }, 6280 { "TunnelPkt", 17, 1 }, 6281 { "RcfPeerFin", 16, 1 }, 6282 { "RcfReasonOut", 12, 4 }, 6283 { "TxCchannel", 10, 2 }, 6284 { "RcfTxChannel", 8, 2 }, 6285 { "RxEchannel", 6, 2 }, 6286 { "RcfRxChannel", 5, 1 }, 6287 { "RcfDataOutSrdy", 4, 1 }, 6288 { "RxDvld", 3, 1 }, 6289 { "RxOoDvld", 2, 1 }, 6290 { "RxCongestion", 1, 1 }, 6291 { "TxCongestion", 0, 1 }, 6292 { NULL } 6293}; 6294 6295static struct field_desc tp_la1[] = { 6296 { "CplCmdIn", 56, 8 }, 6297 { "CplCmdOut", 48, 8 }, 6298 { "ESynOut", 47, 1 }, 6299 { "EAckOut", 46, 1 }, 6300 { "EFinOut", 45, 1 }, 6301 { "ERstOut", 44, 1 }, 6302 { "SynIn", 43, 1 }, 6303 { "AckIn", 42, 1 }, 6304 { "FinIn", 41, 1 }, 6305 { "RstIn", 40, 1 }, 6306 { "DataIn", 39, 1 }, 6307 { "DataInVld", 38, 1 }, 6308 { "PadIn", 37, 1 }, 6309 { "RxBufEmpty", 36, 1 }, 6310 { "RxDdp", 35, 1 }, 6311 { "RxFbCongestion", 34, 1 }, 6312 { "TxFbCongestion", 33, 1 }, 6313 { "TxPktSumSrdy", 32, 1 }, 6314 { "RcfUlpType", 28, 4 }, 6315 { "Eread", 27, 1 }, 6316 { "Ebypass", 26, 1 }, 6317 { "Esave", 25, 1 }, 6318 { "Static0", 24, 1 }, 6319 { "Cread", 23, 1 }, 6320 { "Cbypass", 22, 1 }, 6321 { "Csave", 21, 1 }, 6322 { "CPktOut", 20, 1 }, 6323 { "RxPagePoolFull", 18, 2 }, 6324 { "RxLpbkPkt", 17, 1 }, 6325 { "TxLpbkPkt", 16, 1 }, 6326 { "RxVfValid", 15, 1 }, 6327 { "SynLearned", 14, 1 }, 6328 { "SetDelEntry", 13, 1 }, 6329 { "SetInvEntry", 12, 1 }, 6330 { "CpcmdDvld", 11, 1 }, 6331 { "CpcmdSave", 10, 1 }, 6332 { "RxPstructsFull", 8, 2 }, 6333 { "EpcmdDvld", 7, 1 }, 6334 { "EpcmdFlush", 6, 1 }, 6335 { "EpcmdTrimPrefix", 5, 1 }, 6336 { "EpcmdTrimPostfix", 4, 1 }, 6337 { "ERssIp4Pkt", 3, 1 }, 6338 { "ERssIp6Pkt", 2, 1 }, 6339 { "ERssTcpUdpPkt", 1, 1 }, 6340 { "ERssFceFipPkt", 0, 1 }, 6341 { NULL } 6342}; 6343 6344static struct field_desc tp_la2[] = { 6345 { "CplCmdIn", 56, 8 }, 6346 { "MpsVfVld", 55, 1 }, 6347 { "MpsPf", 52, 3 }, 6348 { "MpsVf", 44, 8 }, 6349 { "SynIn", 43, 1 }, 6350 { "AckIn", 42, 1 }, 6351 { "FinIn", 41, 1 }, 6352 { "RstIn", 40, 1 }, 6353 { "DataIn", 39, 1 }, 6354 { "DataInVld", 38, 1 }, 6355 { "PadIn", 37, 1 }, 6356 { "RxBufEmpty", 36, 1 }, 6357 { "RxDdp", 35, 1 }, 6358 { "RxFbCongestion", 34, 1 }, 6359 { "TxFbCongestion", 33, 1 }, 6360 { "TxPktSumSrdy", 32, 1 }, 6361 { "RcfUlpType", 28, 4 }, 6362 { "Eread", 27, 1 }, 6363 { "Ebypass", 26, 1 }, 6364 { "Esave", 25, 1 }, 6365 { "Static0", 24, 1 }, 6366 { "Cread", 23, 1 }, 6367 { "Cbypass", 22, 1 }, 6368 { "Csave", 21, 1 }, 6369 { "CPktOut", 20, 1 }, 6370 { "RxPagePoolFull", 18, 2 }, 6371 { "RxLpbkPkt", 17, 1 }, 6372 { "TxLpbkPkt", 16, 1 }, 6373 { "RxVfValid", 15, 1 }, 6374 { "SynLearned", 14, 1 }, 6375 { "SetDelEntry", 13, 1 }, 6376 { "SetInvEntry", 12, 1 }, 6377 { "CpcmdDvld", 11, 1 }, 6378 { "CpcmdSave", 10, 1 }, 6379 { "RxPstructsFull", 8, 2 }, 6380 { "EpcmdDvld", 7, 1 }, 6381 { "EpcmdFlush", 6, 1 }, 6382 { "EpcmdTrimPrefix", 5, 1 }, 6383 { "EpcmdTrimPostfix", 4, 1 }, 6384 { "ERssIp4Pkt", 3, 1 }, 6385 { "ERssIp6Pkt", 2, 1 }, 6386 { "ERssTcpUdpPkt", 1, 1 }, 6387 { "ERssFceFipPkt", 0, 1 }, 6388 { NULL } 6389}; 6390 6391static void 6392tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6393{ 6394 6395 field_desc_show(sb, *p, tp_la0); 6396} 6397 6398static void 6399tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6400{ 6401 6402 if (idx) 6403 sbuf_printf(sb, "\n"); 6404 field_desc_show(sb, p[0], tp_la0); 6405 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6406 field_desc_show(sb, p[1], tp_la0); 6407} 6408 6409static void 6410tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6411{ 6412 6413 if (idx) 6414 sbuf_printf(sb, "\n"); 6415 field_desc_show(sb, p[0], tp_la0); 6416 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6417 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6418} 6419 6420static int 6421sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6422{ 6423 struct adapter *sc = arg1; 6424 struct sbuf *sb; 6425 uint64_t *buf, *p; 6426 int rc; 6427 u_int i, inc; 6428 void (*show_func)(struct sbuf *, uint64_t *, int); 6429 6430 rc = sysctl_wire_old_buffer(req, 0); 6431 if (rc != 0) 6432 return (rc); 6433 6434 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6435 if (sb == NULL) 6436 return (ENOMEM); 6437 6438 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6439 6440 t4_tp_read_la(sc, buf, NULL); 6441 p = buf; 6442 6443 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6444 case 2: 6445 inc = 2; 6446 show_func = tp_la_show2; 6447 break; 6448 case 3: 6449 inc = 2; 6450 show_func = tp_la_show3; 6451 break; 6452 default: 6453 inc = 1; 6454 show_func = tp_la_show; 6455 } 6456 6457 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6458 (*show_func)(sb, p, i); 6459 6460 rc = sbuf_finish(sb); 6461 sbuf_delete(sb); 6462 free(buf, M_CXGBE); 6463 return (rc); 6464} 6465 6466static int 6467sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6468{ 6469 struct adapter *sc = arg1; 6470 struct sbuf *sb; 6471 int rc; 6472 u64 nrate[NCHAN], orate[NCHAN]; 6473 6474 rc = sysctl_wire_old_buffer(req, 0); 6475 if (rc != 0) 6476 return (rc); 6477 6478 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6479 if (sb == NULL) 6480 return (ENOMEM); 6481 6482 t4_get_chan_txrate(sc, nrate, orate); 6483 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6484 "channel 3\n"); 6485 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6486 nrate[0], nrate[1], nrate[2], nrate[3]); 6487 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6488 orate[0], orate[1], orate[2], orate[3]); 6489 6490 rc = sbuf_finish(sb); 6491 sbuf_delete(sb); 6492 6493 return (rc); 6494} 6495 6496static int 6497sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6498{ 6499 struct adapter *sc = arg1; 6500 struct sbuf *sb; 6501 uint32_t *buf, *p; 6502 int rc, i; 6503 6504 rc = sysctl_wire_old_buffer(req, 0); 6505 if (rc != 0) 6506 return (rc); 6507 6508 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6509 if (sb == NULL) 6510 return (ENOMEM); 6511 6512 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6513 M_ZERO | M_WAITOK); 6514 6515 t4_ulprx_read_la(sc, buf); 6516 p = buf; 6517 6518 sbuf_printf(sb, " Pcmd Type Message" 6519 " Data"); 6520 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6521 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6522 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6523 } 6524 6525 rc = sbuf_finish(sb); 6526 sbuf_delete(sb); 6527 free(buf, M_CXGBE); 6528 return (rc); 6529} 6530 6531static int 6532sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6533{ 6534 struct adapter *sc = arg1; 6535 struct sbuf *sb; 6536 int rc, v; 6537 6538 rc = sysctl_wire_old_buffer(req, 0); 6539 if (rc != 0) 6540 return (rc); 6541 6542 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6543 if (sb == NULL) 6544 return (ENOMEM); 6545 6546 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6547 if (G_STATSOURCE_T5(v) == 7) { 6548 if (G_STATMODE(v) == 0) { 6549 sbuf_printf(sb, "total %d, incomplete %d", 6550 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6551 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6552 } else if (G_STATMODE(v) == 1) { 6553 sbuf_printf(sb, "total %d, data overflow %d", 6554 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6555 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6556 } 6557 } 6558 rc = sbuf_finish(sb); 6559 sbuf_delete(sb); 6560 6561 return (rc); 6562} 6563#endif 6564 6565static inline void 6566txq_start(struct ifnet *ifp, struct sge_txq *txq) 6567{ 6568 struct buf_ring *br; 6569 struct mbuf *m; 6570 6571 TXQ_LOCK_ASSERT_OWNED(txq); 6572 6573 br = txq->br; 6574 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6575 if (m) 6576 t4_eth_tx(ifp, txq, m); 6577} 6578 6579void 6580t4_tx_callout(void *arg) 6581{ 6582 struct sge_eq *eq = arg; 6583 struct adapter *sc; 6584 6585 if (EQ_TRYLOCK(eq) == 0) 6586 goto reschedule; 6587 6588 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6589 EQ_UNLOCK(eq); 6590reschedule: 6591 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6592 callout_schedule(&eq->tx_callout, 1); 6593 return; 6594 } 6595 6596 EQ_LOCK_ASSERT_OWNED(eq); 6597 6598 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6599 6600 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6601 struct sge_txq *txq = arg; 6602 struct port_info *pi = txq->ifp->if_softc; 6603 6604 sc = pi->adapter; 6605 } else { 6606 struct sge_wrq *wrq = arg; 6607 6608 sc = wrq->adapter; 6609 } 6610 6611 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6612 } 6613 6614 EQ_UNLOCK(eq); 6615} 6616 6617void 6618t4_tx_task(void *arg, int count) 6619{ 6620 struct sge_eq *eq = arg; 6621 6622 EQ_LOCK(eq); 6623 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6624 struct sge_txq *txq = arg; 6625 txq_start(txq->ifp, txq); 6626 } else { 6627 struct sge_wrq *wrq = arg; 6628 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6629 } 6630 EQ_UNLOCK(eq); 6631} 6632 6633static uint32_t 6634fconf_to_mode(uint32_t fconf) 6635{ 6636 uint32_t mode; 6637 6638 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6639 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6640 6641 if (fconf & F_FRAGMENTATION) 6642 mode |= T4_FILTER_IP_FRAGMENT; 6643 6644 if (fconf & F_MPSHITTYPE) 6645 mode |= T4_FILTER_MPS_HIT_TYPE; 6646 6647 if (fconf & F_MACMATCH) 6648 mode |= T4_FILTER_MAC_IDX; 6649 6650 if (fconf & F_ETHERTYPE) 6651 mode |= T4_FILTER_ETH_TYPE; 6652 6653 if (fconf & F_PROTOCOL) 6654 mode |= T4_FILTER_IP_PROTO; 6655 6656 if (fconf & F_TOS) 6657 mode |= T4_FILTER_IP_TOS; 6658 6659 if (fconf & F_VLAN) 6660 mode |= T4_FILTER_VLAN; 6661 6662 if (fconf & F_VNIC_ID) 6663 mode |= T4_FILTER_VNIC; 6664 6665 if (fconf & F_PORT) 6666 mode |= T4_FILTER_PORT; 6667 6668 if (fconf & F_FCOE) 6669 mode |= T4_FILTER_FCoE; 6670 6671 return (mode); 6672} 6673 6674static uint32_t 6675mode_to_fconf(uint32_t mode) 6676{ 6677 uint32_t fconf = 0; 6678 6679 if (mode & T4_FILTER_IP_FRAGMENT) 6680 fconf |= F_FRAGMENTATION; 6681 6682 if (mode & T4_FILTER_MPS_HIT_TYPE) 6683 fconf |= F_MPSHITTYPE; 6684 6685 if (mode & T4_FILTER_MAC_IDX) 6686 fconf |= F_MACMATCH; 6687 6688 if (mode & T4_FILTER_ETH_TYPE) 6689 fconf |= F_ETHERTYPE; 6690 6691 if (mode & T4_FILTER_IP_PROTO) 6692 fconf |= F_PROTOCOL; 6693 6694 if (mode & T4_FILTER_IP_TOS) 6695 fconf |= F_TOS; 6696 6697 if (mode & T4_FILTER_VLAN) 6698 fconf |= F_VLAN; 6699 6700 if (mode & T4_FILTER_VNIC) 6701 fconf |= F_VNIC_ID; 6702 6703 if (mode & T4_FILTER_PORT) 6704 fconf |= F_PORT; 6705 6706 if (mode & T4_FILTER_FCoE) 6707 fconf |= F_FCOE; 6708 6709 return (fconf); 6710} 6711 6712static uint32_t 6713fspec_to_fconf(struct t4_filter_specification *fs) 6714{ 6715 uint32_t fconf = 0; 6716 6717 if (fs->val.frag || fs->mask.frag) 6718 fconf |= F_FRAGMENTATION; 6719 6720 if (fs->val.matchtype || fs->mask.matchtype) 6721 fconf |= F_MPSHITTYPE; 6722 6723 if (fs->val.macidx || fs->mask.macidx) 6724 fconf |= F_MACMATCH; 6725 6726 if (fs->val.ethtype || fs->mask.ethtype) 6727 fconf |= F_ETHERTYPE; 6728 6729 if (fs->val.proto || fs->mask.proto) 6730 fconf |= F_PROTOCOL; 6731 6732 if (fs->val.tos || fs->mask.tos) 6733 fconf |= F_TOS; 6734 6735 if (fs->val.vlan_vld || fs->mask.vlan_vld) 6736 fconf |= F_VLAN; 6737 6738 if (fs->val.vnic_vld || fs->mask.vnic_vld) 6739 fconf |= F_VNIC_ID; 6740 6741 if (fs->val.iport || fs->mask.iport) 6742 fconf |= F_PORT; 6743 6744 if (fs->val.fcoe || fs->mask.fcoe) 6745 fconf |= F_FCOE; 6746 6747 return (fconf); 6748} 6749 6750static int 6751get_filter_mode(struct adapter *sc, uint32_t *mode) 6752{ 6753 int rc; 6754 uint32_t fconf; 6755 6756 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6757 "t4getfm"); 6758 if (rc) 6759 return (rc); 6760 6761 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 6762 A_TP_VLAN_PRI_MAP); 6763 6764 if (sc->params.tp.vlan_pri_map != fconf) { 6765 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 6766 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 6767 fconf); 6768 sc->params.tp.vlan_pri_map = fconf; 6769 } 6770 6771 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 6772 6773 end_synchronized_op(sc, LOCK_HELD); 6774 return (0); 6775} 6776 6777static int 6778set_filter_mode(struct adapter *sc, uint32_t mode) 6779{ 6780 uint32_t fconf; 6781 int rc; 6782 6783 fconf = mode_to_fconf(mode); 6784 6785 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6786 "t4setfm"); 6787 if (rc) 6788 return (rc); 6789 6790 if (sc->tids.ftids_in_use > 0) { 6791 rc = EBUSY; 6792 goto done; 6793 } 6794 6795#ifdef TCP_OFFLOAD 6796 if (sc->offload_map) { 6797 rc = EBUSY; 6798 goto done; 6799 } 6800#endif 6801 6802#ifdef notyet 6803 rc = -t4_set_filter_mode(sc, fconf); 6804 if (rc == 0) 6805 sc->filter_mode = fconf; 6806#else 6807 rc = ENOTSUP; 6808#endif 6809 6810done: 6811 end_synchronized_op(sc, LOCK_HELD); 6812 return (rc); 6813} 6814 6815static inline uint64_t 6816get_filter_hits(struct adapter *sc, uint32_t fid) 6817{ 6818 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6819 uint64_t hits; 6820 6821 memwin_info(sc, 0, &mw_base, NULL); 6822 off = position_memwin(sc, 0, 6823 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 6824 if (is_t4(sc)) { 6825 hits = t4_read_reg64(sc, mw_base + off + 16); 6826 hits = be64toh(hits); 6827 } else { 6828 hits = t4_read_reg(sc, mw_base + off + 24); 6829 hits = be32toh(hits); 6830 } 6831 6832 return (hits); 6833} 6834 6835static int 6836get_filter(struct adapter *sc, struct t4_filter *t) 6837{ 6838 int i, rc, nfilters = sc->tids.nftids; 6839 struct filter_entry *f; 6840 6841 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6842 "t4getf"); 6843 if (rc) 6844 return (rc); 6845 6846 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 6847 t->idx >= nfilters) { 6848 t->idx = 0xffffffff; 6849 goto done; 6850 } 6851 6852 f = &sc->tids.ftid_tab[t->idx]; 6853 for (i = t->idx; i < nfilters; i++, f++) { 6854 if (f->valid) { 6855 t->idx = i; 6856 t->l2tidx = f->l2t ? f->l2t->idx : 0; 6857 t->smtidx = f->smtidx; 6858 if (f->fs.hitcnts) 6859 t->hits = get_filter_hits(sc, t->idx); 6860 else 6861 t->hits = UINT64_MAX; 6862 t->fs = f->fs; 6863 6864 goto done; 6865 } 6866 } 6867 6868 t->idx = 0xffffffff; 6869done: 6870 end_synchronized_op(sc, LOCK_HELD); 6871 return (0); 6872} 6873 6874static int 6875set_filter(struct adapter *sc, struct t4_filter *t) 6876{ 6877 unsigned int nfilters, nports; 6878 struct filter_entry *f; 6879 int i, rc; 6880 6881 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 6882 if (rc) 6883 return (rc); 6884 6885 nfilters = sc->tids.nftids; 6886 nports = sc->params.nports; 6887 6888 if (nfilters == 0) { 6889 rc = ENOTSUP; 6890 goto done; 6891 } 6892 6893 if (!(sc->flags & FULL_INIT_DONE)) { 6894 rc = EAGAIN; 6895 goto done; 6896 } 6897 6898 if (t->idx >= nfilters) { 6899 rc = EINVAL; 6900 goto done; 6901 } 6902 6903 /* Validate against the global filter mode */ 6904 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 6905 sc->params.tp.vlan_pri_map) { 6906 rc = E2BIG; 6907 goto done; 6908 } 6909 6910 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 6911 rc = EINVAL; 6912 goto done; 6913 } 6914 6915 if (t->fs.val.iport >= nports) { 6916 rc = EINVAL; 6917 goto done; 6918 } 6919 6920 /* Can't specify an iq if not steering to it */ 6921 if (!t->fs.dirsteer && t->fs.iq) { 6922 rc = EINVAL; 6923 goto done; 6924 } 6925 6926 /* IPv6 filter idx must be 4 aligned */ 6927 if (t->fs.type == 1 && 6928 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 6929 rc = EINVAL; 6930 goto done; 6931 } 6932 6933 if (sc->tids.ftid_tab == NULL) { 6934 KASSERT(sc->tids.ftids_in_use == 0, 6935 ("%s: no memory allocated but filters_in_use > 0", 6936 __func__)); 6937 6938 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 6939 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 6940 if (sc->tids.ftid_tab == NULL) { 6941 rc = ENOMEM; 6942 goto done; 6943 } 6944 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 6945 } 6946 6947 for (i = 0; i < 4; i++) { 6948 f = &sc->tids.ftid_tab[t->idx + i]; 6949 6950 if (f->pending || f->valid) { 6951 rc = EBUSY; 6952 goto done; 6953 } 6954 if (f->locked) { 6955 rc = EPERM; 6956 goto done; 6957 } 6958 6959 if (t->fs.type == 0) 6960 break; 6961 } 6962 6963 f = &sc->tids.ftid_tab[t->idx]; 6964 f->fs = t->fs; 6965 6966 rc = set_filter_wr(sc, t->idx); 6967done: 6968 end_synchronized_op(sc, 0); 6969 6970 if (rc == 0) { 6971 mtx_lock(&sc->tids.ftid_lock); 6972 for (;;) { 6973 if (f->pending == 0) { 6974 rc = f->valid ? 0 : EIO; 6975 break; 6976 } 6977 6978 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6979 PCATCH, "t4setfw", 0)) { 6980 rc = EINPROGRESS; 6981 break; 6982 } 6983 } 6984 mtx_unlock(&sc->tids.ftid_lock); 6985 } 6986 return (rc); 6987} 6988 6989static int 6990del_filter(struct adapter *sc, struct t4_filter *t) 6991{ 6992 unsigned int nfilters; 6993 struct filter_entry *f; 6994 int rc; 6995 6996 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 6997 if (rc) 6998 return (rc); 6999 7000 nfilters = sc->tids.nftids; 7001 7002 if (nfilters == 0) { 7003 rc = ENOTSUP; 7004 goto done; 7005 } 7006 7007 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7008 t->idx >= nfilters) { 7009 rc = EINVAL; 7010 goto done; 7011 } 7012 7013 if (!(sc->flags & FULL_INIT_DONE)) { 7014 rc = EAGAIN; 7015 goto done; 7016 } 7017 7018 f = &sc->tids.ftid_tab[t->idx]; 7019 7020 if (f->pending) { 7021 rc = EBUSY; 7022 goto done; 7023 } 7024 if (f->locked) { 7025 rc = EPERM; 7026 goto done; 7027 } 7028 7029 if (f->valid) { 7030 t->fs = f->fs; /* extra info for the caller */ 7031 rc = del_filter_wr(sc, t->idx); 7032 } 7033 7034done: 7035 end_synchronized_op(sc, 0); 7036 7037 if (rc == 0) { 7038 mtx_lock(&sc->tids.ftid_lock); 7039 for (;;) { 7040 if (f->pending == 0) { 7041 rc = f->valid ? EIO : 0; 7042 break; 7043 } 7044 7045 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7046 PCATCH, "t4delfw", 0)) { 7047 rc = EINPROGRESS; 7048 break; 7049 } 7050 } 7051 mtx_unlock(&sc->tids.ftid_lock); 7052 } 7053 7054 return (rc); 7055} 7056 7057static void 7058clear_filter(struct filter_entry *f) 7059{ 7060 if (f->l2t) 7061 t4_l2t_release(f->l2t); 7062 7063 bzero(f, sizeof (*f)); 7064} 7065 7066static int 7067set_filter_wr(struct adapter *sc, int fidx) 7068{ 7069 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7070 struct wrqe *wr; 7071 struct fw_filter_wr *fwr; 7072 unsigned int ftid; 7073 7074 ASSERT_SYNCHRONIZED_OP(sc); 7075 7076 if (f->fs.newdmac || f->fs.newvlan) { 7077 /* This filter needs an L2T entry; allocate one. */ 7078 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7079 if (f->l2t == NULL) 7080 return (EAGAIN); 7081 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7082 f->fs.dmac)) { 7083 t4_l2t_release(f->l2t); 7084 f->l2t = NULL; 7085 return (ENOMEM); 7086 } 7087 } 7088 7089 ftid = sc->tids.ftid_base + fidx; 7090 7091 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7092 if (wr == NULL) 7093 return (ENOMEM); 7094 7095 fwr = wrtod(wr); 7096 bzero(fwr, sizeof (*fwr)); 7097 7098 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7099 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7100 fwr->tid_to_iq = 7101 htobe32(V_FW_FILTER_WR_TID(ftid) | 7102 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7103 V_FW_FILTER_WR_NOREPLY(0) | 7104 V_FW_FILTER_WR_IQ(f->fs.iq)); 7105 fwr->del_filter_to_l2tix = 7106 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7107 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7108 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7109 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7110 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7111 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7112 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7113 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7114 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7115 f->fs.newvlan == VLAN_REWRITE) | 7116 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7117 f->fs.newvlan == VLAN_REWRITE) | 7118 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7119 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7120 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7121 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7122 fwr->ethtype = htobe16(f->fs.val.ethtype); 7123 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7124 fwr->frag_to_ovlan_vldm = 7125 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7126 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7127 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7128 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7129 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7130 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7131 fwr->smac_sel = 0; 7132 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7133 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7134 fwr->maci_to_matchtypem = 7135 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7136 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7137 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7138 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7139 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7140 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7141 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7142 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7143 fwr->ptcl = f->fs.val.proto; 7144 fwr->ptclm = f->fs.mask.proto; 7145 fwr->ttyp = f->fs.val.tos; 7146 fwr->ttypm = f->fs.mask.tos; 7147 fwr->ivlan = htobe16(f->fs.val.vlan); 7148 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7149 fwr->ovlan = htobe16(f->fs.val.vnic); 7150 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7151 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7152 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7153 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7154 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7155 fwr->lp = htobe16(f->fs.val.dport); 7156 fwr->lpm = htobe16(f->fs.mask.dport); 7157 fwr->fp = htobe16(f->fs.val.sport); 7158 fwr->fpm = htobe16(f->fs.mask.sport); 7159 if (f->fs.newsmac) 7160 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7161 7162 f->pending = 1; 7163 sc->tids.ftids_in_use++; 7164 7165 t4_wrq_tx(sc, wr); 7166 return (0); 7167} 7168 7169static int 7170del_filter_wr(struct adapter *sc, int fidx) 7171{ 7172 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7173 struct wrqe *wr; 7174 struct fw_filter_wr *fwr; 7175 unsigned int ftid; 7176 7177 ftid = sc->tids.ftid_base + fidx; 7178 7179 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7180 if (wr == NULL) 7181 return (ENOMEM); 7182 fwr = wrtod(wr); 7183 bzero(fwr, sizeof (*fwr)); 7184 7185 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7186 7187 f->pending = 1; 7188 t4_wrq_tx(sc, wr); 7189 return (0); 7190} 7191 7192int 7193t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7194{ 7195 struct adapter *sc = iq->adapter; 7196 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7197 unsigned int idx = GET_TID(rpl); 7198 7199 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7200 rss->opcode)); 7201 7202 if (idx >= sc->tids.ftid_base && 7203 (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 7204 unsigned int rc = G_COOKIE(rpl->cookie); 7205 struct filter_entry *f = &sc->tids.ftid_tab[idx]; 7206 7207 mtx_lock(&sc->tids.ftid_lock); 7208 if (rc == FW_FILTER_WR_FLT_ADDED) { 7209 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7210 __func__, idx)); 7211 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7212 f->pending = 0; /* asynchronous setup completed */ 7213 f->valid = 1; 7214 } else { 7215 if (rc != FW_FILTER_WR_FLT_DELETED) { 7216 /* Add or delete failed, display an error */ 7217 log(LOG_ERR, 7218 "filter %u setup failed with error %u\n", 7219 idx, rc); 7220 } 7221 7222 clear_filter(f); 7223 sc->tids.ftids_in_use--; 7224 } 7225 wakeup(&sc->tids.ftid_tab); 7226 mtx_unlock(&sc->tids.ftid_lock); 7227 } 7228 7229 return (0); 7230} 7231 7232static int 7233get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7234{ 7235 int rc; 7236 7237 if (cntxt->cid > M_CTXTQID) 7238 return (EINVAL); 7239 7240 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7241 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7242 return (EINVAL); 7243 7244 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7245 if (rc) 7246 return (rc); 7247 7248 if (sc->flags & FW_OK) { 7249 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7250 &cntxt->data[0]); 7251 if (rc == 0) 7252 goto done; 7253 } 7254 7255 /* 7256 * Read via firmware failed or wasn't even attempted. Read directly via 7257 * the backdoor. 7258 */ 7259 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7260done: 7261 end_synchronized_op(sc, 0); 7262 return (rc); 7263} 7264 7265static int 7266load_fw(struct adapter *sc, struct t4_data *fw) 7267{ 7268 int rc; 7269 uint8_t *fw_data; 7270 7271 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7272 if (rc) 7273 return (rc); 7274 7275 if (sc->flags & FULL_INIT_DONE) { 7276 rc = EBUSY; 7277 goto done; 7278 } 7279 7280 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7281 if (fw_data == NULL) { 7282 rc = ENOMEM; 7283 goto done; 7284 } 7285 7286 rc = copyin(fw->data, fw_data, fw->len); 7287 if (rc == 0) 7288 rc = -t4_load_fw(sc, fw_data, fw->len); 7289 7290 free(fw_data, M_CXGBE); 7291done: 7292 end_synchronized_op(sc, 0); 7293 return (rc); 7294} 7295 7296static int 7297read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7298{ 7299 uint32_t addr, off, remaining, i, n; 7300 uint32_t *buf, *b; 7301 uint32_t mw_base, mw_aperture; 7302 int rc; 7303 uint8_t *dst; 7304 7305 rc = validate_mem_range(sc, mr->addr, mr->len); 7306 if (rc != 0) 7307 return (rc); 7308 7309 memwin_info(sc, win, &mw_base, &mw_aperture); 7310 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7311 addr = mr->addr; 7312 remaining = mr->len; 7313 dst = (void *)mr->data; 7314 7315 while (remaining) { 7316 off = position_memwin(sc, win, addr); 7317 7318 /* number of bytes that we'll copy in the inner loop */ 7319 n = min(remaining, mw_aperture - off); 7320 for (i = 0; i < n; i += 4) 7321 *b++ = t4_read_reg(sc, mw_base + off + i); 7322 7323 rc = copyout(buf, dst, n); 7324 if (rc != 0) 7325 break; 7326 7327 b = buf; 7328 dst += n; 7329 remaining -= n; 7330 addr += n; 7331 } 7332 7333 free(buf, M_CXGBE); 7334 return (rc); 7335} 7336 7337static int 7338read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7339{ 7340 int rc; 7341 7342 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7343 return (EINVAL); 7344 7345 if (i2cd->len > 1) { 7346 /* XXX: need fw support for longer reads in one go */ 7347 return (ENOTSUP); 7348 } 7349 7350 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7351 if (rc) 7352 return (rc); 7353 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7354 i2cd->offset, &i2cd->data[0]); 7355 end_synchronized_op(sc, 0); 7356 7357 return (rc); 7358} 7359 7360static int 7361in_range(int val, int lo, int hi) 7362{ 7363 7364 return (val < 0 || (val <= hi && val >= lo)); 7365} 7366 7367static int 7368set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7369{ 7370 int fw_subcmd, fw_type, rc; 7371 7372 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7373 if (rc) 7374 return (rc); 7375 7376 if (!(sc->flags & FULL_INIT_DONE)) { 7377 rc = EAGAIN; 7378 goto done; 7379 } 7380 7381 /* 7382 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7383 * sub-command and type are in common locations.) 7384 */ 7385 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7386 fw_subcmd = FW_SCHED_SC_CONFIG; 7387 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7388 fw_subcmd = FW_SCHED_SC_PARAMS; 7389 else { 7390 rc = EINVAL; 7391 goto done; 7392 } 7393 if (p->type == SCHED_CLASS_TYPE_PACKET) 7394 fw_type = FW_SCHED_TYPE_PKTSCHED; 7395 else { 7396 rc = EINVAL; 7397 goto done; 7398 } 7399 7400 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7401 /* Vet our parameters ..*/ 7402 if (p->u.config.minmax < 0) { 7403 rc = EINVAL; 7404 goto done; 7405 } 7406 7407 /* And pass the request to the firmware ...*/ 7408 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax); 7409 goto done; 7410 } 7411 7412 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7413 int fw_level; 7414 int fw_mode; 7415 int fw_rateunit; 7416 int fw_ratemode; 7417 7418 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7419 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7420 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7421 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7422 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7423 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7424 else { 7425 rc = EINVAL; 7426 goto done; 7427 } 7428 7429 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7430 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7431 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7432 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7433 else { 7434 rc = EINVAL; 7435 goto done; 7436 } 7437 7438 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7439 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7440 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7441 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7442 else { 7443 rc = EINVAL; 7444 goto done; 7445 } 7446 7447 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7448 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7449 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7450 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7451 else { 7452 rc = EINVAL; 7453 goto done; 7454 } 7455 7456 /* Vet our parameters ... */ 7457 if (!in_range(p->u.params.channel, 0, 3) || 7458 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7459 !in_range(p->u.params.minrate, 0, 10000000) || 7460 !in_range(p->u.params.maxrate, 0, 10000000) || 7461 !in_range(p->u.params.weight, 0, 100)) { 7462 rc = ERANGE; 7463 goto done; 7464 } 7465 7466 /* 7467 * Translate any unset parameters into the firmware's 7468 * nomenclature and/or fail the call if the parameters 7469 * are required ... 7470 */ 7471 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7472 p->u.params.channel < 0 || p->u.params.cl < 0) { 7473 rc = EINVAL; 7474 goto done; 7475 } 7476 if (p->u.params.minrate < 0) 7477 p->u.params.minrate = 0; 7478 if (p->u.params.maxrate < 0) { 7479 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7480 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7481 rc = EINVAL; 7482 goto done; 7483 } else 7484 p->u.params.maxrate = 0; 7485 } 7486 if (p->u.params.weight < 0) { 7487 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7488 rc = EINVAL; 7489 goto done; 7490 } else 7491 p->u.params.weight = 0; 7492 } 7493 if (p->u.params.pktsize < 0) { 7494 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7495 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7496 rc = EINVAL; 7497 goto done; 7498 } else 7499 p->u.params.pktsize = 0; 7500 } 7501 7502 /* See what the firmware thinks of the request ... */ 7503 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7504 fw_rateunit, fw_ratemode, p->u.params.channel, 7505 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7506 p->u.params.weight, p->u.params.pktsize); 7507 goto done; 7508 } 7509 7510 rc = EINVAL; 7511done: 7512 end_synchronized_op(sc, 0); 7513 return (rc); 7514} 7515 7516static int 7517set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7518{ 7519 struct port_info *pi = NULL; 7520 struct sge_txq *txq; 7521 uint32_t fw_mnem, fw_queue, fw_class; 7522 int i, rc; 7523 7524 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7525 if (rc) 7526 return (rc); 7527 7528 if (!(sc->flags & FULL_INIT_DONE)) { 7529 rc = EAGAIN; 7530 goto done; 7531 } 7532 7533 if (p->port >= sc->params.nports) { 7534 rc = EINVAL; 7535 goto done; 7536 } 7537 7538 pi = sc->port[p->port]; 7539 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7540 rc = EINVAL; 7541 goto done; 7542 } 7543 7544 /* 7545 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7546 * Scheduling Class in this case). 7547 */ 7548 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7549 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7550 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7551 7552 /* 7553 * If op.queue is non-negative, then we're only changing the scheduling 7554 * on a single specified TX queue. 7555 */ 7556 if (p->queue >= 0) { 7557 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7558 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7559 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7560 &fw_class); 7561 goto done; 7562 } 7563 7564 /* 7565 * Change the scheduling on all the TX queues for the 7566 * interface. 7567 */ 7568 for_each_txq(pi, i, txq) { 7569 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7570 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7571 &fw_class); 7572 if (rc) 7573 goto done; 7574 } 7575 7576 rc = 0; 7577done: 7578 end_synchronized_op(sc, 0); 7579 return (rc); 7580} 7581 7582int 7583t4_os_find_pci_capability(struct adapter *sc, int cap) 7584{ 7585 int i; 7586 7587 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7588} 7589 7590int 7591t4_os_pci_save_state(struct adapter *sc) 7592{ 7593 device_t dev; 7594 struct pci_devinfo *dinfo; 7595 7596 dev = sc->dev; 7597 dinfo = device_get_ivars(dev); 7598 7599 pci_cfg_save(dev, dinfo, 0); 7600 return (0); 7601} 7602 7603int 7604t4_os_pci_restore_state(struct adapter *sc) 7605{ 7606 device_t dev; 7607 struct pci_devinfo *dinfo; 7608 7609 dev = sc->dev; 7610 dinfo = device_get_ivars(dev); 7611 7612 pci_cfg_restore(dev, dinfo); 7613 return (0); 7614} 7615 7616void 7617t4_os_portmod_changed(const struct adapter *sc, int idx) 7618{ 7619 struct port_info *pi = sc->port[idx]; 7620 static const char *mod_str[] = { 7621 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7622 }; 7623 7624 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7625 if_printf(pi->ifp, "transceiver unplugged.\n"); 7626 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7627 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7628 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7629 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7630 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7631 if_printf(pi->ifp, "%s transceiver inserted.\n", 7632 mod_str[pi->mod_type]); 7633 } else { 7634 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7635 pi->mod_type); 7636 } 7637} 7638 7639void 7640t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7641{ 7642 struct port_info *pi = sc->port[idx]; 7643 struct ifnet *ifp = pi->ifp; 7644 7645 if (link_stat) { 7646 pi->linkdnrc = -1; 7647 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7648 if_link_state_change(ifp, LINK_STATE_UP); 7649 } else { 7650 if (reason >= 0) 7651 pi->linkdnrc = reason; 7652 if_link_state_change(ifp, LINK_STATE_DOWN); 7653 } 7654} 7655 7656void 7657t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7658{ 7659 struct adapter *sc; 7660 7661 sx_slock(&t4_list_lock); 7662 SLIST_FOREACH(sc, &t4_list, link) { 7663 /* 7664 * func should not make any assumptions about what state sc is 7665 * in - the only guarantee is that sc->sc_lock is a valid lock. 7666 */ 7667 func(sc, arg); 7668 } 7669 sx_sunlock(&t4_list_lock); 7670} 7671 7672static int 7673t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7674{ 7675 return (0); 7676} 7677 7678static int 7679t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7680{ 7681 return (0); 7682} 7683 7684static int 7685t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7686 struct thread *td) 7687{ 7688 int rc; 7689 struct adapter *sc = dev->si_drv1; 7690 7691 rc = priv_check(td, PRIV_DRIVER); 7692 if (rc != 0) 7693 return (rc); 7694 7695 switch (cmd) { 7696 case CHELSIO_T4_GETREG: { 7697 struct t4_reg *edata = (struct t4_reg *)data; 7698 7699 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7700 return (EFAULT); 7701 7702 if (edata->size == 4) 7703 edata->val = t4_read_reg(sc, edata->addr); 7704 else if (edata->size == 8) 7705 edata->val = t4_read_reg64(sc, edata->addr); 7706 else 7707 return (EINVAL); 7708 7709 break; 7710 } 7711 case CHELSIO_T4_SETREG: { 7712 struct t4_reg *edata = (struct t4_reg *)data; 7713 7714 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7715 return (EFAULT); 7716 7717 if (edata->size == 4) { 7718 if (edata->val & 0xffffffff00000000) 7719 return (EINVAL); 7720 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 7721 } else if (edata->size == 8) 7722 t4_write_reg64(sc, edata->addr, edata->val); 7723 else 7724 return (EINVAL); 7725 break; 7726 } 7727 case CHELSIO_T4_REGDUMP: { 7728 struct t4_regdump *regs = (struct t4_regdump *)data; 7729 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 7730 uint8_t *buf; 7731 7732 if (regs->len < reglen) { 7733 regs->len = reglen; /* hint to the caller */ 7734 return (ENOBUFS); 7735 } 7736 7737 regs->len = reglen; 7738 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 7739 t4_get_regs(sc, regs, buf); 7740 rc = copyout(buf, regs->data, reglen); 7741 free(buf, M_CXGBE); 7742 break; 7743 } 7744 case CHELSIO_T4_GET_FILTER_MODE: 7745 rc = get_filter_mode(sc, (uint32_t *)data); 7746 break; 7747 case CHELSIO_T4_SET_FILTER_MODE: 7748 rc = set_filter_mode(sc, *(uint32_t *)data); 7749 break; 7750 case CHELSIO_T4_GET_FILTER: 7751 rc = get_filter(sc, (struct t4_filter *)data); 7752 break; 7753 case CHELSIO_T4_SET_FILTER: 7754 rc = set_filter(sc, (struct t4_filter *)data); 7755 break; 7756 case CHELSIO_T4_DEL_FILTER: 7757 rc = del_filter(sc, (struct t4_filter *)data); 7758 break; 7759 case CHELSIO_T4_GET_SGE_CONTEXT: 7760 rc = get_sge_context(sc, (struct t4_sge_context *)data); 7761 break; 7762 case CHELSIO_T4_LOAD_FW: 7763 rc = load_fw(sc, (struct t4_data *)data); 7764 break; 7765 case CHELSIO_T4_GET_MEM: 7766 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 7767 break; 7768 case CHELSIO_T4_GET_I2C: 7769 rc = read_i2c(sc, (struct t4_i2c_data *)data); 7770 break; 7771 case CHELSIO_T4_CLEAR_STATS: { 7772 int i; 7773 u_int port_id = *(uint32_t *)data; 7774 struct port_info *pi; 7775 7776 if (port_id >= sc->params.nports) 7777 return (EINVAL); 7778 pi = sc->port[port_id]; 7779 7780 /* MAC stats */ 7781 t4_clr_port_stats(sc, pi->tx_chan); 7782 7783 if (pi->flags & PORT_INIT_DONE) { 7784 struct sge_rxq *rxq; 7785 struct sge_txq *txq; 7786 struct sge_wrq *wrq; 7787 7788 for_each_rxq(pi, i, rxq) { 7789#if defined(INET) || defined(INET6) 7790 rxq->lro.lro_queued = 0; 7791 rxq->lro.lro_flushed = 0; 7792#endif 7793 rxq->rxcsum = 0; 7794 rxq->vlan_extraction = 0; 7795 } 7796 7797 for_each_txq(pi, i, txq) { 7798 txq->txcsum = 0; 7799 txq->tso_wrs = 0; 7800 txq->vlan_insertion = 0; 7801 txq->imm_wrs = 0; 7802 txq->sgl_wrs = 0; 7803 txq->txpkt_wrs = 0; 7804 txq->txpkts_wrs = 0; 7805 txq->txpkts_pkts = 0; 7806 txq->br->br_drops = 0; 7807 txq->no_dmamap = 0; 7808 txq->no_desc = 0; 7809 } 7810 7811#ifdef TCP_OFFLOAD 7812 /* nothing to clear for each ofld_rxq */ 7813 7814 for_each_ofld_txq(pi, i, wrq) { 7815 wrq->tx_wrs = 0; 7816 wrq->no_desc = 0; 7817 } 7818#endif 7819 wrq = &sc->sge.ctrlq[pi->port_id]; 7820 wrq->tx_wrs = 0; 7821 wrq->no_desc = 0; 7822 } 7823 break; 7824 } 7825 case CHELSIO_T4_SCHED_CLASS: 7826 rc = set_sched_class(sc, (struct t4_sched_params *)data); 7827 break; 7828 case CHELSIO_T4_SCHED_QUEUE: 7829 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 7830 break; 7831 case CHELSIO_T4_GET_TRACER: 7832 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 7833 break; 7834 case CHELSIO_T4_SET_TRACER: 7835 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 7836 break; 7837 default: 7838 rc = EINVAL; 7839 } 7840 7841 return (rc); 7842} 7843 7844#ifdef TCP_OFFLOAD 7845static int 7846toe_capability(struct port_info *pi, int enable) 7847{ 7848 int rc; 7849 struct adapter *sc = pi->adapter; 7850 7851 ASSERT_SYNCHRONIZED_OP(sc); 7852 7853 if (!is_offload(sc)) 7854 return (ENODEV); 7855 7856 if (enable) { 7857 if (!(sc->flags & FULL_INIT_DONE)) { 7858 rc = cxgbe_init_synchronized(pi); 7859 if (rc) 7860 return (rc); 7861 } 7862 7863 if (isset(&sc->offload_map, pi->port_id)) 7864 return (0); 7865 7866 if (!(sc->flags & TOM_INIT_DONE)) { 7867 rc = t4_activate_uld(sc, ULD_TOM); 7868 if (rc == EAGAIN) { 7869 log(LOG_WARNING, 7870 "You must kldload t4_tom.ko before trying " 7871 "to enable TOE on a cxgbe interface.\n"); 7872 } 7873 if (rc != 0) 7874 return (rc); 7875 KASSERT(sc->tom_softc != NULL, 7876 ("%s: TOM activated but softc NULL", __func__)); 7877 KASSERT(sc->flags & TOM_INIT_DONE, 7878 ("%s: TOM activated but flag not set", __func__)); 7879 } 7880 7881 setbit(&sc->offload_map, pi->port_id); 7882 } else { 7883 if (!isset(&sc->offload_map, pi->port_id)) 7884 return (0); 7885 7886 KASSERT(sc->flags & TOM_INIT_DONE, 7887 ("%s: TOM never initialized?", __func__)); 7888 clrbit(&sc->offload_map, pi->port_id); 7889 } 7890 7891 return (0); 7892} 7893 7894/* 7895 * Add an upper layer driver to the global list. 7896 */ 7897int 7898t4_register_uld(struct uld_info *ui) 7899{ 7900 int rc = 0; 7901 struct uld_info *u; 7902 7903 sx_xlock(&t4_uld_list_lock); 7904 SLIST_FOREACH(u, &t4_uld_list, link) { 7905 if (u->uld_id == ui->uld_id) { 7906 rc = EEXIST; 7907 goto done; 7908 } 7909 } 7910 7911 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 7912 ui->refcount = 0; 7913done: 7914 sx_xunlock(&t4_uld_list_lock); 7915 return (rc); 7916} 7917 7918int 7919t4_unregister_uld(struct uld_info *ui) 7920{ 7921 int rc = EINVAL; 7922 struct uld_info *u; 7923 7924 sx_xlock(&t4_uld_list_lock); 7925 7926 SLIST_FOREACH(u, &t4_uld_list, link) { 7927 if (u == ui) { 7928 if (ui->refcount > 0) { 7929 rc = EBUSY; 7930 goto done; 7931 } 7932 7933 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 7934 rc = 0; 7935 goto done; 7936 } 7937 } 7938done: 7939 sx_xunlock(&t4_uld_list_lock); 7940 return (rc); 7941} 7942 7943int 7944t4_activate_uld(struct adapter *sc, int id) 7945{ 7946 int rc = EAGAIN; 7947 struct uld_info *ui; 7948 7949 ASSERT_SYNCHRONIZED_OP(sc); 7950 7951 sx_slock(&t4_uld_list_lock); 7952 7953 SLIST_FOREACH(ui, &t4_uld_list, link) { 7954 if (ui->uld_id == id) { 7955 rc = ui->activate(sc); 7956 if (rc == 0) 7957 ui->refcount++; 7958 goto done; 7959 } 7960 } 7961done: 7962 sx_sunlock(&t4_uld_list_lock); 7963 7964 return (rc); 7965} 7966 7967int 7968t4_deactivate_uld(struct adapter *sc, int id) 7969{ 7970 int rc = EINVAL; 7971 struct uld_info *ui; 7972 7973 ASSERT_SYNCHRONIZED_OP(sc); 7974 7975 sx_slock(&t4_uld_list_lock); 7976 7977 SLIST_FOREACH(ui, &t4_uld_list, link) { 7978 if (ui->uld_id == id) { 7979 rc = ui->deactivate(sc); 7980 if (rc == 0) 7981 ui->refcount--; 7982 goto done; 7983 } 7984 } 7985done: 7986 sx_sunlock(&t4_uld_list_lock); 7987 7988 return (rc); 7989} 7990#endif 7991 7992/* 7993 * Come up with reasonable defaults for some of the tunables, provided they're 7994 * not set by the user (in which case we'll use the values as is). 7995 */ 7996static void 7997tweak_tunables(void) 7998{ 7999 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8000 8001 if (t4_ntxq10g < 1) 8002 t4_ntxq10g = min(nc, NTXQ_10G); 8003 8004 if (t4_ntxq1g < 1) 8005 t4_ntxq1g = min(nc, NTXQ_1G); 8006 8007 if (t4_nrxq10g < 1) 8008 t4_nrxq10g = min(nc, NRXQ_10G); 8009 8010 if (t4_nrxq1g < 1) 8011 t4_nrxq1g = min(nc, NRXQ_1G); 8012 8013#ifdef TCP_OFFLOAD 8014 if (t4_nofldtxq10g < 1) 8015 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8016 8017 if (t4_nofldtxq1g < 1) 8018 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8019 8020 if (t4_nofldrxq10g < 1) 8021 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8022 8023 if (t4_nofldrxq1g < 1) 8024 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8025 8026 if (t4_toecaps_allowed == -1) 8027 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8028#else 8029 if (t4_toecaps_allowed == -1) 8030 t4_toecaps_allowed = 0; 8031#endif 8032 8033 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8034 t4_tmr_idx_10g = TMR_IDX_10G; 8035 8036 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8037 t4_pktc_idx_10g = PKTC_IDX_10G; 8038 8039 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8040 t4_tmr_idx_1g = TMR_IDX_1G; 8041 8042 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8043 t4_pktc_idx_1g = PKTC_IDX_1G; 8044 8045 if (t4_qsize_txq < 128) 8046 t4_qsize_txq = 128; 8047 8048 if (t4_qsize_rxq < 128) 8049 t4_qsize_rxq = 128; 8050 while (t4_qsize_rxq & 7) 8051 t4_qsize_rxq++; 8052 8053 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8054} 8055 8056static int 8057mod_event(module_t mod, int cmd, void *arg) 8058{ 8059 int rc = 0; 8060 static int loaded = 0; 8061 8062 switch (cmd) { 8063 case MOD_LOAD: 8064 if (atomic_fetchadd_int(&loaded, 1)) 8065 break; 8066 t4_sge_modload(); 8067 sx_init(&t4_list_lock, "T4/T5 adapters"); 8068 SLIST_INIT(&t4_list); 8069#ifdef TCP_OFFLOAD 8070 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8071 SLIST_INIT(&t4_uld_list); 8072#endif 8073 t4_tracer_modload(); 8074 tweak_tunables(); 8075 break; 8076 8077 case MOD_UNLOAD: 8078 if (atomic_fetchadd_int(&loaded, -1) > 1) 8079 break; 8080 t4_tracer_modunload(); 8081#ifdef TCP_OFFLOAD 8082 sx_slock(&t4_uld_list_lock); 8083 if (!SLIST_EMPTY(&t4_uld_list)) { 8084 rc = EBUSY; 8085 sx_sunlock(&t4_uld_list_lock); 8086 break; 8087 } 8088 sx_sunlock(&t4_uld_list_lock); 8089 sx_destroy(&t4_uld_list_lock); 8090#endif 8091 sx_slock(&t4_list_lock); 8092 if (!SLIST_EMPTY(&t4_list)) { 8093 rc = EBUSY; 8094 sx_sunlock(&t4_list_lock); 8095 break; 8096 } 8097 sx_sunlock(&t4_list_lock); 8098 sx_destroy(&t4_list_lock); 8099 break; 8100 } 8101 8102 return (rc); 8103} 8104 8105static devclass_t t4_devclass, t5_devclass; 8106static devclass_t cxgbe_devclass, cxl_devclass; 8107 8108DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8109MODULE_VERSION(t4nex, 1); 8110MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8111 8112DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8113MODULE_VERSION(t5nex, 1); 8114MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8115 8116DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8117MODULE_VERSION(cxgbe, 1); 8118 8119DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8120MODULE_VERSION(cxl, 1); 8121