t4_main.c revision 286898
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 286898 2015-08-18 19:15:20Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/systm.h> 40#include <sys/counter.h> 41#include <sys/module.h> 42#include <sys/malloc.h> 43#include <sys/queue.h> 44#include <sys/taskqueue.h> 45#include <sys/pciio.h> 46#include <dev/pci/pcireg.h> 47#include <dev/pci/pcivar.h> 48#include <dev/pci/pci_private.h> 49#include <sys/firmware.h> 50#include <sys/sbuf.h> 51#include <sys/smp.h> 52#include <sys/socket.h> 53#include <sys/sockio.h> 54#include <sys/sysctl.h> 55#include <net/ethernet.h> 56#include <net/if.h> 57#include <net/if_types.h> 58#include <net/if_dl.h> 59#include <net/if_vlan_var.h> 60#if defined(__i386__) || defined(__amd64__) 61#include <vm/vm.h> 62#include <vm/pmap.h> 63#endif 64 65#include "common/common.h" 66#include "common/t4_msg.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "t4_ioctl.h" 70#include "t4_l2t.h" 71#include "t4_mp_ring.h" 72 73/* T4 bus driver interface */ 74static int t4_probe(device_t); 75static int t4_attach(device_t); 76static int t4_detach(device_t); 77static device_method_t t4_methods[] = { 78 DEVMETHOD(device_probe, t4_probe), 79 DEVMETHOD(device_attach, t4_attach), 80 DEVMETHOD(device_detach, t4_detach), 81 82 DEVMETHOD_END 83}; 84static driver_t t4_driver = { 85 "t4nex", 86 t4_methods, 87 sizeof(struct adapter) 88}; 89 90 91/* T4 port (cxgbe) interface */ 92static int cxgbe_probe(device_t); 93static int cxgbe_attach(device_t); 94static int cxgbe_detach(device_t); 95static device_method_t cxgbe_methods[] = { 96 DEVMETHOD(device_probe, cxgbe_probe), 97 DEVMETHOD(device_attach, cxgbe_attach), 98 DEVMETHOD(device_detach, cxgbe_detach), 99 { 0, 0 } 100}; 101static driver_t cxgbe_driver = { 102 "cxgbe", 103 cxgbe_methods, 104 sizeof(struct port_info) 105}; 106 107static d_ioctl_t t4_ioctl; 108static d_open_t t4_open; 109static d_close_t t4_close; 110 111static struct cdevsw t4_cdevsw = { 112 .d_version = D_VERSION, 113 .d_flags = 0, 114 .d_open = t4_open, 115 .d_close = t4_close, 116 .d_ioctl = t4_ioctl, 117 .d_name = "t4nex", 118}; 119 120/* T5 bus driver interface */ 121static int t5_probe(device_t); 122static device_method_t t5_methods[] = { 123 DEVMETHOD(device_probe, t5_probe), 124 DEVMETHOD(device_attach, t4_attach), 125 DEVMETHOD(device_detach, t4_detach), 126 127 DEVMETHOD_END 128}; 129static driver_t t5_driver = { 130 "t5nex", 131 t5_methods, 132 sizeof(struct adapter) 133}; 134 135 136/* T5 port (cxl) interface */ 137static driver_t cxl_driver = { 138 "cxl", 139 cxgbe_methods, 140 sizeof(struct port_info) 141}; 142 143static struct cdevsw t5_cdevsw = { 144 .d_version = D_VERSION, 145 .d_flags = 0, 146 .d_open = t4_open, 147 .d_close = t4_close, 148 .d_ioctl = t4_ioctl, 149 .d_name = "t5nex", 150}; 151 152/* ifnet + media interface */ 153static void cxgbe_init(void *); 154static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 155static int cxgbe_transmit(struct ifnet *, struct mbuf *); 156static void cxgbe_qflush(struct ifnet *); 157static int cxgbe_media_change(struct ifnet *); 158static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 159 160MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 161 162/* 163 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 164 * then ADAPTER_LOCK, then t4_uld_list_lock. 165 */ 166static struct sx t4_list_lock; 167SLIST_HEAD(, adapter) t4_list; 168#ifdef TCP_OFFLOAD 169static struct sx t4_uld_list_lock; 170SLIST_HEAD(, uld_info) t4_uld_list; 171#endif 172 173/* 174 * Tunables. See tweak_tunables() too. 175 * 176 * Each tunable is set to a default value here if it's known at compile-time. 177 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 178 * provide a reasonable default when the driver is loaded. 179 * 180 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 181 * T5 are under hw.cxl. 182 */ 183 184/* 185 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 186 */ 187#define NTXQ_10G 16 188static int t4_ntxq10g = -1; 189TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 190 191#define NRXQ_10G 8 192static int t4_nrxq10g = -1; 193TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 194 195#define NTXQ_1G 4 196static int t4_ntxq1g = -1; 197TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 198 199#define NRXQ_1G 2 200static int t4_nrxq1g = -1; 201TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 202 203static int t4_rsrv_noflowq = 0; 204TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 205 206#ifdef TCP_OFFLOAD 207#define NOFLDTXQ_10G 8 208static int t4_nofldtxq10g = -1; 209TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 210 211#define NOFLDRXQ_10G 2 212static int t4_nofldrxq10g = -1; 213TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 214 215#define NOFLDTXQ_1G 2 216static int t4_nofldtxq1g = -1; 217TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 218 219#define NOFLDRXQ_1G 1 220static int t4_nofldrxq1g = -1; 221TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 222#endif 223 224#ifdef DEV_NETMAP 225#define NNMTXQ_10G 2 226static int t4_nnmtxq10g = -1; 227TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 228 229#define NNMRXQ_10G 2 230static int t4_nnmrxq10g = -1; 231TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 232 233#define NNMTXQ_1G 1 234static int t4_nnmtxq1g = -1; 235TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 236 237#define NNMRXQ_1G 1 238static int t4_nnmrxq1g = -1; 239TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 240#endif 241 242/* 243 * Holdoff parameters for 10G and 1G ports. 244 */ 245#define TMR_IDX_10G 1 246static int t4_tmr_idx_10g = TMR_IDX_10G; 247TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 248 249#define PKTC_IDX_10G (-1) 250static int t4_pktc_idx_10g = PKTC_IDX_10G; 251TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 252 253#define TMR_IDX_1G 1 254static int t4_tmr_idx_1g = TMR_IDX_1G; 255TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 256 257#define PKTC_IDX_1G (-1) 258static int t4_pktc_idx_1g = PKTC_IDX_1G; 259TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 260 261/* 262 * Size (# of entries) of each tx and rx queue. 263 */ 264static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 266 267static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 268TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 269 270/* 271 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 272 */ 273static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 274TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 275 276/* 277 * Configuration file. 278 */ 279#define DEFAULT_CF "default" 280#define FLASH_CF "flash" 281#define UWIRE_CF "uwire" 282#define FPGA_CF "fpga" 283static char t4_cfg_file[32] = DEFAULT_CF; 284TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 285 286/* 287 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 288 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 289 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 290 * mark or when signalled to do so, 0 to never emit PAUSE. 291 */ 292static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 293TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 294 295/* 296 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 297 * encouraged respectively). 298 */ 299static unsigned int t4_fw_install = 1; 300TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 301 302/* 303 * ASIC features that will be used. Disable the ones you don't want so that the 304 * chip resources aren't wasted on features that will not be used. 305 */ 306static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 307TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 308 309static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 310TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 311 312static int t4_toecaps_allowed = -1; 313TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 314 315static int t4_rdmacaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 317 318static int t4_iscsicaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 320 321static int t4_fcoecaps_allowed = 0; 322TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 323 324static int t5_write_combine = 0; 325TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 326 327struct intrs_and_queues { 328 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 329 uint16_t nirq; /* Total # of vectors */ 330 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 331 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 332 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 333 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 334 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 335 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 336 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 337#ifdef TCP_OFFLOAD 338 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 339 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 340 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 341 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 342#endif 343#ifdef DEV_NETMAP 344 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 345 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 346 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 347 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 348#endif 349}; 350 351struct filter_entry { 352 uint32_t valid:1; /* filter allocated and valid */ 353 uint32_t locked:1; /* filter is administratively locked */ 354 uint32_t pending:1; /* filter action is pending firmware reply */ 355 uint32_t smtidx:8; /* Source MAC Table index for smac */ 356 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 357 358 struct t4_filter_specification fs; 359}; 360 361static int map_bars_0_and_4(struct adapter *); 362static int map_bar_2(struct adapter *); 363static void setup_memwin(struct adapter *); 364static int validate_mem_range(struct adapter *, uint32_t, int); 365static int fwmtype_to_hwmtype(int); 366static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 367 uint32_t *); 368static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 369static uint32_t position_memwin(struct adapter *, int, uint32_t); 370static int cfg_itype_and_nqueues(struct adapter *, int, int, 371 struct intrs_and_queues *); 372static int prep_firmware(struct adapter *); 373static int partition_resources(struct adapter *, const struct firmware *, 374 const char *); 375static int get_params__pre_init(struct adapter *); 376static int get_params__post_init(struct adapter *); 377static int set_params__post_init(struct adapter *); 378static void t4_set_desc(struct adapter *); 379static void build_medialist(struct port_info *, struct ifmedia *); 380static int cxgbe_init_synchronized(struct port_info *); 381static int cxgbe_uninit_synchronized(struct port_info *); 382static int setup_intr_handlers(struct adapter *); 383static void quiesce_txq(struct adapter *, struct sge_txq *); 384static void quiesce_wrq(struct adapter *, struct sge_wrq *); 385static void quiesce_iq(struct adapter *, struct sge_iq *); 386static void quiesce_fl(struct adapter *, struct sge_fl *); 387static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 388 driver_intr_t *, void *, char *); 389static int t4_free_irq(struct adapter *, struct irq *); 390static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 391 unsigned int); 392static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 393static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 394static void cxgbe_tick(void *); 395static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 396static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 397 struct mbuf *); 398static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 399static int fw_msg_not_handled(struct adapter *, const __be64 *); 400static int t4_sysctls(struct adapter *); 401static int cxgbe_sysctls(struct port_info *); 402static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 403static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 404static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 405static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 406static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 407static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 408static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 409static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 410static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 411static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 412static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 413#ifdef SBUF_DRAIN 414static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 415static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 416static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 417static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 418static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 419static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 420static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 422static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 423static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 424static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 425static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 427static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 428static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 429static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 430static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 431static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 432static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 433static int sysctl_tids(SYSCTL_HANDLER_ARGS); 434static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 435static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 436static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 437static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 438static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 439#endif 440static uint32_t fconf_to_mode(uint32_t); 441static uint32_t mode_to_fconf(uint32_t); 442static uint32_t fspec_to_fconf(struct t4_filter_specification *); 443static int get_filter_mode(struct adapter *, uint32_t *); 444static int set_filter_mode(struct adapter *, uint32_t); 445static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 446static int get_filter(struct adapter *, struct t4_filter *); 447static int set_filter(struct adapter *, struct t4_filter *); 448static int del_filter(struct adapter *, struct t4_filter *); 449static void clear_filter(struct filter_entry *); 450static int set_filter_wr(struct adapter *, int); 451static int del_filter_wr(struct adapter *, int); 452static int get_sge_context(struct adapter *, struct t4_sge_context *); 453static int load_fw(struct adapter *, struct t4_data *); 454static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 455static int read_i2c(struct adapter *, struct t4_i2c_data *); 456static int set_sched_class(struct adapter *, struct t4_sched_params *); 457static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 458#ifdef TCP_OFFLOAD 459static int toe_capability(struct port_info *, int); 460#endif 461static int mod_event(module_t, int, void *); 462 463struct { 464 uint16_t device; 465 char *desc; 466} t4_pciids[] = { 467 {0xa000, "Chelsio Terminator 4 FPGA"}, 468 {0x4400, "Chelsio T440-dbg"}, 469 {0x4401, "Chelsio T420-CR"}, 470 {0x4402, "Chelsio T422-CR"}, 471 {0x4403, "Chelsio T440-CR"}, 472 {0x4404, "Chelsio T420-BCH"}, 473 {0x4405, "Chelsio T440-BCH"}, 474 {0x4406, "Chelsio T440-CH"}, 475 {0x4407, "Chelsio T420-SO"}, 476 {0x4408, "Chelsio T420-CX"}, 477 {0x4409, "Chelsio T420-BT"}, 478 {0x440a, "Chelsio T404-BT"}, 479 {0x440e, "Chelsio T440-LP-CR"}, 480}, t5_pciids[] = { 481 {0xb000, "Chelsio Terminator 5 FPGA"}, 482 {0x5400, "Chelsio T580-dbg"}, 483 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 484 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 485 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 486 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 487 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 488 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 489 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 490 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 491 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 492 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 493 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 494 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 495 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 496#ifdef notyet 497 {0x5404, "Chelsio T520-BCH"}, 498 {0x5405, "Chelsio T540-BCH"}, 499 {0x5406, "Chelsio T540-CH"}, 500 {0x5408, "Chelsio T520-CX"}, 501 {0x540b, "Chelsio B520-SR"}, 502 {0x540c, "Chelsio B504-BT"}, 503 {0x540f, "Chelsio Amsterdam"}, 504 {0x5413, "Chelsio T580-CHR"}, 505#endif 506}; 507 508#ifdef TCP_OFFLOAD 509/* 510 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 511 * exactly the same for both rxq and ofld_rxq. 512 */ 513CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 514CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 515#endif 516 517/* No easy way to include t4_msg.h before adapter.h so we check this way */ 518CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 519CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 520 521CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 522 523static int 524t4_probe(device_t dev) 525{ 526 int i; 527 uint16_t v = pci_get_vendor(dev); 528 uint16_t d = pci_get_device(dev); 529 uint8_t f = pci_get_function(dev); 530 531 if (v != PCI_VENDOR_ID_CHELSIO) 532 return (ENXIO); 533 534 /* Attach only to PF0 of the FPGA */ 535 if (d == 0xa000 && f != 0) 536 return (ENXIO); 537 538 for (i = 0; i < nitems(t4_pciids); i++) { 539 if (d == t4_pciids[i].device) { 540 device_set_desc(dev, t4_pciids[i].desc); 541 return (BUS_PROBE_DEFAULT); 542 } 543 } 544 545 return (ENXIO); 546} 547 548static int 549t5_probe(device_t dev) 550{ 551 int i; 552 uint16_t v = pci_get_vendor(dev); 553 uint16_t d = pci_get_device(dev); 554 uint8_t f = pci_get_function(dev); 555 556 if (v != PCI_VENDOR_ID_CHELSIO) 557 return (ENXIO); 558 559 /* Attach only to PF0 of the FPGA */ 560 if (d == 0xb000 && f != 0) 561 return (ENXIO); 562 563 for (i = 0; i < nitems(t5_pciids); i++) { 564 if (d == t5_pciids[i].device) { 565 device_set_desc(dev, t5_pciids[i].desc); 566 return (BUS_PROBE_DEFAULT); 567 } 568 } 569 570 return (ENXIO); 571} 572 573static int 574t4_attach(device_t dev) 575{ 576 struct adapter *sc; 577 int rc = 0, i, n10g, n1g, rqidx, tqidx; 578 struct intrs_and_queues iaq; 579 struct sge *s; 580#ifdef TCP_OFFLOAD 581 int ofld_rqidx, ofld_tqidx; 582#endif 583#ifdef DEV_NETMAP 584 int nm_rqidx, nm_tqidx; 585#endif 586 587 sc = device_get_softc(dev); 588 sc->dev = dev; 589 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 590 591 pci_enable_busmaster(dev); 592 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 593 uint32_t v; 594 595 pci_set_max_read_req(dev, 4096); 596 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 597 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 598 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 599 600 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 601 } 602 603 sc->traceq = -1; 604 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 605 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 606 device_get_nameunit(dev)); 607 608 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 609 device_get_nameunit(dev)); 610 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 611 sx_xlock(&t4_list_lock); 612 SLIST_INSERT_HEAD(&t4_list, sc, link); 613 sx_xunlock(&t4_list_lock); 614 615 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 616 TAILQ_INIT(&sc->sfl); 617 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 618 619 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 620 621 rc = map_bars_0_and_4(sc); 622 if (rc != 0) 623 goto done; /* error message displayed already */ 624 625 /* 626 * This is the real PF# to which we're attaching. Works from within PCI 627 * passthrough environments too, where pci_get_function() could return a 628 * different PF# depending on the passthrough configuration. We need to 629 * use the real PF# in all our communication with the firmware. 630 */ 631 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 632 sc->mbox = sc->pf; 633 634 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 635 sc->an_handler = an_not_handled; 636 for (i = 0; i < nitems(sc->cpl_handler); i++) 637 sc->cpl_handler[i] = cpl_not_handled; 638 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 639 sc->fw_msg_handler[i] = fw_msg_not_handled; 640 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 642 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 643 t4_init_sge_cpl_handlers(sc); 644 645 /* Prepare the adapter for operation */ 646 rc = -t4_prep_adapter(sc); 647 if (rc != 0) { 648 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 649 goto done; 650 } 651 652 /* 653 * Do this really early, with the memory windows set up even before the 654 * character device. The userland tool's register i/o and mem read 655 * will work even in "recovery mode". 656 */ 657 setup_memwin(sc); 658 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 659 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 660 device_get_nameunit(dev)); 661 if (sc->cdev == NULL) 662 device_printf(dev, "failed to create nexus char device.\n"); 663 else 664 sc->cdev->si_drv1 = sc; 665 666 /* Go no further if recovery mode has been requested. */ 667 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 668 device_printf(dev, "recovery mode.\n"); 669 goto done; 670 } 671 672#if defined(__i386__) 673 if ((cpu_feature & CPUID_CX8) == 0) { 674 device_printf(dev, "64 bit atomics not available.\n"); 675 rc = ENOTSUP; 676 goto done; 677 } 678#endif 679 680 /* Prepare the firmware for operation */ 681 rc = prep_firmware(sc); 682 if (rc != 0) 683 goto done; /* error message displayed already */ 684 685 rc = get_params__post_init(sc); 686 if (rc != 0) 687 goto done; /* error message displayed already */ 688 689 rc = set_params__post_init(sc); 690 if (rc != 0) 691 goto done; /* error message displayed already */ 692 693 rc = map_bar_2(sc); 694 if (rc != 0) 695 goto done; /* error message displayed already */ 696 697 rc = t4_create_dma_tag(sc); 698 if (rc != 0) 699 goto done; /* error message displayed already */ 700 701 /* 702 * First pass over all the ports - allocate VIs and initialize some 703 * basic parameters like mac address, port type, etc. We also figure 704 * out whether a port is 10G or 1G and use that information when 705 * calculating how many interrupts to attempt to allocate. 706 */ 707 n10g = n1g = 0; 708 for_each_port(sc, i) { 709 struct port_info *pi; 710 711 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 712 sc->port[i] = pi; 713 714 /* These must be set before t4_port_init */ 715 pi->adapter = sc; 716 pi->port_id = i; 717 718 /* Allocate the vi and initialize parameters like mac addr */ 719 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 720 if (rc != 0) { 721 device_printf(dev, "unable to initialize port %d: %d\n", 722 i, rc); 723 free(pi, M_CXGBE); 724 sc->port[i] = NULL; 725 goto done; 726 } 727 728 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 729 pi->link_cfg.requested_fc |= t4_pause_settings; 730 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 731 pi->link_cfg.fc |= t4_pause_settings; 732 733 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 734 if (rc != 0) { 735 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 736 free(pi, M_CXGBE); 737 sc->port[i] = NULL; 738 goto done; 739 } 740 741 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 742 device_get_nameunit(dev), i); 743 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 744 sc->chan_map[pi->tx_chan] = i; 745 746 if (is_10G_port(pi) || is_40G_port(pi)) { 747 n10g++; 748 pi->tmr_idx = t4_tmr_idx_10g; 749 pi->pktc_idx = t4_pktc_idx_10g; 750 } else { 751 n1g++; 752 pi->tmr_idx = t4_tmr_idx_1g; 753 pi->pktc_idx = t4_pktc_idx_1g; 754 } 755 756 pi->xact_addr_filt = -1; 757 pi->linkdnrc = -1; 758 759 pi->qsize_rxq = t4_qsize_rxq; 760 pi->qsize_txq = t4_qsize_txq; 761 762 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 763 if (pi->dev == NULL) { 764 device_printf(dev, 765 "failed to add device for port %d.\n", i); 766 rc = ENXIO; 767 goto done; 768 } 769 device_set_softc(pi->dev, pi); 770 } 771 772 /* 773 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 774 */ 775 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 776 if (rc != 0) 777 goto done; /* error message displayed already */ 778 779 sc->intr_type = iaq.intr_type; 780 sc->intr_count = iaq.nirq; 781 782 s = &sc->sge; 783 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 784 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 785 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 786 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 787 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 788#ifdef TCP_OFFLOAD 789 if (is_offload(sc)) { 790 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 791 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 792 s->neq += s->nofldtxq + s->nofldrxq; 793 s->niq += s->nofldrxq; 794 795 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 796 M_CXGBE, M_ZERO | M_WAITOK); 797 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 798 M_CXGBE, M_ZERO | M_WAITOK); 799 } 800#endif 801#ifdef DEV_NETMAP 802 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 803 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 804 s->neq += s->nnmtxq + s->nnmrxq; 805 s->niq += s->nnmrxq; 806 807 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 808 M_CXGBE, M_ZERO | M_WAITOK); 809 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 810 M_CXGBE, M_ZERO | M_WAITOK); 811#endif 812 813 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 814 M_ZERO | M_WAITOK); 815 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 816 M_ZERO | M_WAITOK); 817 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 818 M_ZERO | M_WAITOK); 819 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 820 M_ZERO | M_WAITOK); 821 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 822 M_ZERO | M_WAITOK); 823 824 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 825 M_ZERO | M_WAITOK); 826 827 t4_init_l2t(sc, M_WAITOK); 828 829 /* 830 * Second pass over the ports. This time we know the number of rx and 831 * tx queues that each port should get. 832 */ 833 rqidx = tqidx = 0; 834#ifdef TCP_OFFLOAD 835 ofld_rqidx = ofld_tqidx = 0; 836#endif 837#ifdef DEV_NETMAP 838 nm_rqidx = nm_tqidx = 0; 839#endif 840 for_each_port(sc, i) { 841 struct port_info *pi = sc->port[i]; 842 843 if (pi == NULL) 844 continue; 845 846 pi->first_rxq = rqidx; 847 pi->first_txq = tqidx; 848 if (is_10G_port(pi) || is_40G_port(pi)) { 849 pi->flags |= iaq.intr_flags_10g; 850 pi->nrxq = iaq.nrxq10g; 851 pi->ntxq = iaq.ntxq10g; 852 } else { 853 pi->flags |= iaq.intr_flags_1g; 854 pi->nrxq = iaq.nrxq1g; 855 pi->ntxq = iaq.ntxq1g; 856 } 857 858 if (pi->ntxq > 1) 859 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 860 else 861 pi->rsrv_noflowq = 0; 862 863 rqidx += pi->nrxq; 864 tqidx += pi->ntxq; 865#ifdef TCP_OFFLOAD 866 if (is_offload(sc)) { 867 pi->first_ofld_rxq = ofld_rqidx; 868 pi->first_ofld_txq = ofld_tqidx; 869 if (is_10G_port(pi) || is_40G_port(pi)) { 870 pi->nofldrxq = iaq.nofldrxq10g; 871 pi->nofldtxq = iaq.nofldtxq10g; 872 } else { 873 pi->nofldrxq = iaq.nofldrxq1g; 874 pi->nofldtxq = iaq.nofldtxq1g; 875 } 876 ofld_rqidx += pi->nofldrxq; 877 ofld_tqidx += pi->nofldtxq; 878 } 879#endif 880#ifdef DEV_NETMAP 881 pi->first_nm_rxq = nm_rqidx; 882 pi->first_nm_txq = nm_tqidx; 883 if (is_10G_port(pi) || is_40G_port(pi)) { 884 pi->nnmrxq = iaq.nnmrxq10g; 885 pi->nnmtxq = iaq.nnmtxq10g; 886 } else { 887 pi->nnmrxq = iaq.nnmrxq1g; 888 pi->nnmtxq = iaq.nnmtxq1g; 889 } 890 nm_rqidx += pi->nnmrxq; 891 nm_tqidx += pi->nnmtxq; 892#endif 893 } 894 895 rc = setup_intr_handlers(sc); 896 if (rc != 0) { 897 device_printf(dev, 898 "failed to setup interrupt handlers: %d\n", rc); 899 goto done; 900 } 901 902 rc = bus_generic_attach(dev); 903 if (rc != 0) { 904 device_printf(dev, 905 "failed to attach all child ports: %d\n", rc); 906 goto done; 907 } 908 909 device_printf(dev, 910 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 911 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 912 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 913 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 914 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 915 916 t4_set_desc(sc); 917 918done: 919 if (rc != 0 && sc->cdev) { 920 /* cdev was created and so cxgbetool works; recover that way. */ 921 device_printf(dev, 922 "error during attach, adapter is now in recovery mode.\n"); 923 rc = 0; 924 } 925 926 if (rc != 0) 927 t4_detach(dev); 928 else 929 t4_sysctls(sc); 930 931 return (rc); 932} 933 934/* 935 * Idempotent 936 */ 937static int 938t4_detach(device_t dev) 939{ 940 struct adapter *sc; 941 struct port_info *pi; 942 int i, rc; 943 944 sc = device_get_softc(dev); 945 946 if (sc->flags & FULL_INIT_DONE) 947 t4_intr_disable(sc); 948 949 if (sc->cdev) { 950 destroy_dev(sc->cdev); 951 sc->cdev = NULL; 952 } 953 954 rc = bus_generic_detach(dev); 955 if (rc) { 956 device_printf(dev, 957 "failed to detach child devices: %d\n", rc); 958 return (rc); 959 } 960 961 for (i = 0; i < sc->intr_count; i++) 962 t4_free_irq(sc, &sc->irq[i]); 963 964 for (i = 0; i < MAX_NPORTS; i++) { 965 pi = sc->port[i]; 966 if (pi) { 967 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 968 if (pi->dev) 969 device_delete_child(dev, pi->dev); 970 971 mtx_destroy(&pi->pi_lock); 972 free(pi, M_CXGBE); 973 } 974 } 975 976 if (sc->flags & FULL_INIT_DONE) 977 adapter_full_uninit(sc); 978 979 if (sc->flags & FW_OK) 980 t4_fw_bye(sc, sc->mbox); 981 982 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 983 pci_release_msi(dev); 984 985 if (sc->regs_res) 986 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 987 sc->regs_res); 988 989 if (sc->udbs_res) 990 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 991 sc->udbs_res); 992 993 if (sc->msix_res) 994 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 995 sc->msix_res); 996 997 if (sc->l2t) 998 t4_free_l2t(sc->l2t); 999 1000#ifdef TCP_OFFLOAD 1001 free(sc->sge.ofld_rxq, M_CXGBE); 1002 free(sc->sge.ofld_txq, M_CXGBE); 1003#endif 1004#ifdef DEV_NETMAP 1005 free(sc->sge.nm_rxq, M_CXGBE); 1006 free(sc->sge.nm_txq, M_CXGBE); 1007#endif 1008 free(sc->irq, M_CXGBE); 1009 free(sc->sge.rxq, M_CXGBE); 1010 free(sc->sge.txq, M_CXGBE); 1011 free(sc->sge.ctrlq, M_CXGBE); 1012 free(sc->sge.iqmap, M_CXGBE); 1013 free(sc->sge.eqmap, M_CXGBE); 1014 free(sc->tids.ftid_tab, M_CXGBE); 1015 t4_destroy_dma_tag(sc); 1016 if (mtx_initialized(&sc->sc_lock)) { 1017 sx_xlock(&t4_list_lock); 1018 SLIST_REMOVE(&t4_list, sc, adapter, link); 1019 sx_xunlock(&t4_list_lock); 1020 mtx_destroy(&sc->sc_lock); 1021 } 1022 1023 if (mtx_initialized(&sc->tids.ftid_lock)) 1024 mtx_destroy(&sc->tids.ftid_lock); 1025 if (mtx_initialized(&sc->sfl_lock)) 1026 mtx_destroy(&sc->sfl_lock); 1027 if (mtx_initialized(&sc->ifp_lock)) 1028 mtx_destroy(&sc->ifp_lock); 1029 if (mtx_initialized(&sc->regwin_lock)) 1030 mtx_destroy(&sc->regwin_lock); 1031 1032 bzero(sc, sizeof(*sc)); 1033 1034 return (0); 1035} 1036 1037static int 1038cxgbe_probe(device_t dev) 1039{ 1040 char buf[128]; 1041 struct port_info *pi = device_get_softc(dev); 1042 1043 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1044 device_set_desc_copy(dev, buf); 1045 1046 return (BUS_PROBE_DEFAULT); 1047} 1048 1049#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1050 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1051 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1052#define T4_CAP_ENABLE (T4_CAP) 1053 1054static int 1055cxgbe_attach(device_t dev) 1056{ 1057 struct port_info *pi = device_get_softc(dev); 1058 struct ifnet *ifp; 1059 char *s; 1060 int n, o; 1061 1062 /* Allocate an ifnet and set it up */ 1063 ifp = if_alloc(IFT_ETHER); 1064 if (ifp == NULL) { 1065 device_printf(dev, "Cannot allocate ifnet\n"); 1066 return (ENOMEM); 1067 } 1068 pi->ifp = ifp; 1069 ifp->if_softc = pi; 1070 1071 callout_init(&pi->tick, CALLOUT_MPSAFE); 1072 1073 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1074 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1075 1076 ifp->if_init = cxgbe_init; 1077 ifp->if_ioctl = cxgbe_ioctl; 1078 ifp->if_transmit = cxgbe_transmit; 1079 ifp->if_qflush = cxgbe_qflush; 1080 1081 ifp->if_capabilities = T4_CAP; 1082#ifdef TCP_OFFLOAD 1083 if (is_offload(pi->adapter)) 1084 ifp->if_capabilities |= IFCAP_TOE; 1085#endif 1086 ifp->if_capenable = T4_CAP_ENABLE; 1087 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1088 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1089 1090 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1091 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1092 ifp->if_hw_tsomaxsegsize = 65536; 1093 1094 /* Initialize ifmedia for this port */ 1095 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1096 cxgbe_media_status); 1097 build_medialist(pi, &pi->media); 1098 1099 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1100 EVENTHANDLER_PRI_ANY); 1101 1102 ether_ifattach(ifp, pi->hw_addr); 1103 1104 n = 128; 1105 s = malloc(n, M_CXGBE, M_WAITOK); 1106 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1107 MPASS(n > o); 1108#ifdef TCP_OFFLOAD 1109 if (is_offload(pi->adapter)) { 1110 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1111 pi->nofldtxq, pi->nofldrxq); 1112 MPASS(n > o); 1113 } 1114#endif 1115#ifdef DEV_NETMAP 1116 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1117 pi->nnmrxq); 1118 MPASS(n > o); 1119#endif 1120 device_printf(dev, "%s\n", s); 1121 free(s, M_CXGBE); 1122 1123#ifdef DEV_NETMAP 1124 /* nm_media handled here to keep implementation private to this file */ 1125 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1126 cxgbe_media_status); 1127 build_medialist(pi, &pi->nm_media); 1128 create_netmap_ifnet(pi); /* logs errors it something fails */ 1129#endif 1130 cxgbe_sysctls(pi); 1131 1132 return (0); 1133} 1134 1135static int 1136cxgbe_detach(device_t dev) 1137{ 1138 struct port_info *pi = device_get_softc(dev); 1139 struct adapter *sc = pi->adapter; 1140 struct ifnet *ifp = pi->ifp; 1141 1142 /* Tell if_ioctl and if_init that the port is going away */ 1143 ADAPTER_LOCK(sc); 1144 SET_DOOMED(pi); 1145 wakeup(&sc->flags); 1146 while (IS_BUSY(sc)) 1147 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1148 SET_BUSY(sc); 1149#ifdef INVARIANTS 1150 sc->last_op = "t4detach"; 1151 sc->last_op_thr = curthread; 1152#endif 1153 ADAPTER_UNLOCK(sc); 1154 1155 if (pi->flags & HAS_TRACEQ) { 1156 sc->traceq = -1; /* cloner should not create ifnet */ 1157 t4_tracer_port_detach(sc); 1158 } 1159 1160 if (pi->vlan_c) 1161 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1162 1163 PORT_LOCK(pi); 1164 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1165 callout_stop(&pi->tick); 1166 PORT_UNLOCK(pi); 1167 callout_drain(&pi->tick); 1168 1169 /* Let detach proceed even if these fail. */ 1170 cxgbe_uninit_synchronized(pi); 1171 port_full_uninit(pi); 1172 1173 ifmedia_removeall(&pi->media); 1174 ether_ifdetach(pi->ifp); 1175 if_free(pi->ifp); 1176 1177#ifdef DEV_NETMAP 1178 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1179 destroy_netmap_ifnet(pi); 1180#endif 1181 1182 ADAPTER_LOCK(sc); 1183 CLR_BUSY(sc); 1184 wakeup(&sc->flags); 1185 ADAPTER_UNLOCK(sc); 1186 1187 return (0); 1188} 1189 1190static void 1191cxgbe_init(void *arg) 1192{ 1193 struct port_info *pi = arg; 1194 struct adapter *sc = pi->adapter; 1195 1196 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1197 return; 1198 cxgbe_init_synchronized(pi); 1199 end_synchronized_op(sc, 0); 1200} 1201 1202static int 1203cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1204{ 1205 int rc = 0, mtu, flags, can_sleep; 1206 struct port_info *pi = ifp->if_softc; 1207 struct adapter *sc = pi->adapter; 1208 struct ifreq *ifr = (struct ifreq *)data; 1209 uint32_t mask; 1210 1211 switch (cmd) { 1212 case SIOCSIFMTU: 1213 mtu = ifr->ifr_mtu; 1214 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1215 return (EINVAL); 1216 1217 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1218 if (rc) 1219 return (rc); 1220 ifp->if_mtu = mtu; 1221 if (pi->flags & PORT_INIT_DONE) { 1222 t4_update_fl_bufsize(ifp); 1223 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1224 rc = update_mac_settings(ifp, XGMAC_MTU); 1225 } 1226 end_synchronized_op(sc, 0); 1227 break; 1228 1229 case SIOCSIFFLAGS: 1230 can_sleep = 0; 1231redo_sifflags: 1232 rc = begin_synchronized_op(sc, pi, 1233 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1234 if (rc) 1235 return (rc); 1236 1237 if (ifp->if_flags & IFF_UP) { 1238 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1239 flags = pi->if_flags; 1240 if ((ifp->if_flags ^ flags) & 1241 (IFF_PROMISC | IFF_ALLMULTI)) { 1242 if (can_sleep == 1) { 1243 end_synchronized_op(sc, 0); 1244 can_sleep = 0; 1245 goto redo_sifflags; 1246 } 1247 rc = update_mac_settings(ifp, 1248 XGMAC_PROMISC | XGMAC_ALLMULTI); 1249 } 1250 } else { 1251 if (can_sleep == 0) { 1252 end_synchronized_op(sc, LOCK_HELD); 1253 can_sleep = 1; 1254 goto redo_sifflags; 1255 } 1256 rc = cxgbe_init_synchronized(pi); 1257 } 1258 pi->if_flags = ifp->if_flags; 1259 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1260 if (can_sleep == 0) { 1261 end_synchronized_op(sc, LOCK_HELD); 1262 can_sleep = 1; 1263 goto redo_sifflags; 1264 } 1265 rc = cxgbe_uninit_synchronized(pi); 1266 } 1267 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1268 break; 1269 1270 case SIOCADDMULTI: 1271 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1272 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1273 if (rc) 1274 return (rc); 1275 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1276 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1277 end_synchronized_op(sc, LOCK_HELD); 1278 break; 1279 1280 case SIOCSIFCAP: 1281 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1282 if (rc) 1283 return (rc); 1284 1285 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1286 if (mask & IFCAP_TXCSUM) { 1287 ifp->if_capenable ^= IFCAP_TXCSUM; 1288 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1289 1290 if (IFCAP_TSO4 & ifp->if_capenable && 1291 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1292 ifp->if_capenable &= ~IFCAP_TSO4; 1293 if_printf(ifp, 1294 "tso4 disabled due to -txcsum.\n"); 1295 } 1296 } 1297 if (mask & IFCAP_TXCSUM_IPV6) { 1298 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1299 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1300 1301 if (IFCAP_TSO6 & ifp->if_capenable && 1302 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1303 ifp->if_capenable &= ~IFCAP_TSO6; 1304 if_printf(ifp, 1305 "tso6 disabled due to -txcsum6.\n"); 1306 } 1307 } 1308 if (mask & IFCAP_RXCSUM) 1309 ifp->if_capenable ^= IFCAP_RXCSUM; 1310 if (mask & IFCAP_RXCSUM_IPV6) 1311 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1312 1313 /* 1314 * Note that we leave CSUM_TSO alone (it is always set). The 1315 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1316 * sending a TSO request our way, so it's sufficient to toggle 1317 * IFCAP_TSOx only. 1318 */ 1319 if (mask & IFCAP_TSO4) { 1320 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1321 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1322 if_printf(ifp, "enable txcsum first.\n"); 1323 rc = EAGAIN; 1324 goto fail; 1325 } 1326 ifp->if_capenable ^= IFCAP_TSO4; 1327 } 1328 if (mask & IFCAP_TSO6) { 1329 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1330 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1331 if_printf(ifp, "enable txcsum6 first.\n"); 1332 rc = EAGAIN; 1333 goto fail; 1334 } 1335 ifp->if_capenable ^= IFCAP_TSO6; 1336 } 1337 if (mask & IFCAP_LRO) { 1338#if defined(INET) || defined(INET6) 1339 int i; 1340 struct sge_rxq *rxq; 1341 1342 ifp->if_capenable ^= IFCAP_LRO; 1343 for_each_rxq(pi, i, rxq) { 1344 if (ifp->if_capenable & IFCAP_LRO) 1345 rxq->iq.flags |= IQ_LRO_ENABLED; 1346 else 1347 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1348 } 1349#endif 1350 } 1351#ifdef TCP_OFFLOAD 1352 if (mask & IFCAP_TOE) { 1353 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1354 1355 rc = toe_capability(pi, enable); 1356 if (rc != 0) 1357 goto fail; 1358 1359 ifp->if_capenable ^= mask; 1360 } 1361#endif 1362 if (mask & IFCAP_VLAN_HWTAGGING) { 1363 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1364 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1365 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1366 } 1367 if (mask & IFCAP_VLAN_MTU) { 1368 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1369 1370 /* Need to find out how to disable auto-mtu-inflation */ 1371 } 1372 if (mask & IFCAP_VLAN_HWTSO) 1373 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1374 if (mask & IFCAP_VLAN_HWCSUM) 1375 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1376 1377#ifdef VLAN_CAPABILITIES 1378 VLAN_CAPABILITIES(ifp); 1379#endif 1380fail: 1381 end_synchronized_op(sc, 0); 1382 break; 1383 1384 case SIOCSIFMEDIA: 1385 case SIOCGIFMEDIA: 1386 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1387 break; 1388 1389 case SIOCGI2C: { 1390 struct ifi2creq i2c; 1391 1392 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1393 if (rc != 0) 1394 break; 1395 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1396 rc = EPERM; 1397 break; 1398 } 1399 if (i2c.len > sizeof(i2c.data)) { 1400 rc = EINVAL; 1401 break; 1402 } 1403 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1404 if (rc) 1405 return (rc); 1406 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1407 i2c.offset, i2c.len, &i2c.data[0]); 1408 end_synchronized_op(sc, 0); 1409 if (rc == 0) 1410 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1411 break; 1412 } 1413 1414 default: 1415 rc = ether_ioctl(ifp, cmd, data); 1416 } 1417 1418 return (rc); 1419} 1420 1421static int 1422cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1423{ 1424 struct port_info *pi = ifp->if_softc; 1425 struct adapter *sc = pi->adapter; 1426 struct sge_txq *txq; 1427 void *items[1]; 1428 int rc; 1429 1430 M_ASSERTPKTHDR(m); 1431 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1432 1433 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1434 m_freem(m); 1435 return (ENETDOWN); 1436 } 1437 1438 rc = parse_pkt(&m); 1439 if (__predict_false(rc != 0)) { 1440 MPASS(m == NULL); /* was freed already */ 1441 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1442 return (rc); 1443 } 1444 1445 /* Select a txq. */ 1446 txq = &sc->sge.txq[pi->first_txq]; 1447 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1448 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1449 pi->rsrv_noflowq); 1450 1451 items[0] = m; 1452 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1453 if (__predict_false(rc != 0)) 1454 m_freem(m); 1455 1456 return (rc); 1457} 1458 1459static void 1460cxgbe_qflush(struct ifnet *ifp) 1461{ 1462 struct port_info *pi = ifp->if_softc; 1463 struct sge_txq *txq; 1464 int i; 1465 1466 /* queues do not exist if !PORT_INIT_DONE. */ 1467 if (pi->flags & PORT_INIT_DONE) { 1468 for_each_txq(pi, i, txq) { 1469 TXQ_LOCK(txq); 1470 txq->eq.flags &= ~EQ_ENABLED; 1471 TXQ_UNLOCK(txq); 1472 while (!mp_ring_is_idle(txq->r)) { 1473 mp_ring_check_drainage(txq->r, 0); 1474 pause("qflush", 1); 1475 } 1476 } 1477 } 1478 if_qflush(ifp); 1479} 1480 1481static int 1482cxgbe_media_change(struct ifnet *ifp) 1483{ 1484 struct port_info *pi = ifp->if_softc; 1485 1486 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1487 1488 return (EOPNOTSUPP); 1489} 1490 1491static void 1492cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1493{ 1494 struct port_info *pi = ifp->if_softc; 1495 struct ifmedia *media = NULL; 1496 struct ifmedia_entry *cur; 1497 int speed = pi->link_cfg.speed; 1498#ifdef INVARIANTS 1499 int data = (pi->port_type << 8) | pi->mod_type; 1500#endif 1501 1502 if (ifp == pi->ifp) 1503 media = &pi->media; 1504#ifdef DEV_NETMAP 1505 else if (ifp == pi->nm_ifp) 1506 media = &pi->nm_media; 1507#endif 1508 MPASS(media != NULL); 1509 1510 cur = media->ifm_cur; 1511 MPASS(cur->ifm_data == data); 1512 1513 ifmr->ifm_status = IFM_AVALID; 1514 if (!pi->link_cfg.link_ok) 1515 return; 1516 1517 ifmr->ifm_status |= IFM_ACTIVE; 1518 1519 /* active and current will differ iff current media is autoselect. */ 1520 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1521 return; 1522 1523 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1524 if (speed == SPEED_10000) 1525 ifmr->ifm_active |= IFM_10G_T; 1526 else if (speed == SPEED_1000) 1527 ifmr->ifm_active |= IFM_1000_T; 1528 else if (speed == SPEED_100) 1529 ifmr->ifm_active |= IFM_100_TX; 1530 else if (speed == SPEED_10) 1531 ifmr->ifm_active |= IFM_10_T; 1532 else 1533 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1534 speed)); 1535} 1536 1537void 1538t4_fatal_err(struct adapter *sc) 1539{ 1540 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1541 t4_intr_disable(sc); 1542 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1543 device_get_nameunit(sc->dev)); 1544} 1545 1546static int 1547map_bars_0_and_4(struct adapter *sc) 1548{ 1549 sc->regs_rid = PCIR_BAR(0); 1550 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1551 &sc->regs_rid, RF_ACTIVE); 1552 if (sc->regs_res == NULL) { 1553 device_printf(sc->dev, "cannot map registers.\n"); 1554 return (ENXIO); 1555 } 1556 sc->bt = rman_get_bustag(sc->regs_res); 1557 sc->bh = rman_get_bushandle(sc->regs_res); 1558 sc->mmio_len = rman_get_size(sc->regs_res); 1559 setbit(&sc->doorbells, DOORBELL_KDB); 1560 1561 sc->msix_rid = PCIR_BAR(4); 1562 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1563 &sc->msix_rid, RF_ACTIVE); 1564 if (sc->msix_res == NULL) { 1565 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1566 return (ENXIO); 1567 } 1568 1569 return (0); 1570} 1571 1572static int 1573map_bar_2(struct adapter *sc) 1574{ 1575 1576 /* 1577 * T4: only iWARP driver uses the userspace doorbells. There is no need 1578 * to map it if RDMA is disabled. 1579 */ 1580 if (is_t4(sc) && sc->rdmacaps == 0) 1581 return (0); 1582 1583 sc->udbs_rid = PCIR_BAR(2); 1584 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1585 &sc->udbs_rid, RF_ACTIVE); 1586 if (sc->udbs_res == NULL) { 1587 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1588 return (ENXIO); 1589 } 1590 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1591 1592 if (is_t5(sc)) { 1593 setbit(&sc->doorbells, DOORBELL_UDB); 1594#if defined(__i386__) || defined(__amd64__) 1595 if (t5_write_combine) { 1596 int rc; 1597 1598 /* 1599 * Enable write combining on BAR2. This is the 1600 * userspace doorbell BAR and is split into 128B 1601 * (UDBS_SEG_SIZE) doorbell regions, each associated 1602 * with an egress queue. The first 64B has the doorbell 1603 * and the second 64B can be used to submit a tx work 1604 * request with an implicit doorbell. 1605 */ 1606 1607 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1608 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1609 if (rc == 0) { 1610 clrbit(&sc->doorbells, DOORBELL_UDB); 1611 setbit(&sc->doorbells, DOORBELL_WCWR); 1612 setbit(&sc->doorbells, DOORBELL_UDBWC); 1613 } else { 1614 device_printf(sc->dev, 1615 "couldn't enable write combining: %d\n", 1616 rc); 1617 } 1618 1619 t4_write_reg(sc, A_SGE_STAT_CFG, 1620 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1621 } 1622#endif 1623 } 1624 1625 return (0); 1626} 1627 1628static const struct memwin t4_memwin[] = { 1629 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1630 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1631 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1632}; 1633 1634static const struct memwin t5_memwin[] = { 1635 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1636 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1637 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1638}; 1639 1640static void 1641setup_memwin(struct adapter *sc) 1642{ 1643 const struct memwin *mw; 1644 int i, n; 1645 uint32_t bar0; 1646 1647 if (is_t4(sc)) { 1648 /* 1649 * Read low 32b of bar0 indirectly via the hardware backdoor 1650 * mechanism. Works from within PCI passthrough environments 1651 * too, where rman_get_start() can return a different value. We 1652 * need to program the T4 memory window decoders with the actual 1653 * addresses that will be coming across the PCIe link. 1654 */ 1655 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1656 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1657 1658 mw = &t4_memwin[0]; 1659 n = nitems(t4_memwin); 1660 } else { 1661 /* T5 uses the relative offset inside the PCIe BAR */ 1662 bar0 = 0; 1663 1664 mw = &t5_memwin[0]; 1665 n = nitems(t5_memwin); 1666 } 1667 1668 for (i = 0; i < n; i++, mw++) { 1669 t4_write_reg(sc, 1670 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1671 (mw->base + bar0) | V_BIR(0) | 1672 V_WINDOW(ilog2(mw->aperture) - 10)); 1673 } 1674 1675 /* flush */ 1676 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1677} 1678 1679/* 1680 * Verify that the memory range specified by the addr/len pair is valid and lies 1681 * entirely within a single region (EDCx or MCx). 1682 */ 1683static int 1684validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1685{ 1686 uint32_t em, addr_len, maddr, mlen; 1687 1688 /* Memory can only be accessed in naturally aligned 4 byte units */ 1689 if (addr & 3 || len & 3 || len == 0) 1690 return (EINVAL); 1691 1692 /* Enabled memories */ 1693 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1694 if (em & F_EDRAM0_ENABLE) { 1695 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1696 maddr = G_EDRAM0_BASE(addr_len) << 20; 1697 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1698 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1699 addr + len <= maddr + mlen) 1700 return (0); 1701 } 1702 if (em & F_EDRAM1_ENABLE) { 1703 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1704 maddr = G_EDRAM1_BASE(addr_len) << 20; 1705 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1706 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1707 addr + len <= maddr + mlen) 1708 return (0); 1709 } 1710 if (em & F_EXT_MEM_ENABLE) { 1711 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1712 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1713 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1714 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1715 addr + len <= maddr + mlen) 1716 return (0); 1717 } 1718 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1719 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1720 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1721 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1722 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1723 addr + len <= maddr + mlen) 1724 return (0); 1725 } 1726 1727 return (EFAULT); 1728} 1729 1730static int 1731fwmtype_to_hwmtype(int mtype) 1732{ 1733 1734 switch (mtype) { 1735 case FW_MEMTYPE_EDC0: 1736 return (MEM_EDC0); 1737 case FW_MEMTYPE_EDC1: 1738 return (MEM_EDC1); 1739 case FW_MEMTYPE_EXTMEM: 1740 return (MEM_MC0); 1741 case FW_MEMTYPE_EXTMEM1: 1742 return (MEM_MC1); 1743 default: 1744 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1745 } 1746} 1747 1748/* 1749 * Verify that the memory range specified by the memtype/offset/len pair is 1750 * valid and lies entirely within the memtype specified. The global address of 1751 * the start of the range is returned in addr. 1752 */ 1753static int 1754validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1755 uint32_t *addr) 1756{ 1757 uint32_t em, addr_len, maddr, mlen; 1758 1759 /* Memory can only be accessed in naturally aligned 4 byte units */ 1760 if (off & 3 || len & 3 || len == 0) 1761 return (EINVAL); 1762 1763 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1764 switch (fwmtype_to_hwmtype(mtype)) { 1765 case MEM_EDC0: 1766 if (!(em & F_EDRAM0_ENABLE)) 1767 return (EINVAL); 1768 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1769 maddr = G_EDRAM0_BASE(addr_len) << 20; 1770 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1771 break; 1772 case MEM_EDC1: 1773 if (!(em & F_EDRAM1_ENABLE)) 1774 return (EINVAL); 1775 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1776 maddr = G_EDRAM1_BASE(addr_len) << 20; 1777 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1778 break; 1779 case MEM_MC: 1780 if (!(em & F_EXT_MEM_ENABLE)) 1781 return (EINVAL); 1782 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1783 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1784 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1785 break; 1786 case MEM_MC1: 1787 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1788 return (EINVAL); 1789 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1790 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1791 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1792 break; 1793 default: 1794 return (EINVAL); 1795 } 1796 1797 if (mlen > 0 && off < mlen && off + len <= mlen) { 1798 *addr = maddr + off; /* global address */ 1799 return (0); 1800 } 1801 1802 return (EFAULT); 1803} 1804 1805static void 1806memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1807{ 1808 const struct memwin *mw; 1809 1810 if (is_t4(sc)) { 1811 KASSERT(win >= 0 && win < nitems(t4_memwin), 1812 ("%s: incorrect memwin# (%d)", __func__, win)); 1813 mw = &t4_memwin[win]; 1814 } else { 1815 KASSERT(win >= 0 && win < nitems(t5_memwin), 1816 ("%s: incorrect memwin# (%d)", __func__, win)); 1817 mw = &t5_memwin[win]; 1818 } 1819 1820 if (base != NULL) 1821 *base = mw->base; 1822 if (aperture != NULL) 1823 *aperture = mw->aperture; 1824} 1825 1826/* 1827 * Positions the memory window such that it can be used to access the specified 1828 * address in the chip's address space. The return value is the offset of addr 1829 * from the start of the window. 1830 */ 1831static uint32_t 1832position_memwin(struct adapter *sc, int n, uint32_t addr) 1833{ 1834 uint32_t start, pf; 1835 uint32_t reg; 1836 1837 KASSERT(n >= 0 && n <= 3, 1838 ("%s: invalid window %d.", __func__, n)); 1839 KASSERT((addr & 3) == 0, 1840 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1841 1842 if (is_t4(sc)) { 1843 pf = 0; 1844 start = addr & ~0xf; /* start must be 16B aligned */ 1845 } else { 1846 pf = V_PFNUM(sc->pf); 1847 start = addr & ~0x7f; /* start must be 128B aligned */ 1848 } 1849 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1850 1851 t4_write_reg(sc, reg, start | pf); 1852 t4_read_reg(sc, reg); 1853 1854 return (addr - start); 1855} 1856 1857static int 1858cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1859 struct intrs_and_queues *iaq) 1860{ 1861 int rc, itype, navail, nrxq10g, nrxq1g, n; 1862 int nofldrxq10g = 0, nofldrxq1g = 0; 1863 int nnmrxq10g = 0, nnmrxq1g = 0; 1864 1865 bzero(iaq, sizeof(*iaq)); 1866 1867 iaq->ntxq10g = t4_ntxq10g; 1868 iaq->ntxq1g = t4_ntxq1g; 1869 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1870 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1871 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1872#ifdef TCP_OFFLOAD 1873 if (is_offload(sc)) { 1874 iaq->nofldtxq10g = t4_nofldtxq10g; 1875 iaq->nofldtxq1g = t4_nofldtxq1g; 1876 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1877 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1878 } 1879#endif 1880#ifdef DEV_NETMAP 1881 iaq->nnmtxq10g = t4_nnmtxq10g; 1882 iaq->nnmtxq1g = t4_nnmtxq1g; 1883 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1884 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1885#endif 1886 1887 for (itype = INTR_MSIX; itype; itype >>= 1) { 1888 1889 if ((itype & t4_intr_types) == 0) 1890 continue; /* not allowed */ 1891 1892 if (itype == INTR_MSIX) 1893 navail = pci_msix_count(sc->dev); 1894 else if (itype == INTR_MSI) 1895 navail = pci_msi_count(sc->dev); 1896 else 1897 navail = 1; 1898restart: 1899 if (navail == 0) 1900 continue; 1901 1902 iaq->intr_type = itype; 1903 iaq->intr_flags_10g = 0; 1904 iaq->intr_flags_1g = 0; 1905 1906 /* 1907 * Best option: an interrupt vector for errors, one for the 1908 * firmware event queue, and one for every rxq (NIC, TOE, and 1909 * netmap). 1910 */ 1911 iaq->nirq = T4_EXTRA_INTR; 1912 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1913 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1914 if (iaq->nirq <= navail && 1915 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1916 iaq->intr_flags_10g = INTR_ALL; 1917 iaq->intr_flags_1g = INTR_ALL; 1918 goto allocate; 1919 } 1920 1921 /* 1922 * Second best option: a vector for errors, one for the firmware 1923 * event queue, and vectors for either all the NIC rx queues or 1924 * all the TOE rx queues. The queues that don't get vectors 1925 * will forward their interrupts to those that do. 1926 * 1927 * Note: netmap rx queues cannot be created early and so they 1928 * can't be setup to receive forwarded interrupts for others. 1929 */ 1930 iaq->nirq = T4_EXTRA_INTR; 1931 if (nrxq10g >= nofldrxq10g) { 1932 iaq->intr_flags_10g = INTR_RXQ; 1933 iaq->nirq += n10g * nrxq10g; 1934#ifdef DEV_NETMAP 1935 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1936#endif 1937 } else { 1938 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1939 iaq->nirq += n10g * nofldrxq10g; 1940#ifdef DEV_NETMAP 1941 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1942#endif 1943 } 1944 if (nrxq1g >= nofldrxq1g) { 1945 iaq->intr_flags_1g = INTR_RXQ; 1946 iaq->nirq += n1g * nrxq1g; 1947#ifdef DEV_NETMAP 1948 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1949#endif 1950 } else { 1951 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1952 iaq->nirq += n1g * nofldrxq1g; 1953#ifdef DEV_NETMAP 1954 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1955#endif 1956 } 1957 if (iaq->nirq <= navail && 1958 (itype != INTR_MSI || powerof2(iaq->nirq))) 1959 goto allocate; 1960 1961 /* 1962 * Next best option: an interrupt vector for errors, one for the 1963 * firmware event queue, and at least one per port. At this 1964 * point we know we'll have to downsize nrxq and/or nofldrxq 1965 * and/or nnmrxq to fit what's available to us. 1966 */ 1967 iaq->nirq = T4_EXTRA_INTR; 1968 iaq->nirq += n10g + n1g; 1969 if (iaq->nirq <= navail) { 1970 int leftover = navail - iaq->nirq; 1971 1972 if (n10g > 0) { 1973 int target = max(nrxq10g, nofldrxq10g); 1974 1975 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1976 INTR_RXQ : INTR_OFLD_RXQ; 1977 1978 n = 1; 1979 while (n < target && leftover >= n10g) { 1980 leftover -= n10g; 1981 iaq->nirq += n10g; 1982 n++; 1983 } 1984 iaq->nrxq10g = min(n, nrxq10g); 1985#ifdef TCP_OFFLOAD 1986 iaq->nofldrxq10g = min(n, nofldrxq10g); 1987#endif 1988#ifdef DEV_NETMAP 1989 iaq->nnmrxq10g = min(n, nnmrxq10g); 1990#endif 1991 } 1992 1993 if (n1g > 0) { 1994 int target = max(nrxq1g, nofldrxq1g); 1995 1996 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1997 INTR_RXQ : INTR_OFLD_RXQ; 1998 1999 n = 1; 2000 while (n < target && leftover >= n1g) { 2001 leftover -= n1g; 2002 iaq->nirq += n1g; 2003 n++; 2004 } 2005 iaq->nrxq1g = min(n, nrxq1g); 2006#ifdef TCP_OFFLOAD 2007 iaq->nofldrxq1g = min(n, nofldrxq1g); 2008#endif 2009#ifdef DEV_NETMAP 2010 iaq->nnmrxq1g = min(n, nnmrxq1g); 2011#endif 2012 } 2013 2014 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2015 goto allocate; 2016 } 2017 2018 /* 2019 * Least desirable option: one interrupt vector for everything. 2020 */ 2021 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2022 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2023#ifdef TCP_OFFLOAD 2024 if (is_offload(sc)) 2025 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2026#endif 2027#ifdef DEV_NETMAP 2028 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2029#endif 2030 2031allocate: 2032 navail = iaq->nirq; 2033 rc = 0; 2034 if (itype == INTR_MSIX) 2035 rc = pci_alloc_msix(sc->dev, &navail); 2036 else if (itype == INTR_MSI) 2037 rc = pci_alloc_msi(sc->dev, &navail); 2038 2039 if (rc == 0) { 2040 if (navail == iaq->nirq) 2041 return (0); 2042 2043 /* 2044 * Didn't get the number requested. Use whatever number 2045 * the kernel is willing to allocate (it's in navail). 2046 */ 2047 device_printf(sc->dev, "fewer vectors than requested, " 2048 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2049 itype, iaq->nirq, navail); 2050 pci_release_msi(sc->dev); 2051 goto restart; 2052 } 2053 2054 device_printf(sc->dev, 2055 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2056 itype, rc, iaq->nirq, navail); 2057 } 2058 2059 device_printf(sc->dev, 2060 "failed to find a usable interrupt type. " 2061 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2062 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2063 2064 return (ENXIO); 2065} 2066 2067#define FW_VERSION(chip) ( \ 2068 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2069 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2070 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2071 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2072#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2073 2074struct fw_info { 2075 uint8_t chip; 2076 char *kld_name; 2077 char *fw_mod_name; 2078 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2079} fw_info[] = { 2080 { 2081 .chip = CHELSIO_T4, 2082 .kld_name = "t4fw_cfg", 2083 .fw_mod_name = "t4fw", 2084 .fw_hdr = { 2085 .chip = FW_HDR_CHIP_T4, 2086 .fw_ver = htobe32_const(FW_VERSION(T4)), 2087 .intfver_nic = FW_INTFVER(T4, NIC), 2088 .intfver_vnic = FW_INTFVER(T4, VNIC), 2089 .intfver_ofld = FW_INTFVER(T4, OFLD), 2090 .intfver_ri = FW_INTFVER(T4, RI), 2091 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2092 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2093 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2094 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2095 }, 2096 }, { 2097 .chip = CHELSIO_T5, 2098 .kld_name = "t5fw_cfg", 2099 .fw_mod_name = "t5fw", 2100 .fw_hdr = { 2101 .chip = FW_HDR_CHIP_T5, 2102 .fw_ver = htobe32_const(FW_VERSION(T5)), 2103 .intfver_nic = FW_INTFVER(T5, NIC), 2104 .intfver_vnic = FW_INTFVER(T5, VNIC), 2105 .intfver_ofld = FW_INTFVER(T5, OFLD), 2106 .intfver_ri = FW_INTFVER(T5, RI), 2107 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2108 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2109 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2110 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2111 }, 2112 } 2113}; 2114 2115static struct fw_info * 2116find_fw_info(int chip) 2117{ 2118 int i; 2119 2120 for (i = 0; i < nitems(fw_info); i++) { 2121 if (fw_info[i].chip == chip) 2122 return (&fw_info[i]); 2123 } 2124 return (NULL); 2125} 2126 2127/* 2128 * Is the given firmware API compatible with the one the driver was compiled 2129 * with? 2130 */ 2131static int 2132fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2133{ 2134 2135 /* short circuit if it's the exact same firmware version */ 2136 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2137 return (1); 2138 2139 /* 2140 * XXX: Is this too conservative? Perhaps I should limit this to the 2141 * features that are supported in the driver. 2142 */ 2143#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2144 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2145 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2146 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2147 return (1); 2148#undef SAME_INTF 2149 2150 return (0); 2151} 2152 2153/* 2154 * The firmware in the KLD is usable, but should it be installed? This routine 2155 * explains itself in detail if it indicates the KLD firmware should be 2156 * installed. 2157 */ 2158static int 2159should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2160{ 2161 const char *reason; 2162 2163 if (!card_fw_usable) { 2164 reason = "incompatible or unusable"; 2165 goto install; 2166 } 2167 2168 if (k > c) { 2169 reason = "older than the version bundled with this driver"; 2170 goto install; 2171 } 2172 2173 if (t4_fw_install == 2 && k != c) { 2174 reason = "different than the version bundled with this driver"; 2175 goto install; 2176 } 2177 2178 return (0); 2179 2180install: 2181 if (t4_fw_install == 0) { 2182 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2183 "but the driver is prohibited from installing a different " 2184 "firmware on the card.\n", 2185 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2186 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2187 2188 return (0); 2189 } 2190 2191 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2192 "installing firmware %u.%u.%u.%u on card.\n", 2193 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2194 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2195 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2196 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2197 2198 return (1); 2199} 2200/* 2201 * Establish contact with the firmware and determine if we are the master driver 2202 * or not, and whether we are responsible for chip initialization. 2203 */ 2204static int 2205prep_firmware(struct adapter *sc) 2206{ 2207 const struct firmware *fw = NULL, *default_cfg; 2208 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2209 enum dev_state state; 2210 struct fw_info *fw_info; 2211 struct fw_hdr *card_fw; /* fw on the card */ 2212 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2213 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2214 against */ 2215 2216 /* Contact firmware. */ 2217 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2218 if (rc < 0 || state == DEV_STATE_ERR) { 2219 rc = -rc; 2220 device_printf(sc->dev, 2221 "failed to connect to the firmware: %d, %d.\n", rc, state); 2222 return (rc); 2223 } 2224 pf = rc; 2225 if (pf == sc->mbox) 2226 sc->flags |= MASTER_PF; 2227 else if (state == DEV_STATE_UNINIT) { 2228 /* 2229 * We didn't get to be the master so we definitely won't be 2230 * configuring the chip. It's a bug if someone else hasn't 2231 * configured it already. 2232 */ 2233 device_printf(sc->dev, "couldn't be master(%d), " 2234 "device not already initialized either(%d).\n", rc, state); 2235 return (EDOOFUS); 2236 } 2237 2238 /* This is the firmware whose headers the driver was compiled against */ 2239 fw_info = find_fw_info(chip_id(sc)); 2240 if (fw_info == NULL) { 2241 device_printf(sc->dev, 2242 "unable to look up firmware information for chip %d.\n", 2243 chip_id(sc)); 2244 return (EINVAL); 2245 } 2246 drv_fw = &fw_info->fw_hdr; 2247 2248 /* 2249 * The firmware KLD contains many modules. The KLD name is also the 2250 * name of the module that contains the default config file. 2251 */ 2252 default_cfg = firmware_get(fw_info->kld_name); 2253 2254 /* Read the header of the firmware on the card */ 2255 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2256 rc = -t4_read_flash(sc, FLASH_FW_START, 2257 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2258 if (rc == 0) 2259 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2260 else { 2261 device_printf(sc->dev, 2262 "Unable to read card's firmware header: %d\n", rc); 2263 card_fw_usable = 0; 2264 } 2265 2266 /* This is the firmware in the KLD */ 2267 fw = firmware_get(fw_info->fw_mod_name); 2268 if (fw != NULL) { 2269 kld_fw = (const void *)fw->data; 2270 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2271 } else { 2272 kld_fw = NULL; 2273 kld_fw_usable = 0; 2274 } 2275 2276 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2277 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2278 /* 2279 * Common case: the firmware on the card is an exact match and 2280 * the KLD is an exact match too, or the KLD is 2281 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2282 * here -- use cxgbetool loadfw if you want to reinstall the 2283 * same firmware as the one on the card. 2284 */ 2285 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2286 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2287 be32toh(card_fw->fw_ver))) { 2288 2289 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2290 if (rc != 0) { 2291 device_printf(sc->dev, 2292 "failed to install firmware: %d\n", rc); 2293 goto done; 2294 } 2295 2296 /* Installed successfully, update the cached header too. */ 2297 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2298 card_fw_usable = 1; 2299 need_fw_reset = 0; /* already reset as part of load_fw */ 2300 } 2301 2302 if (!card_fw_usable) { 2303 uint32_t d, c, k; 2304 2305 d = ntohl(drv_fw->fw_ver); 2306 c = ntohl(card_fw->fw_ver); 2307 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2308 2309 device_printf(sc->dev, "Cannot find a usable firmware: " 2310 "fw_install %d, chip state %d, " 2311 "driver compiled with %d.%d.%d.%d, " 2312 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2313 t4_fw_install, state, 2314 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2315 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2316 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2317 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2318 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2319 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2320 rc = EINVAL; 2321 goto done; 2322 } 2323 2324 /* We're using whatever's on the card and it's known to be good. */ 2325 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2326 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2327 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2328 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2329 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2330 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2331 t4_get_tp_version(sc, &sc->params.tp_vers); 2332 2333 /* Reset device */ 2334 if (need_fw_reset && 2335 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2336 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2337 if (rc != ETIMEDOUT && rc != EIO) 2338 t4_fw_bye(sc, sc->mbox); 2339 goto done; 2340 } 2341 sc->flags |= FW_OK; 2342 2343 rc = get_params__pre_init(sc); 2344 if (rc != 0) 2345 goto done; /* error message displayed already */ 2346 2347 /* Partition adapter resources as specified in the config file. */ 2348 if (state == DEV_STATE_UNINIT) { 2349 2350 KASSERT(sc->flags & MASTER_PF, 2351 ("%s: trying to change chip settings when not master.", 2352 __func__)); 2353 2354 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2355 if (rc != 0) 2356 goto done; /* error message displayed already */ 2357 2358 t4_tweak_chip_settings(sc); 2359 2360 /* get basic stuff going */ 2361 rc = -t4_fw_initialize(sc, sc->mbox); 2362 if (rc != 0) { 2363 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2364 goto done; 2365 } 2366 } else { 2367 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2368 sc->cfcsum = 0; 2369 } 2370 2371done: 2372 free(card_fw, M_CXGBE); 2373 if (fw != NULL) 2374 firmware_put(fw, FIRMWARE_UNLOAD); 2375 if (default_cfg != NULL) 2376 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2377 2378 return (rc); 2379} 2380 2381#define FW_PARAM_DEV(param) \ 2382 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2383 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2384#define FW_PARAM_PFVF(param) \ 2385 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2386 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2387 2388/* 2389 * Partition chip resources for use between various PFs, VFs, etc. 2390 */ 2391static int 2392partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2393 const char *name_prefix) 2394{ 2395 const struct firmware *cfg = NULL; 2396 int rc = 0; 2397 struct fw_caps_config_cmd caps; 2398 uint32_t mtype, moff, finicsum, cfcsum; 2399 2400 /* 2401 * Figure out what configuration file to use. Pick the default config 2402 * file for the card if the user hasn't specified one explicitly. 2403 */ 2404 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2405 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2406 /* Card specific overrides go here. */ 2407 if (pci_get_device(sc->dev) == 0x440a) 2408 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2409 if (is_fpga(sc)) 2410 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2411 } 2412 2413 /* 2414 * We need to load another module if the profile is anything except 2415 * "default" or "flash". 2416 */ 2417 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2418 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2419 char s[32]; 2420 2421 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2422 cfg = firmware_get(s); 2423 if (cfg == NULL) { 2424 if (default_cfg != NULL) { 2425 device_printf(sc->dev, 2426 "unable to load module \"%s\" for " 2427 "configuration profile \"%s\", will use " 2428 "the default config file instead.\n", 2429 s, sc->cfg_file); 2430 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2431 "%s", DEFAULT_CF); 2432 } else { 2433 device_printf(sc->dev, 2434 "unable to load module \"%s\" for " 2435 "configuration profile \"%s\", will use " 2436 "the config file on the card's flash " 2437 "instead.\n", s, sc->cfg_file); 2438 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2439 "%s", FLASH_CF); 2440 } 2441 } 2442 } 2443 2444 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2445 default_cfg == NULL) { 2446 device_printf(sc->dev, 2447 "default config file not available, will use the config " 2448 "file on the card's flash instead.\n"); 2449 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2450 } 2451 2452 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2453 u_int cflen, i, n; 2454 const uint32_t *cfdata; 2455 uint32_t param, val, addr, off, mw_base, mw_aperture; 2456 2457 KASSERT(cfg != NULL || default_cfg != NULL, 2458 ("%s: no config to upload", __func__)); 2459 2460 /* 2461 * Ask the firmware where it wants us to upload the config file. 2462 */ 2463 param = FW_PARAM_DEV(CF); 2464 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2465 if (rc != 0) { 2466 /* No support for config file? Shouldn't happen. */ 2467 device_printf(sc->dev, 2468 "failed to query config file location: %d.\n", rc); 2469 goto done; 2470 } 2471 mtype = G_FW_PARAMS_PARAM_Y(val); 2472 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2473 2474 /* 2475 * XXX: sheer laziness. We deliberately added 4 bytes of 2476 * useless stuffing/comments at the end of the config file so 2477 * it's ok to simply throw away the last remaining bytes when 2478 * the config file is not an exact multiple of 4. This also 2479 * helps with the validate_mt_off_len check. 2480 */ 2481 if (cfg != NULL) { 2482 cflen = cfg->datasize & ~3; 2483 cfdata = cfg->data; 2484 } else { 2485 cflen = default_cfg->datasize & ~3; 2486 cfdata = default_cfg->data; 2487 } 2488 2489 if (cflen > FLASH_CFG_MAX_SIZE) { 2490 device_printf(sc->dev, 2491 "config file too long (%d, max allowed is %d). " 2492 "Will try to use the config on the card, if any.\n", 2493 cflen, FLASH_CFG_MAX_SIZE); 2494 goto use_config_on_flash; 2495 } 2496 2497 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2498 if (rc != 0) { 2499 device_printf(sc->dev, 2500 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2501 "Will try to use the config on the card, if any.\n", 2502 __func__, mtype, moff, cflen, rc); 2503 goto use_config_on_flash; 2504 } 2505 2506 memwin_info(sc, 2, &mw_base, &mw_aperture); 2507 while (cflen) { 2508 off = position_memwin(sc, 2, addr); 2509 n = min(cflen, mw_aperture - off); 2510 for (i = 0; i < n; i += 4) 2511 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2512 cflen -= n; 2513 addr += n; 2514 } 2515 } else { 2516use_config_on_flash: 2517 mtype = FW_MEMTYPE_FLASH; 2518 moff = t4_flash_cfg_addr(sc); 2519 } 2520 2521 bzero(&caps, sizeof(caps)); 2522 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2523 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2524 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2525 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2526 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2527 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2528 if (rc != 0) { 2529 device_printf(sc->dev, 2530 "failed to pre-process config file: %d " 2531 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2532 goto done; 2533 } 2534 2535 finicsum = be32toh(caps.finicsum); 2536 cfcsum = be32toh(caps.cfcsum); 2537 if (finicsum != cfcsum) { 2538 device_printf(sc->dev, 2539 "WARNING: config file checksum mismatch: %08x %08x\n", 2540 finicsum, cfcsum); 2541 } 2542 sc->cfcsum = cfcsum; 2543 2544#define LIMIT_CAPS(x) do { \ 2545 caps.x &= htobe16(t4_##x##_allowed); \ 2546} while (0) 2547 2548 /* 2549 * Let the firmware know what features will (not) be used so it can tune 2550 * things accordingly. 2551 */ 2552 LIMIT_CAPS(linkcaps); 2553 LIMIT_CAPS(niccaps); 2554 LIMIT_CAPS(toecaps); 2555 LIMIT_CAPS(rdmacaps); 2556 LIMIT_CAPS(iscsicaps); 2557 LIMIT_CAPS(fcoecaps); 2558#undef LIMIT_CAPS 2559 2560 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2561 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2562 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2563 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2564 if (rc != 0) { 2565 device_printf(sc->dev, 2566 "failed to process config file: %d.\n", rc); 2567 } 2568done: 2569 if (cfg != NULL) 2570 firmware_put(cfg, FIRMWARE_UNLOAD); 2571 return (rc); 2572} 2573 2574/* 2575 * Retrieve parameters that are needed (or nice to have) very early. 2576 */ 2577static int 2578get_params__pre_init(struct adapter *sc) 2579{ 2580 int rc; 2581 uint32_t param[2], val[2]; 2582 struct fw_devlog_cmd cmd; 2583 struct devlog_params *dlog = &sc->params.devlog; 2584 2585 param[0] = FW_PARAM_DEV(PORTVEC); 2586 param[1] = FW_PARAM_DEV(CCLK); 2587 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2588 if (rc != 0) { 2589 device_printf(sc->dev, 2590 "failed to query parameters (pre_init): %d.\n", rc); 2591 return (rc); 2592 } 2593 2594 sc->params.portvec = val[0]; 2595 sc->params.nports = bitcount32(val[0]); 2596 sc->params.vpd.cclk = val[1]; 2597 2598 /* Read device log parameters. */ 2599 bzero(&cmd, sizeof(cmd)); 2600 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2601 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2602 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2603 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2604 if (rc != 0) { 2605 device_printf(sc->dev, 2606 "failed to get devlog parameters: %d.\n", rc); 2607 bzero(dlog, sizeof (*dlog)); 2608 rc = 0; /* devlog isn't critical for device operation */ 2609 } else { 2610 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2611 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2612 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2613 dlog->size = be32toh(cmd.memsize_devlog); 2614 } 2615 2616 return (rc); 2617} 2618 2619/* 2620 * Retrieve various parameters that are of interest to the driver. The device 2621 * has been initialized by the firmware at this point. 2622 */ 2623static int 2624get_params__post_init(struct adapter *sc) 2625{ 2626 int rc; 2627 uint32_t param[7], val[7]; 2628 struct fw_caps_config_cmd caps; 2629 2630 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2631 param[1] = FW_PARAM_PFVF(EQ_START); 2632 param[2] = FW_PARAM_PFVF(FILTER_START); 2633 param[3] = FW_PARAM_PFVF(FILTER_END); 2634 param[4] = FW_PARAM_PFVF(L2T_START); 2635 param[5] = FW_PARAM_PFVF(L2T_END); 2636 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2637 if (rc != 0) { 2638 device_printf(sc->dev, 2639 "failed to query parameters (post_init): %d.\n", rc); 2640 return (rc); 2641 } 2642 2643 sc->sge.iq_start = val[0]; 2644 sc->sge.eq_start = val[1]; 2645 sc->tids.ftid_base = val[2]; 2646 sc->tids.nftids = val[3] - val[2] + 1; 2647 sc->params.ftid_min = val[2]; 2648 sc->params.ftid_max = val[3]; 2649 sc->vres.l2t.start = val[4]; 2650 sc->vres.l2t.size = val[5] - val[4] + 1; 2651 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2652 ("%s: L2 table size (%u) larger than expected (%u)", 2653 __func__, sc->vres.l2t.size, L2T_SIZE)); 2654 2655 /* get capabilites */ 2656 bzero(&caps, sizeof(caps)); 2657 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2658 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2659 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2660 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2661 if (rc != 0) { 2662 device_printf(sc->dev, 2663 "failed to get card capabilities: %d.\n", rc); 2664 return (rc); 2665 } 2666 2667#define READ_CAPS(x) do { \ 2668 sc->x = htobe16(caps.x); \ 2669} while (0) 2670 READ_CAPS(linkcaps); 2671 READ_CAPS(niccaps); 2672 READ_CAPS(toecaps); 2673 READ_CAPS(rdmacaps); 2674 READ_CAPS(iscsicaps); 2675 READ_CAPS(fcoecaps); 2676 2677 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2678 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2679 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2680 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2681 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2682 if (rc != 0) { 2683 device_printf(sc->dev, 2684 "failed to query NIC parameters: %d.\n", rc); 2685 return (rc); 2686 } 2687 sc->tids.etid_base = val[0]; 2688 sc->params.etid_min = val[0]; 2689 sc->tids.netids = val[1] - val[0] + 1; 2690 sc->params.netids = sc->tids.netids; 2691 sc->params.eo_wr_cred = val[2]; 2692 sc->params.ethoffload = 1; 2693 } 2694 2695 if (sc->toecaps) { 2696 /* query offload-related parameters */ 2697 param[0] = FW_PARAM_DEV(NTID); 2698 param[1] = FW_PARAM_PFVF(SERVER_START); 2699 param[2] = FW_PARAM_PFVF(SERVER_END); 2700 param[3] = FW_PARAM_PFVF(TDDP_START); 2701 param[4] = FW_PARAM_PFVF(TDDP_END); 2702 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2703 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2704 if (rc != 0) { 2705 device_printf(sc->dev, 2706 "failed to query TOE parameters: %d.\n", rc); 2707 return (rc); 2708 } 2709 sc->tids.ntids = val[0]; 2710 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2711 sc->tids.stid_base = val[1]; 2712 sc->tids.nstids = val[2] - val[1] + 1; 2713 sc->vres.ddp.start = val[3]; 2714 sc->vres.ddp.size = val[4] - val[3] + 1; 2715 sc->params.ofldq_wr_cred = val[5]; 2716 sc->params.offload = 1; 2717 } 2718 if (sc->rdmacaps) { 2719 param[0] = FW_PARAM_PFVF(STAG_START); 2720 param[1] = FW_PARAM_PFVF(STAG_END); 2721 param[2] = FW_PARAM_PFVF(RQ_START); 2722 param[3] = FW_PARAM_PFVF(RQ_END); 2723 param[4] = FW_PARAM_PFVF(PBL_START); 2724 param[5] = FW_PARAM_PFVF(PBL_END); 2725 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2726 if (rc != 0) { 2727 device_printf(sc->dev, 2728 "failed to query RDMA parameters(1): %d.\n", rc); 2729 return (rc); 2730 } 2731 sc->vres.stag.start = val[0]; 2732 sc->vres.stag.size = val[1] - val[0] + 1; 2733 sc->vres.rq.start = val[2]; 2734 sc->vres.rq.size = val[3] - val[2] + 1; 2735 sc->vres.pbl.start = val[4]; 2736 sc->vres.pbl.size = val[5] - val[4] + 1; 2737 2738 param[0] = FW_PARAM_PFVF(SQRQ_START); 2739 param[1] = FW_PARAM_PFVF(SQRQ_END); 2740 param[2] = FW_PARAM_PFVF(CQ_START); 2741 param[3] = FW_PARAM_PFVF(CQ_END); 2742 param[4] = FW_PARAM_PFVF(OCQ_START); 2743 param[5] = FW_PARAM_PFVF(OCQ_END); 2744 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2745 if (rc != 0) { 2746 device_printf(sc->dev, 2747 "failed to query RDMA parameters(2): %d.\n", rc); 2748 return (rc); 2749 } 2750 sc->vres.qp.start = val[0]; 2751 sc->vres.qp.size = val[1] - val[0] + 1; 2752 sc->vres.cq.start = val[2]; 2753 sc->vres.cq.size = val[3] - val[2] + 1; 2754 sc->vres.ocq.start = val[4]; 2755 sc->vres.ocq.size = val[5] - val[4] + 1; 2756 } 2757 if (sc->iscsicaps) { 2758 param[0] = FW_PARAM_PFVF(ISCSI_START); 2759 param[1] = FW_PARAM_PFVF(ISCSI_END); 2760 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2761 if (rc != 0) { 2762 device_printf(sc->dev, 2763 "failed to query iSCSI parameters: %d.\n", rc); 2764 return (rc); 2765 } 2766 sc->vres.iscsi.start = val[0]; 2767 sc->vres.iscsi.size = val[1] - val[0] + 1; 2768 } 2769 2770 /* 2771 * We've got the params we wanted to query via the firmware. Now grab 2772 * some others directly from the chip. 2773 */ 2774 rc = t4_read_chip_settings(sc); 2775 2776 return (rc); 2777} 2778 2779static int 2780set_params__post_init(struct adapter *sc) 2781{ 2782 uint32_t param, val; 2783 2784 /* ask for encapsulated CPLs */ 2785 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2786 val = 1; 2787 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2788 2789 return (0); 2790} 2791 2792#undef FW_PARAM_PFVF 2793#undef FW_PARAM_DEV 2794 2795static void 2796t4_set_desc(struct adapter *sc) 2797{ 2798 char buf[128]; 2799 struct adapter_params *p = &sc->params; 2800 2801 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2802 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2803 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2804 2805 device_set_desc_copy(sc->dev, buf); 2806} 2807 2808static void 2809build_medialist(struct port_info *pi, struct ifmedia *media) 2810{ 2811 int data, m; 2812 2813 PORT_LOCK(pi); 2814 2815 ifmedia_removeall(media); 2816 2817 m = IFM_ETHER | IFM_FDX; 2818 data = (pi->port_type << 8) | pi->mod_type; 2819 2820 switch(pi->port_type) { 2821 case FW_PORT_TYPE_BT_XFI: 2822 case FW_PORT_TYPE_BT_XAUI: 2823 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2824 /* fall through */ 2825 2826 case FW_PORT_TYPE_BT_SGMII: 2827 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2828 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2829 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2830 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2831 break; 2832 2833 case FW_PORT_TYPE_CX4: 2834 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2835 ifmedia_set(media, m | IFM_10G_CX4); 2836 break; 2837 2838 case FW_PORT_TYPE_QSFP_10G: 2839 case FW_PORT_TYPE_SFP: 2840 case FW_PORT_TYPE_FIBER_XFI: 2841 case FW_PORT_TYPE_FIBER_XAUI: 2842 switch (pi->mod_type) { 2843 2844 case FW_PORT_MOD_TYPE_LR: 2845 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2846 ifmedia_set(media, m | IFM_10G_LR); 2847 break; 2848 2849 case FW_PORT_MOD_TYPE_SR: 2850 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2851 ifmedia_set(media, m | IFM_10G_SR); 2852 break; 2853 2854 case FW_PORT_MOD_TYPE_LRM: 2855 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2856 ifmedia_set(media, m | IFM_10G_LRM); 2857 break; 2858 2859 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2860 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2861 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2862 ifmedia_set(media, m | IFM_10G_TWINAX); 2863 break; 2864 2865 case FW_PORT_MOD_TYPE_NONE: 2866 m &= ~IFM_FDX; 2867 ifmedia_add(media, m | IFM_NONE, data, NULL); 2868 ifmedia_set(media, m | IFM_NONE); 2869 break; 2870 2871 case FW_PORT_MOD_TYPE_NA: 2872 case FW_PORT_MOD_TYPE_ER: 2873 default: 2874 device_printf(pi->dev, 2875 "unknown port_type (%d), mod_type (%d)\n", 2876 pi->port_type, pi->mod_type); 2877 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2878 ifmedia_set(media, m | IFM_UNKNOWN); 2879 break; 2880 } 2881 break; 2882 2883 case FW_PORT_TYPE_QSFP: 2884 switch (pi->mod_type) { 2885 2886 case FW_PORT_MOD_TYPE_LR: 2887 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2888 ifmedia_set(media, m | IFM_40G_LR4); 2889 break; 2890 2891 case FW_PORT_MOD_TYPE_SR: 2892 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2893 ifmedia_set(media, m | IFM_40G_SR4); 2894 break; 2895 2896 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2897 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2898 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2899 ifmedia_set(media, m | IFM_40G_CR4); 2900 break; 2901 2902 case FW_PORT_MOD_TYPE_NONE: 2903 m &= ~IFM_FDX; 2904 ifmedia_add(media, m | IFM_NONE, data, NULL); 2905 ifmedia_set(media, m | IFM_NONE); 2906 break; 2907 2908 default: 2909 device_printf(pi->dev, 2910 "unknown port_type (%d), mod_type (%d)\n", 2911 pi->port_type, pi->mod_type); 2912 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2913 ifmedia_set(media, m | IFM_UNKNOWN); 2914 break; 2915 } 2916 break; 2917 2918 default: 2919 device_printf(pi->dev, 2920 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2921 pi->mod_type); 2922 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2923 ifmedia_set(media, m | IFM_UNKNOWN); 2924 break; 2925 } 2926 2927 PORT_UNLOCK(pi); 2928} 2929 2930#define FW_MAC_EXACT_CHUNK 7 2931 2932/* 2933 * Program the port's XGMAC based on parameters in ifnet. The caller also 2934 * indicates which parameters should be programmed (the rest are left alone). 2935 */ 2936int 2937update_mac_settings(struct ifnet *ifp, int flags) 2938{ 2939 int rc = 0; 2940 struct port_info *pi = ifp->if_softc; 2941 struct adapter *sc = pi->adapter; 2942 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2943 uint16_t viid = 0xffff; 2944 int16_t *xact_addr_filt = NULL; 2945 2946 ASSERT_SYNCHRONIZED_OP(sc); 2947 KASSERT(flags, ("%s: not told what to update.", __func__)); 2948 2949 if (ifp == pi->ifp) { 2950 viid = pi->viid; 2951 xact_addr_filt = &pi->xact_addr_filt; 2952 } 2953#ifdef DEV_NETMAP 2954 else if (ifp == pi->nm_ifp) { 2955 viid = pi->nm_viid; 2956 xact_addr_filt = &pi->nm_xact_addr_filt; 2957 } 2958#endif 2959 if (flags & XGMAC_MTU) 2960 mtu = ifp->if_mtu; 2961 2962 if (flags & XGMAC_PROMISC) 2963 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2964 2965 if (flags & XGMAC_ALLMULTI) 2966 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2967 2968 if (flags & XGMAC_VLANEX) 2969 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2970 2971 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2972 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2973 1, vlanex, false); 2974 if (rc) { 2975 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2976 rc); 2977 return (rc); 2978 } 2979 } 2980 2981 if (flags & XGMAC_UCADDR) { 2982 uint8_t ucaddr[ETHER_ADDR_LEN]; 2983 2984 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2985 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2986 true, true); 2987 if (rc < 0) { 2988 rc = -rc; 2989 if_printf(ifp, "change_mac failed: %d\n", rc); 2990 return (rc); 2991 } else { 2992 *xact_addr_filt = rc; 2993 rc = 0; 2994 } 2995 } 2996 2997 if (flags & XGMAC_MCADDRS) { 2998 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2999 int del = 1; 3000 uint64_t hash = 0; 3001 struct ifmultiaddr *ifma; 3002 int i = 0, j; 3003 3004 if_maddr_rlock(ifp); 3005 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3006 if (ifma->ifma_addr->sa_family != AF_LINK) 3007 continue; 3008 mcaddr[i] = 3009 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3010 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3011 i++; 3012 3013 if (i == FW_MAC_EXACT_CHUNK) { 3014 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3015 i, mcaddr, NULL, &hash, 0); 3016 if (rc < 0) { 3017 rc = -rc; 3018 for (j = 0; j < i; j++) { 3019 if_printf(ifp, 3020 "failed to add mc address" 3021 " %02x:%02x:%02x:" 3022 "%02x:%02x:%02x rc=%d\n", 3023 mcaddr[j][0], mcaddr[j][1], 3024 mcaddr[j][2], mcaddr[j][3], 3025 mcaddr[j][4], mcaddr[j][5], 3026 rc); 3027 } 3028 goto mcfail; 3029 } 3030 del = 0; 3031 i = 0; 3032 } 3033 } 3034 if (i > 0) { 3035 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3036 mcaddr, NULL, &hash, 0); 3037 if (rc < 0) { 3038 rc = -rc; 3039 for (j = 0; j < i; j++) { 3040 if_printf(ifp, 3041 "failed to add mc address" 3042 " %02x:%02x:%02x:" 3043 "%02x:%02x:%02x rc=%d\n", 3044 mcaddr[j][0], mcaddr[j][1], 3045 mcaddr[j][2], mcaddr[j][3], 3046 mcaddr[j][4], mcaddr[j][5], 3047 rc); 3048 } 3049 goto mcfail; 3050 } 3051 } 3052 3053 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3054 if (rc != 0) 3055 if_printf(ifp, "failed to set mc address hash: %d", rc); 3056mcfail: 3057 if_maddr_runlock(ifp); 3058 } 3059 3060 return (rc); 3061} 3062 3063/* 3064 * {begin|end}_synchronized_op must be called from the same thread. 3065 */ 3066int 3067begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3068 char *wmesg) 3069{ 3070 int rc, pri; 3071 3072#ifdef WITNESS 3073 /* the caller thinks it's ok to sleep, but is it really? */ 3074 if (flags & SLEEP_OK) 3075 pause("t4slptst", 1); 3076#endif 3077 3078 if (INTR_OK) 3079 pri = PCATCH; 3080 else 3081 pri = 0; 3082 3083 ADAPTER_LOCK(sc); 3084 for (;;) { 3085 3086 if (pi && IS_DOOMED(pi)) { 3087 rc = ENXIO; 3088 goto done; 3089 } 3090 3091 if (!IS_BUSY(sc)) { 3092 rc = 0; 3093 break; 3094 } 3095 3096 if (!(flags & SLEEP_OK)) { 3097 rc = EBUSY; 3098 goto done; 3099 } 3100 3101 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3102 rc = EINTR; 3103 goto done; 3104 } 3105 } 3106 3107 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3108 SET_BUSY(sc); 3109#ifdef INVARIANTS 3110 sc->last_op = wmesg; 3111 sc->last_op_thr = curthread; 3112#endif 3113 3114done: 3115 if (!(flags & HOLD_LOCK) || rc) 3116 ADAPTER_UNLOCK(sc); 3117 3118 return (rc); 3119} 3120 3121/* 3122 * {begin|end}_synchronized_op must be called from the same thread. 3123 */ 3124void 3125end_synchronized_op(struct adapter *sc, int flags) 3126{ 3127 3128 if (flags & LOCK_HELD) 3129 ADAPTER_LOCK_ASSERT_OWNED(sc); 3130 else 3131 ADAPTER_LOCK(sc); 3132 3133 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3134 CLR_BUSY(sc); 3135 wakeup(&sc->flags); 3136 ADAPTER_UNLOCK(sc); 3137} 3138 3139static int 3140cxgbe_init_synchronized(struct port_info *pi) 3141{ 3142 struct adapter *sc = pi->adapter; 3143 struct ifnet *ifp = pi->ifp; 3144 int rc = 0, i; 3145 struct sge_txq *txq; 3146 3147 ASSERT_SYNCHRONIZED_OP(sc); 3148 3149 if (isset(&sc->open_device_map, pi->port_id)) { 3150 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3151 ("mismatch between open_device_map and if_drv_flags")); 3152 return (0); /* already running */ 3153 } 3154 3155 if (!(sc->flags & FULL_INIT_DONE) && 3156 ((rc = adapter_full_init(sc)) != 0)) 3157 return (rc); /* error message displayed already */ 3158 3159 if (!(pi->flags & PORT_INIT_DONE) && 3160 ((rc = port_full_init(pi)) != 0)) 3161 return (rc); /* error message displayed already */ 3162 3163 rc = update_mac_settings(ifp, XGMAC_ALL); 3164 if (rc) 3165 goto done; /* error message displayed already */ 3166 3167 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3168 if (rc != 0) { 3169 if_printf(ifp, "enable_vi failed: %d\n", rc); 3170 goto done; 3171 } 3172 3173 /* 3174 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3175 * if this changes. 3176 */ 3177 3178 for_each_txq(pi, i, txq) { 3179 TXQ_LOCK(txq); 3180 txq->eq.flags |= EQ_ENABLED; 3181 TXQ_UNLOCK(txq); 3182 } 3183 3184 /* 3185 * The first iq of the first port to come up is used for tracing. 3186 */ 3187 if (sc->traceq < 0) { 3188 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3189 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3190 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3191 V_QUEUENUMBER(sc->traceq)); 3192 pi->flags |= HAS_TRACEQ; 3193 } 3194 3195 /* all ok */ 3196 setbit(&sc->open_device_map, pi->port_id); 3197 PORT_LOCK(pi); 3198 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3199 PORT_UNLOCK(pi); 3200 3201 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3202done: 3203 if (rc != 0) 3204 cxgbe_uninit_synchronized(pi); 3205 3206 return (rc); 3207} 3208 3209/* 3210 * Idempotent. 3211 */ 3212static int 3213cxgbe_uninit_synchronized(struct port_info *pi) 3214{ 3215 struct adapter *sc = pi->adapter; 3216 struct ifnet *ifp = pi->ifp; 3217 int rc, i; 3218 struct sge_txq *txq; 3219 3220 ASSERT_SYNCHRONIZED_OP(sc); 3221 3222 /* 3223 * Disable the VI so that all its data in either direction is discarded 3224 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3225 * tick) intact as the TP can deliver negative advice or data that it's 3226 * holding in its RAM (for an offloaded connection) even after the VI is 3227 * disabled. 3228 */ 3229 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3230 if (rc) { 3231 if_printf(ifp, "disable_vi failed: %d\n", rc); 3232 return (rc); 3233 } 3234 3235 for_each_txq(pi, i, txq) { 3236 TXQ_LOCK(txq); 3237 txq->eq.flags &= ~EQ_ENABLED; 3238 TXQ_UNLOCK(txq); 3239 } 3240 3241 clrbit(&sc->open_device_map, pi->port_id); 3242 PORT_LOCK(pi); 3243 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3244 PORT_UNLOCK(pi); 3245 3246 pi->link_cfg.link_ok = 0; 3247 pi->link_cfg.speed = 0; 3248 pi->linkdnrc = -1; 3249 t4_os_link_changed(sc, pi->port_id, 0, -1); 3250 3251 return (0); 3252} 3253 3254/* 3255 * It is ok for this function to fail midway and return right away. t4_detach 3256 * will walk the entire sc->irq list and clean up whatever is valid. 3257 */ 3258static int 3259setup_intr_handlers(struct adapter *sc) 3260{ 3261 int rc, rid, p, q; 3262 char s[8]; 3263 struct irq *irq; 3264 struct port_info *pi; 3265 struct sge_rxq *rxq; 3266#ifdef TCP_OFFLOAD 3267 struct sge_ofld_rxq *ofld_rxq; 3268#endif 3269#ifdef DEV_NETMAP 3270 struct sge_nm_rxq *nm_rxq; 3271#endif 3272 3273 /* 3274 * Setup interrupts. 3275 */ 3276 irq = &sc->irq[0]; 3277 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3278 if (sc->intr_count == 1) 3279 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3280 3281 /* Multiple interrupts. */ 3282 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3283 ("%s: too few intr.", __func__)); 3284 3285 /* The first one is always error intr */ 3286 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3287 if (rc != 0) 3288 return (rc); 3289 irq++; 3290 rid++; 3291 3292 /* The second one is always the firmware event queue */ 3293 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3294 if (rc != 0) 3295 return (rc); 3296 irq++; 3297 rid++; 3298 3299 for_each_port(sc, p) { 3300 pi = sc->port[p]; 3301 3302 if (pi->flags & INTR_RXQ) { 3303 for_each_rxq(pi, q, rxq) { 3304 snprintf(s, sizeof(s), "%d.%d", p, q); 3305 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3306 s); 3307 if (rc != 0) 3308 return (rc); 3309 irq++; 3310 rid++; 3311 } 3312 } 3313#ifdef TCP_OFFLOAD 3314 if (pi->flags & INTR_OFLD_RXQ) { 3315 for_each_ofld_rxq(pi, q, ofld_rxq) { 3316 snprintf(s, sizeof(s), "%d,%d", p, q); 3317 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3318 ofld_rxq, s); 3319 if (rc != 0) 3320 return (rc); 3321 irq++; 3322 rid++; 3323 } 3324 } 3325#endif 3326#ifdef DEV_NETMAP 3327 if (pi->flags & INTR_NM_RXQ) { 3328 for_each_nm_rxq(pi, q, nm_rxq) { 3329 snprintf(s, sizeof(s), "%d-%d", p, q); 3330 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3331 nm_rxq, s); 3332 if (rc != 0) 3333 return (rc); 3334 irq++; 3335 rid++; 3336 } 3337 } 3338#endif 3339 } 3340 MPASS(irq == &sc->irq[sc->intr_count]); 3341 3342 return (0); 3343} 3344 3345int 3346adapter_full_init(struct adapter *sc) 3347{ 3348 int rc, i; 3349 3350 ASSERT_SYNCHRONIZED_OP(sc); 3351 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3352 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3353 ("%s: FULL_INIT_DONE already", __func__)); 3354 3355 /* 3356 * queues that belong to the adapter (not any particular port). 3357 */ 3358 rc = t4_setup_adapter_queues(sc); 3359 if (rc != 0) 3360 goto done; 3361 3362 for (i = 0; i < nitems(sc->tq); i++) { 3363 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3364 taskqueue_thread_enqueue, &sc->tq[i]); 3365 if (sc->tq[i] == NULL) { 3366 device_printf(sc->dev, 3367 "failed to allocate task queue %d\n", i); 3368 rc = ENOMEM; 3369 goto done; 3370 } 3371 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3372 device_get_nameunit(sc->dev), i); 3373 } 3374 3375 t4_intr_enable(sc); 3376 sc->flags |= FULL_INIT_DONE; 3377done: 3378 if (rc != 0) 3379 adapter_full_uninit(sc); 3380 3381 return (rc); 3382} 3383 3384int 3385adapter_full_uninit(struct adapter *sc) 3386{ 3387 int i; 3388 3389 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3390 3391 t4_teardown_adapter_queues(sc); 3392 3393 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3394 taskqueue_free(sc->tq[i]); 3395 sc->tq[i] = NULL; 3396 } 3397 3398 sc->flags &= ~FULL_INIT_DONE; 3399 3400 return (0); 3401} 3402 3403int 3404port_full_init(struct port_info *pi) 3405{ 3406 struct adapter *sc = pi->adapter; 3407 struct ifnet *ifp = pi->ifp; 3408 uint16_t *rss; 3409 struct sge_rxq *rxq; 3410 int rc, i, j; 3411 3412 ASSERT_SYNCHRONIZED_OP(sc); 3413 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3414 ("%s: PORT_INIT_DONE already", __func__)); 3415 3416 sysctl_ctx_init(&pi->ctx); 3417 pi->flags |= PORT_SYSCTL_CTX; 3418 3419 /* 3420 * Allocate tx/rx/fl queues for this port. 3421 */ 3422 rc = t4_setup_port_queues(pi); 3423 if (rc != 0) 3424 goto done; /* error message displayed already */ 3425 3426 /* 3427 * Setup RSS for this port. Save a copy of the RSS table for later use. 3428 */ 3429 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3430 for (i = 0; i < pi->rss_size;) { 3431 for_each_rxq(pi, j, rxq) { 3432 rss[i++] = rxq->iq.abs_id; 3433 if (i == pi->rss_size) 3434 break; 3435 } 3436 } 3437 3438 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3439 pi->rss_size); 3440 if (rc != 0) { 3441 if_printf(ifp, "rss_config failed: %d\n", rc); 3442 goto done; 3443 } 3444 3445 pi->rss = rss; 3446 pi->flags |= PORT_INIT_DONE; 3447done: 3448 if (rc != 0) 3449 port_full_uninit(pi); 3450 3451 return (rc); 3452} 3453 3454/* 3455 * Idempotent. 3456 */ 3457int 3458port_full_uninit(struct port_info *pi) 3459{ 3460 struct adapter *sc = pi->adapter; 3461 int i; 3462 struct sge_rxq *rxq; 3463 struct sge_txq *txq; 3464#ifdef TCP_OFFLOAD 3465 struct sge_ofld_rxq *ofld_rxq; 3466 struct sge_wrq *ofld_txq; 3467#endif 3468 3469 if (pi->flags & PORT_INIT_DONE) { 3470 3471 /* Need to quiesce queues. */ 3472 3473 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3474 3475 for_each_txq(pi, i, txq) { 3476 quiesce_txq(sc, txq); 3477 } 3478 3479#ifdef TCP_OFFLOAD 3480 for_each_ofld_txq(pi, i, ofld_txq) { 3481 quiesce_wrq(sc, ofld_txq); 3482 } 3483#endif 3484 3485 for_each_rxq(pi, i, rxq) { 3486 quiesce_iq(sc, &rxq->iq); 3487 quiesce_fl(sc, &rxq->fl); 3488 } 3489 3490#ifdef TCP_OFFLOAD 3491 for_each_ofld_rxq(pi, i, ofld_rxq) { 3492 quiesce_iq(sc, &ofld_rxq->iq); 3493 quiesce_fl(sc, &ofld_rxq->fl); 3494 } 3495#endif 3496 free(pi->rss, M_CXGBE); 3497 } 3498 3499 t4_teardown_port_queues(pi); 3500 pi->flags &= ~PORT_INIT_DONE; 3501 3502 return (0); 3503} 3504 3505static void 3506quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3507{ 3508 struct sge_eq *eq = &txq->eq; 3509 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3510 3511 (void) sc; /* unused */ 3512 3513#ifdef INVARIANTS 3514 TXQ_LOCK(txq); 3515 MPASS((eq->flags & EQ_ENABLED) == 0); 3516 TXQ_UNLOCK(txq); 3517#endif 3518 3519 /* Wait for the mp_ring to empty. */ 3520 while (!mp_ring_is_idle(txq->r)) { 3521 mp_ring_check_drainage(txq->r, 0); 3522 pause("rquiesce", 1); 3523 } 3524 3525 /* Then wait for the hardware to finish. */ 3526 while (spg->cidx != htobe16(eq->pidx)) 3527 pause("equiesce", 1); 3528 3529 /* Finally, wait for the driver to reclaim all descriptors. */ 3530 while (eq->cidx != eq->pidx) 3531 pause("dquiesce", 1); 3532} 3533 3534static void 3535quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3536{ 3537 3538 /* XXXTX */ 3539} 3540 3541static void 3542quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3543{ 3544 (void) sc; /* unused */ 3545 3546 /* Synchronize with the interrupt handler */ 3547 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3548 pause("iqfree", 1); 3549} 3550 3551static void 3552quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3553{ 3554 mtx_lock(&sc->sfl_lock); 3555 FL_LOCK(fl); 3556 fl->flags |= FL_DOOMED; 3557 FL_UNLOCK(fl); 3558 mtx_unlock(&sc->sfl_lock); 3559 3560 callout_drain(&sc->sfl_callout); 3561 KASSERT((fl->flags & FL_STARVING) == 0, 3562 ("%s: still starving", __func__)); 3563} 3564 3565static int 3566t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3567 driver_intr_t *handler, void *arg, char *name) 3568{ 3569 int rc; 3570 3571 irq->rid = rid; 3572 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3573 RF_SHAREABLE | RF_ACTIVE); 3574 if (irq->res == NULL) { 3575 device_printf(sc->dev, 3576 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3577 return (ENOMEM); 3578 } 3579 3580 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3581 NULL, handler, arg, &irq->tag); 3582 if (rc != 0) { 3583 device_printf(sc->dev, 3584 "failed to setup interrupt for rid %d, name %s: %d\n", 3585 rid, name, rc); 3586 } else if (name) 3587 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3588 3589 return (rc); 3590} 3591 3592static int 3593t4_free_irq(struct adapter *sc, struct irq *irq) 3594{ 3595 if (irq->tag) 3596 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3597 if (irq->res) 3598 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3599 3600 bzero(irq, sizeof(*irq)); 3601 3602 return (0); 3603} 3604 3605static void 3606reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3607 unsigned int end) 3608{ 3609 uint32_t *p = (uint32_t *)(buf + start); 3610 3611 for ( ; start <= end; start += sizeof(uint32_t)) 3612 *p++ = t4_read_reg(sc, start); 3613} 3614 3615static void 3616t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3617{ 3618 int i, n; 3619 const unsigned int *reg_ranges; 3620 static const unsigned int t4_reg_ranges[] = { 3621 0x1008, 0x1108, 3622 0x1180, 0x11b4, 3623 0x11fc, 0x123c, 3624 0x1300, 0x173c, 3625 0x1800, 0x18fc, 3626 0x3000, 0x30d8, 3627 0x30e0, 0x5924, 3628 0x5960, 0x59d4, 3629 0x5a00, 0x5af8, 3630 0x6000, 0x6098, 3631 0x6100, 0x6150, 3632 0x6200, 0x6208, 3633 0x6240, 0x6248, 3634 0x6280, 0x6338, 3635 0x6370, 0x638c, 3636 0x6400, 0x643c, 3637 0x6500, 0x6524, 3638 0x6a00, 0x6a38, 3639 0x6a60, 0x6a78, 3640 0x6b00, 0x6b84, 3641 0x6bf0, 0x6c84, 3642 0x6cf0, 0x6d84, 3643 0x6df0, 0x6e84, 3644 0x6ef0, 0x6f84, 3645 0x6ff0, 0x7084, 3646 0x70f0, 0x7184, 3647 0x71f0, 0x7284, 3648 0x72f0, 0x7384, 3649 0x73f0, 0x7450, 3650 0x7500, 0x7530, 3651 0x7600, 0x761c, 3652 0x7680, 0x76cc, 3653 0x7700, 0x7798, 3654 0x77c0, 0x77fc, 3655 0x7900, 0x79fc, 3656 0x7b00, 0x7c38, 3657 0x7d00, 0x7efc, 3658 0x8dc0, 0x8e1c, 3659 0x8e30, 0x8e78, 3660 0x8ea0, 0x8f6c, 3661 0x8fc0, 0x9074, 3662 0x90fc, 0x90fc, 3663 0x9400, 0x9458, 3664 0x9600, 0x96bc, 3665 0x9800, 0x9808, 3666 0x9820, 0x983c, 3667 0x9850, 0x9864, 3668 0x9c00, 0x9c6c, 3669 0x9c80, 0x9cec, 3670 0x9d00, 0x9d6c, 3671 0x9d80, 0x9dec, 3672 0x9e00, 0x9e6c, 3673 0x9e80, 0x9eec, 3674 0x9f00, 0x9f6c, 3675 0x9f80, 0x9fec, 3676 0xd004, 0xd03c, 3677 0xdfc0, 0xdfe0, 3678 0xe000, 0xea7c, 3679 0xf000, 0x11110, 3680 0x11118, 0x11190, 3681 0x19040, 0x1906c, 3682 0x19078, 0x19080, 3683 0x1908c, 0x19124, 3684 0x19150, 0x191b0, 3685 0x191d0, 0x191e8, 3686 0x19238, 0x1924c, 3687 0x193f8, 0x19474, 3688 0x19490, 0x194f8, 3689 0x19800, 0x19f30, 3690 0x1a000, 0x1a06c, 3691 0x1a0b0, 0x1a120, 3692 0x1a128, 0x1a138, 3693 0x1a190, 0x1a1c4, 3694 0x1a1fc, 0x1a1fc, 3695 0x1e040, 0x1e04c, 3696 0x1e284, 0x1e28c, 3697 0x1e2c0, 0x1e2c0, 3698 0x1e2e0, 0x1e2e0, 3699 0x1e300, 0x1e384, 3700 0x1e3c0, 0x1e3c8, 3701 0x1e440, 0x1e44c, 3702 0x1e684, 0x1e68c, 3703 0x1e6c0, 0x1e6c0, 3704 0x1e6e0, 0x1e6e0, 3705 0x1e700, 0x1e784, 3706 0x1e7c0, 0x1e7c8, 3707 0x1e840, 0x1e84c, 3708 0x1ea84, 0x1ea8c, 3709 0x1eac0, 0x1eac0, 3710 0x1eae0, 0x1eae0, 3711 0x1eb00, 0x1eb84, 3712 0x1ebc0, 0x1ebc8, 3713 0x1ec40, 0x1ec4c, 3714 0x1ee84, 0x1ee8c, 3715 0x1eec0, 0x1eec0, 3716 0x1eee0, 0x1eee0, 3717 0x1ef00, 0x1ef84, 3718 0x1efc0, 0x1efc8, 3719 0x1f040, 0x1f04c, 3720 0x1f284, 0x1f28c, 3721 0x1f2c0, 0x1f2c0, 3722 0x1f2e0, 0x1f2e0, 3723 0x1f300, 0x1f384, 3724 0x1f3c0, 0x1f3c8, 3725 0x1f440, 0x1f44c, 3726 0x1f684, 0x1f68c, 3727 0x1f6c0, 0x1f6c0, 3728 0x1f6e0, 0x1f6e0, 3729 0x1f700, 0x1f784, 3730 0x1f7c0, 0x1f7c8, 3731 0x1f840, 0x1f84c, 3732 0x1fa84, 0x1fa8c, 3733 0x1fac0, 0x1fac0, 3734 0x1fae0, 0x1fae0, 3735 0x1fb00, 0x1fb84, 3736 0x1fbc0, 0x1fbc8, 3737 0x1fc40, 0x1fc4c, 3738 0x1fe84, 0x1fe8c, 3739 0x1fec0, 0x1fec0, 3740 0x1fee0, 0x1fee0, 3741 0x1ff00, 0x1ff84, 3742 0x1ffc0, 0x1ffc8, 3743 0x20000, 0x2002c, 3744 0x20100, 0x2013c, 3745 0x20190, 0x201c8, 3746 0x20200, 0x20318, 3747 0x20400, 0x20528, 3748 0x20540, 0x20614, 3749 0x21000, 0x21040, 3750 0x2104c, 0x21060, 3751 0x210c0, 0x210ec, 3752 0x21200, 0x21268, 3753 0x21270, 0x21284, 3754 0x212fc, 0x21388, 3755 0x21400, 0x21404, 3756 0x21500, 0x21518, 3757 0x2152c, 0x2153c, 3758 0x21550, 0x21554, 3759 0x21600, 0x21600, 3760 0x21608, 0x21628, 3761 0x21630, 0x2163c, 3762 0x21700, 0x2171c, 3763 0x21780, 0x2178c, 3764 0x21800, 0x21c38, 3765 0x21c80, 0x21d7c, 3766 0x21e00, 0x21e04, 3767 0x22000, 0x2202c, 3768 0x22100, 0x2213c, 3769 0x22190, 0x221c8, 3770 0x22200, 0x22318, 3771 0x22400, 0x22528, 3772 0x22540, 0x22614, 3773 0x23000, 0x23040, 3774 0x2304c, 0x23060, 3775 0x230c0, 0x230ec, 3776 0x23200, 0x23268, 3777 0x23270, 0x23284, 3778 0x232fc, 0x23388, 3779 0x23400, 0x23404, 3780 0x23500, 0x23518, 3781 0x2352c, 0x2353c, 3782 0x23550, 0x23554, 3783 0x23600, 0x23600, 3784 0x23608, 0x23628, 3785 0x23630, 0x2363c, 3786 0x23700, 0x2371c, 3787 0x23780, 0x2378c, 3788 0x23800, 0x23c38, 3789 0x23c80, 0x23d7c, 3790 0x23e00, 0x23e04, 3791 0x24000, 0x2402c, 3792 0x24100, 0x2413c, 3793 0x24190, 0x241c8, 3794 0x24200, 0x24318, 3795 0x24400, 0x24528, 3796 0x24540, 0x24614, 3797 0x25000, 0x25040, 3798 0x2504c, 0x25060, 3799 0x250c0, 0x250ec, 3800 0x25200, 0x25268, 3801 0x25270, 0x25284, 3802 0x252fc, 0x25388, 3803 0x25400, 0x25404, 3804 0x25500, 0x25518, 3805 0x2552c, 0x2553c, 3806 0x25550, 0x25554, 3807 0x25600, 0x25600, 3808 0x25608, 0x25628, 3809 0x25630, 0x2563c, 3810 0x25700, 0x2571c, 3811 0x25780, 0x2578c, 3812 0x25800, 0x25c38, 3813 0x25c80, 0x25d7c, 3814 0x25e00, 0x25e04, 3815 0x26000, 0x2602c, 3816 0x26100, 0x2613c, 3817 0x26190, 0x261c8, 3818 0x26200, 0x26318, 3819 0x26400, 0x26528, 3820 0x26540, 0x26614, 3821 0x27000, 0x27040, 3822 0x2704c, 0x27060, 3823 0x270c0, 0x270ec, 3824 0x27200, 0x27268, 3825 0x27270, 0x27284, 3826 0x272fc, 0x27388, 3827 0x27400, 0x27404, 3828 0x27500, 0x27518, 3829 0x2752c, 0x2753c, 3830 0x27550, 0x27554, 3831 0x27600, 0x27600, 3832 0x27608, 0x27628, 3833 0x27630, 0x2763c, 3834 0x27700, 0x2771c, 3835 0x27780, 0x2778c, 3836 0x27800, 0x27c38, 3837 0x27c80, 0x27d7c, 3838 0x27e00, 0x27e04 3839 }; 3840 static const unsigned int t5_reg_ranges[] = { 3841 0x1008, 0x1148, 3842 0x1180, 0x11b4, 3843 0x11fc, 0x123c, 3844 0x1280, 0x173c, 3845 0x1800, 0x18fc, 3846 0x3000, 0x3028, 3847 0x3060, 0x30d8, 3848 0x30e0, 0x30fc, 3849 0x3140, 0x357c, 3850 0x35a8, 0x35cc, 3851 0x35ec, 0x35ec, 3852 0x3600, 0x5624, 3853 0x56cc, 0x575c, 3854 0x580c, 0x5814, 3855 0x5890, 0x58bc, 3856 0x5940, 0x59dc, 3857 0x59fc, 0x5a18, 3858 0x5a60, 0x5a9c, 3859 0x5b94, 0x5bfc, 3860 0x6000, 0x6040, 3861 0x6058, 0x614c, 3862 0x7700, 0x7798, 3863 0x77c0, 0x78fc, 3864 0x7b00, 0x7c54, 3865 0x7d00, 0x7efc, 3866 0x8dc0, 0x8de0, 3867 0x8df8, 0x8e84, 3868 0x8ea0, 0x8f84, 3869 0x8fc0, 0x90f8, 3870 0x9400, 0x9470, 3871 0x9600, 0x96f4, 3872 0x9800, 0x9808, 3873 0x9820, 0x983c, 3874 0x9850, 0x9864, 3875 0x9c00, 0x9c6c, 3876 0x9c80, 0x9cec, 3877 0x9d00, 0x9d6c, 3878 0x9d80, 0x9dec, 3879 0x9e00, 0x9e6c, 3880 0x9e80, 0x9eec, 3881 0x9f00, 0x9f6c, 3882 0x9f80, 0xa020, 3883 0xd004, 0xd03c, 3884 0xdfc0, 0xdfe0, 3885 0xe000, 0x11088, 3886 0x1109c, 0x11110, 3887 0x11118, 0x1117c, 3888 0x11190, 0x11204, 3889 0x19040, 0x1906c, 3890 0x19078, 0x19080, 3891 0x1908c, 0x19124, 3892 0x19150, 0x191b0, 3893 0x191d0, 0x191e8, 3894 0x19238, 0x19290, 3895 0x193f8, 0x19474, 3896 0x19490, 0x194cc, 3897 0x194f0, 0x194f8, 3898 0x19c00, 0x19c60, 3899 0x19c94, 0x19e10, 3900 0x19e50, 0x19f34, 3901 0x19f40, 0x19f50, 3902 0x19f90, 0x19fe4, 3903 0x1a000, 0x1a06c, 3904 0x1a0b0, 0x1a120, 3905 0x1a128, 0x1a138, 3906 0x1a190, 0x1a1c4, 3907 0x1a1fc, 0x1a1fc, 3908 0x1e008, 0x1e00c, 3909 0x1e040, 0x1e04c, 3910 0x1e284, 0x1e290, 3911 0x1e2c0, 0x1e2c0, 3912 0x1e2e0, 0x1e2e0, 3913 0x1e300, 0x1e384, 3914 0x1e3c0, 0x1e3c8, 3915 0x1e408, 0x1e40c, 3916 0x1e440, 0x1e44c, 3917 0x1e684, 0x1e690, 3918 0x1e6c0, 0x1e6c0, 3919 0x1e6e0, 0x1e6e0, 3920 0x1e700, 0x1e784, 3921 0x1e7c0, 0x1e7c8, 3922 0x1e808, 0x1e80c, 3923 0x1e840, 0x1e84c, 3924 0x1ea84, 0x1ea90, 3925 0x1eac0, 0x1eac0, 3926 0x1eae0, 0x1eae0, 3927 0x1eb00, 0x1eb84, 3928 0x1ebc0, 0x1ebc8, 3929 0x1ec08, 0x1ec0c, 3930 0x1ec40, 0x1ec4c, 3931 0x1ee84, 0x1ee90, 3932 0x1eec0, 0x1eec0, 3933 0x1eee0, 0x1eee0, 3934 0x1ef00, 0x1ef84, 3935 0x1efc0, 0x1efc8, 3936 0x1f008, 0x1f00c, 3937 0x1f040, 0x1f04c, 3938 0x1f284, 0x1f290, 3939 0x1f2c0, 0x1f2c0, 3940 0x1f2e0, 0x1f2e0, 3941 0x1f300, 0x1f384, 3942 0x1f3c0, 0x1f3c8, 3943 0x1f408, 0x1f40c, 3944 0x1f440, 0x1f44c, 3945 0x1f684, 0x1f690, 3946 0x1f6c0, 0x1f6c0, 3947 0x1f6e0, 0x1f6e0, 3948 0x1f700, 0x1f784, 3949 0x1f7c0, 0x1f7c8, 3950 0x1f808, 0x1f80c, 3951 0x1f840, 0x1f84c, 3952 0x1fa84, 0x1fa90, 3953 0x1fac0, 0x1fac0, 3954 0x1fae0, 0x1fae0, 3955 0x1fb00, 0x1fb84, 3956 0x1fbc0, 0x1fbc8, 3957 0x1fc08, 0x1fc0c, 3958 0x1fc40, 0x1fc4c, 3959 0x1fe84, 0x1fe90, 3960 0x1fec0, 0x1fec0, 3961 0x1fee0, 0x1fee0, 3962 0x1ff00, 0x1ff84, 3963 0x1ffc0, 0x1ffc8, 3964 0x30000, 0x30030, 3965 0x30100, 0x30144, 3966 0x30190, 0x301d0, 3967 0x30200, 0x30318, 3968 0x30400, 0x3052c, 3969 0x30540, 0x3061c, 3970 0x30800, 0x30834, 3971 0x308c0, 0x30908, 3972 0x30910, 0x309ac, 3973 0x30a00, 0x30a2c, 3974 0x30a44, 0x30a50, 3975 0x30a74, 0x30c24, 3976 0x30d00, 0x30d00, 3977 0x30d08, 0x30d14, 3978 0x30d1c, 0x30d20, 3979 0x30d3c, 0x30d50, 3980 0x31200, 0x3120c, 3981 0x31220, 0x31220, 3982 0x31240, 0x31240, 3983 0x31600, 0x3160c, 3984 0x31a00, 0x31a1c, 3985 0x31e00, 0x31e20, 3986 0x31e38, 0x31e3c, 3987 0x31e80, 0x31e80, 3988 0x31e88, 0x31ea8, 3989 0x31eb0, 0x31eb4, 3990 0x31ec8, 0x31ed4, 3991 0x31fb8, 0x32004, 3992 0x32200, 0x32200, 3993 0x32208, 0x32240, 3994 0x32248, 0x32280, 3995 0x32288, 0x322c0, 3996 0x322c8, 0x322fc, 3997 0x32600, 0x32630, 3998 0x32a00, 0x32abc, 3999 0x32b00, 0x32b70, 4000 0x33000, 0x33048, 4001 0x33060, 0x3309c, 4002 0x330f0, 0x33148, 4003 0x33160, 0x3319c, 4004 0x331f0, 0x332e4, 4005 0x332f8, 0x333e4, 4006 0x333f8, 0x33448, 4007 0x33460, 0x3349c, 4008 0x334f0, 0x33548, 4009 0x33560, 0x3359c, 4010 0x335f0, 0x336e4, 4011 0x336f8, 0x337e4, 4012 0x337f8, 0x337fc, 4013 0x33814, 0x33814, 4014 0x3382c, 0x3382c, 4015 0x33880, 0x3388c, 4016 0x338e8, 0x338ec, 4017 0x33900, 0x33948, 4018 0x33960, 0x3399c, 4019 0x339f0, 0x33ae4, 4020 0x33af8, 0x33b10, 4021 0x33b28, 0x33b28, 4022 0x33b3c, 0x33b50, 4023 0x33bf0, 0x33c10, 4024 0x33c28, 0x33c28, 4025 0x33c3c, 0x33c50, 4026 0x33cf0, 0x33cfc, 4027 0x34000, 0x34030, 4028 0x34100, 0x34144, 4029 0x34190, 0x341d0, 4030 0x34200, 0x34318, 4031 0x34400, 0x3452c, 4032 0x34540, 0x3461c, 4033 0x34800, 0x34834, 4034 0x348c0, 0x34908, 4035 0x34910, 0x349ac, 4036 0x34a00, 0x34a2c, 4037 0x34a44, 0x34a50, 4038 0x34a74, 0x34c24, 4039 0x34d00, 0x34d00, 4040 0x34d08, 0x34d14, 4041 0x34d1c, 0x34d20, 4042 0x34d3c, 0x34d50, 4043 0x35200, 0x3520c, 4044 0x35220, 0x35220, 4045 0x35240, 0x35240, 4046 0x35600, 0x3560c, 4047 0x35a00, 0x35a1c, 4048 0x35e00, 0x35e20, 4049 0x35e38, 0x35e3c, 4050 0x35e80, 0x35e80, 4051 0x35e88, 0x35ea8, 4052 0x35eb0, 0x35eb4, 4053 0x35ec8, 0x35ed4, 4054 0x35fb8, 0x36004, 4055 0x36200, 0x36200, 4056 0x36208, 0x36240, 4057 0x36248, 0x36280, 4058 0x36288, 0x362c0, 4059 0x362c8, 0x362fc, 4060 0x36600, 0x36630, 4061 0x36a00, 0x36abc, 4062 0x36b00, 0x36b70, 4063 0x37000, 0x37048, 4064 0x37060, 0x3709c, 4065 0x370f0, 0x37148, 4066 0x37160, 0x3719c, 4067 0x371f0, 0x372e4, 4068 0x372f8, 0x373e4, 4069 0x373f8, 0x37448, 4070 0x37460, 0x3749c, 4071 0x374f0, 0x37548, 4072 0x37560, 0x3759c, 4073 0x375f0, 0x376e4, 4074 0x376f8, 0x377e4, 4075 0x377f8, 0x377fc, 4076 0x37814, 0x37814, 4077 0x3782c, 0x3782c, 4078 0x37880, 0x3788c, 4079 0x378e8, 0x378ec, 4080 0x37900, 0x37948, 4081 0x37960, 0x3799c, 4082 0x379f0, 0x37ae4, 4083 0x37af8, 0x37b10, 4084 0x37b28, 0x37b28, 4085 0x37b3c, 0x37b50, 4086 0x37bf0, 0x37c10, 4087 0x37c28, 0x37c28, 4088 0x37c3c, 0x37c50, 4089 0x37cf0, 0x37cfc, 4090 0x38000, 0x38030, 4091 0x38100, 0x38144, 4092 0x38190, 0x381d0, 4093 0x38200, 0x38318, 4094 0x38400, 0x3852c, 4095 0x38540, 0x3861c, 4096 0x38800, 0x38834, 4097 0x388c0, 0x38908, 4098 0x38910, 0x389ac, 4099 0x38a00, 0x38a2c, 4100 0x38a44, 0x38a50, 4101 0x38a74, 0x38c24, 4102 0x38d00, 0x38d00, 4103 0x38d08, 0x38d14, 4104 0x38d1c, 0x38d20, 4105 0x38d3c, 0x38d50, 4106 0x39200, 0x3920c, 4107 0x39220, 0x39220, 4108 0x39240, 0x39240, 4109 0x39600, 0x3960c, 4110 0x39a00, 0x39a1c, 4111 0x39e00, 0x39e20, 4112 0x39e38, 0x39e3c, 4113 0x39e80, 0x39e80, 4114 0x39e88, 0x39ea8, 4115 0x39eb0, 0x39eb4, 4116 0x39ec8, 0x39ed4, 4117 0x39fb8, 0x3a004, 4118 0x3a200, 0x3a200, 4119 0x3a208, 0x3a240, 4120 0x3a248, 0x3a280, 4121 0x3a288, 0x3a2c0, 4122 0x3a2c8, 0x3a2fc, 4123 0x3a600, 0x3a630, 4124 0x3aa00, 0x3aabc, 4125 0x3ab00, 0x3ab70, 4126 0x3b000, 0x3b048, 4127 0x3b060, 0x3b09c, 4128 0x3b0f0, 0x3b148, 4129 0x3b160, 0x3b19c, 4130 0x3b1f0, 0x3b2e4, 4131 0x3b2f8, 0x3b3e4, 4132 0x3b3f8, 0x3b448, 4133 0x3b460, 0x3b49c, 4134 0x3b4f0, 0x3b548, 4135 0x3b560, 0x3b59c, 4136 0x3b5f0, 0x3b6e4, 4137 0x3b6f8, 0x3b7e4, 4138 0x3b7f8, 0x3b7fc, 4139 0x3b814, 0x3b814, 4140 0x3b82c, 0x3b82c, 4141 0x3b880, 0x3b88c, 4142 0x3b8e8, 0x3b8ec, 4143 0x3b900, 0x3b948, 4144 0x3b960, 0x3b99c, 4145 0x3b9f0, 0x3bae4, 4146 0x3baf8, 0x3bb10, 4147 0x3bb28, 0x3bb28, 4148 0x3bb3c, 0x3bb50, 4149 0x3bbf0, 0x3bc10, 4150 0x3bc28, 0x3bc28, 4151 0x3bc3c, 0x3bc50, 4152 0x3bcf0, 0x3bcfc, 4153 0x3c000, 0x3c030, 4154 0x3c100, 0x3c144, 4155 0x3c190, 0x3c1d0, 4156 0x3c200, 0x3c318, 4157 0x3c400, 0x3c52c, 4158 0x3c540, 0x3c61c, 4159 0x3c800, 0x3c834, 4160 0x3c8c0, 0x3c908, 4161 0x3c910, 0x3c9ac, 4162 0x3ca00, 0x3ca2c, 4163 0x3ca44, 0x3ca50, 4164 0x3ca74, 0x3cc24, 4165 0x3cd00, 0x3cd00, 4166 0x3cd08, 0x3cd14, 4167 0x3cd1c, 0x3cd20, 4168 0x3cd3c, 0x3cd50, 4169 0x3d200, 0x3d20c, 4170 0x3d220, 0x3d220, 4171 0x3d240, 0x3d240, 4172 0x3d600, 0x3d60c, 4173 0x3da00, 0x3da1c, 4174 0x3de00, 0x3de20, 4175 0x3de38, 0x3de3c, 4176 0x3de80, 0x3de80, 4177 0x3de88, 0x3dea8, 4178 0x3deb0, 0x3deb4, 4179 0x3dec8, 0x3ded4, 4180 0x3dfb8, 0x3e004, 4181 0x3e200, 0x3e200, 4182 0x3e208, 0x3e240, 4183 0x3e248, 0x3e280, 4184 0x3e288, 0x3e2c0, 4185 0x3e2c8, 0x3e2fc, 4186 0x3e600, 0x3e630, 4187 0x3ea00, 0x3eabc, 4188 0x3eb00, 0x3eb70, 4189 0x3f000, 0x3f048, 4190 0x3f060, 0x3f09c, 4191 0x3f0f0, 0x3f148, 4192 0x3f160, 0x3f19c, 4193 0x3f1f0, 0x3f2e4, 4194 0x3f2f8, 0x3f3e4, 4195 0x3f3f8, 0x3f448, 4196 0x3f460, 0x3f49c, 4197 0x3f4f0, 0x3f548, 4198 0x3f560, 0x3f59c, 4199 0x3f5f0, 0x3f6e4, 4200 0x3f6f8, 0x3f7e4, 4201 0x3f7f8, 0x3f7fc, 4202 0x3f814, 0x3f814, 4203 0x3f82c, 0x3f82c, 4204 0x3f880, 0x3f88c, 4205 0x3f8e8, 0x3f8ec, 4206 0x3f900, 0x3f948, 4207 0x3f960, 0x3f99c, 4208 0x3f9f0, 0x3fae4, 4209 0x3faf8, 0x3fb10, 4210 0x3fb28, 0x3fb28, 4211 0x3fb3c, 0x3fb50, 4212 0x3fbf0, 0x3fc10, 4213 0x3fc28, 0x3fc28, 4214 0x3fc3c, 0x3fc50, 4215 0x3fcf0, 0x3fcfc, 4216 0x40000, 0x4000c, 4217 0x40040, 0x40068, 4218 0x4007c, 0x40144, 4219 0x40180, 0x4018c, 4220 0x40200, 0x40298, 4221 0x402ac, 0x4033c, 4222 0x403f8, 0x403fc, 4223 0x41304, 0x413c4, 4224 0x41400, 0x4141c, 4225 0x41480, 0x414d0, 4226 0x44000, 0x44078, 4227 0x440c0, 0x44278, 4228 0x442c0, 0x44478, 4229 0x444c0, 0x44678, 4230 0x446c0, 0x44878, 4231 0x448c0, 0x449fc, 4232 0x45000, 0x45068, 4233 0x45080, 0x45084, 4234 0x450a0, 0x450b0, 4235 0x45200, 0x45268, 4236 0x45280, 0x45284, 4237 0x452a0, 0x452b0, 4238 0x460c0, 0x460e4, 4239 0x47000, 0x4708c, 4240 0x47200, 0x47250, 4241 0x47400, 0x47420, 4242 0x47600, 0x47618, 4243 0x47800, 0x47814, 4244 0x48000, 0x4800c, 4245 0x48040, 0x48068, 4246 0x4807c, 0x48144, 4247 0x48180, 0x4818c, 4248 0x48200, 0x48298, 4249 0x482ac, 0x4833c, 4250 0x483f8, 0x483fc, 4251 0x49304, 0x493c4, 4252 0x49400, 0x4941c, 4253 0x49480, 0x494d0, 4254 0x4c000, 0x4c078, 4255 0x4c0c0, 0x4c278, 4256 0x4c2c0, 0x4c478, 4257 0x4c4c0, 0x4c678, 4258 0x4c6c0, 0x4c878, 4259 0x4c8c0, 0x4c9fc, 4260 0x4d000, 0x4d068, 4261 0x4d080, 0x4d084, 4262 0x4d0a0, 0x4d0b0, 4263 0x4d200, 0x4d268, 4264 0x4d280, 0x4d284, 4265 0x4d2a0, 0x4d2b0, 4266 0x4e0c0, 0x4e0e4, 4267 0x4f000, 0x4f08c, 4268 0x4f200, 0x4f250, 4269 0x4f400, 0x4f420, 4270 0x4f600, 0x4f618, 4271 0x4f800, 0x4f814, 4272 0x50000, 0x500cc, 4273 0x50400, 0x50400, 4274 0x50800, 0x508cc, 4275 0x50c00, 0x50c00, 4276 0x51000, 0x5101c, 4277 0x51300, 0x51308, 4278 }; 4279 4280 if (is_t4(sc)) { 4281 reg_ranges = &t4_reg_ranges[0]; 4282 n = nitems(t4_reg_ranges); 4283 } else { 4284 reg_ranges = &t5_reg_ranges[0]; 4285 n = nitems(t5_reg_ranges); 4286 } 4287 4288 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4289 for (i = 0; i < n; i += 2) 4290 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4291} 4292 4293static void 4294cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4295{ 4296 struct ifnet *ifp = pi->ifp; 4297 struct sge_txq *txq; 4298 int i, drops; 4299 struct port_stats *s = &pi->stats; 4300 struct timeval tv; 4301 const struct timeval interval = {0, 250000}; /* 250ms */ 4302 4303 getmicrotime(&tv); 4304 timevalsub(&tv, &interval); 4305 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4306 return; 4307 4308 t4_get_port_stats(sc, pi->tx_chan, s); 4309 4310 ifp->if_opackets = s->tx_frames - s->tx_pause; 4311 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4312 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4313 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4314 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4315 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4316 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4317 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4318 s->rx_trunc3; 4319 for (i = 0; i < NCHAN; i++) { 4320 if (pi->rx_chan_map & (1 << i)) { 4321 uint32_t v; 4322 4323 mtx_lock(&sc->regwin_lock); 4324 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4325 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4326 mtx_unlock(&sc->regwin_lock); 4327 ifp->if_iqdrops += v; 4328 } 4329 } 4330 4331 drops = s->tx_drop; 4332 for_each_txq(pi, i, txq) 4333 drops += counter_u64_fetch(txq->r->drops); 4334 ifp->if_snd.ifq_drops = drops; 4335 4336 ifp->if_oerrors = s->tx_error_frames; 4337 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4338 s->rx_fcs_err + s->rx_len_err; 4339 4340 getmicrotime(&pi->last_refreshed); 4341} 4342 4343static void 4344cxgbe_tick(void *arg) 4345{ 4346 struct port_info *pi = arg; 4347 struct adapter *sc = pi->adapter; 4348 struct ifnet *ifp = pi->ifp; 4349 4350 PORT_LOCK(pi); 4351 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4352 PORT_UNLOCK(pi); 4353 return; /* without scheduling another callout */ 4354 } 4355 4356 cxgbe_refresh_stats(sc, pi); 4357 4358 callout_schedule(&pi->tick, hz); 4359 PORT_UNLOCK(pi); 4360} 4361 4362static void 4363cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4364{ 4365 struct ifnet *vlan; 4366 4367 if (arg != ifp || ifp->if_type != IFT_ETHER) 4368 return; 4369 4370 vlan = VLAN_DEVAT(ifp, vid); 4371 VLAN_SETCOOKIE(vlan, ifp); 4372} 4373 4374static int 4375cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4376{ 4377 4378#ifdef INVARIANTS 4379 panic("%s: opcode 0x%02x on iq %p with payload %p", 4380 __func__, rss->opcode, iq, m); 4381#else 4382 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4383 __func__, rss->opcode, iq, m); 4384 m_freem(m); 4385#endif 4386 return (EDOOFUS); 4387} 4388 4389int 4390t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4391{ 4392 uintptr_t *loc, new; 4393 4394 if (opcode >= nitems(sc->cpl_handler)) 4395 return (EINVAL); 4396 4397 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4398 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4399 atomic_store_rel_ptr(loc, new); 4400 4401 return (0); 4402} 4403 4404static int 4405an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4406{ 4407 4408#ifdef INVARIANTS 4409 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4410#else 4411 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4412 __func__, iq, ctrl); 4413#endif 4414 return (EDOOFUS); 4415} 4416 4417int 4418t4_register_an_handler(struct adapter *sc, an_handler_t h) 4419{ 4420 uintptr_t *loc, new; 4421 4422 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4423 loc = (uintptr_t *) &sc->an_handler; 4424 atomic_store_rel_ptr(loc, new); 4425 4426 return (0); 4427} 4428 4429static int 4430fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4431{ 4432 const struct cpl_fw6_msg *cpl = 4433 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4434 4435#ifdef INVARIANTS 4436 panic("%s: fw_msg type %d", __func__, cpl->type); 4437#else 4438 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4439#endif 4440 return (EDOOFUS); 4441} 4442 4443int 4444t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4445{ 4446 uintptr_t *loc, new; 4447 4448 if (type >= nitems(sc->fw_msg_handler)) 4449 return (EINVAL); 4450 4451 /* 4452 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4453 * handler dispatch table. Reject any attempt to install a handler for 4454 * this subtype. 4455 */ 4456 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4457 return (EINVAL); 4458 4459 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4460 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4461 atomic_store_rel_ptr(loc, new); 4462 4463 return (0); 4464} 4465 4466static int 4467t4_sysctls(struct adapter *sc) 4468{ 4469 struct sysctl_ctx_list *ctx; 4470 struct sysctl_oid *oid; 4471 struct sysctl_oid_list *children, *c0; 4472 static char *caps[] = { 4473 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4474 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4475 "\6HASHFILTER\7ETHOFLD", 4476 "\20\1TOE", /* caps[2] toecaps */ 4477 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4478 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4479 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4480 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4481 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4482 "\4PO_INITIAOR\5PO_TARGET" 4483 }; 4484 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4485 4486 ctx = device_get_sysctl_ctx(sc->dev); 4487 4488 /* 4489 * dev.t4nex.X. 4490 */ 4491 oid = device_get_sysctl_tree(sc->dev); 4492 c0 = children = SYSCTL_CHILDREN(oid); 4493 4494 sc->sc_do_rxcopy = 1; 4495 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4496 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4497 4498 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4499 sc->params.nports, "# of ports"); 4500 4501 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4502 NULL, chip_rev(sc), "chip hardware revision"); 4503 4504 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4505 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4506 4507 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4508 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4509 4510 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4511 sc->cfcsum, "config file checksum"); 4512 4513 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4514 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4515 sysctl_bitfield, "A", "available doorbells"); 4516 4517 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4518 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4519 sysctl_bitfield, "A", "available link capabilities"); 4520 4521 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4522 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4523 sysctl_bitfield, "A", "available NIC capabilities"); 4524 4525 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4526 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4527 sysctl_bitfield, "A", "available TCP offload capabilities"); 4528 4529 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4530 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4531 sysctl_bitfield, "A", "available RDMA capabilities"); 4532 4533 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4534 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4535 sysctl_bitfield, "A", "available iSCSI capabilities"); 4536 4537 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4538 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4539 sysctl_bitfield, "A", "available FCoE capabilities"); 4540 4541 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4542 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4543 4544 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4545 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4546 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4547 "interrupt holdoff timer values (us)"); 4548 4549 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4550 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4551 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4552 "interrupt holdoff packet counter values"); 4553 4554 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4555 NULL, sc->tids.nftids, "number of filters"); 4556 4557 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4558 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4559 "chip temperature (in Celsius)"); 4560 4561 t4_sge_sysctls(sc, ctx, children); 4562 4563 sc->lro_timeout = 100; 4564 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4565 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4566 4567 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4568 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4569 4570#ifdef SBUF_DRAIN 4571 /* 4572 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4573 */ 4574 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4575 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4576 "logs and miscellaneous information"); 4577 children = SYSCTL_CHILDREN(oid); 4578 4579 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4580 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4581 sysctl_cctrl, "A", "congestion control"); 4582 4583 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4584 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4585 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4586 4587 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4588 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4589 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4590 4591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4592 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4593 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4594 4595 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4596 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4597 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4598 4599 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4600 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4601 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4602 4603 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4604 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4605 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4606 4607 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4608 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4609 sysctl_cim_la, "A", "CIM logic analyzer"); 4610 4611 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4612 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4613 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4614 4615 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4616 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4617 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4618 4619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4620 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4621 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4622 4623 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4624 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4625 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4626 4627 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4628 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4629 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4630 4631 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4632 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4633 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4634 4635 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4636 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4637 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4638 4639 if (is_t5(sc)) { 4640 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4641 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4642 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4643 4644 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4645 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4646 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4647 } 4648 4649 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4650 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4651 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4652 4653 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4654 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4655 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4656 4657 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4658 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4659 sysctl_cpl_stats, "A", "CPL statistics"); 4660 4661 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4662 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4663 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4664 4665 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4666 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4667 sysctl_devlog, "A", "firmware's device log"); 4668 4669 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4670 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4671 sysctl_fcoe_stats, "A", "FCoE statistics"); 4672 4673 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4674 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4675 sysctl_hw_sched, "A", "hardware scheduler "); 4676 4677 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4678 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4679 sysctl_l2t, "A", "hardware L2 table"); 4680 4681 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4682 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4683 sysctl_lb_stats, "A", "loopback statistics"); 4684 4685 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4686 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4687 sysctl_meminfo, "A", "memory regions"); 4688 4689 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4690 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4691 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4692 4693 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4694 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4695 sysctl_path_mtus, "A", "path MTUs"); 4696 4697 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4698 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4699 sysctl_pm_stats, "A", "PM statistics"); 4700 4701 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4702 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4703 sysctl_rdma_stats, "A", "RDMA statistics"); 4704 4705 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4706 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4707 sysctl_tcp_stats, "A", "TCP statistics"); 4708 4709 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4710 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4711 sysctl_tids, "A", "TID information"); 4712 4713 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4714 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4715 sysctl_tp_err_stats, "A", "TP error statistics"); 4716 4717 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4718 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4719 sysctl_tp_la, "A", "TP logic analyzer"); 4720 4721 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4722 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4723 sysctl_tx_rate, "A", "Tx rate"); 4724 4725 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4726 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4727 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4728 4729 if (is_t5(sc)) { 4730 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4731 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4732 sysctl_wcwr_stats, "A", "write combined work requests"); 4733 } 4734#endif 4735 4736#ifdef TCP_OFFLOAD 4737 if (is_offload(sc)) { 4738 /* 4739 * dev.t4nex.X.toe. 4740 */ 4741 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4742 NULL, "TOE parameters"); 4743 children = SYSCTL_CHILDREN(oid); 4744 4745 sc->tt.sndbuf = 256 * 1024; 4746 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4747 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4748 4749 sc->tt.ddp = 0; 4750 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4751 &sc->tt.ddp, 0, "DDP allowed"); 4752 4753 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4754 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4755 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4756 4757 sc->tt.ddp_thres = 4758 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4759 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4760 &sc->tt.ddp_thres, 0, "DDP threshold"); 4761 4762 sc->tt.rx_coalesce = 1; 4763 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4764 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4765 4766 sc->tt.tx_align = 1; 4767 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4768 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4769 } 4770#endif 4771 4772 4773 return (0); 4774} 4775 4776static int 4777cxgbe_sysctls(struct port_info *pi) 4778{ 4779 struct sysctl_ctx_list *ctx; 4780 struct sysctl_oid *oid; 4781 struct sysctl_oid_list *children; 4782 struct adapter *sc = pi->adapter; 4783 4784 ctx = device_get_sysctl_ctx(pi->dev); 4785 4786 /* 4787 * dev.cxgbe.X. 4788 */ 4789 oid = device_get_sysctl_tree(pi->dev); 4790 children = SYSCTL_CHILDREN(oid); 4791 4792 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4793 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4794 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4795 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4796 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4797 "PHY temperature (in Celsius)"); 4798 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4799 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4800 "PHY firmware version"); 4801 } 4802 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4803 &pi->nrxq, 0, "# of rx queues"); 4804 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4805 &pi->ntxq, 0, "# of tx queues"); 4806 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4807 &pi->first_rxq, 0, "index of first rx queue"); 4808 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4809 &pi->first_txq, 0, "index of first tx queue"); 4810 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4811 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4812 "Reserve queue 0 for non-flowid packets"); 4813 4814#ifdef TCP_OFFLOAD 4815 if (is_offload(sc)) { 4816 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4817 &pi->nofldrxq, 0, 4818 "# of rx queues for offloaded TCP connections"); 4819 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4820 &pi->nofldtxq, 0, 4821 "# of tx queues for offloaded TCP connections"); 4822 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4823 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4824 "index of first TOE rx queue"); 4825 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4826 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4827 "index of first TOE tx queue"); 4828 } 4829#endif 4830#ifdef DEV_NETMAP 4831 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4832 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4833 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4834 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4835 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4836 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4837 "index of first netmap rx queue"); 4838 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4839 CTLFLAG_RD, &pi->first_nm_txq, 0, 4840 "index of first netmap tx queue"); 4841#endif 4842 4843 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4844 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4845 "holdoff timer index"); 4846 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4847 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4848 "holdoff packet counter index"); 4849 4850 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4851 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4852 "rx queue size"); 4853 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4854 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4855 "tx queue size"); 4856 4857 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4858 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4859 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4860 4861 /* 4862 * dev.cxgbe.X.stats. 4863 */ 4864 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4865 NULL, "port statistics"); 4866 children = SYSCTL_CHILDREN(oid); 4867 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4868 &pi->tx_parse_error, 0, 4869 "# of tx packets with invalid length or # of segments"); 4870 4871#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4872 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4873 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4874 sysctl_handle_t4_reg64, "QU", desc) 4875 4876 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4877 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4878 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4879 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4880 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4881 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4882 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4883 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4884 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4885 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4886 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4887 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4888 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4889 "# of tx frames in this range", 4890 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4891 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4892 "# of tx frames in this range", 4893 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4894 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4895 "# of tx frames in this range", 4896 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4897 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4898 "# of tx frames in this range", 4899 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4900 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4901 "# of tx frames in this range", 4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4903 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4904 "# of tx frames in this range", 4905 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4906 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4907 "# of tx frames in this range", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4917 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4918 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4919 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4920 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4921 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4922 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4923 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4925 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4926 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4927 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4928 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4929 4930 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4931 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4932 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4937 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4938 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4943 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4944 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4945 "# of frames received with bad FCS", 4946 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4947 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4948 "# of frames received with length error", 4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4950 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4954 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4955 "# of rx frames in this range", 4956 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4957 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4958 "# of rx frames in this range", 4959 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4960 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4961 "# of rx frames in this range", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4964 "# of rx frames in this range", 4965 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4966 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4967 "# of rx frames in this range", 4968 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4969 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4970 "# of rx frames in this range", 4971 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4972 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4973 "# of rx frames in this range", 4974 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4975 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4976 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4977 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4978 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4979 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4980 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4981 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4982 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4983 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4984 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4985 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4986 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4987 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4988 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4989 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4990 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4991 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4992 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4993 4994#undef SYSCTL_ADD_T4_REG64 4995 4996#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4997 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4998 &pi->stats.name, desc) 4999 5000 /* We get these from port_stats and they may be stale by upto 1s */ 5001 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5002 "# drops due to buffer-group 0 overflows"); 5003 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5004 "# drops due to buffer-group 1 overflows"); 5005 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5006 "# drops due to buffer-group 2 overflows"); 5007 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5008 "# drops due to buffer-group 3 overflows"); 5009 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5010 "# of buffer-group 0 truncated packets"); 5011 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5012 "# of buffer-group 1 truncated packets"); 5013 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5014 "# of buffer-group 2 truncated packets"); 5015 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5016 "# of buffer-group 3 truncated packets"); 5017 5018#undef SYSCTL_ADD_T4_PORTSTAT 5019 5020 return (0); 5021} 5022 5023static int 5024sysctl_int_array(SYSCTL_HANDLER_ARGS) 5025{ 5026 int rc, *i, space = 0; 5027 struct sbuf sb; 5028 5029 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5030 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5031 if (space) 5032 sbuf_printf(&sb, " "); 5033 sbuf_printf(&sb, "%d", *i); 5034 space = 1; 5035 } 5036 sbuf_finish(&sb); 5037 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5038 sbuf_delete(&sb); 5039 return (rc); 5040} 5041 5042static int 5043sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5044{ 5045 int rc; 5046 struct sbuf *sb; 5047 5048 rc = sysctl_wire_old_buffer(req, 0); 5049 if (rc != 0) 5050 return(rc); 5051 5052 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5053 if (sb == NULL) 5054 return (ENOMEM); 5055 5056 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5057 rc = sbuf_finish(sb); 5058 sbuf_delete(sb); 5059 5060 return (rc); 5061} 5062 5063static int 5064sysctl_btphy(SYSCTL_HANDLER_ARGS) 5065{ 5066 struct port_info *pi = arg1; 5067 int op = arg2; 5068 struct adapter *sc = pi->adapter; 5069 u_int v; 5070 int rc; 5071 5072 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5073 if (rc) 5074 return (rc); 5075 /* XXX: magic numbers */ 5076 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5077 &v); 5078 end_synchronized_op(sc, 0); 5079 if (rc) 5080 return (rc); 5081 if (op == 0) 5082 v /= 256; 5083 5084 rc = sysctl_handle_int(oidp, &v, 0, req); 5085 return (rc); 5086} 5087 5088static int 5089sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5090{ 5091 struct port_info *pi = arg1; 5092 int rc, val; 5093 5094 val = pi->rsrv_noflowq; 5095 rc = sysctl_handle_int(oidp, &val, 0, req); 5096 if (rc != 0 || req->newptr == NULL) 5097 return (rc); 5098 5099 if ((val >= 1) && (pi->ntxq > 1)) 5100 pi->rsrv_noflowq = 1; 5101 else 5102 pi->rsrv_noflowq = 0; 5103 5104 return (rc); 5105} 5106 5107static int 5108sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5109{ 5110 struct port_info *pi = arg1; 5111 struct adapter *sc = pi->adapter; 5112 int idx, rc, i; 5113 struct sge_rxq *rxq; 5114#ifdef TCP_OFFLOAD 5115 struct sge_ofld_rxq *ofld_rxq; 5116#endif 5117 uint8_t v; 5118 5119 idx = pi->tmr_idx; 5120 5121 rc = sysctl_handle_int(oidp, &idx, 0, req); 5122 if (rc != 0 || req->newptr == NULL) 5123 return (rc); 5124 5125 if (idx < 0 || idx >= SGE_NTIMERS) 5126 return (EINVAL); 5127 5128 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5129 "t4tmr"); 5130 if (rc) 5131 return (rc); 5132 5133 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5134 for_each_rxq(pi, i, rxq) { 5135#ifdef atomic_store_rel_8 5136 atomic_store_rel_8(&rxq->iq.intr_params, v); 5137#else 5138 rxq->iq.intr_params = v; 5139#endif 5140 } 5141#ifdef TCP_OFFLOAD 5142 for_each_ofld_rxq(pi, i, ofld_rxq) { 5143#ifdef atomic_store_rel_8 5144 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5145#else 5146 ofld_rxq->iq.intr_params = v; 5147#endif 5148 } 5149#endif 5150 pi->tmr_idx = idx; 5151 5152 end_synchronized_op(sc, LOCK_HELD); 5153 return (0); 5154} 5155 5156static int 5157sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5158{ 5159 struct port_info *pi = arg1; 5160 struct adapter *sc = pi->adapter; 5161 int idx, rc; 5162 5163 idx = pi->pktc_idx; 5164 5165 rc = sysctl_handle_int(oidp, &idx, 0, req); 5166 if (rc != 0 || req->newptr == NULL) 5167 return (rc); 5168 5169 if (idx < -1 || idx >= SGE_NCOUNTERS) 5170 return (EINVAL); 5171 5172 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5173 "t4pktc"); 5174 if (rc) 5175 return (rc); 5176 5177 if (pi->flags & PORT_INIT_DONE) 5178 rc = EBUSY; /* cannot be changed once the queues are created */ 5179 else 5180 pi->pktc_idx = idx; 5181 5182 end_synchronized_op(sc, LOCK_HELD); 5183 return (rc); 5184} 5185 5186static int 5187sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5188{ 5189 struct port_info *pi = arg1; 5190 struct adapter *sc = pi->adapter; 5191 int qsize, rc; 5192 5193 qsize = pi->qsize_rxq; 5194 5195 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5196 if (rc != 0 || req->newptr == NULL) 5197 return (rc); 5198 5199 if (qsize < 128 || (qsize & 7)) 5200 return (EINVAL); 5201 5202 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5203 "t4rxqs"); 5204 if (rc) 5205 return (rc); 5206 5207 if (pi->flags & PORT_INIT_DONE) 5208 rc = EBUSY; /* cannot be changed once the queues are created */ 5209 else 5210 pi->qsize_rxq = qsize; 5211 5212 end_synchronized_op(sc, LOCK_HELD); 5213 return (rc); 5214} 5215 5216static int 5217sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5218{ 5219 struct port_info *pi = arg1; 5220 struct adapter *sc = pi->adapter; 5221 int qsize, rc; 5222 5223 qsize = pi->qsize_txq; 5224 5225 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5226 if (rc != 0 || req->newptr == NULL) 5227 return (rc); 5228 5229 if (qsize < 128 || qsize > 65536) 5230 return (EINVAL); 5231 5232 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5233 "t4txqs"); 5234 if (rc) 5235 return (rc); 5236 5237 if (pi->flags & PORT_INIT_DONE) 5238 rc = EBUSY; /* cannot be changed once the queues are created */ 5239 else 5240 pi->qsize_txq = qsize; 5241 5242 end_synchronized_op(sc, LOCK_HELD); 5243 return (rc); 5244} 5245 5246static int 5247sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5248{ 5249 struct port_info *pi = arg1; 5250 struct adapter *sc = pi->adapter; 5251 struct link_config *lc = &pi->link_cfg; 5252 int rc; 5253 5254 if (req->newptr == NULL) { 5255 struct sbuf *sb; 5256 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5257 5258 rc = sysctl_wire_old_buffer(req, 0); 5259 if (rc != 0) 5260 return(rc); 5261 5262 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5263 if (sb == NULL) 5264 return (ENOMEM); 5265 5266 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5267 rc = sbuf_finish(sb); 5268 sbuf_delete(sb); 5269 } else { 5270 char s[2]; 5271 int n; 5272 5273 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5274 s[1] = 0; 5275 5276 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5277 if (rc != 0) 5278 return(rc); 5279 5280 if (s[1] != 0) 5281 return (EINVAL); 5282 if (s[0] < '0' || s[0] > '9') 5283 return (EINVAL); /* not a number */ 5284 n = s[0] - '0'; 5285 if (n & ~(PAUSE_TX | PAUSE_RX)) 5286 return (EINVAL); /* some other bit is set too */ 5287 5288 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5289 if (rc) 5290 return (rc); 5291 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5292 int link_ok = lc->link_ok; 5293 5294 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5295 lc->requested_fc |= n; 5296 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5297 lc->link_ok = link_ok; /* restore */ 5298 } 5299 end_synchronized_op(sc, 0); 5300 } 5301 5302 return (rc); 5303} 5304 5305static int 5306sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5307{ 5308 struct adapter *sc = arg1; 5309 int reg = arg2; 5310 uint64_t val; 5311 5312 val = t4_read_reg64(sc, reg); 5313 5314 return (sysctl_handle_64(oidp, &val, 0, req)); 5315} 5316 5317static int 5318sysctl_temperature(SYSCTL_HANDLER_ARGS) 5319{ 5320 struct adapter *sc = arg1; 5321 int rc, t; 5322 uint32_t param, val; 5323 5324 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5325 if (rc) 5326 return (rc); 5327 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5328 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5329 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5330 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5331 end_synchronized_op(sc, 0); 5332 if (rc) 5333 return (rc); 5334 5335 /* unknown is returned as 0 but we display -1 in that case */ 5336 t = val == 0 ? -1 : val; 5337 5338 rc = sysctl_handle_int(oidp, &t, 0, req); 5339 return (rc); 5340} 5341 5342#ifdef SBUF_DRAIN 5343static int 5344sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5345{ 5346 struct adapter *sc = arg1; 5347 struct sbuf *sb; 5348 int rc, i; 5349 uint16_t incr[NMTUS][NCCTRL_WIN]; 5350 static const char *dec_fac[] = { 5351 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5352 "0.9375" 5353 }; 5354 5355 rc = sysctl_wire_old_buffer(req, 0); 5356 if (rc != 0) 5357 return (rc); 5358 5359 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5360 if (sb == NULL) 5361 return (ENOMEM); 5362 5363 t4_read_cong_tbl(sc, incr); 5364 5365 for (i = 0; i < NCCTRL_WIN; ++i) { 5366 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5367 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5368 incr[5][i], incr[6][i], incr[7][i]); 5369 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5370 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5371 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5372 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5373 } 5374 5375 rc = sbuf_finish(sb); 5376 sbuf_delete(sb); 5377 5378 return (rc); 5379} 5380 5381static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5382 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5383 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5384 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5385}; 5386 5387static int 5388sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5389{ 5390 struct adapter *sc = arg1; 5391 struct sbuf *sb; 5392 int rc, i, n, qid = arg2; 5393 uint32_t *buf, *p; 5394 char *qtype; 5395 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5396 5397 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5398 ("%s: bad qid %d\n", __func__, qid)); 5399 5400 if (qid < CIM_NUM_IBQ) { 5401 /* inbound queue */ 5402 qtype = "IBQ"; 5403 n = 4 * CIM_IBQ_SIZE; 5404 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5405 rc = t4_read_cim_ibq(sc, qid, buf, n); 5406 } else { 5407 /* outbound queue */ 5408 qtype = "OBQ"; 5409 qid -= CIM_NUM_IBQ; 5410 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5411 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5412 rc = t4_read_cim_obq(sc, qid, buf, n); 5413 } 5414 5415 if (rc < 0) { 5416 rc = -rc; 5417 goto done; 5418 } 5419 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5420 5421 rc = sysctl_wire_old_buffer(req, 0); 5422 if (rc != 0) 5423 goto done; 5424 5425 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5426 if (sb == NULL) { 5427 rc = ENOMEM; 5428 goto done; 5429 } 5430 5431 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5432 for (i = 0, p = buf; i < n; i += 16, p += 4) 5433 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5434 p[2], p[3]); 5435 5436 rc = sbuf_finish(sb); 5437 sbuf_delete(sb); 5438done: 5439 free(buf, M_CXGBE); 5440 return (rc); 5441} 5442 5443static int 5444sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5445{ 5446 struct adapter *sc = arg1; 5447 u_int cfg; 5448 struct sbuf *sb; 5449 uint32_t *buf, *p; 5450 int rc; 5451 5452 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5453 if (rc != 0) 5454 return (rc); 5455 5456 rc = sysctl_wire_old_buffer(req, 0); 5457 if (rc != 0) 5458 return (rc); 5459 5460 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5461 if (sb == NULL) 5462 return (ENOMEM); 5463 5464 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5465 M_ZERO | M_WAITOK); 5466 5467 rc = -t4_cim_read_la(sc, buf, NULL); 5468 if (rc != 0) 5469 goto done; 5470 5471 sbuf_printf(sb, "Status Data PC%s", 5472 cfg & F_UPDBGLACAPTPCONLY ? "" : 5473 " LS0Stat LS0Addr LS0Data"); 5474 5475 KASSERT((sc->params.cim_la_size & 7) == 0, 5476 ("%s: p will walk off the end of buf", __func__)); 5477 5478 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5479 if (cfg & F_UPDBGLACAPTPCONLY) { 5480 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5481 p[6], p[7]); 5482 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5483 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5484 p[4] & 0xff, p[5] >> 8); 5485 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5486 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5487 p[1] & 0xf, p[2] >> 4); 5488 } else { 5489 sbuf_printf(sb, 5490 "\n %02x %x%07x %x%07x %08x %08x " 5491 "%08x%08x%08x%08x", 5492 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5493 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5494 p[6], p[7]); 5495 } 5496 } 5497 5498 rc = sbuf_finish(sb); 5499 sbuf_delete(sb); 5500done: 5501 free(buf, M_CXGBE); 5502 return (rc); 5503} 5504 5505static int 5506sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5507{ 5508 struct adapter *sc = arg1; 5509 u_int i; 5510 struct sbuf *sb; 5511 uint32_t *buf, *p; 5512 int rc; 5513 5514 rc = sysctl_wire_old_buffer(req, 0); 5515 if (rc != 0) 5516 return (rc); 5517 5518 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5519 if (sb == NULL) 5520 return (ENOMEM); 5521 5522 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5523 M_ZERO | M_WAITOK); 5524 5525 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5526 p = buf; 5527 5528 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5529 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5530 p[1], p[0]); 5531 } 5532 5533 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5534 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5535 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5536 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5537 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5538 (p[1] >> 2) | ((p[2] & 3) << 30), 5539 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5540 p[0] & 1); 5541 } 5542 5543 rc = sbuf_finish(sb); 5544 sbuf_delete(sb); 5545 free(buf, M_CXGBE); 5546 return (rc); 5547} 5548 5549static int 5550sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5551{ 5552 struct adapter *sc = arg1; 5553 u_int i; 5554 struct sbuf *sb; 5555 uint32_t *buf, *p; 5556 int rc; 5557 5558 rc = sysctl_wire_old_buffer(req, 0); 5559 if (rc != 0) 5560 return (rc); 5561 5562 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5563 if (sb == NULL) 5564 return (ENOMEM); 5565 5566 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5567 M_ZERO | M_WAITOK); 5568 5569 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5570 p = buf; 5571 5572 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5573 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5574 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5575 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5576 p[4], p[3], p[2], p[1], p[0]); 5577 } 5578 5579 sbuf_printf(sb, "\n\nCntl ID Data"); 5580 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5581 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5582 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5583 } 5584 5585 rc = sbuf_finish(sb); 5586 sbuf_delete(sb); 5587 free(buf, M_CXGBE); 5588 return (rc); 5589} 5590 5591static int 5592sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5593{ 5594 struct adapter *sc = arg1; 5595 struct sbuf *sb; 5596 int rc, i; 5597 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5598 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5599 uint16_t thres[CIM_NUM_IBQ]; 5600 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5601 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5602 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5603 5604 if (is_t4(sc)) { 5605 cim_num_obq = CIM_NUM_OBQ; 5606 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5607 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5608 } else { 5609 cim_num_obq = CIM_NUM_OBQ_T5; 5610 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5611 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5612 } 5613 nq = CIM_NUM_IBQ + cim_num_obq; 5614 5615 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5616 if (rc == 0) 5617 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5618 if (rc != 0) 5619 return (rc); 5620 5621 t4_read_cimq_cfg(sc, base, size, thres); 5622 5623 rc = sysctl_wire_old_buffer(req, 0); 5624 if (rc != 0) 5625 return (rc); 5626 5627 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5628 if (sb == NULL) 5629 return (ENOMEM); 5630 5631 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5632 5633 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5634 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5635 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5636 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5637 G_QUEREMFLITS(p[2]) * 16); 5638 for ( ; i < nq; i++, p += 4, wr += 2) 5639 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5640 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5641 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5642 G_QUEREMFLITS(p[2]) * 16); 5643 5644 rc = sbuf_finish(sb); 5645 sbuf_delete(sb); 5646 5647 return (rc); 5648} 5649 5650static int 5651sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5652{ 5653 struct adapter *sc = arg1; 5654 struct sbuf *sb; 5655 int rc; 5656 struct tp_cpl_stats stats; 5657 5658 rc = sysctl_wire_old_buffer(req, 0); 5659 if (rc != 0) 5660 return (rc); 5661 5662 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5663 if (sb == NULL) 5664 return (ENOMEM); 5665 5666 t4_tp_get_cpl_stats(sc, &stats); 5667 5668 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5669 "channel 3\n"); 5670 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5671 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5672 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5673 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5674 5675 rc = sbuf_finish(sb); 5676 sbuf_delete(sb); 5677 5678 return (rc); 5679} 5680 5681static int 5682sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5683{ 5684 struct adapter *sc = arg1; 5685 struct sbuf *sb; 5686 int rc; 5687 struct tp_usm_stats stats; 5688 5689 rc = sysctl_wire_old_buffer(req, 0); 5690 if (rc != 0) 5691 return(rc); 5692 5693 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5694 if (sb == NULL) 5695 return (ENOMEM); 5696 5697 t4_get_usm_stats(sc, &stats); 5698 5699 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5700 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5701 sbuf_printf(sb, "Drops: %u", stats.drops); 5702 5703 rc = sbuf_finish(sb); 5704 sbuf_delete(sb); 5705 5706 return (rc); 5707} 5708 5709const char *devlog_level_strings[] = { 5710 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5711 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5712 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5713 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5714 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5715 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5716}; 5717 5718const char *devlog_facility_strings[] = { 5719 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5720 [FW_DEVLOG_FACILITY_CF] = "CF", 5721 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5722 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5723 [FW_DEVLOG_FACILITY_RES] = "RES", 5724 [FW_DEVLOG_FACILITY_HW] = "HW", 5725 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5726 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5727 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5728 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5729 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5730 [FW_DEVLOG_FACILITY_VI] = "VI", 5731 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5732 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5733 [FW_DEVLOG_FACILITY_TM] = "TM", 5734 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5735 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5736 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5737 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5738 [FW_DEVLOG_FACILITY_RI] = "RI", 5739 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5740 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5741 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5742 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5743}; 5744 5745static int 5746sysctl_devlog(SYSCTL_HANDLER_ARGS) 5747{ 5748 struct adapter *sc = arg1; 5749 struct devlog_params *dparams = &sc->params.devlog; 5750 struct fw_devlog_e *buf, *e; 5751 int i, j, rc, nentries, first = 0, m; 5752 struct sbuf *sb; 5753 uint64_t ftstamp = UINT64_MAX; 5754 5755 if (dparams->start == 0) { 5756 dparams->memtype = FW_MEMTYPE_EDC0; 5757 dparams->start = 0x84000; 5758 dparams->size = 32768; 5759 } 5760 5761 nentries = dparams->size / sizeof(struct fw_devlog_e); 5762 5763 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5764 if (buf == NULL) 5765 return (ENOMEM); 5766 5767 m = fwmtype_to_hwmtype(dparams->memtype); 5768 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5769 if (rc != 0) 5770 goto done; 5771 5772 for (i = 0; i < nentries; i++) { 5773 e = &buf[i]; 5774 5775 if (e->timestamp == 0) 5776 break; /* end */ 5777 5778 e->timestamp = be64toh(e->timestamp); 5779 e->seqno = be32toh(e->seqno); 5780 for (j = 0; j < 8; j++) 5781 e->params[j] = be32toh(e->params[j]); 5782 5783 if (e->timestamp < ftstamp) { 5784 ftstamp = e->timestamp; 5785 first = i; 5786 } 5787 } 5788 5789 if (buf[first].timestamp == 0) 5790 goto done; /* nothing in the log */ 5791 5792 rc = sysctl_wire_old_buffer(req, 0); 5793 if (rc != 0) 5794 goto done; 5795 5796 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5797 if (sb == NULL) { 5798 rc = ENOMEM; 5799 goto done; 5800 } 5801 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5802 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5803 5804 i = first; 5805 do { 5806 e = &buf[i]; 5807 if (e->timestamp == 0) 5808 break; /* end */ 5809 5810 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5811 e->seqno, e->timestamp, 5812 (e->level < nitems(devlog_level_strings) ? 5813 devlog_level_strings[e->level] : "UNKNOWN"), 5814 (e->facility < nitems(devlog_facility_strings) ? 5815 devlog_facility_strings[e->facility] : "UNKNOWN")); 5816 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5817 e->params[2], e->params[3], e->params[4], 5818 e->params[5], e->params[6], e->params[7]); 5819 5820 if (++i == nentries) 5821 i = 0; 5822 } while (i != first); 5823 5824 rc = sbuf_finish(sb); 5825 sbuf_delete(sb); 5826done: 5827 free(buf, M_CXGBE); 5828 return (rc); 5829} 5830 5831static int 5832sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5833{ 5834 struct adapter *sc = arg1; 5835 struct sbuf *sb; 5836 int rc; 5837 struct tp_fcoe_stats stats[4]; 5838 5839 rc = sysctl_wire_old_buffer(req, 0); 5840 if (rc != 0) 5841 return (rc); 5842 5843 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5844 if (sb == NULL) 5845 return (ENOMEM); 5846 5847 t4_get_fcoe_stats(sc, 0, &stats[0]); 5848 t4_get_fcoe_stats(sc, 1, &stats[1]); 5849 t4_get_fcoe_stats(sc, 2, &stats[2]); 5850 t4_get_fcoe_stats(sc, 3, &stats[3]); 5851 5852 sbuf_printf(sb, " channel 0 channel 1 " 5853 "channel 2 channel 3\n"); 5854 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5855 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5856 stats[3].octetsDDP); 5857 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5858 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5859 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5860 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5861 stats[3].framesDrop); 5862 5863 rc = sbuf_finish(sb); 5864 sbuf_delete(sb); 5865 5866 return (rc); 5867} 5868 5869static int 5870sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5871{ 5872 struct adapter *sc = arg1; 5873 struct sbuf *sb; 5874 int rc, i; 5875 unsigned int map, kbps, ipg, mode; 5876 unsigned int pace_tab[NTX_SCHED]; 5877 5878 rc = sysctl_wire_old_buffer(req, 0); 5879 if (rc != 0) 5880 return (rc); 5881 5882 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5883 if (sb == NULL) 5884 return (ENOMEM); 5885 5886 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5887 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5888 t4_read_pace_tbl(sc, pace_tab); 5889 5890 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5891 "Class IPG (0.1 ns) Flow IPG (us)"); 5892 5893 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5894 t4_get_tx_sched(sc, i, &kbps, &ipg); 5895 sbuf_printf(sb, "\n %u %-5s %u ", i, 5896 (mode & (1 << i)) ? "flow" : "class", map & 3); 5897 if (kbps) 5898 sbuf_printf(sb, "%9u ", kbps); 5899 else 5900 sbuf_printf(sb, " disabled "); 5901 5902 if (ipg) 5903 sbuf_printf(sb, "%13u ", ipg); 5904 else 5905 sbuf_printf(sb, " disabled "); 5906 5907 if (pace_tab[i]) 5908 sbuf_printf(sb, "%10u", pace_tab[i]); 5909 else 5910 sbuf_printf(sb, " disabled"); 5911 } 5912 5913 rc = sbuf_finish(sb); 5914 sbuf_delete(sb); 5915 5916 return (rc); 5917} 5918 5919static int 5920sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5921{ 5922 struct adapter *sc = arg1; 5923 struct sbuf *sb; 5924 int rc, i, j; 5925 uint64_t *p0, *p1; 5926 struct lb_port_stats s[2]; 5927 static const char *stat_name[] = { 5928 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5929 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5930 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5931 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5932 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5933 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5934 "BG2FramesTrunc:", "BG3FramesTrunc:" 5935 }; 5936 5937 rc = sysctl_wire_old_buffer(req, 0); 5938 if (rc != 0) 5939 return (rc); 5940 5941 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5942 if (sb == NULL) 5943 return (ENOMEM); 5944 5945 memset(s, 0, sizeof(s)); 5946 5947 for (i = 0; i < 4; i += 2) { 5948 t4_get_lb_stats(sc, i, &s[0]); 5949 t4_get_lb_stats(sc, i + 1, &s[1]); 5950 5951 p0 = &s[0].octets; 5952 p1 = &s[1].octets; 5953 sbuf_printf(sb, "%s Loopback %u" 5954 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5955 5956 for (j = 0; j < nitems(stat_name); j++) 5957 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5958 *p0++, *p1++); 5959 } 5960 5961 rc = sbuf_finish(sb); 5962 sbuf_delete(sb); 5963 5964 return (rc); 5965} 5966 5967static int 5968sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5969{ 5970 int rc = 0; 5971 struct port_info *pi = arg1; 5972 struct sbuf *sb; 5973 static const char *linkdnreasons[] = { 5974 "non-specific", "remote fault", "autoneg failed", "reserved3", 5975 "PHY overheated", "unknown", "rx los", "reserved7" 5976 }; 5977 5978 rc = sysctl_wire_old_buffer(req, 0); 5979 if (rc != 0) 5980 return(rc); 5981 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5982 if (sb == NULL) 5983 return (ENOMEM); 5984 5985 if (pi->linkdnrc < 0) 5986 sbuf_printf(sb, "n/a"); 5987 else if (pi->linkdnrc < nitems(linkdnreasons)) 5988 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5989 else 5990 sbuf_printf(sb, "%d", pi->linkdnrc); 5991 5992 rc = sbuf_finish(sb); 5993 sbuf_delete(sb); 5994 5995 return (rc); 5996} 5997 5998struct mem_desc { 5999 unsigned int base; 6000 unsigned int limit; 6001 unsigned int idx; 6002}; 6003 6004static int 6005mem_desc_cmp(const void *a, const void *b) 6006{ 6007 return ((const struct mem_desc *)a)->base - 6008 ((const struct mem_desc *)b)->base; 6009} 6010 6011static void 6012mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6013 unsigned int to) 6014{ 6015 unsigned int size; 6016 6017 size = to - from + 1; 6018 if (size == 0) 6019 return; 6020 6021 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6022 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6023} 6024 6025static int 6026sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6027{ 6028 struct adapter *sc = arg1; 6029 struct sbuf *sb; 6030 int rc, i, n; 6031 uint32_t lo, hi, used, alloc; 6032 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6033 static const char *region[] = { 6034 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6035 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6036 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6037 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6038 "RQUDP region:", "PBL region:", "TXPBL region:", 6039 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6040 "On-chip queues:" 6041 }; 6042 struct mem_desc avail[4]; 6043 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6044 struct mem_desc *md = mem; 6045 6046 rc = sysctl_wire_old_buffer(req, 0); 6047 if (rc != 0) 6048 return (rc); 6049 6050 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6051 if (sb == NULL) 6052 return (ENOMEM); 6053 6054 for (i = 0; i < nitems(mem); i++) { 6055 mem[i].limit = 0; 6056 mem[i].idx = i; 6057 } 6058 6059 /* Find and sort the populated memory ranges */ 6060 i = 0; 6061 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6062 if (lo & F_EDRAM0_ENABLE) { 6063 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6064 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6065 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6066 avail[i].idx = 0; 6067 i++; 6068 } 6069 if (lo & F_EDRAM1_ENABLE) { 6070 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6071 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6072 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6073 avail[i].idx = 1; 6074 i++; 6075 } 6076 if (lo & F_EXT_MEM_ENABLE) { 6077 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6078 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6079 avail[i].limit = avail[i].base + 6080 (G_EXT_MEM_SIZE(hi) << 20); 6081 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6082 i++; 6083 } 6084 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6085 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6086 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6087 avail[i].limit = avail[i].base + 6088 (G_EXT_MEM1_SIZE(hi) << 20); 6089 avail[i].idx = 4; 6090 i++; 6091 } 6092 if (!i) /* no memory available */ 6093 return 0; 6094 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6095 6096 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6097 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6098 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6099 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6100 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6101 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6102 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6103 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6104 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6105 6106 /* the next few have explicit upper bounds */ 6107 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6108 md->limit = md->base - 1 + 6109 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6110 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6111 md++; 6112 6113 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6114 md->limit = md->base - 1 + 6115 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6116 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6117 md++; 6118 6119 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6120 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6121 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6122 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6123 } else { 6124 md->base = 0; 6125 md->idx = nitems(region); /* hide it */ 6126 } 6127 md++; 6128 6129#define ulp_region(reg) \ 6130 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6131 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6132 6133 ulp_region(RX_ISCSI); 6134 ulp_region(RX_TDDP); 6135 ulp_region(TX_TPT); 6136 ulp_region(RX_STAG); 6137 ulp_region(RX_RQ); 6138 ulp_region(RX_RQUDP); 6139 ulp_region(RX_PBL); 6140 ulp_region(TX_PBL); 6141#undef ulp_region 6142 6143 md->base = 0; 6144 md->idx = nitems(region); 6145 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6146 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6147 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6148 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6149 } 6150 md++; 6151 6152 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6153 md->limit = md->base + sc->tids.ntids - 1; 6154 md++; 6155 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6156 md->limit = md->base + sc->tids.ntids - 1; 6157 md++; 6158 6159 md->base = sc->vres.ocq.start; 6160 if (sc->vres.ocq.size) 6161 md->limit = md->base + sc->vres.ocq.size - 1; 6162 else 6163 md->idx = nitems(region); /* hide it */ 6164 md++; 6165 6166 /* add any address-space holes, there can be up to 3 */ 6167 for (n = 0; n < i - 1; n++) 6168 if (avail[n].limit < avail[n + 1].base) 6169 (md++)->base = avail[n].limit; 6170 if (avail[n].limit) 6171 (md++)->base = avail[n].limit; 6172 6173 n = md - mem; 6174 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6175 6176 for (lo = 0; lo < i; lo++) 6177 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6178 avail[lo].limit - 1); 6179 6180 sbuf_printf(sb, "\n"); 6181 for (i = 0; i < n; i++) { 6182 if (mem[i].idx >= nitems(region)) 6183 continue; /* skip holes */ 6184 if (!mem[i].limit) 6185 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6186 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6187 mem[i].limit); 6188 } 6189 6190 sbuf_printf(sb, "\n"); 6191 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6192 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6193 mem_region_show(sb, "uP RAM:", lo, hi); 6194 6195 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6196 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6197 mem_region_show(sb, "uP Extmem2:", lo, hi); 6198 6199 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6200 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6201 G_PMRXMAXPAGE(lo), 6202 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6203 (lo & F_PMRXNUMCHN) ? 2 : 1); 6204 6205 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6206 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6207 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6208 G_PMTXMAXPAGE(lo), 6209 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6210 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6211 sbuf_printf(sb, "%u p-structs\n", 6212 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6213 6214 for (i = 0; i < 4; i++) { 6215 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6216 if (is_t4(sc)) { 6217 used = G_USED(lo); 6218 alloc = G_ALLOC(lo); 6219 } else { 6220 used = G_T5_USED(lo); 6221 alloc = G_T5_ALLOC(lo); 6222 } 6223 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6224 i, used, alloc); 6225 } 6226 for (i = 0; i < 4; i++) { 6227 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6228 if (is_t4(sc)) { 6229 used = G_USED(lo); 6230 alloc = G_ALLOC(lo); 6231 } else { 6232 used = G_T5_USED(lo); 6233 alloc = G_T5_ALLOC(lo); 6234 } 6235 sbuf_printf(sb, 6236 "\nLoopback %d using %u pages out of %u allocated", 6237 i, used, alloc); 6238 } 6239 6240 rc = sbuf_finish(sb); 6241 sbuf_delete(sb); 6242 6243 return (rc); 6244} 6245 6246static inline void 6247tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6248{ 6249 *mask = x | y; 6250 y = htobe64(y); 6251 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6252} 6253 6254static int 6255sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6256{ 6257 struct adapter *sc = arg1; 6258 struct sbuf *sb; 6259 int rc, i, n; 6260 6261 rc = sysctl_wire_old_buffer(req, 0); 6262 if (rc != 0) 6263 return (rc); 6264 6265 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6266 if (sb == NULL) 6267 return (ENOMEM); 6268 6269 sbuf_printf(sb, 6270 "Idx Ethernet address Mask Vld Ports PF" 6271 " VF Replication P0 P1 P2 P3 ML"); 6272 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6273 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6274 for (i = 0; i < n; i++) { 6275 uint64_t tcamx, tcamy, mask; 6276 uint32_t cls_lo, cls_hi; 6277 uint8_t addr[ETHER_ADDR_LEN]; 6278 6279 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6280 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6281 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6282 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6283 6284 if (tcamx & tcamy) 6285 continue; 6286 6287 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6288 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6289 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6290 addr[3], addr[4], addr[5], (uintmax_t)mask, 6291 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6292 G_PORTMAP(cls_hi), G_PF(cls_lo), 6293 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6294 6295 if (cls_lo & F_REPLICATE) { 6296 struct fw_ldst_cmd ldst_cmd; 6297 6298 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6299 ldst_cmd.op_to_addrspace = 6300 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6301 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6302 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6303 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6304 ldst_cmd.u.mps.rplc.fid_idx = 6305 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6306 V_FW_LDST_CMD_IDX(i)); 6307 6308 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6309 "t4mps"); 6310 if (rc) 6311 break; 6312 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6313 sizeof(ldst_cmd), &ldst_cmd); 6314 end_synchronized_op(sc, 0); 6315 6316 if (rc != 0) { 6317 sbuf_printf(sb, 6318 " ------------ error %3u ------------", rc); 6319 rc = 0; 6320 } else { 6321 sbuf_printf(sb, " %08x %08x %08x %08x", 6322 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6323 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6324 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6325 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6326 } 6327 } else 6328 sbuf_printf(sb, "%36s", ""); 6329 6330 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6331 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6332 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6333 } 6334 6335 if (rc) 6336 (void) sbuf_finish(sb); 6337 else 6338 rc = sbuf_finish(sb); 6339 sbuf_delete(sb); 6340 6341 return (rc); 6342} 6343 6344static int 6345sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6346{ 6347 struct adapter *sc = arg1; 6348 struct sbuf *sb; 6349 int rc; 6350 uint16_t mtus[NMTUS]; 6351 6352 rc = sysctl_wire_old_buffer(req, 0); 6353 if (rc != 0) 6354 return (rc); 6355 6356 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6357 if (sb == NULL) 6358 return (ENOMEM); 6359 6360 t4_read_mtu_tbl(sc, mtus, NULL); 6361 6362 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6363 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6364 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6365 mtus[14], mtus[15]); 6366 6367 rc = sbuf_finish(sb); 6368 sbuf_delete(sb); 6369 6370 return (rc); 6371} 6372 6373static int 6374sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6375{ 6376 struct adapter *sc = arg1; 6377 struct sbuf *sb; 6378 int rc, i; 6379 uint32_t cnt[PM_NSTATS]; 6380 uint64_t cyc[PM_NSTATS]; 6381 static const char *rx_stats[] = { 6382 "Read:", "Write bypass:", "Write mem:", "Flush:" 6383 }; 6384 static const char *tx_stats[] = { 6385 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6386 }; 6387 6388 rc = sysctl_wire_old_buffer(req, 0); 6389 if (rc != 0) 6390 return (rc); 6391 6392 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6393 if (sb == NULL) 6394 return (ENOMEM); 6395 6396 t4_pmtx_get_stats(sc, cnt, cyc); 6397 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6398 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6399 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6400 cyc[i]); 6401 6402 t4_pmrx_get_stats(sc, cnt, cyc); 6403 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6404 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6405 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6406 cyc[i]); 6407 6408 rc = sbuf_finish(sb); 6409 sbuf_delete(sb); 6410 6411 return (rc); 6412} 6413 6414static int 6415sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6416{ 6417 struct adapter *sc = arg1; 6418 struct sbuf *sb; 6419 int rc; 6420 struct tp_rdma_stats stats; 6421 6422 rc = sysctl_wire_old_buffer(req, 0); 6423 if (rc != 0) 6424 return (rc); 6425 6426 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6427 if (sb == NULL) 6428 return (ENOMEM); 6429 6430 t4_tp_get_rdma_stats(sc, &stats); 6431 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6432 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6433 6434 rc = sbuf_finish(sb); 6435 sbuf_delete(sb); 6436 6437 return (rc); 6438} 6439 6440static int 6441sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6442{ 6443 struct adapter *sc = arg1; 6444 struct sbuf *sb; 6445 int rc; 6446 struct tp_tcp_stats v4, v6; 6447 6448 rc = sysctl_wire_old_buffer(req, 0); 6449 if (rc != 0) 6450 return (rc); 6451 6452 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6453 if (sb == NULL) 6454 return (ENOMEM); 6455 6456 t4_tp_get_tcp_stats(sc, &v4, &v6); 6457 sbuf_printf(sb, 6458 " IP IPv6\n"); 6459 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6460 v4.tcpOutRsts, v6.tcpOutRsts); 6461 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6462 v4.tcpInSegs, v6.tcpInSegs); 6463 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6464 v4.tcpOutSegs, v6.tcpOutSegs); 6465 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6466 v4.tcpRetransSegs, v6.tcpRetransSegs); 6467 6468 rc = sbuf_finish(sb); 6469 sbuf_delete(sb); 6470 6471 return (rc); 6472} 6473 6474static int 6475sysctl_tids(SYSCTL_HANDLER_ARGS) 6476{ 6477 struct adapter *sc = arg1; 6478 struct sbuf *sb; 6479 int rc; 6480 struct tid_info *t = &sc->tids; 6481 6482 rc = sysctl_wire_old_buffer(req, 0); 6483 if (rc != 0) 6484 return (rc); 6485 6486 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6487 if (sb == NULL) 6488 return (ENOMEM); 6489 6490 if (t->natids) { 6491 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6492 t->atids_in_use); 6493 } 6494 6495 if (t->ntids) { 6496 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6497 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6498 6499 if (b) { 6500 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6501 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6502 t->ntids - 1); 6503 } else { 6504 sbuf_printf(sb, "TID range: %u-%u", 6505 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6506 t->ntids - 1); 6507 } 6508 } else 6509 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6510 sbuf_printf(sb, ", in use: %u\n", 6511 atomic_load_acq_int(&t->tids_in_use)); 6512 } 6513 6514 if (t->nstids) { 6515 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6516 t->stid_base + t->nstids - 1, t->stids_in_use); 6517 } 6518 6519 if (t->nftids) { 6520 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6521 t->ftid_base + t->nftids - 1); 6522 } 6523 6524 if (t->netids) { 6525 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6526 t->etid_base + t->netids - 1); 6527 } 6528 6529 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6530 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6531 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6532 6533 rc = sbuf_finish(sb); 6534 sbuf_delete(sb); 6535 6536 return (rc); 6537} 6538 6539static int 6540sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6541{ 6542 struct adapter *sc = arg1; 6543 struct sbuf *sb; 6544 int rc; 6545 struct tp_err_stats stats; 6546 6547 rc = sysctl_wire_old_buffer(req, 0); 6548 if (rc != 0) 6549 return (rc); 6550 6551 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6552 if (sb == NULL) 6553 return (ENOMEM); 6554 6555 t4_tp_get_err_stats(sc, &stats); 6556 6557 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6558 "channel 3\n"); 6559 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6560 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6561 stats.macInErrs[3]); 6562 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6563 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6564 stats.hdrInErrs[3]); 6565 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6566 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6567 stats.tcpInErrs[3]); 6568 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6569 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6570 stats.tcp6InErrs[3]); 6571 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6572 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6573 stats.tnlCongDrops[3]); 6574 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6575 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6576 stats.tnlTxDrops[3]); 6577 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6578 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6579 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6580 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6581 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6582 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6583 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6584 stats.ofldNoNeigh, stats.ofldCongDefer); 6585 6586 rc = sbuf_finish(sb); 6587 sbuf_delete(sb); 6588 6589 return (rc); 6590} 6591 6592struct field_desc { 6593 const char *name; 6594 u_int start; 6595 u_int width; 6596}; 6597 6598static void 6599field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6600{ 6601 char buf[32]; 6602 int line_size = 0; 6603 6604 while (f->name) { 6605 uint64_t mask = (1ULL << f->width) - 1; 6606 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6607 ((uintmax_t)v >> f->start) & mask); 6608 6609 if (line_size + len >= 79) { 6610 line_size = 8; 6611 sbuf_printf(sb, "\n "); 6612 } 6613 sbuf_printf(sb, "%s ", buf); 6614 line_size += len + 1; 6615 f++; 6616 } 6617 sbuf_printf(sb, "\n"); 6618} 6619 6620static struct field_desc tp_la0[] = { 6621 { "RcfOpCodeOut", 60, 4 }, 6622 { "State", 56, 4 }, 6623 { "WcfState", 52, 4 }, 6624 { "RcfOpcSrcOut", 50, 2 }, 6625 { "CRxError", 49, 1 }, 6626 { "ERxError", 48, 1 }, 6627 { "SanityFailed", 47, 1 }, 6628 { "SpuriousMsg", 46, 1 }, 6629 { "FlushInputMsg", 45, 1 }, 6630 { "FlushInputCpl", 44, 1 }, 6631 { "RssUpBit", 43, 1 }, 6632 { "RssFilterHit", 42, 1 }, 6633 { "Tid", 32, 10 }, 6634 { "InitTcb", 31, 1 }, 6635 { "LineNumber", 24, 7 }, 6636 { "Emsg", 23, 1 }, 6637 { "EdataOut", 22, 1 }, 6638 { "Cmsg", 21, 1 }, 6639 { "CdataOut", 20, 1 }, 6640 { "EreadPdu", 19, 1 }, 6641 { "CreadPdu", 18, 1 }, 6642 { "TunnelPkt", 17, 1 }, 6643 { "RcfPeerFin", 16, 1 }, 6644 { "RcfReasonOut", 12, 4 }, 6645 { "TxCchannel", 10, 2 }, 6646 { "RcfTxChannel", 8, 2 }, 6647 { "RxEchannel", 6, 2 }, 6648 { "RcfRxChannel", 5, 1 }, 6649 { "RcfDataOutSrdy", 4, 1 }, 6650 { "RxDvld", 3, 1 }, 6651 { "RxOoDvld", 2, 1 }, 6652 { "RxCongestion", 1, 1 }, 6653 { "TxCongestion", 0, 1 }, 6654 { NULL } 6655}; 6656 6657static struct field_desc tp_la1[] = { 6658 { "CplCmdIn", 56, 8 }, 6659 { "CplCmdOut", 48, 8 }, 6660 { "ESynOut", 47, 1 }, 6661 { "EAckOut", 46, 1 }, 6662 { "EFinOut", 45, 1 }, 6663 { "ERstOut", 44, 1 }, 6664 { "SynIn", 43, 1 }, 6665 { "AckIn", 42, 1 }, 6666 { "FinIn", 41, 1 }, 6667 { "RstIn", 40, 1 }, 6668 { "DataIn", 39, 1 }, 6669 { "DataInVld", 38, 1 }, 6670 { "PadIn", 37, 1 }, 6671 { "RxBufEmpty", 36, 1 }, 6672 { "RxDdp", 35, 1 }, 6673 { "RxFbCongestion", 34, 1 }, 6674 { "TxFbCongestion", 33, 1 }, 6675 { "TxPktSumSrdy", 32, 1 }, 6676 { "RcfUlpType", 28, 4 }, 6677 { "Eread", 27, 1 }, 6678 { "Ebypass", 26, 1 }, 6679 { "Esave", 25, 1 }, 6680 { "Static0", 24, 1 }, 6681 { "Cread", 23, 1 }, 6682 { "Cbypass", 22, 1 }, 6683 { "Csave", 21, 1 }, 6684 { "CPktOut", 20, 1 }, 6685 { "RxPagePoolFull", 18, 2 }, 6686 { "RxLpbkPkt", 17, 1 }, 6687 { "TxLpbkPkt", 16, 1 }, 6688 { "RxVfValid", 15, 1 }, 6689 { "SynLearned", 14, 1 }, 6690 { "SetDelEntry", 13, 1 }, 6691 { "SetInvEntry", 12, 1 }, 6692 { "CpcmdDvld", 11, 1 }, 6693 { "CpcmdSave", 10, 1 }, 6694 { "RxPstructsFull", 8, 2 }, 6695 { "EpcmdDvld", 7, 1 }, 6696 { "EpcmdFlush", 6, 1 }, 6697 { "EpcmdTrimPrefix", 5, 1 }, 6698 { "EpcmdTrimPostfix", 4, 1 }, 6699 { "ERssIp4Pkt", 3, 1 }, 6700 { "ERssIp6Pkt", 2, 1 }, 6701 { "ERssTcpUdpPkt", 1, 1 }, 6702 { "ERssFceFipPkt", 0, 1 }, 6703 { NULL } 6704}; 6705 6706static struct field_desc tp_la2[] = { 6707 { "CplCmdIn", 56, 8 }, 6708 { "MpsVfVld", 55, 1 }, 6709 { "MpsPf", 52, 3 }, 6710 { "MpsVf", 44, 8 }, 6711 { "SynIn", 43, 1 }, 6712 { "AckIn", 42, 1 }, 6713 { "FinIn", 41, 1 }, 6714 { "RstIn", 40, 1 }, 6715 { "DataIn", 39, 1 }, 6716 { "DataInVld", 38, 1 }, 6717 { "PadIn", 37, 1 }, 6718 { "RxBufEmpty", 36, 1 }, 6719 { "RxDdp", 35, 1 }, 6720 { "RxFbCongestion", 34, 1 }, 6721 { "TxFbCongestion", 33, 1 }, 6722 { "TxPktSumSrdy", 32, 1 }, 6723 { "RcfUlpType", 28, 4 }, 6724 { "Eread", 27, 1 }, 6725 { "Ebypass", 26, 1 }, 6726 { "Esave", 25, 1 }, 6727 { "Static0", 24, 1 }, 6728 { "Cread", 23, 1 }, 6729 { "Cbypass", 22, 1 }, 6730 { "Csave", 21, 1 }, 6731 { "CPktOut", 20, 1 }, 6732 { "RxPagePoolFull", 18, 2 }, 6733 { "RxLpbkPkt", 17, 1 }, 6734 { "TxLpbkPkt", 16, 1 }, 6735 { "RxVfValid", 15, 1 }, 6736 { "SynLearned", 14, 1 }, 6737 { "SetDelEntry", 13, 1 }, 6738 { "SetInvEntry", 12, 1 }, 6739 { "CpcmdDvld", 11, 1 }, 6740 { "CpcmdSave", 10, 1 }, 6741 { "RxPstructsFull", 8, 2 }, 6742 { "EpcmdDvld", 7, 1 }, 6743 { "EpcmdFlush", 6, 1 }, 6744 { "EpcmdTrimPrefix", 5, 1 }, 6745 { "EpcmdTrimPostfix", 4, 1 }, 6746 { "ERssIp4Pkt", 3, 1 }, 6747 { "ERssIp6Pkt", 2, 1 }, 6748 { "ERssTcpUdpPkt", 1, 1 }, 6749 { "ERssFceFipPkt", 0, 1 }, 6750 { NULL } 6751}; 6752 6753static void 6754tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6755{ 6756 6757 field_desc_show(sb, *p, tp_la0); 6758} 6759 6760static void 6761tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6762{ 6763 6764 if (idx) 6765 sbuf_printf(sb, "\n"); 6766 field_desc_show(sb, p[0], tp_la0); 6767 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6768 field_desc_show(sb, p[1], tp_la0); 6769} 6770 6771static void 6772tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6773{ 6774 6775 if (idx) 6776 sbuf_printf(sb, "\n"); 6777 field_desc_show(sb, p[0], tp_la0); 6778 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6779 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6780} 6781 6782static int 6783sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6784{ 6785 struct adapter *sc = arg1; 6786 struct sbuf *sb; 6787 uint64_t *buf, *p; 6788 int rc; 6789 u_int i, inc; 6790 void (*show_func)(struct sbuf *, uint64_t *, int); 6791 6792 rc = sysctl_wire_old_buffer(req, 0); 6793 if (rc != 0) 6794 return (rc); 6795 6796 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6797 if (sb == NULL) 6798 return (ENOMEM); 6799 6800 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6801 6802 t4_tp_read_la(sc, buf, NULL); 6803 p = buf; 6804 6805 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6806 case 2: 6807 inc = 2; 6808 show_func = tp_la_show2; 6809 break; 6810 case 3: 6811 inc = 2; 6812 show_func = tp_la_show3; 6813 break; 6814 default: 6815 inc = 1; 6816 show_func = tp_la_show; 6817 } 6818 6819 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6820 (*show_func)(sb, p, i); 6821 6822 rc = sbuf_finish(sb); 6823 sbuf_delete(sb); 6824 free(buf, M_CXGBE); 6825 return (rc); 6826} 6827 6828static int 6829sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6830{ 6831 struct adapter *sc = arg1; 6832 struct sbuf *sb; 6833 int rc; 6834 u64 nrate[NCHAN], orate[NCHAN]; 6835 6836 rc = sysctl_wire_old_buffer(req, 0); 6837 if (rc != 0) 6838 return (rc); 6839 6840 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6841 if (sb == NULL) 6842 return (ENOMEM); 6843 6844 t4_get_chan_txrate(sc, nrate, orate); 6845 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6846 "channel 3\n"); 6847 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6848 nrate[0], nrate[1], nrate[2], nrate[3]); 6849 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6850 orate[0], orate[1], orate[2], orate[3]); 6851 6852 rc = sbuf_finish(sb); 6853 sbuf_delete(sb); 6854 6855 return (rc); 6856} 6857 6858static int 6859sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6860{ 6861 struct adapter *sc = arg1; 6862 struct sbuf *sb; 6863 uint32_t *buf, *p; 6864 int rc, i; 6865 6866 rc = sysctl_wire_old_buffer(req, 0); 6867 if (rc != 0) 6868 return (rc); 6869 6870 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6871 if (sb == NULL) 6872 return (ENOMEM); 6873 6874 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6875 M_ZERO | M_WAITOK); 6876 6877 t4_ulprx_read_la(sc, buf); 6878 p = buf; 6879 6880 sbuf_printf(sb, " Pcmd Type Message" 6881 " Data"); 6882 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6883 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6884 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6885 } 6886 6887 rc = sbuf_finish(sb); 6888 sbuf_delete(sb); 6889 free(buf, M_CXGBE); 6890 return (rc); 6891} 6892 6893static int 6894sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6895{ 6896 struct adapter *sc = arg1; 6897 struct sbuf *sb; 6898 int rc, v; 6899 6900 rc = sysctl_wire_old_buffer(req, 0); 6901 if (rc != 0) 6902 return (rc); 6903 6904 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6905 if (sb == NULL) 6906 return (ENOMEM); 6907 6908 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6909 if (G_STATSOURCE_T5(v) == 7) { 6910 if (G_STATMODE(v) == 0) { 6911 sbuf_printf(sb, "total %d, incomplete %d", 6912 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6913 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6914 } else if (G_STATMODE(v) == 1) { 6915 sbuf_printf(sb, "total %d, data overflow %d", 6916 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6917 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6918 } 6919 } 6920 rc = sbuf_finish(sb); 6921 sbuf_delete(sb); 6922 6923 return (rc); 6924} 6925#endif 6926 6927static uint32_t 6928fconf_to_mode(uint32_t fconf) 6929{ 6930 uint32_t mode; 6931 6932 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6933 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6934 6935 if (fconf & F_FRAGMENTATION) 6936 mode |= T4_FILTER_IP_FRAGMENT; 6937 6938 if (fconf & F_MPSHITTYPE) 6939 mode |= T4_FILTER_MPS_HIT_TYPE; 6940 6941 if (fconf & F_MACMATCH) 6942 mode |= T4_FILTER_MAC_IDX; 6943 6944 if (fconf & F_ETHERTYPE) 6945 mode |= T4_FILTER_ETH_TYPE; 6946 6947 if (fconf & F_PROTOCOL) 6948 mode |= T4_FILTER_IP_PROTO; 6949 6950 if (fconf & F_TOS) 6951 mode |= T4_FILTER_IP_TOS; 6952 6953 if (fconf & F_VLAN) 6954 mode |= T4_FILTER_VLAN; 6955 6956 if (fconf & F_VNIC_ID) 6957 mode |= T4_FILTER_VNIC; 6958 6959 if (fconf & F_PORT) 6960 mode |= T4_FILTER_PORT; 6961 6962 if (fconf & F_FCOE) 6963 mode |= T4_FILTER_FCoE; 6964 6965 return (mode); 6966} 6967 6968static uint32_t 6969mode_to_fconf(uint32_t mode) 6970{ 6971 uint32_t fconf = 0; 6972 6973 if (mode & T4_FILTER_IP_FRAGMENT) 6974 fconf |= F_FRAGMENTATION; 6975 6976 if (mode & T4_FILTER_MPS_HIT_TYPE) 6977 fconf |= F_MPSHITTYPE; 6978 6979 if (mode & T4_FILTER_MAC_IDX) 6980 fconf |= F_MACMATCH; 6981 6982 if (mode & T4_FILTER_ETH_TYPE) 6983 fconf |= F_ETHERTYPE; 6984 6985 if (mode & T4_FILTER_IP_PROTO) 6986 fconf |= F_PROTOCOL; 6987 6988 if (mode & T4_FILTER_IP_TOS) 6989 fconf |= F_TOS; 6990 6991 if (mode & T4_FILTER_VLAN) 6992 fconf |= F_VLAN; 6993 6994 if (mode & T4_FILTER_VNIC) 6995 fconf |= F_VNIC_ID; 6996 6997 if (mode & T4_FILTER_PORT) 6998 fconf |= F_PORT; 6999 7000 if (mode & T4_FILTER_FCoE) 7001 fconf |= F_FCOE; 7002 7003 return (fconf); 7004} 7005 7006static uint32_t 7007fspec_to_fconf(struct t4_filter_specification *fs) 7008{ 7009 uint32_t fconf = 0; 7010 7011 if (fs->val.frag || fs->mask.frag) 7012 fconf |= F_FRAGMENTATION; 7013 7014 if (fs->val.matchtype || fs->mask.matchtype) 7015 fconf |= F_MPSHITTYPE; 7016 7017 if (fs->val.macidx || fs->mask.macidx) 7018 fconf |= F_MACMATCH; 7019 7020 if (fs->val.ethtype || fs->mask.ethtype) 7021 fconf |= F_ETHERTYPE; 7022 7023 if (fs->val.proto || fs->mask.proto) 7024 fconf |= F_PROTOCOL; 7025 7026 if (fs->val.tos || fs->mask.tos) 7027 fconf |= F_TOS; 7028 7029 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7030 fconf |= F_VLAN; 7031 7032 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7033 fconf |= F_VNIC_ID; 7034 7035 if (fs->val.iport || fs->mask.iport) 7036 fconf |= F_PORT; 7037 7038 if (fs->val.fcoe || fs->mask.fcoe) 7039 fconf |= F_FCOE; 7040 7041 return (fconf); 7042} 7043 7044static int 7045get_filter_mode(struct adapter *sc, uint32_t *mode) 7046{ 7047 int rc; 7048 uint32_t fconf; 7049 7050 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7051 "t4getfm"); 7052 if (rc) 7053 return (rc); 7054 7055 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7056 A_TP_VLAN_PRI_MAP); 7057 7058 if (sc->params.tp.vlan_pri_map != fconf) { 7059 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7060 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7061 fconf); 7062 } 7063 7064 *mode = fconf_to_mode(fconf); 7065 7066 end_synchronized_op(sc, LOCK_HELD); 7067 return (0); 7068} 7069 7070static int 7071set_filter_mode(struct adapter *sc, uint32_t mode) 7072{ 7073 uint32_t fconf; 7074 int rc; 7075 7076 fconf = mode_to_fconf(mode); 7077 7078 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7079 "t4setfm"); 7080 if (rc) 7081 return (rc); 7082 7083 if (sc->tids.ftids_in_use > 0) { 7084 rc = EBUSY; 7085 goto done; 7086 } 7087 7088#ifdef TCP_OFFLOAD 7089 if (uld_active(sc, ULD_TOM)) { 7090 rc = EBUSY; 7091 goto done; 7092 } 7093#endif 7094 7095 rc = -t4_set_filter_mode(sc, fconf); 7096done: 7097 end_synchronized_op(sc, LOCK_HELD); 7098 return (rc); 7099} 7100 7101static inline uint64_t 7102get_filter_hits(struct adapter *sc, uint32_t fid) 7103{ 7104 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7105 uint64_t hits; 7106 7107 memwin_info(sc, 0, &mw_base, NULL); 7108 off = position_memwin(sc, 0, 7109 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7110 if (is_t4(sc)) { 7111 hits = t4_read_reg64(sc, mw_base + off + 16); 7112 hits = be64toh(hits); 7113 } else { 7114 hits = t4_read_reg(sc, mw_base + off + 24); 7115 hits = be32toh(hits); 7116 } 7117 7118 return (hits); 7119} 7120 7121static int 7122get_filter(struct adapter *sc, struct t4_filter *t) 7123{ 7124 int i, rc, nfilters = sc->tids.nftids; 7125 struct filter_entry *f; 7126 7127 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7128 "t4getf"); 7129 if (rc) 7130 return (rc); 7131 7132 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7133 t->idx >= nfilters) { 7134 t->idx = 0xffffffff; 7135 goto done; 7136 } 7137 7138 f = &sc->tids.ftid_tab[t->idx]; 7139 for (i = t->idx; i < nfilters; i++, f++) { 7140 if (f->valid) { 7141 t->idx = i; 7142 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7143 t->smtidx = f->smtidx; 7144 if (f->fs.hitcnts) 7145 t->hits = get_filter_hits(sc, t->idx); 7146 else 7147 t->hits = UINT64_MAX; 7148 t->fs = f->fs; 7149 7150 goto done; 7151 } 7152 } 7153 7154 t->idx = 0xffffffff; 7155done: 7156 end_synchronized_op(sc, LOCK_HELD); 7157 return (0); 7158} 7159 7160static int 7161set_filter(struct adapter *sc, struct t4_filter *t) 7162{ 7163 unsigned int nfilters, nports; 7164 struct filter_entry *f; 7165 int i, rc; 7166 7167 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7168 if (rc) 7169 return (rc); 7170 7171 nfilters = sc->tids.nftids; 7172 nports = sc->params.nports; 7173 7174 if (nfilters == 0) { 7175 rc = ENOTSUP; 7176 goto done; 7177 } 7178 7179 if (!(sc->flags & FULL_INIT_DONE)) { 7180 rc = EAGAIN; 7181 goto done; 7182 } 7183 7184 if (t->idx >= nfilters) { 7185 rc = EINVAL; 7186 goto done; 7187 } 7188 7189 /* Validate against the global filter mode */ 7190 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7191 sc->params.tp.vlan_pri_map) { 7192 rc = E2BIG; 7193 goto done; 7194 } 7195 7196 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7197 rc = EINVAL; 7198 goto done; 7199 } 7200 7201 if (t->fs.val.iport >= nports) { 7202 rc = EINVAL; 7203 goto done; 7204 } 7205 7206 /* Can't specify an iq if not steering to it */ 7207 if (!t->fs.dirsteer && t->fs.iq) { 7208 rc = EINVAL; 7209 goto done; 7210 } 7211 7212 /* IPv6 filter idx must be 4 aligned */ 7213 if (t->fs.type == 1 && 7214 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7215 rc = EINVAL; 7216 goto done; 7217 } 7218 7219 if (sc->tids.ftid_tab == NULL) { 7220 KASSERT(sc->tids.ftids_in_use == 0, 7221 ("%s: no memory allocated but filters_in_use > 0", 7222 __func__)); 7223 7224 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7225 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7226 if (sc->tids.ftid_tab == NULL) { 7227 rc = ENOMEM; 7228 goto done; 7229 } 7230 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7231 } 7232 7233 for (i = 0; i < 4; i++) { 7234 f = &sc->tids.ftid_tab[t->idx + i]; 7235 7236 if (f->pending || f->valid) { 7237 rc = EBUSY; 7238 goto done; 7239 } 7240 if (f->locked) { 7241 rc = EPERM; 7242 goto done; 7243 } 7244 7245 if (t->fs.type == 0) 7246 break; 7247 } 7248 7249 f = &sc->tids.ftid_tab[t->idx]; 7250 f->fs = t->fs; 7251 7252 rc = set_filter_wr(sc, t->idx); 7253done: 7254 end_synchronized_op(sc, 0); 7255 7256 if (rc == 0) { 7257 mtx_lock(&sc->tids.ftid_lock); 7258 for (;;) { 7259 if (f->pending == 0) { 7260 rc = f->valid ? 0 : EIO; 7261 break; 7262 } 7263 7264 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7265 PCATCH, "t4setfw", 0)) { 7266 rc = EINPROGRESS; 7267 break; 7268 } 7269 } 7270 mtx_unlock(&sc->tids.ftid_lock); 7271 } 7272 return (rc); 7273} 7274 7275static int 7276del_filter(struct adapter *sc, struct t4_filter *t) 7277{ 7278 unsigned int nfilters; 7279 struct filter_entry *f; 7280 int rc; 7281 7282 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7283 if (rc) 7284 return (rc); 7285 7286 nfilters = sc->tids.nftids; 7287 7288 if (nfilters == 0) { 7289 rc = ENOTSUP; 7290 goto done; 7291 } 7292 7293 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7294 t->idx >= nfilters) { 7295 rc = EINVAL; 7296 goto done; 7297 } 7298 7299 if (!(sc->flags & FULL_INIT_DONE)) { 7300 rc = EAGAIN; 7301 goto done; 7302 } 7303 7304 f = &sc->tids.ftid_tab[t->idx]; 7305 7306 if (f->pending) { 7307 rc = EBUSY; 7308 goto done; 7309 } 7310 if (f->locked) { 7311 rc = EPERM; 7312 goto done; 7313 } 7314 7315 if (f->valid) { 7316 t->fs = f->fs; /* extra info for the caller */ 7317 rc = del_filter_wr(sc, t->idx); 7318 } 7319 7320done: 7321 end_synchronized_op(sc, 0); 7322 7323 if (rc == 0) { 7324 mtx_lock(&sc->tids.ftid_lock); 7325 for (;;) { 7326 if (f->pending == 0) { 7327 rc = f->valid ? EIO : 0; 7328 break; 7329 } 7330 7331 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7332 PCATCH, "t4delfw", 0)) { 7333 rc = EINPROGRESS; 7334 break; 7335 } 7336 } 7337 mtx_unlock(&sc->tids.ftid_lock); 7338 } 7339 7340 return (rc); 7341} 7342 7343static void 7344clear_filter(struct filter_entry *f) 7345{ 7346 if (f->l2t) 7347 t4_l2t_release(f->l2t); 7348 7349 bzero(f, sizeof (*f)); 7350} 7351 7352static int 7353set_filter_wr(struct adapter *sc, int fidx) 7354{ 7355 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7356 struct fw_filter_wr *fwr; 7357 unsigned int ftid; 7358 struct wrq_cookie cookie; 7359 7360 ASSERT_SYNCHRONIZED_OP(sc); 7361 7362 if (f->fs.newdmac || f->fs.newvlan) { 7363 /* This filter needs an L2T entry; allocate one. */ 7364 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7365 if (f->l2t == NULL) 7366 return (EAGAIN); 7367 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7368 f->fs.dmac)) { 7369 t4_l2t_release(f->l2t); 7370 f->l2t = NULL; 7371 return (ENOMEM); 7372 } 7373 } 7374 7375 ftid = sc->tids.ftid_base + fidx; 7376 7377 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7378 if (fwr == NULL) 7379 return (ENOMEM); 7380 bzero(fwr, sizeof(*fwr)); 7381 7382 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7383 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7384 fwr->tid_to_iq = 7385 htobe32(V_FW_FILTER_WR_TID(ftid) | 7386 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7387 V_FW_FILTER_WR_NOREPLY(0) | 7388 V_FW_FILTER_WR_IQ(f->fs.iq)); 7389 fwr->del_filter_to_l2tix = 7390 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7391 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7392 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7393 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7394 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7395 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7396 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7397 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7398 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7399 f->fs.newvlan == VLAN_REWRITE) | 7400 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7401 f->fs.newvlan == VLAN_REWRITE) | 7402 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7403 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7404 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7405 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7406 fwr->ethtype = htobe16(f->fs.val.ethtype); 7407 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7408 fwr->frag_to_ovlan_vldm = 7409 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7410 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7411 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7412 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7413 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7414 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7415 fwr->smac_sel = 0; 7416 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7417 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7418 fwr->maci_to_matchtypem = 7419 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7420 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7421 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7422 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7423 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7424 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7425 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7426 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7427 fwr->ptcl = f->fs.val.proto; 7428 fwr->ptclm = f->fs.mask.proto; 7429 fwr->ttyp = f->fs.val.tos; 7430 fwr->ttypm = f->fs.mask.tos; 7431 fwr->ivlan = htobe16(f->fs.val.vlan); 7432 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7433 fwr->ovlan = htobe16(f->fs.val.vnic); 7434 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7435 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7436 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7437 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7438 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7439 fwr->lp = htobe16(f->fs.val.dport); 7440 fwr->lpm = htobe16(f->fs.mask.dport); 7441 fwr->fp = htobe16(f->fs.val.sport); 7442 fwr->fpm = htobe16(f->fs.mask.sport); 7443 if (f->fs.newsmac) 7444 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7445 7446 f->pending = 1; 7447 sc->tids.ftids_in_use++; 7448 7449 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7450 return (0); 7451} 7452 7453static int 7454del_filter_wr(struct adapter *sc, int fidx) 7455{ 7456 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7457 struct fw_filter_wr *fwr; 7458 unsigned int ftid; 7459 struct wrq_cookie cookie; 7460 7461 ftid = sc->tids.ftid_base + fidx; 7462 7463 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7464 if (fwr == NULL) 7465 return (ENOMEM); 7466 bzero(fwr, sizeof (*fwr)); 7467 7468 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7469 7470 f->pending = 1; 7471 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7472 return (0); 7473} 7474 7475int 7476t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7477{ 7478 struct adapter *sc = iq->adapter; 7479 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7480 unsigned int idx = GET_TID(rpl); 7481 unsigned int rc; 7482 struct filter_entry *f; 7483 7484 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7485 rss->opcode)); 7486 7487 if (is_ftid(sc, idx)) { 7488 7489 idx -= sc->tids.ftid_base; 7490 f = &sc->tids.ftid_tab[idx]; 7491 rc = G_COOKIE(rpl->cookie); 7492 7493 mtx_lock(&sc->tids.ftid_lock); 7494 if (rc == FW_FILTER_WR_FLT_ADDED) { 7495 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7496 __func__, idx)); 7497 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7498 f->pending = 0; /* asynchronous setup completed */ 7499 f->valid = 1; 7500 } else { 7501 if (rc != FW_FILTER_WR_FLT_DELETED) { 7502 /* Add or delete failed, display an error */ 7503 log(LOG_ERR, 7504 "filter %u setup failed with error %u\n", 7505 idx, rc); 7506 } 7507 7508 clear_filter(f); 7509 sc->tids.ftids_in_use--; 7510 } 7511 wakeup(&sc->tids.ftid_tab); 7512 mtx_unlock(&sc->tids.ftid_lock); 7513 } 7514 7515 return (0); 7516} 7517 7518static int 7519get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7520{ 7521 int rc; 7522 7523 if (cntxt->cid > M_CTXTQID) 7524 return (EINVAL); 7525 7526 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7527 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7528 return (EINVAL); 7529 7530 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7531 if (rc) 7532 return (rc); 7533 7534 if (sc->flags & FW_OK) { 7535 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7536 &cntxt->data[0]); 7537 if (rc == 0) 7538 goto done; 7539 } 7540 7541 /* 7542 * Read via firmware failed or wasn't even attempted. Read directly via 7543 * the backdoor. 7544 */ 7545 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7546done: 7547 end_synchronized_op(sc, 0); 7548 return (rc); 7549} 7550 7551static int 7552load_fw(struct adapter *sc, struct t4_data *fw) 7553{ 7554 int rc; 7555 uint8_t *fw_data; 7556 7557 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7558 if (rc) 7559 return (rc); 7560 7561 if (sc->flags & FULL_INIT_DONE) { 7562 rc = EBUSY; 7563 goto done; 7564 } 7565 7566 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7567 if (fw_data == NULL) { 7568 rc = ENOMEM; 7569 goto done; 7570 } 7571 7572 rc = copyin(fw->data, fw_data, fw->len); 7573 if (rc == 0) 7574 rc = -t4_load_fw(sc, fw_data, fw->len); 7575 7576 free(fw_data, M_CXGBE); 7577done: 7578 end_synchronized_op(sc, 0); 7579 return (rc); 7580} 7581 7582static int 7583read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7584{ 7585 uint32_t addr, off, remaining, i, n; 7586 uint32_t *buf, *b; 7587 uint32_t mw_base, mw_aperture; 7588 int rc; 7589 uint8_t *dst; 7590 7591 rc = validate_mem_range(sc, mr->addr, mr->len); 7592 if (rc != 0) 7593 return (rc); 7594 7595 memwin_info(sc, win, &mw_base, &mw_aperture); 7596 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7597 addr = mr->addr; 7598 remaining = mr->len; 7599 dst = (void *)mr->data; 7600 7601 while (remaining) { 7602 off = position_memwin(sc, win, addr); 7603 7604 /* number of bytes that we'll copy in the inner loop */ 7605 n = min(remaining, mw_aperture - off); 7606 for (i = 0; i < n; i += 4) 7607 *b++ = t4_read_reg(sc, mw_base + off + i); 7608 7609 rc = copyout(buf, dst, n); 7610 if (rc != 0) 7611 break; 7612 7613 b = buf; 7614 dst += n; 7615 remaining -= n; 7616 addr += n; 7617 } 7618 7619 free(buf, M_CXGBE); 7620 return (rc); 7621} 7622 7623static int 7624read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7625{ 7626 int rc; 7627 7628 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7629 return (EINVAL); 7630 7631 if (i2cd->len > sizeof(i2cd->data)) 7632 return (EFBIG); 7633 7634 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7635 if (rc) 7636 return (rc); 7637 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7638 i2cd->offset, i2cd->len, &i2cd->data[0]); 7639 end_synchronized_op(sc, 0); 7640 7641 return (rc); 7642} 7643 7644static int 7645in_range(int val, int lo, int hi) 7646{ 7647 7648 return (val < 0 || (val <= hi && val >= lo)); 7649} 7650 7651static int 7652set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7653{ 7654 int fw_subcmd, fw_type, rc; 7655 7656 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7657 if (rc) 7658 return (rc); 7659 7660 if (!(sc->flags & FULL_INIT_DONE)) { 7661 rc = EAGAIN; 7662 goto done; 7663 } 7664 7665 /* 7666 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7667 * sub-command and type are in common locations.) 7668 */ 7669 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7670 fw_subcmd = FW_SCHED_SC_CONFIG; 7671 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7672 fw_subcmd = FW_SCHED_SC_PARAMS; 7673 else { 7674 rc = EINVAL; 7675 goto done; 7676 } 7677 if (p->type == SCHED_CLASS_TYPE_PACKET) 7678 fw_type = FW_SCHED_TYPE_PKTSCHED; 7679 else { 7680 rc = EINVAL; 7681 goto done; 7682 } 7683 7684 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7685 /* Vet our parameters ..*/ 7686 if (p->u.config.minmax < 0) { 7687 rc = EINVAL; 7688 goto done; 7689 } 7690 7691 /* And pass the request to the firmware ...*/ 7692 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7693 goto done; 7694 } 7695 7696 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7697 int fw_level; 7698 int fw_mode; 7699 int fw_rateunit; 7700 int fw_ratemode; 7701 7702 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7703 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7704 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7705 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7706 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7707 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7708 else { 7709 rc = EINVAL; 7710 goto done; 7711 } 7712 7713 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7714 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7715 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7716 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7717 else { 7718 rc = EINVAL; 7719 goto done; 7720 } 7721 7722 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7723 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7724 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7725 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7726 else { 7727 rc = EINVAL; 7728 goto done; 7729 } 7730 7731 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7732 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7733 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7734 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7735 else { 7736 rc = EINVAL; 7737 goto done; 7738 } 7739 7740 /* Vet our parameters ... */ 7741 if (!in_range(p->u.params.channel, 0, 3) || 7742 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7743 !in_range(p->u.params.minrate, 0, 10000000) || 7744 !in_range(p->u.params.maxrate, 0, 10000000) || 7745 !in_range(p->u.params.weight, 0, 100)) { 7746 rc = ERANGE; 7747 goto done; 7748 } 7749 7750 /* 7751 * Translate any unset parameters into the firmware's 7752 * nomenclature and/or fail the call if the parameters 7753 * are required ... 7754 */ 7755 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7756 p->u.params.channel < 0 || p->u.params.cl < 0) { 7757 rc = EINVAL; 7758 goto done; 7759 } 7760 if (p->u.params.minrate < 0) 7761 p->u.params.minrate = 0; 7762 if (p->u.params.maxrate < 0) { 7763 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7764 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7765 rc = EINVAL; 7766 goto done; 7767 } else 7768 p->u.params.maxrate = 0; 7769 } 7770 if (p->u.params.weight < 0) { 7771 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7772 rc = EINVAL; 7773 goto done; 7774 } else 7775 p->u.params.weight = 0; 7776 } 7777 if (p->u.params.pktsize < 0) { 7778 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7779 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7780 rc = EINVAL; 7781 goto done; 7782 } else 7783 p->u.params.pktsize = 0; 7784 } 7785 7786 /* See what the firmware thinks of the request ... */ 7787 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7788 fw_rateunit, fw_ratemode, p->u.params.channel, 7789 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7790 p->u.params.weight, p->u.params.pktsize, 1); 7791 goto done; 7792 } 7793 7794 rc = EINVAL; 7795done: 7796 end_synchronized_op(sc, 0); 7797 return (rc); 7798} 7799 7800static int 7801set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7802{ 7803 struct port_info *pi = NULL; 7804 struct sge_txq *txq; 7805 uint32_t fw_mnem, fw_queue, fw_class; 7806 int i, rc; 7807 7808 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7809 if (rc) 7810 return (rc); 7811 7812 if (!(sc->flags & FULL_INIT_DONE)) { 7813 rc = EAGAIN; 7814 goto done; 7815 } 7816 7817 if (p->port >= sc->params.nports) { 7818 rc = EINVAL; 7819 goto done; 7820 } 7821 7822 pi = sc->port[p->port]; 7823 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7824 rc = EINVAL; 7825 goto done; 7826 } 7827 7828 /* 7829 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7830 * Scheduling Class in this case). 7831 */ 7832 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7833 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7834 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7835 7836 /* 7837 * If op.queue is non-negative, then we're only changing the scheduling 7838 * on a single specified TX queue. 7839 */ 7840 if (p->queue >= 0) { 7841 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7842 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7843 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7844 &fw_class); 7845 goto done; 7846 } 7847 7848 /* 7849 * Change the scheduling on all the TX queues for the 7850 * interface. 7851 */ 7852 for_each_txq(pi, i, txq) { 7853 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7854 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7855 &fw_class); 7856 if (rc) 7857 goto done; 7858 } 7859 7860 rc = 0; 7861done: 7862 end_synchronized_op(sc, 0); 7863 return (rc); 7864} 7865 7866int 7867t4_os_find_pci_capability(struct adapter *sc, int cap) 7868{ 7869 int i; 7870 7871 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7872} 7873 7874int 7875t4_os_pci_save_state(struct adapter *sc) 7876{ 7877 device_t dev; 7878 struct pci_devinfo *dinfo; 7879 7880 dev = sc->dev; 7881 dinfo = device_get_ivars(dev); 7882 7883 pci_cfg_save(dev, dinfo, 0); 7884 return (0); 7885} 7886 7887int 7888t4_os_pci_restore_state(struct adapter *sc) 7889{ 7890 device_t dev; 7891 struct pci_devinfo *dinfo; 7892 7893 dev = sc->dev; 7894 dinfo = device_get_ivars(dev); 7895 7896 pci_cfg_restore(dev, dinfo); 7897 return (0); 7898} 7899 7900void 7901t4_os_portmod_changed(const struct adapter *sc, int idx) 7902{ 7903 struct port_info *pi = sc->port[idx]; 7904 static const char *mod_str[] = { 7905 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7906 }; 7907 7908 build_medialist(pi, &pi->media); 7909#ifdef DEV_NETMAP 7910 build_medialist(pi, &pi->nm_media); 7911#endif 7912 7913 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7914 if_printf(pi->ifp, "transceiver unplugged.\n"); 7915 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7916 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7917 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7918 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7919 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7920 if_printf(pi->ifp, "%s transceiver inserted.\n", 7921 mod_str[pi->mod_type]); 7922 } else { 7923 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7924 pi->mod_type); 7925 } 7926} 7927 7928void 7929t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7930{ 7931 struct port_info *pi = sc->port[idx]; 7932 struct ifnet *ifp = pi->ifp; 7933 7934 if (link_stat) { 7935 pi->linkdnrc = -1; 7936 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7937 if_link_state_change(ifp, LINK_STATE_UP); 7938 } else { 7939 if (reason >= 0) 7940 pi->linkdnrc = reason; 7941 if_link_state_change(ifp, LINK_STATE_DOWN); 7942 } 7943} 7944 7945void 7946t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7947{ 7948 struct adapter *sc; 7949 7950 sx_slock(&t4_list_lock); 7951 SLIST_FOREACH(sc, &t4_list, link) { 7952 /* 7953 * func should not make any assumptions about what state sc is 7954 * in - the only guarantee is that sc->sc_lock is a valid lock. 7955 */ 7956 func(sc, arg); 7957 } 7958 sx_sunlock(&t4_list_lock); 7959} 7960 7961static int 7962t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7963{ 7964 return (0); 7965} 7966 7967static int 7968t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7969{ 7970 return (0); 7971} 7972 7973static int 7974t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7975 struct thread *td) 7976{ 7977 int rc; 7978 struct adapter *sc = dev->si_drv1; 7979 7980 rc = priv_check(td, PRIV_DRIVER); 7981 if (rc != 0) 7982 return (rc); 7983 7984 switch (cmd) { 7985 case CHELSIO_T4_GETREG: { 7986 struct t4_reg *edata = (struct t4_reg *)data; 7987 7988 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7989 return (EFAULT); 7990 7991 if (edata->size == 4) 7992 edata->val = t4_read_reg(sc, edata->addr); 7993 else if (edata->size == 8) 7994 edata->val = t4_read_reg64(sc, edata->addr); 7995 else 7996 return (EINVAL); 7997 7998 break; 7999 } 8000 case CHELSIO_T4_SETREG: { 8001 struct t4_reg *edata = (struct t4_reg *)data; 8002 8003 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8004 return (EFAULT); 8005 8006 if (edata->size == 4) { 8007 if (edata->val & 0xffffffff00000000) 8008 return (EINVAL); 8009 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8010 } else if (edata->size == 8) 8011 t4_write_reg64(sc, edata->addr, edata->val); 8012 else 8013 return (EINVAL); 8014 break; 8015 } 8016 case CHELSIO_T4_REGDUMP: { 8017 struct t4_regdump *regs = (struct t4_regdump *)data; 8018 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8019 uint8_t *buf; 8020 8021 if (regs->len < reglen) { 8022 regs->len = reglen; /* hint to the caller */ 8023 return (ENOBUFS); 8024 } 8025 8026 regs->len = reglen; 8027 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8028 t4_get_regs(sc, regs, buf); 8029 rc = copyout(buf, regs->data, reglen); 8030 free(buf, M_CXGBE); 8031 break; 8032 } 8033 case CHELSIO_T4_GET_FILTER_MODE: 8034 rc = get_filter_mode(sc, (uint32_t *)data); 8035 break; 8036 case CHELSIO_T4_SET_FILTER_MODE: 8037 rc = set_filter_mode(sc, *(uint32_t *)data); 8038 break; 8039 case CHELSIO_T4_GET_FILTER: 8040 rc = get_filter(sc, (struct t4_filter *)data); 8041 break; 8042 case CHELSIO_T4_SET_FILTER: 8043 rc = set_filter(sc, (struct t4_filter *)data); 8044 break; 8045 case CHELSIO_T4_DEL_FILTER: 8046 rc = del_filter(sc, (struct t4_filter *)data); 8047 break; 8048 case CHELSIO_T4_GET_SGE_CONTEXT: 8049 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8050 break; 8051 case CHELSIO_T4_LOAD_FW: 8052 rc = load_fw(sc, (struct t4_data *)data); 8053 break; 8054 case CHELSIO_T4_GET_MEM: 8055 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8056 break; 8057 case CHELSIO_T4_GET_I2C: 8058 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8059 break; 8060 case CHELSIO_T4_CLEAR_STATS: { 8061 int i; 8062 u_int port_id = *(uint32_t *)data; 8063 struct port_info *pi; 8064 8065 if (port_id >= sc->params.nports) 8066 return (EINVAL); 8067 pi = sc->port[port_id]; 8068 8069 /* MAC stats */ 8070 t4_clr_port_stats(sc, pi->tx_chan); 8071 pi->tx_parse_error = 0; 8072 8073 if (pi->flags & PORT_INIT_DONE) { 8074 struct sge_rxq *rxq; 8075 struct sge_txq *txq; 8076 struct sge_wrq *wrq; 8077 8078 for_each_rxq(pi, i, rxq) { 8079#if defined(INET) || defined(INET6) 8080 rxq->lro.lro_queued = 0; 8081 rxq->lro.lro_flushed = 0; 8082#endif 8083 rxq->rxcsum = 0; 8084 rxq->vlan_extraction = 0; 8085 } 8086 8087 for_each_txq(pi, i, txq) { 8088 txq->txcsum = 0; 8089 txq->tso_wrs = 0; 8090 txq->vlan_insertion = 0; 8091 txq->imm_wrs = 0; 8092 txq->sgl_wrs = 0; 8093 txq->txpkt_wrs = 0; 8094 txq->txpkts0_wrs = 0; 8095 txq->txpkts1_wrs = 0; 8096 txq->txpkts0_pkts = 0; 8097 txq->txpkts1_pkts = 0; 8098 mp_ring_reset_stats(txq->r); 8099 } 8100 8101#ifdef TCP_OFFLOAD 8102 /* nothing to clear for each ofld_rxq */ 8103 8104 for_each_ofld_txq(pi, i, wrq) { 8105 wrq->tx_wrs_direct = 0; 8106 wrq->tx_wrs_copied = 0; 8107 } 8108#endif 8109 wrq = &sc->sge.ctrlq[pi->port_id]; 8110 wrq->tx_wrs_direct = 0; 8111 wrq->tx_wrs_copied = 0; 8112 } 8113 break; 8114 } 8115 case CHELSIO_T4_SCHED_CLASS: 8116 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8117 break; 8118 case CHELSIO_T4_SCHED_QUEUE: 8119 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8120 break; 8121 case CHELSIO_T4_GET_TRACER: 8122 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8123 break; 8124 case CHELSIO_T4_SET_TRACER: 8125 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8126 break; 8127 default: 8128 rc = EINVAL; 8129 } 8130 8131 return (rc); 8132} 8133 8134#ifdef TCP_OFFLOAD 8135void 8136t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8137 const unsigned int *pgsz_order) 8138{ 8139 struct port_info *pi = ifp->if_softc; 8140 struct adapter *sc = pi->adapter; 8141 8142 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8143 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8144 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8145 V_HPZ3(pgsz_order[3])); 8146} 8147 8148static int 8149toe_capability(struct port_info *pi, int enable) 8150{ 8151 int rc; 8152 struct adapter *sc = pi->adapter; 8153 8154 ASSERT_SYNCHRONIZED_OP(sc); 8155 8156 if (!is_offload(sc)) 8157 return (ENODEV); 8158 8159 if (enable) { 8160 /* 8161 * We need the port's queues around so that we're able to send 8162 * and receive CPLs to/from the TOE even if the ifnet for this 8163 * port has never been UP'd administratively. 8164 */ 8165 if (!(pi->flags & PORT_INIT_DONE)) { 8166 rc = cxgbe_init_synchronized(pi); 8167 if (rc) 8168 return (rc); 8169 } 8170 8171 if (isset(&sc->offload_map, pi->port_id)) 8172 return (0); 8173 8174 if (!uld_active(sc, ULD_TOM)) { 8175 rc = t4_activate_uld(sc, ULD_TOM); 8176 if (rc == EAGAIN) { 8177 log(LOG_WARNING, 8178 "You must kldload t4_tom.ko before trying " 8179 "to enable TOE on a cxgbe interface.\n"); 8180 } 8181 if (rc != 0) 8182 return (rc); 8183 KASSERT(sc->tom_softc != NULL, 8184 ("%s: TOM activated but softc NULL", __func__)); 8185 KASSERT(uld_active(sc, ULD_TOM), 8186 ("%s: TOM activated but flag not set", __func__)); 8187 } 8188 8189 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8190 if (!uld_active(sc, ULD_IWARP)) 8191 (void) t4_activate_uld(sc, ULD_IWARP); 8192 if (!uld_active(sc, ULD_ISCSI)) 8193 (void) t4_activate_uld(sc, ULD_ISCSI); 8194 8195 setbit(&sc->offload_map, pi->port_id); 8196 } else { 8197 if (!isset(&sc->offload_map, pi->port_id)) 8198 return (0); 8199 8200 KASSERT(uld_active(sc, ULD_TOM), 8201 ("%s: TOM never initialized?", __func__)); 8202 clrbit(&sc->offload_map, pi->port_id); 8203 } 8204 8205 return (0); 8206} 8207 8208/* 8209 * Add an upper layer driver to the global list. 8210 */ 8211int 8212t4_register_uld(struct uld_info *ui) 8213{ 8214 int rc = 0; 8215 struct uld_info *u; 8216 8217 sx_xlock(&t4_uld_list_lock); 8218 SLIST_FOREACH(u, &t4_uld_list, link) { 8219 if (u->uld_id == ui->uld_id) { 8220 rc = EEXIST; 8221 goto done; 8222 } 8223 } 8224 8225 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8226 ui->refcount = 0; 8227done: 8228 sx_xunlock(&t4_uld_list_lock); 8229 return (rc); 8230} 8231 8232int 8233t4_unregister_uld(struct uld_info *ui) 8234{ 8235 int rc = EINVAL; 8236 struct uld_info *u; 8237 8238 sx_xlock(&t4_uld_list_lock); 8239 8240 SLIST_FOREACH(u, &t4_uld_list, link) { 8241 if (u == ui) { 8242 if (ui->refcount > 0) { 8243 rc = EBUSY; 8244 goto done; 8245 } 8246 8247 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8248 rc = 0; 8249 goto done; 8250 } 8251 } 8252done: 8253 sx_xunlock(&t4_uld_list_lock); 8254 return (rc); 8255} 8256 8257int 8258t4_activate_uld(struct adapter *sc, int id) 8259{ 8260 int rc; 8261 struct uld_info *ui; 8262 8263 ASSERT_SYNCHRONIZED_OP(sc); 8264 8265 if (id < 0 || id > ULD_MAX) 8266 return (EINVAL); 8267 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8268 8269 sx_slock(&t4_uld_list_lock); 8270 8271 SLIST_FOREACH(ui, &t4_uld_list, link) { 8272 if (ui->uld_id == id) { 8273 if (!(sc->flags & FULL_INIT_DONE)) { 8274 rc = adapter_full_init(sc); 8275 if (rc != 0) 8276 break; 8277 } 8278 8279 rc = ui->activate(sc); 8280 if (rc == 0) { 8281 setbit(&sc->active_ulds, id); 8282 ui->refcount++; 8283 } 8284 break; 8285 } 8286 } 8287 8288 sx_sunlock(&t4_uld_list_lock); 8289 8290 return (rc); 8291} 8292 8293int 8294t4_deactivate_uld(struct adapter *sc, int id) 8295{ 8296 int rc; 8297 struct uld_info *ui; 8298 8299 ASSERT_SYNCHRONIZED_OP(sc); 8300 8301 if (id < 0 || id > ULD_MAX) 8302 return (EINVAL); 8303 rc = ENXIO; 8304 8305 sx_slock(&t4_uld_list_lock); 8306 8307 SLIST_FOREACH(ui, &t4_uld_list, link) { 8308 if (ui->uld_id == id) { 8309 rc = ui->deactivate(sc); 8310 if (rc == 0) { 8311 clrbit(&sc->active_ulds, id); 8312 ui->refcount--; 8313 } 8314 break; 8315 } 8316 } 8317 8318 sx_sunlock(&t4_uld_list_lock); 8319 8320 return (rc); 8321} 8322 8323int 8324uld_active(struct adapter *sc, int uld_id) 8325{ 8326 8327 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8328 8329 return (isset(&sc->active_ulds, uld_id)); 8330} 8331#endif 8332 8333/* 8334 * Come up with reasonable defaults for some of the tunables, provided they're 8335 * not set by the user (in which case we'll use the values as is). 8336 */ 8337static void 8338tweak_tunables(void) 8339{ 8340 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8341 8342 if (t4_ntxq10g < 1) 8343 t4_ntxq10g = min(nc, NTXQ_10G); 8344 8345 if (t4_ntxq1g < 1) 8346 t4_ntxq1g = min(nc, NTXQ_1G); 8347 8348 if (t4_nrxq10g < 1) 8349 t4_nrxq10g = min(nc, NRXQ_10G); 8350 8351 if (t4_nrxq1g < 1) 8352 t4_nrxq1g = min(nc, NRXQ_1G); 8353 8354#ifdef TCP_OFFLOAD 8355 if (t4_nofldtxq10g < 1) 8356 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8357 8358 if (t4_nofldtxq1g < 1) 8359 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8360 8361 if (t4_nofldrxq10g < 1) 8362 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8363 8364 if (t4_nofldrxq1g < 1) 8365 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8366 8367 if (t4_toecaps_allowed == -1) 8368 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8369#else 8370 if (t4_toecaps_allowed == -1) 8371 t4_toecaps_allowed = 0; 8372#endif 8373 8374#ifdef DEV_NETMAP 8375 if (t4_nnmtxq10g < 1) 8376 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8377 8378 if (t4_nnmtxq1g < 1) 8379 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8380 8381 if (t4_nnmrxq10g < 1) 8382 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8383 8384 if (t4_nnmrxq1g < 1) 8385 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8386#endif 8387 8388 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8389 t4_tmr_idx_10g = TMR_IDX_10G; 8390 8391 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8392 t4_pktc_idx_10g = PKTC_IDX_10G; 8393 8394 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8395 t4_tmr_idx_1g = TMR_IDX_1G; 8396 8397 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8398 t4_pktc_idx_1g = PKTC_IDX_1G; 8399 8400 if (t4_qsize_txq < 128) 8401 t4_qsize_txq = 128; 8402 8403 if (t4_qsize_rxq < 128) 8404 t4_qsize_rxq = 128; 8405 while (t4_qsize_rxq & 7) 8406 t4_qsize_rxq++; 8407 8408 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8409} 8410 8411static struct sx mlu; /* mod load unload */ 8412SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8413 8414static int 8415mod_event(module_t mod, int cmd, void *arg) 8416{ 8417 int rc = 0; 8418 static int loaded = 0; 8419 8420 switch (cmd) { 8421 case MOD_LOAD: 8422 sx_xlock(&mlu); 8423 if (loaded++ == 0) { 8424 t4_sge_modload(); 8425 sx_init(&t4_list_lock, "T4/T5 adapters"); 8426 SLIST_INIT(&t4_list); 8427#ifdef TCP_OFFLOAD 8428 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8429 SLIST_INIT(&t4_uld_list); 8430#endif 8431 t4_tracer_modload(); 8432 tweak_tunables(); 8433 } 8434 sx_xunlock(&mlu); 8435 break; 8436 8437 case MOD_UNLOAD: 8438 sx_xlock(&mlu); 8439 if (--loaded == 0) { 8440 int tries; 8441 8442 sx_slock(&t4_list_lock); 8443 if (!SLIST_EMPTY(&t4_list)) { 8444 rc = EBUSY; 8445 sx_sunlock(&t4_list_lock); 8446 goto done_unload; 8447 } 8448#ifdef TCP_OFFLOAD 8449 sx_slock(&t4_uld_list_lock); 8450 if (!SLIST_EMPTY(&t4_uld_list)) { 8451 rc = EBUSY; 8452 sx_sunlock(&t4_uld_list_lock); 8453 sx_sunlock(&t4_list_lock); 8454 goto done_unload; 8455 } 8456#endif 8457 tries = 0; 8458 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8459 uprintf("%ju clusters with custom free routine " 8460 "still is use.\n", t4_sge_extfree_refs()); 8461 pause("t4unload", 2 * hz); 8462 } 8463#ifdef TCP_OFFLOAD 8464 sx_sunlock(&t4_uld_list_lock); 8465#endif 8466 sx_sunlock(&t4_list_lock); 8467 8468 if (t4_sge_extfree_refs() == 0) { 8469 t4_tracer_modunload(); 8470#ifdef TCP_OFFLOAD 8471 sx_destroy(&t4_uld_list_lock); 8472#endif 8473 sx_destroy(&t4_list_lock); 8474 t4_sge_modunload(); 8475 loaded = 0; 8476 } else { 8477 rc = EBUSY; 8478 loaded++; /* undo earlier decrement */ 8479 } 8480 } 8481done_unload: 8482 sx_xunlock(&mlu); 8483 break; 8484 } 8485 8486 return (rc); 8487} 8488 8489static devclass_t t4_devclass, t5_devclass; 8490static devclass_t cxgbe_devclass, cxl_devclass; 8491 8492DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8493MODULE_VERSION(t4nex, 1); 8494MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8495 8496DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8497MODULE_VERSION(t5nex, 1); 8498MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8499 8500DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8501MODULE_VERSION(cxgbe, 1); 8502 8503DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8504MODULE_VERSION(cxl, 1); 8505